84 #define tun_debug(level, tun, fmt, args...)			\  argument
86 	if (tun->debug)						\
87 		netdev_printk(level, tun->dev, fmt, ##args);	\
95 #define tun_debug(level, tun, fmt, args...)			\  argument
98 		netdev_printk(level, tun->dev, fmt, ##args);	\
163 	struct tun_struct __rcu *tun;  member
189 	struct tun_struct *tun;  member
310 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,  in tun_napi_init()  argument
316 		netif_napi_add(tun->dev, &tfile->napi, tun_napi_poll,  in tun_napi_init()
340 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)  in tun_legacy_is_little_endian()  argument
342 	return tun->flags & TUN_VNET_BE ? false :  in tun_legacy_is_little_endian()
346 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)  in tun_get_vnet_be()  argument
348 	int be = !!(tun->flags & TUN_VNET_BE);  in tun_get_vnet_be()
356 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)  in tun_set_vnet_be()  argument
364 		tun->flags |= TUN_VNET_BE;  in tun_set_vnet_be()
366 		tun->flags &= ~TUN_VNET_BE;  in tun_set_vnet_be()
371 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)  in tun_legacy_is_little_endian()  argument
376 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)  in tun_get_vnet_be()  argument
381 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)  in tun_set_vnet_be()  argument
387 static inline bool tun_is_little_endian(struct tun_struct *tun)  in tun_is_little_endian()  argument
389 	return tun->flags & TUN_VNET_LE ||  in tun_is_little_endian()
390 		tun_legacy_is_little_endian(tun);  in tun_is_little_endian()
393 static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)  in tun16_to_cpu()  argument
395 	return __virtio16_to_cpu(tun_is_little_endian(tun), val);  in tun16_to_cpu()
398 static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)  in cpu_to_tun16()  argument
400 	return __cpu_to_virtio16(tun_is_little_endian(tun), val);  in cpu_to_tun16()
419 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,  in tun_flow_create()  argument
426 		tun_debug(KERN_INFO, tun, "create flow: hash %u index %u\n",  in tun_flow_create()
432 		e->tun = tun;  in tun_flow_create()
434 		++tun->flow_count;  in tun_flow_create()
439 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)  in tun_flow_delete()  argument
441 	tun_debug(KERN_INFO, tun, "delete flow: hash %u index %u\n",  in tun_flow_delete()
445 	--tun->flow_count;  in tun_flow_delete()
448 static void tun_flow_flush(struct tun_struct *tun)  in tun_flow_flush()  argument
452 	spin_lock_bh(&tun->lock);  in tun_flow_flush()
457 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)  in tun_flow_flush()
458 			tun_flow_delete(tun, e);  in tun_flow_flush()
460 	spin_unlock_bh(&tun->lock);  in tun_flow_flush()
463 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)  in tun_flow_delete_by_queue()  argument
467 	spin_lock_bh(&tun->lock);  in tun_flow_delete_by_queue()
472 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {  in tun_flow_delete_by_queue()
474 				tun_flow_delete(tun, e);  in tun_flow_delete_by_queue()
477 	spin_unlock_bh(&tun->lock);  in tun_flow_delete_by_queue()
482 	struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);  in tun_flow_cleanup()  local
483 	unsigned long delay = tun->ageing_time;  in tun_flow_cleanup()
488 	tun_debug(KERN_INFO, tun, "tun_flow_cleanup\n");  in tun_flow_cleanup()
490 	spin_lock(&tun->lock);  in tun_flow_cleanup()
495 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {  in tun_flow_cleanup()
500 				tun_flow_delete(tun, e);  in tun_flow_cleanup()
510 		mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));  in tun_flow_cleanup()
511 	spin_unlock(&tun->lock);  in tun_flow_cleanup()
514 static void tun_flow_update(struct tun_struct *tun, u32 rxhash,  in tun_flow_update()  argument
519 	unsigned long delay = tun->ageing_time;  in tun_flow_update()
522 	head = &tun->flows[tun_hashfn(rxhash)];  in tun_flow_update()
535 		spin_lock_bh(&tun->lock);  in tun_flow_update()
537 		    tun->flow_count < MAX_TAP_FLOWS)  in tun_flow_update()
538 			tun_flow_create(tun, head, rxhash, queue_index);  in tun_flow_update()
540 		if (!timer_pending(&tun->flow_gc_timer))  in tun_flow_update()
541 			mod_timer(&tun->flow_gc_timer,  in tun_flow_update()
543 		spin_unlock_bh(&tun->lock);  in tun_flow_update()
565 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)  in tun_automq_select_queue()  argument
571 	numqueues = READ_ONCE(tun->numqueues);  in tun_automq_select_queue()
574 	e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);  in tun_automq_select_queue()
586 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)  in tun_ebpf_select_queue()  argument
592 	numqueues = READ_ONCE(tun->numqueues);  in tun_ebpf_select_queue()
596 	prog = rcu_dereference(tun->steering_prog);  in tun_ebpf_select_queue()
606 	struct tun_struct *tun = netdev_priv(dev);  in tun_select_queue()  local
610 	if (rcu_dereference(tun->steering_prog))  in tun_select_queue()
611 		ret = tun_ebpf_select_queue(tun, skb);  in tun_select_queue()
613 		ret = tun_automq_select_queue(tun, skb);  in tun_select_queue()
619 static inline bool tun_not_capable(struct tun_struct *tun)  in tun_not_capable()  argument
622 	struct net *net = dev_net(tun->dev);  in tun_not_capable()
624 	return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||  in tun_not_capable()
625 		  (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&  in tun_not_capable()
629 static void tun_set_real_num_queues(struct tun_struct *tun)  in tun_set_real_num_queues()  argument
631 	netif_set_real_num_tx_queues(tun->dev, tun->numqueues);  in tun_set_real_num_queues()
632 	netif_set_real_num_rx_queues(tun->dev, tun->numqueues);  in tun_set_real_num_queues()
635 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)  in tun_disable_queue()  argument
637 	tfile->detached = tun;  in tun_disable_queue()
638 	list_add_tail(&tfile->next, &tun->disabled);  in tun_disable_queue()
639 	++tun->numdisabled;  in tun_disable_queue()
644 	struct tun_struct *tun = tfile->detached;  in tun_enable_queue()  local
648 	--tun->numdisabled;  in tun_enable_queue()
649 	return tun;  in tun_enable_queue()
680 	struct tun_struct *tun;  in __tun_detach()  local
682 	tun = rtnl_dereference(tfile->tun);  in __tun_detach()
684 	if (tun && clean) {  in __tun_detach()
689 	if (tun && !tfile->detached) {  in __tun_detach()
691 		BUG_ON(index >= tun->numqueues);  in __tun_detach()
693 		rcu_assign_pointer(tun->tfiles[index],  in __tun_detach()
694 				   tun->tfiles[tun->numqueues - 1]);  in __tun_detach()
695 		ntfile = rtnl_dereference(tun->tfiles[index]);  in __tun_detach()
697 		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],  in __tun_detach()
700 		--tun->numqueues;  in __tun_detach()
702 			RCU_INIT_POINTER(tfile->tun, NULL);  in __tun_detach()
705 			tun_disable_queue(tun, tfile);  in __tun_detach()
708 		tun_flow_delete_by_queue(tun, tun->numqueues + 1);  in __tun_detach()
711 		tun_set_real_num_queues(tun);  in __tun_detach()
713 		tun = tun_enable_queue(tfile);  in __tun_detach()
718 		if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {  in __tun_detach()
719 			netif_carrier_off(tun->dev);  in __tun_detach()
721 			if (!(tun->flags & IFF_PERSIST) &&  in __tun_detach()
722 			    tun->dev->reg_state == NETREG_REGISTERED)  in __tun_detach()
723 				unregister_netdevice(tun->dev);  in __tun_detach()
725 		if (tun)  in __tun_detach()
734 	struct tun_struct *tun;  in tun_detach()  local
738 	tun = rtnl_dereference(tfile->tun);  in tun_detach()
739 	dev = tun ? tun->dev : NULL;  in tun_detach()
748 	struct tun_struct *tun = netdev_priv(dev);  in tun_detach_all()  local
750 	int i, n = tun->numqueues;  in tun_detach_all()
753 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_all()
758 		RCU_INIT_POINTER(tfile->tun, NULL);  in tun_detach_all()
759 		--tun->numqueues;  in tun_detach_all()
761 	list_for_each_entry(tfile, &tun->disabled, next) {  in tun_detach_all()
764 		RCU_INIT_POINTER(tfile->tun, NULL);  in tun_detach_all()
766 	BUG_ON(tun->numqueues != 0);  in tun_detach_all()
770 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_all()
777 	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {  in tun_detach_all()
783 	BUG_ON(tun->numdisabled != 0);  in tun_detach_all()
785 	if (tun->flags & IFF_PERSIST)  in tun_detach_all()
789 static int tun_attach(struct tun_struct *tun, struct file *file,  in tun_attach()  argument
794 	struct net_device *dev = tun->dev;  in tun_attach()
797 	err = security_tun_dev_attach(tfile->socket.sk, tun->security);  in tun_attach()
802 	if (rtnl_dereference(tfile->tun) && !tfile->detached)  in tun_attach()
806 	if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)  in tun_attach()
811 	    tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)  in tun_attach()
817 	if (!skip_filter && (tun->filter_attached == true)) {  in tun_attach()
819 		err = sk_attach_filter(&tun->fprog, tfile->socket.sk);  in tun_attach()
832 	tfile->queue_index = tun->numqueues;  in tun_attach()
844 				       tun->dev, tfile->queue_index);  in tun_attach()
860 		tun_napi_init(tun, tfile, napi, napi_frags);  in tun_attach()
863 	if (rtnl_dereference(tun->xdp_prog))  in tun_attach()
875 		rcu_assign_pointer(tfile->tun, tun);  in tun_attach()
876 	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);  in tun_attach()
877 	tun->numqueues++;  in tun_attach()
878 	tun_set_real_num_queues(tun);  in tun_attach()
885 	struct tun_struct *tun;  in tun_get()  local
888 	tun = rcu_dereference(tfile->tun);  in tun_get()
889 	if (tun)  in tun_get()
890 		dev_hold(tun->dev);  in tun_get()
893 	return tun;  in tun_get()
896 static void tun_put(struct tun_struct *tun)  in tun_put()  argument
898 	dev_put(tun->dev);  in tun_put()
1031 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)  in tun_automq_xmit()  argument
1034 	if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {  in tun_automq_xmit()
1042 		e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);  in tun_automq_xmit()
1049 static unsigned int run_ebpf_filter(struct tun_struct *tun,  in run_ebpf_filter()  argument
1053 	struct tun_prog *prog = rcu_dereference(tun->filter_prog);  in run_ebpf_filter()
1064 	struct tun_struct *tun = netdev_priv(dev);  in tun_net_xmit()  local
1070 	tfile = rcu_dereference(tun->tfiles[txq]);  in tun_net_xmit()
1076 	if (!rcu_dereference(tun->steering_prog))  in tun_net_xmit()
1077 		tun_automq_xmit(tun, skb);  in tun_net_xmit()
1079 	tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len);  in tun_net_xmit()
1086 	if (!check_filter(&tun->txflt, skb))  in tun_net_xmit()
1093 	len = run_ebpf_filter(tun, skb, len);  in tun_net_xmit()
1121 	this_cpu_inc(tun->pcpu_stats->tx_dropped);  in tun_net_xmit()
1140 	struct tun_struct *tun = netdev_priv(dev);  in tun_net_fix_features()  local
1142 	return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);  in tun_net_fix_features()
1147 	struct tun_struct *tun = netdev_priv(dev);  in tun_set_headroom()  local
1152 	tun->align = new_hr;  in tun_set_headroom()
1159 	struct tun_struct *tun = netdev_priv(dev);  in tun_net_get_stats64()  local
1167 		p = per_cpu_ptr(tun->pcpu_stats, i);  in tun_net_get_stats64()
1194 	struct tun_struct *tun = netdev_priv(dev);  in tun_xdp_set()  local
1199 	old_prog = rtnl_dereference(tun->xdp_prog);  in tun_xdp_set()
1200 	rcu_assign_pointer(tun->xdp_prog, prog);  in tun_xdp_set()
1204 	for (i = 0; i < tun->numqueues; i++) {  in tun_xdp_set()
1205 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_xdp_set()
1211 	list_for_each_entry(tfile, &tun->disabled, next) {  in tun_xdp_set()
1223 	struct tun_struct *tun = netdev_priv(dev);  in tun_xdp_query()  local
1226 	xdp_prog = rtnl_dereference(tun->xdp_prog);  in tun_xdp_query()
1249 		struct tun_struct *tun = netdev_priv(dev);  in tun_net_change_carrier()  local
1251 		if (!tun->numqueues)  in tun_net_change_carrier()
1284 	struct tun_struct *tun = netdev_priv(dev);  in tun_xdp_xmit()  local
1297 	numqueues = READ_ONCE(tun->numqueues);  in tun_xdp_xmit()
1303 	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %  in tun_xdp_xmit()
1317 			this_cpu_inc(tun->pcpu_stats->tx_dropped);  in tun_xdp_xmit()
1359 static void tun_flow_init(struct tun_struct *tun)  in tun_flow_init()  argument
1364 		INIT_HLIST_HEAD(&tun->flows[i]);  in tun_flow_init()
1366 	tun->ageing_time = TUN_FLOW_EXPIRE;  in tun_flow_init()
1367 	timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);  in tun_flow_init()
1368 	mod_timer(&tun->flow_gc_timer,  in tun_flow_init()
1369 		  round_jiffies_up(jiffies + tun->ageing_time));  in tun_flow_init()
1372 static void tun_flow_uninit(struct tun_struct *tun)  in tun_flow_uninit()  argument
1374 	del_timer_sync(&tun->flow_gc_timer);  in tun_flow_uninit()
1375 	tun_flow_flush(tun);  in tun_flow_uninit()
1384 	struct tun_struct *tun = netdev_priv(dev);  in tun_net_init()  local
1386 	switch (tun->flags & TUN_TYPE_MASK) {  in tun_net_init()
1416 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)  in tun_sock_writeable()  argument
1420 	return (tun->dev->flags & IFF_UP) && sock_writeable(sk);  in tun_sock_writeable()
1429 	struct tun_struct *tun = tun_get(tfile);  in tun_chr_poll()  local
1433 	if (!tun)  in tun_chr_poll()
1438 	tun_debug(KERN_INFO, tun, "tun_chr_poll\n");  in tun_chr_poll()
1450 	if (tun_sock_writeable(tun, tfile) ||  in tun_chr_poll()
1452 	     tun_sock_writeable(tun, tfile)))  in tun_chr_poll()
1455 	if (tun->dev->reg_state != NETREG_REGISTERED)  in tun_chr_poll()
1458 	tun_put(tun);  in tun_chr_poll()
1542 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,  in tun_rx_batched()  argument
1547 	u32 rx_batched = tun->rx_batched;  in tun_rx_batched()
1582 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,  in tun_can_build_skb()  argument
1585 	if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)  in tun_can_build_skb()
1623 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,  in tun_xdp_act()  argument
1630 		err = xdp_do_redirect(tun->dev, xdp, xdp_prog);  in tun_xdp_act()
1635 		err = tun_xdp_tx(tun->dev, xdp);  in tun_xdp_act()
1645 		trace_xdp_exception(tun->dev, xdp_prog, act);  in tun_xdp_act()
1648 		this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_xdp_act()
1655 static struct sk_buff *tun_build_skb(struct tun_struct *tun,  in tun_build_skb()  argument
1670 	xdp_prog = rcu_dereference(tun->xdp_prog);  in tun_build_skb()
1701 	xdp_prog = rcu_dereference(tun->xdp_prog);  in tun_build_skb()
1717 		err = tun_xdp_act(tun, xdp_prog, &xdp, act);  in tun_build_skb()
1742 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,  in tun_get_user()  argument
1749 	size_t len = total_len, align = tun->align, linear;  in tun_get_user()
1760 	if (!(tun->flags & IFF_NO_PI)) {  in tun_get_user()
1769 	if (tun->flags & IFF_VNET_HDR) {  in tun_get_user()
1770 		int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);  in tun_get_user()
1780 …tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso…  in tun_get_user()
1781 …gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_off…  in tun_get_user()
1783 		if (tun16_to_cpu(tun, gso.hdr_len) > len)  in tun_get_user()
1788 	if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {  in tun_get_user()
1791 			     (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))  in tun_get_user()
1804 		copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;  in tun_get_user()
1813 	if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {  in tun_get_user()
1818 		skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);  in tun_get_user()
1820 			this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_get_user()
1828 			if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)  in tun_get_user()
1831 				linear = tun16_to_cpu(tun, gso.hdr_len);  in tun_get_user()
1849 				this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_get_user()
1863 			this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_get_user()
1874 	if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {  in tun_get_user()
1875 		this_cpu_inc(tun->pcpu_stats->rx_frame_errors);  in tun_get_user()
1885 	switch (tun->flags & TUN_TYPE_MASK) {  in tun_get_user()
1887 		if (tun->flags & IFF_NO_PI) {  in tun_get_user()
1898 				this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_get_user()
1906 		skb->dev = tun->dev;  in tun_get_user()
1910 			skb->protocol = eth_type_trans(skb, tun->dev);  in tun_get_user()
1933 		xdp_prog = rcu_dereference(tun->xdp_prog);  in tun_get_user()
1950 	if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&  in tun_get_user()
1955 	if (unlikely(!(tun->dev->flags & IFF_UP))) {  in tun_get_user()
1963 		u32 headlen = eth_get_headlen(tun->dev, skb->data,  in tun_get_user()
1967 			this_cpu_inc(tun->pcpu_stats->rx_dropped);  in tun_get_user()
1993 		tun_rx_batched(tun, tfile, skb, more);  in tun_get_user()
1999 	stats = get_cpu_ptr(tun->pcpu_stats);  in tun_get_user()
2007 		tun_flow_update(tun, rxhash, tfile);  in tun_get_user()
2016 	struct tun_struct *tun = tun_get(tfile);  in tun_chr_write_iter()  local
2019 	if (!tun)  in tun_chr_write_iter()
2022 	result = tun_get_user(tun, tfile, NULL, from,  in tun_chr_write_iter()
2025 	tun_put(tun);  in tun_chr_write_iter()
2029 static ssize_t tun_put_user_xdp(struct tun_struct *tun,  in tun_put_user_xdp()  argument
2039 	if (tun->flags & IFF_VNET_HDR) {  in tun_put_user_xdp()
2042 		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);  in tun_put_user_xdp()
2053 	stats = get_cpu_ptr(tun->pcpu_stats);  in tun_put_user_xdp()
2058 	put_cpu_ptr(tun->pcpu_stats);  in tun_put_user_xdp()
2064 static ssize_t tun_put_user(struct tun_struct *tun,  in tun_put_user()  argument
2079 	if (tun->flags & IFF_VNET_HDR)  in tun_put_user()
2080 		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);  in tun_put_user()
2084 	if (!(tun->flags & IFF_NO_PI)) {  in tun_put_user()
2105 					    tun_is_little_endian(tun), true,  in tun_put_user()
2110 			       sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),  in tun_put_user()
2111 			       tun16_to_cpu(tun, gso.hdr_len));  in tun_put_user()
2115 				       min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);  in tun_put_user()
2148 	stats = get_cpu_ptr(tun->pcpu_stats);  in tun_put_user()
2153 	put_cpu_ptr(tun->pcpu_stats);  in tun_put_user()
2199 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,  in tun_do_read()  argument
2206 	tun_debug(KERN_INFO, tun, "tun_do_read\n");  in tun_do_read()
2223 		ret = tun_put_user_xdp(tun, tfile, xdpf, to);  in tun_do_read()
2228 		ret = tun_put_user(tun, tfile, skb, to);  in tun_do_read()
2242 	struct tun_struct *tun = tun_get(tfile);  in tun_chr_read_iter()  local
2245 	if (!tun)  in tun_chr_read_iter()
2247 	ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK, NULL);  in tun_chr_read_iter()
2251 	tun_put(tun);  in tun_chr_read_iter()
2263 static int __tun_set_ebpf(struct tun_struct *tun,  in __tun_set_ebpf()  argument
2276 	spin_lock_bh(&tun->lock);  in __tun_set_ebpf()
2278 					lockdep_is_held(&tun->lock));  in __tun_set_ebpf()
2280 	spin_unlock_bh(&tun->lock);  in __tun_set_ebpf()
2290 	struct tun_struct *tun = netdev_priv(dev);  in tun_free_netdev()  local
2292 	BUG_ON(!(list_empty(&tun->disabled)));  in tun_free_netdev()
2293 	free_percpu(tun->pcpu_stats);  in tun_free_netdev()
2294 	tun_flow_uninit(tun);  in tun_free_netdev()
2295 	security_tun_dev_free_security(tun->security);  in tun_free_netdev()
2296 	__tun_set_ebpf(tun, &tun->steering_prog, NULL);  in tun_free_netdev()
2297 	__tun_set_ebpf(tun, &tun->filter_prog, NULL);  in tun_free_netdev()
2302 	struct tun_struct *tun = netdev_priv(dev);  in tun_setup()  local
2304 	tun->owner = INVALID_UID;  in tun_setup()
2305 	tun->group = INVALID_GID;  in tun_setup()
2306 	tun_default_link_ksettings(dev, &tun->link_ksettings);  in tun_setup()
2345 	struct tun_struct *tun = netdev_priv(dev);  in tun_fill_info()  local
2347 	if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))  in tun_fill_info()
2349 	if (uid_valid(tun->owner) &&  in tun_fill_info()
2351 			from_kuid_munged(current_user_ns(), tun->owner)))  in tun_fill_info()
2353 	if (gid_valid(tun->group) &&  in tun_fill_info()
2355 			from_kgid_munged(current_user_ns(), tun->group)))  in tun_fill_info()
2357 	if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))  in tun_fill_info()
2359 	if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))  in tun_fill_info()
2361 	if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))  in tun_fill_info()
2364 		       !!(tun->flags & IFF_MULTI_QUEUE)))  in tun_fill_info()
2366 	if (tun->flags & IFF_MULTI_QUEUE) {  in tun_fill_info()
2367 		if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))  in tun_fill_info()
2370 				tun->numdisabled))  in tun_fill_info()
2415 static int tun_xdp_one(struct tun_struct *tun,  in tun_xdp_one()  argument
2432 	xdp_prog = rcu_dereference(tun->xdp_prog);  in tun_xdp_one()
2442 		err = tun_xdp_act(tun, xdp_prog, xdp, act);  in tun_xdp_one()
2479 	if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) {  in tun_xdp_one()
2480 		this_cpu_inc(tun->pcpu_stats->rx_frame_errors);  in tun_xdp_one()
2486 	skb->protocol = eth_type_trans(skb, tun->dev);  in tun_xdp_one()
2496 	if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&  in tun_xdp_one()
2506 	stats = this_cpu_ptr(tun->pcpu_stats);  in tun_xdp_one()
2513 		tun_flow_update(tun, rxhash, tfile);  in tun_xdp_one()
2523 	struct tun_struct *tun = tun_get(tfile);  in tun_sendmsg()  local
2527 	if (!tun)  in tun_sendmsg()
2542 			tun_xdp_one(tun, tfile, xdp, &flush, &tpage);  in tun_sendmsg()
2557 	ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,  in tun_sendmsg()
2561 	tun_put(tun);  in tun_sendmsg()
2569 	struct tun_struct *tun = tun_get(tfile);  in tun_recvmsg()  local
2573 	if (!tun) {  in tun_recvmsg()
2587 	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);  in tun_recvmsg()
2593 	tun_put(tun);  in tun_recvmsg()
2597 	tun_put(tun);  in tun_recvmsg()
2620 	struct tun_struct *tun;  in tun_peek_len()  local
2623 	tun = tun_get(tfile);  in tun_peek_len()
2624 	if (!tun)  in tun_peek_len()
2628 	tun_put(tun);  in tun_peek_len()
2646 static int tun_flags(struct tun_struct *tun)  in tun_flags()  argument
2648 	return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);  in tun_flags()
2654 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));  in tun_show_flags()  local
2655 	return sprintf(buf, "0x%x\n", tun_flags(tun));  in tun_show_flags()
2661 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));  in tun_show_owner()  local
2662 	return uid_valid(tun->owner)?  in tun_show_owner()
2664 			from_kuid_munged(current_user_ns(), tun->owner)):  in tun_show_owner()
2671 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));  in tun_show_group()  local
2672 	return gid_valid(tun->group) ?  in tun_show_group()
2674 			from_kgid_munged(current_user_ns(), tun->group)):  in tun_show_group()
2695 	struct tun_struct *tun;  in tun_set_iff()  local
2717 			tun = netdev_priv(dev);  in tun_set_iff()
2719 			tun = netdev_priv(dev);  in tun_set_iff()
2724 		    !!(tun->flags & IFF_MULTI_QUEUE))  in tun_set_iff()
2727 		if (tun_not_capable(tun))  in tun_set_iff()
2729 		err = security_tun_dev_open(tun->security);  in tun_set_iff()
2733 		err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,  in tun_set_iff()
2739 		if (tun->flags & IFF_MULTI_QUEUE &&  in tun_set_iff()
2740 		    (tun->numqueues + tun->numdisabled > 1)) {  in tun_set_iff()
2748 		tun->flags = (tun->flags & ~TUN_FEATURES) |  in tun_set_iff()
2794 		tun = netdev_priv(dev);  in tun_set_iff()
2795 		tun->dev = dev;  in tun_set_iff()
2796 		tun->flags = flags;  in tun_set_iff()
2797 		tun->txflt.count = 0;  in tun_set_iff()
2798 		tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);  in tun_set_iff()
2800 		tun->align = NET_SKB_PAD;  in tun_set_iff()
2801 		tun->filter_attached = false;  in tun_set_iff()
2802 		tun->sndbuf = tfile->socket.sk->sk_sndbuf;  in tun_set_iff()
2803 		tun->rx_batched = 0;  in tun_set_iff()
2804 		RCU_INIT_POINTER(tun->steering_prog, NULL);  in tun_set_iff()
2806 		tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);  in tun_set_iff()
2807 		if (!tun->pcpu_stats) {  in tun_set_iff()
2812 		spin_lock_init(&tun->lock);  in tun_set_iff()
2814 		err = security_tun_dev_alloc_security(&tun->security);  in tun_set_iff()
2819 		tun_flow_init(tun);  in tun_set_iff()
2829 		tun->flags = (tun->flags & ~TUN_FEATURES) |  in tun_set_iff()
2832 		INIT_LIST_HEAD(&tun->disabled);  in tun_set_iff()
2833 		err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI,  in tun_set_iff()
2838 		err = register_netdevice(tun->dev);  in tun_set_iff()
2844 		rcu_assign_pointer(tfile->tun, tun);  in tun_set_iff()
2847 	netif_carrier_on(tun->dev);  in tun_set_iff()
2849 	tun_debug(KERN_INFO, tun, "tun_set_iff\n");  in tun_set_iff()
2854 	if (netif_running(tun->dev))  in tun_set_iff()
2855 		netif_tx_wake_all_queues(tun->dev);  in tun_set_iff()
2857 	strcpy(ifr->ifr_name, tun->dev->name);  in tun_set_iff()
2866 	tun_flow_uninit(tun);  in tun_set_iff()
2867 	security_tun_dev_free_security(tun->security);  in tun_set_iff()
2869 	free_percpu(tun->pcpu_stats);  in tun_set_iff()
2875 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)  in tun_get_iff()  argument
2877 	tun_debug(KERN_INFO, tun, "tun_get_iff\n");  in tun_get_iff()
2879 	strcpy(ifr->ifr_name, tun->dev->name);  in tun_get_iff()
2881 	ifr->ifr_flags = tun_flags(tun);  in tun_get_iff()
2887 static int set_offload(struct tun_struct *tun, unsigned long arg)  in set_offload()  argument
2915 	tun->set_features = features;  in set_offload()
2916 	tun->dev->wanted_features &= ~TUN_USER_FEATURES;  in set_offload()
2917 	tun->dev->wanted_features |= features;  in set_offload()
2918 	netdev_update_features(tun->dev);  in set_offload()
2923 static void tun_detach_filter(struct tun_struct *tun, int n)  in tun_detach_filter()  argument
2929 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_filter()
2935 	tun->filter_attached = false;  in tun_detach_filter()
2938 static int tun_attach_filter(struct tun_struct *tun)  in tun_attach_filter()  argument
2943 	for (i = 0; i < tun->numqueues; i++) {  in tun_attach_filter()
2944 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_attach_filter()
2946 		ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);  in tun_attach_filter()
2949 			tun_detach_filter(tun, i);  in tun_attach_filter()
2954 	tun->filter_attached = true;  in tun_attach_filter()
2958 static void tun_set_sndbuf(struct tun_struct *tun)  in tun_set_sndbuf()  argument
2963 	for (i = 0; i < tun->numqueues; i++) {  in tun_set_sndbuf()
2964 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_set_sndbuf()
2965 		tfile->socket.sk->sk_sndbuf = tun->sndbuf;  in tun_set_sndbuf()
2972 	struct tun_struct *tun;  in tun_set_queue()  local
2978 		tun = tfile->detached;  in tun_set_queue()
2979 		if (!tun) {  in tun_set_queue()
2983 		ret = security_tun_dev_attach_queue(tun->security);  in tun_set_queue()
2986 		ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,  in tun_set_queue()
2987 				 tun->flags & IFF_NAPI_FRAGS, true);  in tun_set_queue()
2989 		tun = rtnl_dereference(tfile->tun);  in tun_set_queue()
2990 		if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)  in tun_set_queue()
2998 		netdev_state_change(tun->dev);  in tun_set_queue()
3005 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog **prog_p,  in tun_set_ebpf()  argument
3022 	return __tun_set_ebpf(tun, prog_p, prog);  in tun_set_ebpf()
3030 	struct tun_struct *tun;  in __tun_chr_ioctl()  local
3067 	tun = tun_get(tfile);  in __tun_chr_ioctl()
3070 		if (tun)  in __tun_chr_ioctl()
3086 		if (tun)  in __tun_chr_ioctl()
3099 	if (!tun)  in __tun_chr_ioctl()
3102 	tun_debug(KERN_INFO, tun, "tun_chr_ioctl cmd %u\n", cmd);  in __tun_chr_ioctl()
3104 	net = dev_net(tun->dev);  in __tun_chr_ioctl()
3108 		tun_get_iff(tun, &ifr);  in __tun_chr_ioctl()
3123 		tun_debug(KERN_INFO, tun, "ignored: set checksum %s\n",  in __tun_chr_ioctl()
3131 		if (arg && !(tun->flags & IFF_PERSIST)) {  in __tun_chr_ioctl()
3132 			tun->flags |= IFF_PERSIST;  in __tun_chr_ioctl()
3136 		if (!arg && (tun->flags & IFF_PERSIST)) {  in __tun_chr_ioctl()
3137 			tun->flags &= ~IFF_PERSIST;  in __tun_chr_ioctl()
3142 		tun_debug(KERN_INFO, tun, "persist %s\n",  in __tun_chr_ioctl()
3153 		tun->owner = owner;  in __tun_chr_ioctl()
3155 		tun_debug(KERN_INFO, tun, "owner set to %u\n",  in __tun_chr_ioctl()
3156 			  from_kuid(&init_user_ns, tun->owner));  in __tun_chr_ioctl()
3166 		tun->group = group;  in __tun_chr_ioctl()
3168 		tun_debug(KERN_INFO, tun, "group set to %u\n",  in __tun_chr_ioctl()
3169 			  from_kgid(&init_user_ns, tun->group));  in __tun_chr_ioctl()
3174 		if (tun->dev->flags & IFF_UP) {  in __tun_chr_ioctl()
3175 			tun_debug(KERN_INFO, tun,  in __tun_chr_ioctl()
3179 			tun->dev->type = (int) arg;  in __tun_chr_ioctl()
3180 			tun_debug(KERN_INFO, tun, "linktype set to %d\n",  in __tun_chr_ioctl()
3181 				  tun->dev->type);  in __tun_chr_ioctl()
3188 		tun->debug = arg;  in __tun_chr_ioctl()
3192 		ret = set_offload(tun, arg);  in __tun_chr_ioctl()
3198 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)  in __tun_chr_ioctl()
3200 		ret = update_filter(&tun->txflt, (void __user *)arg);  in __tun_chr_ioctl()
3205 		memcpy(ifr.ifr_hwaddr.sa_data, tun->dev->dev_addr, ETH_ALEN);  in __tun_chr_ioctl()
3206 		ifr.ifr_hwaddr.sa_family = tun->dev->type;  in __tun_chr_ioctl()
3213 		tun_debug(KERN_DEBUG, tun, "set hw address: %pM\n",  in __tun_chr_ioctl()
3216 		ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr, NULL);  in __tun_chr_ioctl()
3235 		tun->sndbuf = sndbuf;  in __tun_chr_ioctl()
3236 		tun_set_sndbuf(tun);  in __tun_chr_ioctl()
3240 		vnet_hdr_sz = tun->vnet_hdr_sz;  in __tun_chr_ioctl()
3255 		tun->vnet_hdr_sz = vnet_hdr_sz;  in __tun_chr_ioctl()
3259 		le = !!(tun->flags & TUN_VNET_LE);  in __tun_chr_ioctl()
3270 			tun->flags |= TUN_VNET_LE;  in __tun_chr_ioctl()
3272 			tun->flags &= ~TUN_VNET_LE;  in __tun_chr_ioctl()
3276 		ret = tun_get_vnet_be(tun, argp);  in __tun_chr_ioctl()
3280 		ret = tun_set_vnet_be(tun, argp);  in __tun_chr_ioctl()
3286 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)  in __tun_chr_ioctl()
3289 		if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))  in __tun_chr_ioctl()
3292 		ret = tun_attach_filter(tun);  in __tun_chr_ioctl()
3298 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)  in __tun_chr_ioctl()
3301 		tun_detach_filter(tun, tun->numqueues);  in __tun_chr_ioctl()
3306 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)  in __tun_chr_ioctl()
3309 		if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))  in __tun_chr_ioctl()
3315 		ret = tun_set_ebpf(tun, &tun->steering_prog, argp);  in __tun_chr_ioctl()
3319 		ret = tun_set_ebpf(tun, &tun->filter_prog, argp);  in __tun_chr_ioctl()
3327 		ret = tun_net_change_carrier(tun->dev, (bool)carrier);  in __tun_chr_ioctl()
3343 		netdev_state_change(tun->dev);  in __tun_chr_ioctl()
3347 	if (tun)  in __tun_chr_ioctl()
3348 		tun_put(tun);  in __tun_chr_ioctl()
3422 	RCU_INIT_POINTER(tfile->tun, NULL);  in tun_chr_open()
3457 	struct tun_struct *tun;  in tun_chr_show_fdinfo()  local
3463 	tun = tun_get(tfile);  in tun_chr_show_fdinfo()
3464 	if (tun)  in tun_chr_show_fdinfo()
3465 		tun_get_iff(tun, &ifr);  in tun_chr_show_fdinfo()
3468 	if (tun)  in tun_chr_show_fdinfo()
3469 		tun_put(tun);  in tun_chr_show_fdinfo()
3517 	struct tun_struct *tun = netdev_priv(dev);  in tun_get_link_ksettings()  local
3519 	memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));  in tun_get_link_ksettings()
3526 	struct tun_struct *tun = netdev_priv(dev);  in tun_set_link_ksettings()  local
3528 	memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));  in tun_set_link_ksettings()
3534 	struct tun_struct *tun = netdev_priv(dev);  in tun_get_drvinfo()  local
3539 	switch (tun->flags & TUN_TYPE_MASK) {  in tun_get_drvinfo()
3552 	struct tun_struct *tun = netdev_priv(dev);  in tun_get_msglevel()  local
3553 	return tun->debug;  in tun_get_msglevel()
3562 	struct tun_struct *tun = netdev_priv(dev);  in tun_set_msglevel()  local
3563 	tun->debug = value;  in tun_set_msglevel()
3570 	struct tun_struct *tun = netdev_priv(dev);  in tun_get_coalesce()  local
3572 	ec->rx_max_coalesced_frames = tun->rx_batched;  in tun_get_coalesce()
3580 	struct tun_struct *tun = netdev_priv(dev);  in tun_set_coalesce()  local
3583 		tun->rx_batched = NAPI_POLL_WEIGHT;  in tun_set_coalesce()
3585 		tun->rx_batched = ec->rx_max_coalesced_frames;  in tun_set_coalesce()
3602 static int tun_queue_resize(struct tun_struct *tun)  in tun_queue_resize()  argument
3604 	struct net_device *dev = tun->dev;  in tun_queue_resize()
3607 	int n = tun->numqueues + tun->numdisabled;  in tun_queue_resize()
3614 	for (i = 0; i < tun->numqueues; i++) {  in tun_queue_resize()
3615 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_queue_resize()
3618 	list_for_each_entry(tfile, &tun->disabled, next)  in tun_queue_resize()
3633 	struct tun_struct *tun = netdev_priv(dev);  in tun_device_event()  local
3641 		if (tun_queue_resize(tun))  in tun_device_event()
3645 		for (i = 0; i < tun->numqueues; i++) {  in tun_device_event()
3648 			tfile = rtnl_dereference(tun->tfiles[i]);  in tun_device_event()