1002 	struct napi_struct *napi;  in dev_get_by_napi_id()  local
1009 	napi = napi_by_id(napi_id);  in dev_get_by_napi_id()
1011 	return napi ? napi->dev : NULL;  in dev_get_by_napi_id()
4242 				     struct napi_struct *napi)  in ____napi_schedule()  argument
4244 	list_add_tail(&napi->poll_list, &sd->poll_list);  in ____napi_schedule()
5714 static void gro_normal_list(struct napi_struct *napi)  in gro_normal_list()  argument
5716 	if (!napi->rx_count)  in gro_normal_list()
5718 	netif_receive_skb_list_internal(&napi->rx_list);  in gro_normal_list()
5719 	INIT_LIST_HEAD(&napi->rx_list);  in gro_normal_list()
5720 	napi->rx_count = 0;  in gro_normal_list()
5726 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb)  in gro_normal_one()  argument
5728 	list_add_tail(&skb->list, &napi->rx_list);  in gro_normal_one()
5729 	if (++napi->rx_count >= gro_normal_batch)  in gro_normal_one()
5730 		gro_normal_list(napi);  in gro_normal_one()
5735 static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)  in napi_gro_complete()  argument
5768 	gro_normal_one(napi, skb);  in napi_gro_complete()
5772 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,  in __napi_gro_flush_chain()  argument
5775 	struct list_head *head = &napi->gro_hash[index].list;  in __napi_gro_flush_chain()
5782 		napi_gro_complete(napi, skb);  in __napi_gro_flush_chain()
5783 		napi->gro_hash[index].count--;  in __napi_gro_flush_chain()
5786 	if (!napi->gro_hash[index].count)  in __napi_gro_flush_chain()
5787 		__clear_bit(index, &napi->gro_bitmask);  in __napi_gro_flush_chain()
5794 void napi_gro_flush(struct napi_struct *napi, bool flush_old)  in napi_gro_flush()  argument
5796 	unsigned long bitmask = napi->gro_bitmask;  in napi_gro_flush()
5802 		__napi_gro_flush_chain(napi, base, flush_old);  in napi_gro_flush()
5807 static struct list_head *gro_list_prepare(struct napi_struct *napi,  in gro_list_prepare()  argument
5815 	head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;  in gro_list_prepare()
5884 static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)  in gro_flush_oldest()  argument
5900 	napi_gro_complete(napi, oldest);  in gro_flush_oldest()
5907 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in dev_gro_receive()  argument
5922 	gro_head = gro_list_prepare(napi, skb);  in dev_gro_receive()
5976 		napi_gro_complete(napi, pp);  in dev_gro_receive()
5977 		napi->gro_hash[hash].count--;  in dev_gro_receive()
5986 	if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {  in dev_gro_receive()
5987 		gro_flush_oldest(napi, gro_head);  in dev_gro_receive()
5989 		napi->gro_hash[hash].count++;  in dev_gro_receive()
6003 	if (napi->gro_hash[hash].count) {  in dev_gro_receive()
6004 		if (!test_bit(hash, &napi->gro_bitmask))  in dev_gro_receive()
6005 			__set_bit(hash, &napi->gro_bitmask);  in dev_gro_receive()
6006 	} else if (test_bit(hash, &napi->gro_bitmask)) {  in dev_gro_receive()
6007 		__clear_bit(hash, &napi->gro_bitmask);  in dev_gro_receive()
6052 static gro_result_t napi_skb_finish(struct napi_struct *napi,  in napi_skb_finish()  argument
6058 		gro_normal_one(napi, skb);  in napi_skb_finish()
6081 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in napi_gro_receive()  argument
6085 	skb_mark_napi_id(skb, napi);  in napi_gro_receive()
6090 	ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));  in napi_gro_receive()
6097 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)  in napi_reuse_skb()  argument
6107 	skb->dev = napi->dev;  in napi_reuse_skb()
6118 	napi->skb = skb;  in napi_reuse_skb()
6121 struct sk_buff *napi_get_frags(struct napi_struct *napi)  in napi_get_frags()  argument
6123 	struct sk_buff *skb = napi->skb;  in napi_get_frags()
6126 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);  in napi_get_frags()
6128 			napi->skb = skb;  in napi_get_frags()
6129 			skb_mark_napi_id(skb, napi);  in napi_get_frags()
6136 static gro_result_t napi_frags_finish(struct napi_struct *napi,  in napi_frags_finish()  argument
6146 			gro_normal_one(napi, skb);  in napi_frags_finish()
6150 		napi_reuse_skb(napi, skb);  in napi_frags_finish()
6157 			napi_reuse_skb(napi, skb);  in napi_frags_finish()
6172 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)  in napi_frags_skb()  argument
6174 	struct sk_buff *skb = napi->skb;  in napi_frags_skb()
6178 	napi->skb = NULL;  in napi_frags_skb()
6187 					     __func__, napi->dev->name);  in napi_frags_skb()
6188 			napi_reuse_skb(napi, skb);  in napi_frags_skb()
6209 gro_result_t napi_gro_frags(struct napi_struct *napi)  in napi_gro_frags()  argument
6212 	struct sk_buff *skb = napi_frags_skb(napi);  in napi_gro_frags()
6219 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));  in napi_gro_frags()
6295 static int process_backlog(struct napi_struct *napi, int quota)  in process_backlog()  argument
6297 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);  in process_backlog()
6309 	napi->weight = dev_rx_weight;  in process_backlog()
6334 			napi->state = 0;  in process_backlog()
6483 	struct napi_struct *napi;  in napi_by_id()  local
6485 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)  in napi_by_id()
6486 		if (napi->napi_id == napi_id)  in napi_by_id()
6487 			return napi;  in napi_by_id()
6496 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)  in busy_poll_stop()  argument
6509 	clear_bit(NAPI_STATE_MISSED, &napi->state);  in busy_poll_stop()
6510 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);  in busy_poll_stop()
6517 	rc = napi->poll(napi, BUSY_POLL_BUDGET);  in busy_poll_stop()
6522 	trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);  in busy_poll_stop()
6528 		gro_normal_list(napi);  in busy_poll_stop()
6529 		__napi_schedule(napi);  in busy_poll_stop()
6539 	int (*napi_poll)(struct napi_struct *napi, int budget);  in napi_busy_loop()
6541 	struct napi_struct *napi;  in napi_busy_loop()  local
6548 	napi = napi_by_id(napi_id);  in napi_busy_loop()
6549 	if (!napi)  in napi_busy_loop()
6558 			unsigned long val = READ_ONCE(napi->state);  in napi_busy_loop()
6566 			if (cmpxchg(&napi->state, val,  in napi_busy_loop()
6570 			have_poll_lock = netpoll_poll_lock(napi);  in napi_busy_loop()
6571 			napi_poll = napi->poll;  in napi_busy_loop()
6573 		work = napi_poll(napi, BUSY_POLL_BUDGET);  in napi_busy_loop()
6574 		trace_napi_poll(napi, work, BUSY_POLL_BUDGET);  in napi_busy_loop()
6575 		gro_normal_list(napi);  in napi_busy_loop()
6578 			__NET_ADD_STATS(dev_net(napi->dev),  in napi_busy_loop()
6587 				busy_poll_stop(napi, have_poll_lock);  in napi_busy_loop()
6598 		busy_poll_stop(napi, have_poll_lock);  in napi_busy_loop()
6607 static void napi_hash_add(struct napi_struct *napi)  in napi_hash_add()  argument
6609 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))  in napi_hash_add()
6619 	napi->napi_id = napi_gen_id;  in napi_hash_add()
6621 	hlist_add_head_rcu(&napi->napi_hash_node,  in napi_hash_add()
6622 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);  in napi_hash_add()
6630 static void napi_hash_del(struct napi_struct *napi)  in napi_hash_del()  argument
6634 	hlist_del_init_rcu(&napi->napi_hash_node);  in napi_hash_del()
6641 	struct napi_struct *napi;  in napi_watchdog()  local
6643 	napi = container_of(timer, struct napi_struct, timer);  in napi_watchdog()
6648 	if (!napi_disable_pending(napi) &&  in napi_watchdog()
6649 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))  in napi_watchdog()
6650 		__napi_schedule_irqoff(napi);  in napi_watchdog()
6655 static void init_gro_hash(struct napi_struct *napi)  in init_gro_hash()  argument
6660 		INIT_LIST_HEAD(&napi->gro_hash[i].list);  in init_gro_hash()
6661 		napi->gro_hash[i].count = 0;  in init_gro_hash()
6663 	napi->gro_bitmask = 0;  in init_gro_hash()
6666 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,  in netif_napi_add()  argument
6669 	if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))  in netif_napi_add()
6672 	INIT_LIST_HEAD(&napi->poll_list);  in netif_napi_add()
6673 	INIT_HLIST_NODE(&napi->napi_hash_node);  in netif_napi_add()
6674 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);  in netif_napi_add()
6675 	napi->timer.function = napi_watchdog;  in netif_napi_add()
6676 	init_gro_hash(napi);  in netif_napi_add()
6677 	napi->skb = NULL;  in netif_napi_add()
6678 	INIT_LIST_HEAD(&napi->rx_list);  in netif_napi_add()
6679 	napi->rx_count = 0;  in netif_napi_add()
6680 	napi->poll = poll;  in netif_napi_add()
6684 	napi->weight = weight;  in netif_napi_add()
6685 	napi->dev = dev;  in netif_napi_add()
6687 	napi->poll_owner = -1;  in netif_napi_add()
6689 	set_bit(NAPI_STATE_SCHED, &napi->state);  in netif_napi_add()
6690 	set_bit(NAPI_STATE_NPSVC, &napi->state);  in netif_napi_add()
6691 	list_add_rcu(&napi->dev_list, &dev->napi_list);  in netif_napi_add()
6692 	napi_hash_add(napi);  in netif_napi_add()
6712 static void flush_gro_hash(struct napi_struct *napi)  in flush_gro_hash()  argument
6719 		list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)  in flush_gro_hash()
6721 		napi->gro_hash[i].count = 0;  in flush_gro_hash()
6726 void __netif_napi_del(struct napi_struct *napi)  in __netif_napi_del()  argument
6728 	if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))  in __netif_napi_del()
6731 	napi_hash_del(napi);  in __netif_napi_del()
6732 	list_del_rcu(&napi->dev_list);  in __netif_napi_del()
6733 	napi_free_frags(napi);  in __netif_napi_del()
6735 	flush_gro_hash(napi);  in __netif_napi_del()
6736 	napi->gro_bitmask = 0;  in __netif_napi_del()
10832 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,  in dev_cpu_dead()  local
10836 		list_del_init(&napi->poll_list);  in dev_cpu_dead()
10837 		if (napi->poll == process_backlog)  in dev_cpu_dead()
10838 			napi->state = 0;  in dev_cpu_dead()
10840 			____napi_schedule(sd, napi);  in dev_cpu_dead()