802 	struct napi_struct *napi;  in dev_get_by_napi_id()  local
809 	napi = napi_by_id(napi_id);  in dev_get_by_napi_id()
811 	return napi ? napi->dev : NULL;  in dev_get_by_napi_id()
3894 				     struct napi_struct *napi)  in ____napi_schedule()  argument
3896 	list_add_tail(&napi->poll_list, &sd->poll_list);  in ____napi_schedule()
5311 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,  in __napi_gro_flush_chain()  argument
5314 	struct list_head *head = &napi->gro_hash[index].list;  in __napi_gro_flush_chain()
5322 		napi->gro_hash[index].count--;  in __napi_gro_flush_chain()
5325 	if (!napi->gro_hash[index].count)  in __napi_gro_flush_chain()
5326 		__clear_bit(index, &napi->gro_bitmask);  in __napi_gro_flush_chain()
5333 void napi_gro_flush(struct napi_struct *napi, bool flush_old)  in napi_gro_flush()  argument
5335 	unsigned long bitmask = napi->gro_bitmask;  in napi_gro_flush()
5341 		__napi_gro_flush_chain(napi, base, flush_old);  in napi_gro_flush()
5346 static struct list_head *gro_list_prepare(struct napi_struct *napi,  in gro_list_prepare()  argument
5354 	head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;  in gro_list_prepare()
5447 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in dev_gro_receive()  argument
5462 	gro_head = gro_list_prepare(napi, skb);  in dev_gro_receive()
5517 		napi->gro_hash[hash].count--;  in dev_gro_receive()
5526 	if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {  in dev_gro_receive()
5529 		napi->gro_hash[hash].count++;  in dev_gro_receive()
5543 	if (napi->gro_hash[hash].count) {  in dev_gro_receive()
5544 		if (!test_bit(hash, &napi->gro_bitmask))  in dev_gro_receive()
5545 			__set_bit(hash, &napi->gro_bitmask);  in dev_gro_receive()
5546 	} else if (test_bit(hash, &napi->gro_bitmask)) {  in dev_gro_receive()
5547 		__clear_bit(hash, &napi->gro_bitmask);  in dev_gro_receive()
5620 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)  in napi_gro_receive()  argument
5624 	skb_mark_napi_id(skb, napi);  in napi_gro_receive()
5629 	ret = napi_skb_finish(dev_gro_receive(napi, skb), skb);  in napi_gro_receive()
5636 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)  in napi_reuse_skb()  argument
5646 	skb->dev = napi->dev;  in napi_reuse_skb()
5657 	napi->skb = skb;  in napi_reuse_skb()
5660 struct sk_buff *napi_get_frags(struct napi_struct *napi)  in napi_get_frags()  argument
5662 	struct sk_buff *skb = napi->skb;  in napi_get_frags()
5665 		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);  in napi_get_frags()
5667 			napi->skb = skb;  in napi_get_frags()
5668 			skb_mark_napi_id(skb, napi);  in napi_get_frags()
5676 static void gro_normal_list(struct napi_struct *napi)  in gro_normal_list()  argument
5678 	if (!napi->rx_count)  in gro_normal_list()
5680 	netif_receive_skb_list_internal(&napi->rx_list);  in gro_normal_list()
5681 	INIT_LIST_HEAD(&napi->rx_list);  in gro_normal_list()
5682 	napi->rx_count = 0;  in gro_normal_list()
5688 static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb)  in gro_normal_one()  argument
5690 	list_add_tail(&skb->list, &napi->rx_list);  in gro_normal_one()
5691 	if (++napi->rx_count >= gro_normal_batch)  in gro_normal_one()
5692 		gro_normal_list(napi);  in gro_normal_one()
5695 static gro_result_t napi_frags_finish(struct napi_struct *napi,  in napi_frags_finish()  argument
5705 			gro_normal_one(napi, skb);  in napi_frags_finish()
5709 		napi_reuse_skb(napi, skb);  in napi_frags_finish()
5716 			napi_reuse_skb(napi, skb);  in napi_frags_finish()
5731 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)  in napi_frags_skb()  argument
5733 	struct sk_buff *skb = napi->skb;  in napi_frags_skb()
5737 	napi->skb = NULL;  in napi_frags_skb()
5746 					     __func__, napi->dev->name);  in napi_frags_skb()
5747 			napi_reuse_skb(napi, skb);  in napi_frags_skb()
5768 gro_result_t napi_gro_frags(struct napi_struct *napi)  in napi_gro_frags()  argument
5771 	struct sk_buff *skb = napi_frags_skb(napi);  in napi_gro_frags()
5778 	ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));  in napi_gro_frags()
5854 static int process_backlog(struct napi_struct *napi, int quota)  in process_backlog()  argument
5856 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);  in process_backlog()
5868 	napi->weight = dev_rx_weight;  in process_backlog()
5893 			napi->state = 0;  in process_backlog()
6034 	struct napi_struct *napi;  in napi_by_id()  local
6036 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)  in napi_by_id()
6037 		if (napi->napi_id == napi_id)  in napi_by_id()
6038 			return napi;  in napi_by_id()
6047 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)  in busy_poll_stop()  argument
6060 	clear_bit(NAPI_STATE_MISSED, &napi->state);  in busy_poll_stop()
6061 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);  in busy_poll_stop()
6068 	rc = napi->poll(napi, BUSY_POLL_BUDGET);  in busy_poll_stop()
6073 	trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);  in busy_poll_stop()
6079 		gro_normal_list(napi);  in busy_poll_stop()
6080 		__napi_schedule(napi);  in busy_poll_stop()
6090 	int (*napi_poll)(struct napi_struct *napi, int budget);  in napi_busy_loop()
6092 	struct napi_struct *napi;  in napi_busy_loop()  local
6099 	napi = napi_by_id(napi_id);  in napi_busy_loop()
6100 	if (!napi)  in napi_busy_loop()
6109 			unsigned long val = READ_ONCE(napi->state);  in napi_busy_loop()
6117 			if (cmpxchg(&napi->state, val,  in napi_busy_loop()
6121 			have_poll_lock = netpoll_poll_lock(napi);  in napi_busy_loop()
6122 			napi_poll = napi->poll;  in napi_busy_loop()
6124 		work = napi_poll(napi, BUSY_POLL_BUDGET);  in napi_busy_loop()
6125 		trace_napi_poll(napi, work, BUSY_POLL_BUDGET);  in napi_busy_loop()
6126 		gro_normal_list(napi);  in napi_busy_loop()
6129 			__NET_ADD_STATS(dev_net(napi->dev),  in napi_busy_loop()
6138 				busy_poll_stop(napi, have_poll_lock);  in napi_busy_loop()
6149 		busy_poll_stop(napi, have_poll_lock);  in napi_busy_loop()
6158 static void napi_hash_add(struct napi_struct *napi)  in napi_hash_add()  argument
6160 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||  in napi_hash_add()
6161 	    test_and_set_bit(NAPI_STATE_HASHED, &napi->state))  in napi_hash_add()
6171 	napi->napi_id = napi_gen_id;  in napi_hash_add()
6173 	hlist_add_head_rcu(&napi->napi_hash_node,  in napi_hash_add()
6174 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);  in napi_hash_add()
6182 bool napi_hash_del(struct napi_struct *napi)  in napi_hash_del()  argument
6188 	if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {  in napi_hash_del()
6190 		hlist_del_rcu(&napi->napi_hash_node);  in napi_hash_del()
6199 	struct napi_struct *napi;  in napi_watchdog()  local
6201 	napi = container_of(timer, struct napi_struct, timer);  in napi_watchdog()
6206 	if (napi->gro_bitmask && !napi_disable_pending(napi) &&  in napi_watchdog()
6207 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))  in napi_watchdog()
6208 		__napi_schedule_irqoff(napi);  in napi_watchdog()
6213 static void init_gro_hash(struct napi_struct *napi)  in init_gro_hash()  argument
6218 		INIT_LIST_HEAD(&napi->gro_hash[i].list);  in init_gro_hash()
6219 		napi->gro_hash[i].count = 0;  in init_gro_hash()
6221 	napi->gro_bitmask = 0;  in init_gro_hash()
6224 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,  in netif_napi_add()  argument
6227 	INIT_LIST_HEAD(&napi->poll_list);  in netif_napi_add()
6228 	hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);  in netif_napi_add()
6229 	napi->timer.function = napi_watchdog;  in netif_napi_add()
6230 	init_gro_hash(napi);  in netif_napi_add()
6231 	napi->skb = NULL;  in netif_napi_add()
6232 	INIT_LIST_HEAD(&napi->rx_list);  in netif_napi_add()
6233 	napi->rx_count = 0;  in netif_napi_add()
6234 	napi->poll = poll;  in netif_napi_add()
6238 	napi->weight = weight;  in netif_napi_add()
6239 	list_add(&napi->dev_list, &dev->napi_list);  in netif_napi_add()
6240 	napi->dev = dev;  in netif_napi_add()
6242 	napi->poll_owner = -1;  in netif_napi_add()
6244 	set_bit(NAPI_STATE_SCHED, &napi->state);  in netif_napi_add()
6245 	napi_hash_add(napi);  in netif_napi_add()
6265 static void flush_gro_hash(struct napi_struct *napi)  in flush_gro_hash()  argument
6272 		list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)  in flush_gro_hash()
6274 		napi->gro_hash[i].count = 0;  in flush_gro_hash()
6279 void netif_napi_del(struct napi_struct *napi)  in netif_napi_del()  argument
6282 	if (napi_hash_del(napi))  in netif_napi_del()
6284 	list_del_init(&napi->dev_list);  in netif_napi_del()
6285 	napi_free_frags(napi);  in netif_napi_del()
6287 	flush_gro_hash(napi);  in netif_napi_del()
6288 	napi->gro_bitmask = 0;  in netif_napi_del()
9857 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,  in dev_cpu_dead()  local
9861 		list_del_init(&napi->poll_list);  in dev_cpu_dead()
9862 		if (napi->poll == process_backlog)  in dev_cpu_dead()
9863 			napi->state = 0;  in dev_cpu_dead()
9865 			____napi_schedule(sd, napi);  in dev_cpu_dead()