Lines Matching refs:napi
883 struct napi_struct *napi; in dev_get_by_napi_id() local
890 napi = napi_by_id(napi_id); in dev_get_by_napi_id()
892 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
3924 struct napi_struct *napi) in ____napi_schedule() argument
3926 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
5308 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, in __napi_gro_flush_chain() argument
5311 struct list_head *head = &napi->gro_hash[index].list; in __napi_gro_flush_chain()
5320 napi->gro_hash[index].count--; in __napi_gro_flush_chain()
5323 if (!napi->gro_hash[index].count) in __napi_gro_flush_chain()
5324 __clear_bit(index, &napi->gro_bitmask); in __napi_gro_flush_chain()
5331 void napi_gro_flush(struct napi_struct *napi, bool flush_old) in napi_gro_flush() argument
5336 if (test_bit(i, &napi->gro_bitmask)) in napi_gro_flush()
5337 __napi_gro_flush_chain(napi, i, flush_old); in napi_gro_flush()
5342 static struct list_head *gro_list_prepare(struct napi_struct *napi, in gro_list_prepare() argument
5350 head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list; in gro_list_prepare()
5437 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in dev_gro_receive() argument
5452 gro_head = gro_list_prepare(napi, skb); in dev_gro_receive()
5506 napi->gro_hash[hash].count--; in dev_gro_receive()
5515 if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) { in dev_gro_receive()
5518 napi->gro_hash[hash].count++; in dev_gro_receive()
5532 if (napi->gro_hash[hash].count) { in dev_gro_receive()
5533 if (!test_bit(hash, &napi->gro_bitmask)) in dev_gro_receive()
5534 __set_bit(hash, &napi->gro_bitmask); in dev_gro_receive()
5535 } else if (test_bit(hash, &napi->gro_bitmask)) { in dev_gro_receive()
5536 __clear_bit(hash, &napi->gro_bitmask); in dev_gro_receive()
5609 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) in napi_gro_receive() argument
5611 skb_mark_napi_id(skb, napi); in napi_gro_receive()
5616 return napi_skb_finish(dev_gro_receive(napi, skb), skb); in napi_gro_receive()
5620 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) in napi_reuse_skb() argument
5630 skb->dev = napi->dev; in napi_reuse_skb()
5637 napi->skb = skb; in napi_reuse_skb()
5640 struct sk_buff *napi_get_frags(struct napi_struct *napi) in napi_get_frags() argument
5642 struct sk_buff *skb = napi->skb; in napi_get_frags()
5645 skb = napi_alloc_skb(napi, GRO_MAX_HEAD); in napi_get_frags()
5647 napi->skb = skb; in napi_get_frags()
5648 skb_mark_napi_id(skb, napi); in napi_get_frags()
5655 static gro_result_t napi_frags_finish(struct napi_struct *napi, in napi_frags_finish() argument
5669 napi_reuse_skb(napi, skb); in napi_frags_finish()
5676 napi_reuse_skb(napi, skb); in napi_frags_finish()
5691 static struct sk_buff *napi_frags_skb(struct napi_struct *napi) in napi_frags_skb() argument
5693 struct sk_buff *skb = napi->skb; in napi_frags_skb()
5697 napi->skb = NULL; in napi_frags_skb()
5707 __func__, napi->dev->name); in napi_frags_skb()
5708 napi_reuse_skb(napi, skb); in napi_frags_skb()
5728 gro_result_t napi_gro_frags(struct napi_struct *napi) in napi_gro_frags() argument
5730 struct sk_buff *skb = napi_frags_skb(napi); in napi_gro_frags()
5737 return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); in napi_gro_frags()
5809 static int process_backlog(struct napi_struct *napi, int quota) in process_backlog() argument
5811 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); in process_backlog()
5823 napi->weight = dev_rx_weight; in process_backlog()
5848 napi->state = 0; in process_backlog()
5984 struct napi_struct *napi; in napi_by_id() local
5986 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) in napi_by_id()
5987 if (napi->napi_id == napi_id) in napi_by_id()
5988 return napi; in napi_by_id()
5997 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock) in busy_poll_stop() argument
6010 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6011 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6018 rc = napi->poll(napi, BUSY_POLL_BUDGET); in busy_poll_stop()
6019 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET); in busy_poll_stop()
6022 __napi_schedule(napi); in busy_poll_stop()
6031 int (*napi_poll)(struct napi_struct *napi, int budget); in napi_busy_loop()
6033 struct napi_struct *napi; in napi_busy_loop() local
6040 napi = napi_by_id(napi_id); in napi_busy_loop()
6041 if (!napi) in napi_busy_loop()
6050 unsigned long val = READ_ONCE(napi->state); in napi_busy_loop()
6058 if (cmpxchg(&napi->state, val, in napi_busy_loop()
6062 have_poll_lock = netpoll_poll_lock(napi); in napi_busy_loop()
6063 napi_poll = napi->poll; in napi_busy_loop()
6065 work = napi_poll(napi, BUSY_POLL_BUDGET); in napi_busy_loop()
6066 trace_napi_poll(napi, work, BUSY_POLL_BUDGET); in napi_busy_loop()
6069 __NET_ADD_STATS(dev_net(napi->dev), in napi_busy_loop()
6078 busy_poll_stop(napi, have_poll_lock); in napi_busy_loop()
6089 busy_poll_stop(napi, have_poll_lock); in napi_busy_loop()
6098 static void napi_hash_add(struct napi_struct *napi) in napi_hash_add() argument
6100 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) || in napi_hash_add()
6101 test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) in napi_hash_add()
6111 napi->napi_id = napi_gen_id; in napi_hash_add()
6113 hlist_add_head_rcu(&napi->napi_hash_node, in napi_hash_add()
6114 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in napi_hash_add()
6122 bool napi_hash_del(struct napi_struct *napi) in napi_hash_del() argument
6128 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) { in napi_hash_del()
6130 hlist_del_rcu(&napi->napi_hash_node); in napi_hash_del()
6139 struct napi_struct *napi; in napi_watchdog() local
6141 napi = container_of(timer, struct napi_struct, timer); in napi_watchdog()
6146 if (napi->gro_bitmask && !napi_disable_pending(napi) && in napi_watchdog()
6147 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) in napi_watchdog()
6148 __napi_schedule_irqoff(napi); in napi_watchdog()
6153 static void init_gro_hash(struct napi_struct *napi) in init_gro_hash() argument
6158 INIT_LIST_HEAD(&napi->gro_hash[i].list); in init_gro_hash()
6159 napi->gro_hash[i].count = 0; in init_gro_hash()
6161 napi->gro_bitmask = 0; in init_gro_hash()
6164 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, in netif_napi_add() argument
6167 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add()
6168 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add()
6169 napi->timer.function = napi_watchdog; in netif_napi_add()
6170 init_gro_hash(napi); in netif_napi_add()
6171 napi->skb = NULL; in netif_napi_add()
6172 napi->poll = poll; in netif_napi_add()
6176 napi->weight = weight; in netif_napi_add()
6177 list_add(&napi->dev_list, &dev->napi_list); in netif_napi_add()
6178 napi->dev = dev; in netif_napi_add()
6180 napi->poll_owner = -1; in netif_napi_add()
6182 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add()
6183 napi_hash_add(napi); in netif_napi_add()
6203 static void flush_gro_hash(struct napi_struct *napi) in flush_gro_hash() argument
6210 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) in flush_gro_hash()
6212 napi->gro_hash[i].count = 0; in flush_gro_hash()
6217 void netif_napi_del(struct napi_struct *napi) in netif_napi_del() argument
6220 if (napi_hash_del(napi)) in netif_napi_del()
6222 list_del_init(&napi->dev_list); in netif_napi_del()
6223 napi_free_frags(napi); in netif_napi_del()
6225 flush_gro_hash(napi); in netif_napi_del()
6226 napi->gro_bitmask = 0; in netif_napi_del()
9263 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead() local
9267 list_del_init(&napi->poll_list); in dev_cpu_dead()
9268 if (napi->poll == process_backlog) in dev_cpu_dead()
9269 napi->state = 0; in dev_cpu_dead()
9271 ____napi_schedule(sd, napi); in dev_cpu_dead()