Lines Matching refs:po
192 static void *packet_previous_frame(struct packet_sock *po,
241 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
242 static void __fanout_link(struct sock *sk, struct packet_sock *po);
249 static struct net_device *packet_cached_dev_get(struct packet_sock *po) in packet_cached_dev_get() argument
254 dev = rcu_dereference(po->cached_dev); in packet_cached_dev_get()
261 static void packet_cached_dev_assign(struct packet_sock *po, in packet_cached_dev_assign() argument
264 rcu_assign_pointer(po->cached_dev, dev); in packet_cached_dev_assign()
267 static void packet_cached_dev_reset(struct packet_sock *po) in packet_cached_dev_reset() argument
269 RCU_INIT_POINTER(po->cached_dev, NULL); in packet_cached_dev_reset()
272 static bool packet_use_direct_xmit(const struct packet_sock *po) in packet_use_direct_xmit() argument
274 return po->xmit == packet_direct_xmit; in packet_use_direct_xmit()
304 struct packet_sock *po = pkt_sk(sk); in __register_prot_hook() local
306 if (!po->running) { in __register_prot_hook()
307 if (po->fanout) in __register_prot_hook()
308 __fanout_link(sk, po); in __register_prot_hook()
310 dev_add_pack(&po->prot_hook); in __register_prot_hook()
313 po->running = 1; in __register_prot_hook()
331 struct packet_sock *po = pkt_sk(sk); in __unregister_prot_hook() local
333 lockdep_assert_held_once(&po->bind_lock); in __unregister_prot_hook()
335 po->running = 0; in __unregister_prot_hook()
337 if (po->fanout) in __unregister_prot_hook()
338 __fanout_unlink(sk, po); in __unregister_prot_hook()
340 __dev_remove_pack(&po->prot_hook); in __unregister_prot_hook()
345 spin_unlock(&po->bind_lock); in __unregister_prot_hook()
347 spin_lock(&po->bind_lock); in __unregister_prot_hook()
353 struct packet_sock *po = pkt_sk(sk); in unregister_prot_hook() local
355 if (po->running) in unregister_prot_hook()
366 static void __packet_set_status(struct packet_sock *po, void *frame, int status) in __packet_set_status() argument
371 switch (po->tp_version) { in __packet_set_status()
392 static int __packet_get_status(const struct packet_sock *po, void *frame) in __packet_get_status() argument
399 switch (po->tp_version) { in __packet_get_status()
433 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame, in __packet_set_timestamp() argument
440 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in __packet_set_timestamp()
451 switch (po->tp_version) { in __packet_set_timestamp()
476 static void *packet_lookup_frame(const struct packet_sock *po, in packet_lookup_frame() argument
490 if (status != __packet_get_status(po, h.raw)) in packet_lookup_frame()
496 static void *packet_current_frame(struct packet_sock *po, in packet_current_frame() argument
500 return packet_lookup_frame(po, rb, rb->head, status); in packet_current_frame()
508 static void prb_shutdown_retire_blk_timer(struct packet_sock *po, in prb_shutdown_retire_blk_timer() argument
513 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_shutdown_retire_blk_timer()
522 static void prb_setup_retire_blk_timer(struct packet_sock *po) in prb_setup_retire_blk_timer() argument
526 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_setup_retire_blk_timer()
532 static int prb_calc_retire_blk_tmo(struct packet_sock *po, in prb_calc_retire_blk_tmo() argument
541 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); in prb_calc_retire_blk_tmo()
575 static void init_prb_bdqc(struct packet_sock *po, in init_prb_bdqc() argument
591 p1->hdrlen = po->tp_hdrlen; in init_prb_bdqc()
592 p1->version = po->tp_version; in init_prb_bdqc()
594 po->stats.stats3.tp_freeze_q_cnt = 0; in init_prb_bdqc()
598 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po, in init_prb_bdqc()
606 prb_setup_retire_blk_timer(po); in init_prb_bdqc()
645 struct packet_sock *po = in prb_retire_rx_blk_timer_expired() local
646 from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer); in prb_retire_rx_blk_timer_expired()
647 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_retire_rx_blk_timer_expired()
651 spin_lock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
680 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO); in prb_retire_rx_blk_timer_expired()
681 if (!prb_dispatch_next_block(pkc, po)) in prb_retire_rx_blk_timer_expired()
713 spin_unlock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
761 struct packet_sock *po, unsigned int stat) in prb_close_block() argument
767 struct sock *sk = &po->sk; in prb_close_block()
769 if (atomic_read(&po->tp_drops)) in prb_close_block()
874 struct packet_sock *po) in prb_freeze_queue() argument
877 po->stats.stats3.tp_freeze_q_cnt++; in prb_freeze_queue()
889 struct packet_sock *po) in prb_dispatch_next_block() argument
900 prb_freeze_queue(pkc, po); in prb_dispatch_next_block()
914 struct packet_sock *po, unsigned int status) in prb_retire_current_block() argument
934 prb_close_block(pkc, pbd, po, status); in prb_retire_current_block()
1014 static void *__packet_lookup_frame_in_block(struct packet_sock *po, in __packet_lookup_frame_in_block() argument
1023 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in __packet_lookup_frame_in_block()
1058 prb_retire_current_block(pkc, po, 0); in __packet_lookup_frame_in_block()
1061 curr = (char *)prb_dispatch_next_block(pkc, po); in __packet_lookup_frame_in_block()
1075 static void *packet_current_rx_frame(struct packet_sock *po, in packet_current_rx_frame() argument
1080 switch (po->tp_version) { in packet_current_rx_frame()
1083 curr = packet_lookup_frame(po, &po->rx_ring, in packet_current_rx_frame()
1084 po->rx_ring.head, status); in packet_current_rx_frame()
1087 return __packet_lookup_frame_in_block(po, skb, len); in packet_current_rx_frame()
1095 static void *prb_lookup_block(const struct packet_sock *po, in prb_lookup_block() argument
1119 static void *__prb_previous_block(struct packet_sock *po, in __prb_previous_block() argument
1124 return prb_lookup_block(po, rb, previous, status); in __prb_previous_block()
1127 static void *packet_previous_rx_frame(struct packet_sock *po, in packet_previous_rx_frame() argument
1131 if (po->tp_version <= TPACKET_V2) in packet_previous_rx_frame()
1132 return packet_previous_frame(po, rb, status); in packet_previous_rx_frame()
1134 return __prb_previous_block(po, rb, status); in packet_previous_rx_frame()
1137 static void packet_increment_rx_head(struct packet_sock *po, in packet_increment_rx_head() argument
1140 switch (po->tp_version) { in packet_increment_rx_head()
1152 static void *packet_previous_frame(struct packet_sock *po, in packet_previous_frame() argument
1157 return packet_lookup_frame(po, rb, previous, status); in packet_previous_frame()
1190 static int packet_alloc_pending(struct packet_sock *po) in packet_alloc_pending() argument
1192 po->rx_ring.pending_refcnt = NULL; in packet_alloc_pending()
1194 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); in packet_alloc_pending()
1195 if (unlikely(po->tx_ring.pending_refcnt == NULL)) in packet_alloc_pending()
1201 static void packet_free_pending(struct packet_sock *po) in packet_free_pending() argument
1203 free_percpu(po->tx_ring.pending_refcnt); in packet_free_pending()
1211 static bool __tpacket_has_room(const struct packet_sock *po, int pow_off) in __tpacket_has_room() argument
1215 len = READ_ONCE(po->rx_ring.frame_max) + 1; in __tpacket_has_room()
1216 idx = READ_ONCE(po->rx_ring.head); in __tpacket_has_room()
1221 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_has_room()
1224 static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off) in __tpacket_v3_has_room() argument
1228 len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks); in __tpacket_v3_has_room()
1229 idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num); in __tpacket_v3_has_room()
1234 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_v3_has_room()
1237 static int __packet_rcv_has_room(const struct packet_sock *po, in __packet_rcv_has_room() argument
1240 const struct sock *sk = &po->sk; in __packet_rcv_has_room()
1243 if (po->prot_hook.func != tpacket_rcv) { in __packet_rcv_has_room()
1256 if (po->tp_version == TPACKET_V3) { in __packet_rcv_has_room()
1257 if (__tpacket_v3_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1259 else if (__tpacket_v3_has_room(po, 0)) in __packet_rcv_has_room()
1262 if (__tpacket_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1264 else if (__tpacket_has_room(po, 0)) in __packet_rcv_has_room()
1271 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) in packet_rcv_has_room() argument
1275 ret = __packet_rcv_has_room(po, skb); in packet_rcv_has_room()
1278 if (READ_ONCE(po->pressure) != pressure) in packet_rcv_has_room()
1279 WRITE_ONCE(po->pressure, pressure); in packet_rcv_has_room()
1284 static void packet_rcv_try_clear_pressure(struct packet_sock *po) in packet_rcv_try_clear_pressure() argument
1286 if (READ_ONCE(po->pressure) && in packet_rcv_try_clear_pressure()
1287 __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) in packet_rcv_try_clear_pressure()
1288 WRITE_ONCE(po->pressure, 0); in packet_rcv_try_clear_pressure()
1306 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb) in fanout_flow_is_huge() argument
1308 u32 *history = po->rollover->history; in fanout_flow_is_huge()
1361 struct packet_sock *po, *po_next, *po_skip = NULL; in fanout_demux_rollover() local
1364 po = pkt_sk(rcu_dereference(f->arr[idx])); in fanout_demux_rollover()
1367 room = packet_rcv_has_room(po, skb); in fanout_demux_rollover()
1369 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb))) in fanout_demux_rollover()
1371 po_skip = po; in fanout_demux_rollover()
1374 i = j = min_t(int, po->rollover->sock, num - 1); in fanout_demux_rollover()
1380 po->rollover->sock = i; in fanout_demux_rollover()
1381 atomic_long_inc(&po->rollover->num); in fanout_demux_rollover()
1383 atomic_long_inc(&po->rollover->num_huge); in fanout_demux_rollover()
1391 atomic_long_inc(&po->rollover->num_failed); in fanout_demux_rollover()
1429 struct packet_sock *po; in packet_rcv_fanout() local
1471 po = pkt_sk(rcu_dereference(f->arr[idx])); in packet_rcv_fanout()
1472 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); in packet_rcv_fanout()
1480 static void __fanout_link(struct sock *sk, struct packet_sock *po) in __fanout_link() argument
1482 struct packet_fanout *f = po->fanout; in __fanout_link()
1493 static void __fanout_unlink(struct sock *sk, struct packet_sock *po) in __fanout_unlink() argument
1495 struct packet_fanout *f = po->fanout; in __fanout_unlink()
1550 static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data, in fanout_set_data_cbpf() argument
1557 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_cbpf()
1568 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_cbpf()
1572 static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data, in fanout_set_data_ebpf() argument
1578 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_ebpf()
1589 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_ebpf()
1593 static int fanout_set_data(struct packet_sock *po, sockptr_t data, in fanout_set_data() argument
1596 switch (po->fanout->type) { in fanout_set_data()
1598 return fanout_set_data_cbpf(po, data, len); in fanout_set_data()
1600 return fanout_set_data_ebpf(po, data, len); in fanout_set_data()
1648 struct packet_sock *po = pkt_sk(sk); in fanout_add() local
1676 if (po->fanout) in fanout_add()
1737 match->prot_hook.type = po->prot_hook.type; in fanout_add()
1738 match->prot_hook.dev = po->prot_hook.dev; in fanout_add()
1747 spin_lock(&po->bind_lock); in fanout_add()
1748 if (po->running && in fanout_add()
1750 match->prot_hook.type == po->prot_hook.type && in fanout_add()
1751 match->prot_hook.dev == po->prot_hook.dev) { in fanout_add()
1754 __dev_remove_pack(&po->prot_hook); in fanout_add()
1755 po->fanout = match; in fanout_add()
1756 po->rollover = rollover; in fanout_add()
1759 __fanout_link(sk, po); in fanout_add()
1763 spin_unlock(&po->bind_lock); in fanout_add()
1783 struct packet_sock *po = pkt_sk(sk); in fanout_release() local
1787 f = po->fanout; in fanout_release()
1789 po->fanout = NULL; in fanout_release()
2075 struct packet_sock *po; in packet_rcv() local
2085 po = pkt_sk(sk); in packet_rcv()
2137 if (unlikely(po->origdev)) in packet_rcv()
2160 po->stats.stats1.tp_packets++; in packet_rcv()
2169 atomic_inc(&po->tp_drops); in packet_rcv()
2189 struct packet_sock *po; in tpacket_rcv() local
2216 po = pkt_sk(sk); in tpacket_rcv()
2237 if (__packet_rcv_has_room(po, skb) == ROOM_NONE) { in tpacket_rcv()
2238 atomic_inc(&po->tp_drops); in tpacket_rcv()
2253 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + in tpacket_rcv()
2254 po->tp_reserve; in tpacket_rcv()
2257 netoff = TPACKET_ALIGN(po->tp_hdrlen + in tpacket_rcv()
2259 po->tp_reserve; in tpacket_rcv()
2260 if (po->has_vnet_hdr) { in tpacket_rcv()
2267 atomic_inc(&po->tp_drops); in tpacket_rcv()
2270 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2271 if (macoff + snaplen > po->rx_ring.frame_size) { in tpacket_rcv()
2272 if (po->copy_thresh && in tpacket_rcv()
2283 snaplen = po->rx_ring.frame_size - macoff; in tpacket_rcv()
2290 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) { in tpacket_rcv()
2293 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff; in tpacket_rcv()
2299 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len; in tpacket_rcv()
2304 h.raw = packet_current_rx_frame(po, skb, in tpacket_rcv()
2309 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2310 slot_id = po->rx_ring.head; in tpacket_rcv()
2311 if (test_bit(slot_id, po->rx_ring.rx_owner_map)) in tpacket_rcv()
2313 __set_bit(slot_id, po->rx_ring.rx_owner_map); in tpacket_rcv()
2320 if (po->tp_version == TPACKET_V3) in tpacket_rcv()
2321 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
2325 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2326 packet_increment_rx_head(po, &po->rx_ring); in tpacket_rcv()
2333 if (atomic_read(&po->tp_drops)) in tpacket_rcv()
2337 po->stats.stats1.tp_packets++; in tpacket_rcv()
2350 po->tp_tstamp | SOF_TIMESTAMPING_SOFTWARE); in tpacket_rcv()
2356 switch (po->tp_version) { in tpacket_rcv()
2408 if (unlikely(po->origdev)) in tpacket_rcv()
2416 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2428 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2430 __packet_set_status(po, h.raw, status); in tpacket_rcv()
2431 __clear_bit(slot_id, po->rx_ring.rx_owner_map); in tpacket_rcv()
2434 } else if (po->tp_version == TPACKET_V3) { in tpacket_rcv()
2435 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
2452 atomic_inc(&po->tp_drops); in tpacket_rcv()
2462 struct packet_sock *po = pkt_sk(skb->sk); in tpacket_destruct_skb() local
2464 if (likely(po->tx_ring.pg_vec)) { in tpacket_destruct_skb()
2469 packet_dec_pending(&po->tx_ring); in tpacket_destruct_skb()
2471 ts = __packet_set_timestamp(po, ph, skb); in tpacket_destruct_skb()
2472 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); in tpacket_destruct_skb()
2474 if (!packet_read_pending(&po->tx_ring)) in tpacket_destruct_skb()
2475 complete(&po->skb_completion); in tpacket_destruct_skb()
2510 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, in tpacket_fill_skb() argument
2517 struct socket *sock = po->sk.sk_socket; in tpacket_fill_skb()
2525 skb->priority = po->sk.sk_priority; in tpacket_fill_skb()
2526 skb->mark = po->sk.sk_mark; in tpacket_fill_skb()
2563 refcount_add(to_write, &po->sk.sk_wmem_alloc); in tpacket_fill_skb()
2590 static int tpacket_parse_header(struct packet_sock *po, void *frame, in tpacket_parse_header() argument
2598 switch (po->tp_version) { in tpacket_parse_header()
2618 if (unlikely(po->tp_tx_has_off)) { in tpacket_parse_header()
2621 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2622 off_max = po->tx_ring.frame_size - tp_len; in tpacket_parse_header()
2623 if (po->sk.sk_type == SOCK_DGRAM) { in tpacket_parse_header()
2624 switch (po->tp_version) { in tpacket_parse_header()
2636 switch (po->tp_version) { in tpacket_parse_header()
2651 off = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2658 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) in tpacket_snd() argument
2677 mutex_lock(&po->pg_vec_lock); in tpacket_snd()
2682 if (unlikely(!po->tx_ring.pg_vec)) { in tpacket_snd()
2687 dev = packet_cached_dev_get(po); in tpacket_snd()
2688 proto = READ_ONCE(po->num); in tpacket_snd()
2698 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); in tpacket_snd()
2699 if (po->sk.sk_socket->type == SOCK_DGRAM) { in tpacket_snd()
2714 sockcm_init(&sockc, &po->sk); in tpacket_snd()
2716 err = sock_cmsg_send(&po->sk, msg, &sockc); in tpacket_snd()
2721 if (po->sk.sk_socket->type == SOCK_RAW) in tpacket_snd()
2723 size_max = po->tx_ring.frame_size in tpacket_snd()
2724 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); in tpacket_snd()
2726 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr) in tpacket_snd()
2729 reinit_completion(&po->skb_completion); in tpacket_snd()
2732 ph = packet_current_frame(po, &po->tx_ring, in tpacket_snd()
2736 timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT); in tpacket_snd()
2737 timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo); in tpacket_snd()
2748 tp_len = tpacket_parse_header(po, ph, size_max, &data); in tpacket_snd()
2755 if (po->has_vnet_hdr) { in tpacket_snd()
2768 skb = sock_alloc_send_skb(&po->sk, in tpacket_snd()
2779 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, in tpacket_snd()
2783 !po->has_vnet_hdr && in tpacket_snd()
2789 if (po->tp_loss) { in tpacket_snd()
2790 __packet_set_status(po, ph, in tpacket_snd()
2792 packet_increment_head(&po->tx_ring); in tpacket_snd()
2802 if (po->has_vnet_hdr) { in tpacket_snd()
2811 __packet_set_status(po, ph, TP_STATUS_SENDING); in tpacket_snd()
2812 packet_inc_pending(&po->tx_ring); in tpacket_snd()
2815 err = po->xmit(skb); in tpacket_snd()
2818 if (err && __packet_get_status(po, ph) == in tpacket_snd()
2830 packet_increment_head(&po->tx_ring); in tpacket_snd()
2839 (need_wait && packet_read_pending(&po->tx_ring)))); in tpacket_snd()
2845 __packet_set_status(po, ph, status); in tpacket_snd()
2850 mutex_unlock(&po->pg_vec_lock); in tpacket_snd()
2890 struct packet_sock *po = pkt_sk(sk); in packet_snd() local
2900 dev = packet_cached_dev_get(po); in packet_snd()
2901 proto = READ_ONCE(po->num); in packet_snd()
2935 if (po->has_vnet_hdr) { in packet_snd()
3017 err = po->xmit(skb); in packet_snd()
3036 struct packet_sock *po = pkt_sk(sk); in packet_sendmsg() local
3041 if (data_race(po->tx_ring.pg_vec)) in packet_sendmsg()
3042 return tpacket_snd(po, msg); in packet_sendmsg()
3055 struct packet_sock *po; in packet_release() local
3064 po = pkt_sk(sk); in packet_release()
3074 spin_lock(&po->bind_lock); in packet_release()
3076 packet_cached_dev_reset(po); in packet_release()
3078 if (po->prot_hook.dev) { in packet_release()
3079 dev_put(po->prot_hook.dev); in packet_release()
3080 po->prot_hook.dev = NULL; in packet_release()
3082 spin_unlock(&po->bind_lock); in packet_release()
3087 if (po->rx_ring.pg_vec) { in packet_release()
3092 if (po->tx_ring.pg_vec) { in packet_release()
3102 kfree(po->rollover); in packet_release()
3116 packet_free_pending(po); in packet_release()
3130 struct packet_sock *po = pkt_sk(sk); in packet_do_bind() local
3139 spin_lock(&po->bind_lock); in packet_do_bind()
3142 if (po->fanout) { in packet_do_bind()
3163 proto_curr = po->prot_hook.type; in packet_do_bind()
3164 dev_curr = po->prot_hook.dev; in packet_do_bind()
3169 if (po->running) { in packet_do_bind()
3174 WRITE_ONCE(po->num, 0); in packet_do_bind()
3177 dev_curr = po->prot_hook.dev; in packet_do_bind()
3183 BUG_ON(po->running); in packet_do_bind()
3184 WRITE_ONCE(po->num, proto); in packet_do_bind()
3185 po->prot_hook.type = proto; in packet_do_bind()
3189 po->prot_hook.dev = NULL; in packet_do_bind()
3190 WRITE_ONCE(po->ifindex, -1); in packet_do_bind()
3191 packet_cached_dev_reset(po); in packet_do_bind()
3193 po->prot_hook.dev = dev; in packet_do_bind()
3194 WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0); in packet_do_bind()
3195 packet_cached_dev_assign(po, dev); in packet_do_bind()
3213 spin_unlock(&po->bind_lock); in packet_do_bind()
3275 struct packet_sock *po; in packet_create() local
3298 po = pkt_sk(sk); in packet_create()
3299 init_completion(&po->skb_completion); in packet_create()
3301 po->num = proto; in packet_create()
3302 po->xmit = dev_queue_xmit; in packet_create()
3304 err = packet_alloc_pending(po); in packet_create()
3308 packet_cached_dev_reset(po); in packet_create()
3317 spin_lock_init(&po->bind_lock); in packet_create()
3318 mutex_init(&po->pg_vec_lock); in packet_create()
3319 po->rollover = NULL; in packet_create()
3320 po->prot_hook.func = packet_rcv; in packet_create()
3323 po->prot_hook.func = packet_rcv_spkt; in packet_create()
3325 po->prot_hook.af_packet_priv = sk; in packet_create()
3328 po->prot_hook.type = proto; in packet_create()
3520 struct packet_sock *po = pkt_sk(sk); in packet_getname() local
3527 ifindex = READ_ONCE(po->ifindex); in packet_getname()
3530 sll->sll_protocol = READ_ONCE(po->num); in packet_getname()
3594 struct packet_sock *po = pkt_sk(sk); in packet_mc_add() local
3616 for (ml = po->mclist; ml; ml = ml->next) { in packet_mc_add()
3634 i->next = po->mclist; in packet_mc_add()
3635 po->mclist = i; in packet_mc_add()
3638 po->mclist = i->next; in packet_mc_add()
3675 struct packet_sock *po = pkt_sk(sk); in packet_flush_mclist() local
3678 if (!po->mclist) in packet_flush_mclist()
3682 while ((ml = po->mclist) != NULL) { in packet_flush_mclist()
3685 po->mclist = ml->next; in packet_flush_mclist()
3699 struct packet_sock *po = pkt_sk(sk); in packet_setsockopt() local
3734 switch (po->tp_version) { in packet_setsockopt()
3785 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3788 po->tp_version = val; in packet_setsockopt()
3805 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3808 po->tp_reserve = val; in packet_setsockopt()
3824 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3827 po->tp_loss = !!val; in packet_setsockopt()
3843 po->auxdata = !!val; in packet_setsockopt()
3857 po->origdev = !!val; in packet_setsockopt()
3873 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3876 po->has_vnet_hdr = !!val; in packet_setsockopt()
3891 po->tp_tstamp = val; in packet_setsockopt()
3907 if (!po->fanout) in packet_setsockopt()
3910 return fanout_set_data(po, optval, optlen); in packet_setsockopt()
3923 po->prot_hook.ignore_outgoing = !!val; in packet_setsockopt()
3936 if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec) in packet_setsockopt()
3937 po->tp_tx_has_off = !!val; in packet_setsockopt()
3951 po->xmit = val ? packet_direct_xmit : dev_queue_xmit; in packet_setsockopt()
3965 struct packet_sock *po = pkt_sk(sk); in packet_getsockopt() local
3983 memcpy(&st, &po->stats, sizeof(st)); in packet_getsockopt()
3984 memset(&po->stats, 0, sizeof(po->stats)); in packet_getsockopt()
3986 drops = atomic_xchg(&po->tp_drops, 0); in packet_getsockopt()
3988 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
4002 val = po->auxdata; in packet_getsockopt()
4005 val = po->origdev; in packet_getsockopt()
4008 val = po->has_vnet_hdr; in packet_getsockopt()
4011 val = po->tp_version; in packet_getsockopt()
4035 val = po->tp_reserve; in packet_getsockopt()
4038 val = po->tp_loss; in packet_getsockopt()
4041 val = po->tp_tstamp; in packet_getsockopt()
4044 val = (po->fanout ? in packet_getsockopt()
4045 ((u32)po->fanout->id | in packet_getsockopt()
4046 ((u32)po->fanout->type << 16) | in packet_getsockopt()
4047 ((u32)po->fanout->flags << 24)) : in packet_getsockopt()
4051 val = po->prot_hook.ignore_outgoing; in packet_getsockopt()
4054 if (!po->rollover) in packet_getsockopt()
4056 rstats.tp_all = atomic_long_read(&po->rollover->num); in packet_getsockopt()
4057 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge); in packet_getsockopt()
4058 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed); in packet_getsockopt()
4063 val = po->tp_tx_has_off; in packet_getsockopt()
4066 val = packet_use_direct_xmit(po); in packet_getsockopt()
4090 struct packet_sock *po = pkt_sk(sk); in packet_notifier() local
4094 if (po->mclist) in packet_notifier()
4095 packet_dev_mclist_delete(dev, &po->mclist); in packet_notifier()
4099 if (dev->ifindex == po->ifindex) { in packet_notifier()
4100 spin_lock(&po->bind_lock); in packet_notifier()
4101 if (po->running) { in packet_notifier()
4108 packet_cached_dev_reset(po); in packet_notifier()
4109 WRITE_ONCE(po->ifindex, -1); in packet_notifier()
4110 dev_put(po->prot_hook.dev); in packet_notifier()
4111 po->prot_hook.dev = NULL; in packet_notifier()
4113 spin_unlock(&po->bind_lock); in packet_notifier()
4117 if (dev->ifindex == po->ifindex) { in packet_notifier()
4118 spin_lock(&po->bind_lock); in packet_notifier()
4119 if (po->num) in packet_notifier()
4121 spin_unlock(&po->bind_lock); in packet_notifier()
4183 struct packet_sock *po = pkt_sk(sk); in packet_poll() local
4187 if (po->rx_ring.pg_vec) { in packet_poll()
4188 if (!packet_previous_rx_frame(po, &po->rx_ring, in packet_poll()
4192 packet_rcv_try_clear_pressure(po); in packet_poll()
4195 if (po->tx_ring.pg_vec) { in packet_poll()
4196 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) in packet_poll()
4305 struct packet_sock *po = pkt_sk(sk); in packet_set_ring() local
4315 rb = tx_ring ? &po->tx_ring : &po->rx_ring; in packet_set_ring()
4320 if (atomic_read(&po->mapped)) in packet_set_ring()
4334 switch (po->tp_version) { in packet_set_ring()
4336 po->tp_hdrlen = TPACKET_HDRLEN; in packet_set_ring()
4339 po->tp_hdrlen = TPACKET2_HDRLEN; in packet_set_ring()
4342 po->tp_hdrlen = TPACKET3_HDRLEN; in packet_set_ring()
4351 min_frame_size = po->tp_hdrlen + po->tp_reserve; in packet_set_ring()
4352 if (po->tp_version >= TPACKET_V3 && in packet_set_ring()
4375 switch (po->tp_version) { in packet_set_ring()
4379 init_prb_bdqc(po, rb, pg_vec, req_u); in packet_set_ring()
4410 spin_lock(&po->bind_lock); in packet_set_ring()
4411 was_running = po->running; in packet_set_ring()
4412 num = po->num; in packet_set_ring()
4414 WRITE_ONCE(po->num, 0); in packet_set_ring()
4417 spin_unlock(&po->bind_lock); in packet_set_ring()
4422 mutex_lock(&po->pg_vec_lock); in packet_set_ring()
4423 if (closing || atomic_read(&po->mapped) == 0) { in packet_set_ring()
4427 if (po->tp_version <= TPACKET_V2) in packet_set_ring()
4438 po->prot_hook.func = (po->rx_ring.pg_vec) ? in packet_set_ring()
4441 if (atomic_read(&po->mapped)) in packet_set_ring()
4443 atomic_read(&po->mapped)); in packet_set_ring()
4445 mutex_unlock(&po->pg_vec_lock); in packet_set_ring()
4447 spin_lock(&po->bind_lock); in packet_set_ring()
4449 WRITE_ONCE(po->num, num); in packet_set_ring()
4452 spin_unlock(&po->bind_lock); in packet_set_ring()
4453 if (pg_vec && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
4456 prb_shutdown_retire_blk_timer(po, rb_queue); in packet_set_ring()
4471 struct packet_sock *po = pkt_sk(sk); in packet_mmap() local
4481 mutex_lock(&po->pg_vec_lock); in packet_mmap()
4484 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4500 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4520 atomic_inc(&po->mapped); in packet_mmap()
4525 mutex_unlock(&po->pg_vec_lock); in packet_mmap()
4612 const struct packet_sock *po = pkt_sk(s); in packet_seq_show() local
4619 ntohs(READ_ONCE(po->num)), in packet_seq_show()
4620 READ_ONCE(po->ifindex), in packet_seq_show()
4621 po->running, in packet_seq_show()