Lines Matching refs:po
186 static void *packet_previous_frame(struct packet_sock *po,
235 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
236 static void __fanout_link(struct sock *sk, struct packet_sock *po);
243 static struct net_device *packet_cached_dev_get(struct packet_sock *po) in packet_cached_dev_get() argument
248 dev = rcu_dereference(po->cached_dev); in packet_cached_dev_get()
256 static void packet_cached_dev_assign(struct packet_sock *po, in packet_cached_dev_assign() argument
259 rcu_assign_pointer(po->cached_dev, dev); in packet_cached_dev_assign()
262 static void packet_cached_dev_reset(struct packet_sock *po) in packet_cached_dev_reset() argument
264 RCU_INIT_POINTER(po->cached_dev, NULL); in packet_cached_dev_reset()
267 static bool packet_use_direct_xmit(const struct packet_sock *po) in packet_use_direct_xmit() argument
269 return po->xmit == packet_direct_xmit; in packet_use_direct_xmit()
299 struct packet_sock *po = pkt_sk(sk); in __register_prot_hook() local
301 if (!po->running) { in __register_prot_hook()
302 if (po->fanout) in __register_prot_hook()
303 __fanout_link(sk, po); in __register_prot_hook()
305 dev_add_pack(&po->prot_hook); in __register_prot_hook()
308 po->running = 1; in __register_prot_hook()
326 struct packet_sock *po = pkt_sk(sk); in __unregister_prot_hook() local
328 lockdep_assert_held_once(&po->bind_lock); in __unregister_prot_hook()
330 po->running = 0; in __unregister_prot_hook()
332 if (po->fanout) in __unregister_prot_hook()
333 __fanout_unlink(sk, po); in __unregister_prot_hook()
335 __dev_remove_pack(&po->prot_hook); in __unregister_prot_hook()
340 spin_unlock(&po->bind_lock); in __unregister_prot_hook()
342 spin_lock(&po->bind_lock); in __unregister_prot_hook()
348 struct packet_sock *po = pkt_sk(sk); in unregister_prot_hook() local
350 if (po->running) in unregister_prot_hook()
361 static void __packet_set_status(struct packet_sock *po, void *frame, int status) in __packet_set_status() argument
366 switch (po->tp_version) { in __packet_set_status()
387 static int __packet_get_status(const struct packet_sock *po, void *frame) in __packet_get_status() argument
394 switch (po->tp_version) { in __packet_get_status()
427 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame, in __packet_set_timestamp() argument
434 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in __packet_set_timestamp()
438 switch (po->tp_version) { in __packet_set_timestamp()
463 static void *packet_lookup_frame(const struct packet_sock *po, in packet_lookup_frame() argument
477 if (status != __packet_get_status(po, h.raw)) in packet_lookup_frame()
483 static void *packet_current_frame(struct packet_sock *po, in packet_current_frame() argument
487 return packet_lookup_frame(po, rb, rb->head, status); in packet_current_frame()
495 static void prb_shutdown_retire_blk_timer(struct packet_sock *po, in prb_shutdown_retire_blk_timer() argument
500 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_shutdown_retire_blk_timer()
509 static void prb_setup_retire_blk_timer(struct packet_sock *po) in prb_setup_retire_blk_timer() argument
513 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_setup_retire_blk_timer()
519 static int prb_calc_retire_blk_tmo(struct packet_sock *po, in prb_calc_retire_blk_tmo() argument
528 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); in prb_calc_retire_blk_tmo()
567 static void init_prb_bdqc(struct packet_sock *po, in init_prb_bdqc() argument
583 p1->hdrlen = po->tp_hdrlen; in init_prb_bdqc()
584 p1->version = po->tp_version; in init_prb_bdqc()
586 po->stats.stats3.tp_freeze_q_cnt = 0; in init_prb_bdqc()
590 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po, in init_prb_bdqc()
597 prb_setup_retire_blk_timer(po); in init_prb_bdqc()
636 struct packet_sock *po = in prb_retire_rx_blk_timer_expired() local
637 from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer); in prb_retire_rx_blk_timer_expired()
638 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_retire_rx_blk_timer_expired()
642 spin_lock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
672 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO); in prb_retire_rx_blk_timer_expired()
673 if (!prb_dispatch_next_block(pkc, po)) in prb_retire_rx_blk_timer_expired()
705 spin_unlock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
753 struct packet_sock *po, unsigned int stat) in prb_close_block() argument
759 struct sock *sk = &po->sk; in prb_close_block()
761 if (atomic_read(&po->tp_drops)) in prb_close_block()
866 struct packet_sock *po) in prb_freeze_queue() argument
869 po->stats.stats3.tp_freeze_q_cnt++; in prb_freeze_queue()
881 struct packet_sock *po) in prb_dispatch_next_block() argument
892 prb_freeze_queue(pkc, po); in prb_dispatch_next_block()
906 struct packet_sock *po, unsigned int status) in prb_retire_current_block() argument
927 prb_close_block(pkc, pbd, po, status); in prb_retire_current_block()
1004 static void *__packet_lookup_frame_in_block(struct packet_sock *po, in __packet_lookup_frame_in_block() argument
1013 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in __packet_lookup_frame_in_block()
1048 prb_retire_current_block(pkc, po, 0); in __packet_lookup_frame_in_block()
1051 curr = (char *)prb_dispatch_next_block(pkc, po); in __packet_lookup_frame_in_block()
1065 static void *packet_current_rx_frame(struct packet_sock *po, in packet_current_rx_frame() argument
1070 switch (po->tp_version) { in packet_current_rx_frame()
1073 curr = packet_lookup_frame(po, &po->rx_ring, in packet_current_rx_frame()
1074 po->rx_ring.head, status); in packet_current_rx_frame()
1077 return __packet_lookup_frame_in_block(po, skb, len); in packet_current_rx_frame()
1085 static void *prb_lookup_block(const struct packet_sock *po, in prb_lookup_block() argument
1109 static void *__prb_previous_block(struct packet_sock *po, in __prb_previous_block() argument
1114 return prb_lookup_block(po, rb, previous, status); in __prb_previous_block()
1117 static void *packet_previous_rx_frame(struct packet_sock *po, in packet_previous_rx_frame() argument
1121 if (po->tp_version <= TPACKET_V2) in packet_previous_rx_frame()
1122 return packet_previous_frame(po, rb, status); in packet_previous_rx_frame()
1124 return __prb_previous_block(po, rb, status); in packet_previous_rx_frame()
1127 static void packet_increment_rx_head(struct packet_sock *po, in packet_increment_rx_head() argument
1130 switch (po->tp_version) { in packet_increment_rx_head()
1142 static void *packet_previous_frame(struct packet_sock *po, in packet_previous_frame() argument
1147 return packet_lookup_frame(po, rb, previous, status); in packet_previous_frame()
1180 static int packet_alloc_pending(struct packet_sock *po) in packet_alloc_pending() argument
1182 po->rx_ring.pending_refcnt = NULL; in packet_alloc_pending()
1184 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); in packet_alloc_pending()
1185 if (unlikely(po->tx_ring.pending_refcnt == NULL)) in packet_alloc_pending()
1191 static void packet_free_pending(struct packet_sock *po) in packet_free_pending() argument
1193 free_percpu(po->tx_ring.pending_refcnt); in packet_free_pending()
1201 static bool __tpacket_has_room(const struct packet_sock *po, int pow_off) in __tpacket_has_room() argument
1205 len = READ_ONCE(po->rx_ring.frame_max) + 1; in __tpacket_has_room()
1206 idx = READ_ONCE(po->rx_ring.head); in __tpacket_has_room()
1211 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_has_room()
1214 static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off) in __tpacket_v3_has_room() argument
1218 len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks); in __tpacket_v3_has_room()
1219 idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num); in __tpacket_v3_has_room()
1224 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL); in __tpacket_v3_has_room()
1227 static int __packet_rcv_has_room(const struct packet_sock *po, in __packet_rcv_has_room() argument
1230 const struct sock *sk = &po->sk; in __packet_rcv_has_room()
1233 if (po->prot_hook.func != tpacket_rcv) { in __packet_rcv_has_room()
1246 if (po->tp_version == TPACKET_V3) { in __packet_rcv_has_room()
1247 if (__tpacket_v3_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1249 else if (__tpacket_v3_has_room(po, 0)) in __packet_rcv_has_room()
1252 if (__tpacket_has_room(po, ROOM_POW_OFF)) in __packet_rcv_has_room()
1254 else if (__tpacket_has_room(po, 0)) in __packet_rcv_has_room()
1261 static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) in packet_rcv_has_room() argument
1265 ret = __packet_rcv_has_room(po, skb); in packet_rcv_has_room()
1268 if (READ_ONCE(po->pressure) != pressure) in packet_rcv_has_room()
1269 WRITE_ONCE(po->pressure, pressure); in packet_rcv_has_room()
1274 static void packet_rcv_try_clear_pressure(struct packet_sock *po) in packet_rcv_try_clear_pressure() argument
1276 if (READ_ONCE(po->pressure) && in packet_rcv_try_clear_pressure()
1277 __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) in packet_rcv_try_clear_pressure()
1278 WRITE_ONCE(po->pressure, 0); in packet_rcv_try_clear_pressure()
1296 static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb) in fanout_flow_is_huge() argument
1303 if (po->rollover->history[i] == rxhash) in fanout_flow_is_huge()
1306 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash; in fanout_flow_is_huge()
1345 struct packet_sock *po, *po_next, *po_skip = NULL; in fanout_demux_rollover() local
1348 po = pkt_sk(f->arr[idx]); in fanout_demux_rollover()
1351 room = packet_rcv_has_room(po, skb); in fanout_demux_rollover()
1353 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb))) in fanout_demux_rollover()
1355 po_skip = po; in fanout_demux_rollover()
1358 i = j = min_t(int, po->rollover->sock, num - 1); in fanout_demux_rollover()
1364 po->rollover->sock = i; in fanout_demux_rollover()
1365 atomic_long_inc(&po->rollover->num); in fanout_demux_rollover()
1367 atomic_long_inc(&po->rollover->num_huge); in fanout_demux_rollover()
1375 atomic_long_inc(&po->rollover->num_failed); in fanout_demux_rollover()
1413 struct packet_sock *po; in packet_rcv_fanout() local
1455 po = pkt_sk(f->arr[idx]); in packet_rcv_fanout()
1456 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); in packet_rcv_fanout()
1464 static void __fanout_link(struct sock *sk, struct packet_sock *po) in __fanout_link() argument
1466 struct packet_fanout *f = po->fanout; in __fanout_link()
1477 static void __fanout_unlink(struct sock *sk, struct packet_sock *po) in __fanout_unlink() argument
1479 struct packet_fanout *f = po->fanout; in __fanout_unlink()
1531 static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data, in fanout_set_data_cbpf() argument
1538 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_cbpf()
1549 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_cbpf()
1553 static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data, in fanout_set_data_ebpf() argument
1559 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) in fanout_set_data_ebpf()
1570 __fanout_set_data_bpf(po->fanout, new); in fanout_set_data_ebpf()
1574 static int fanout_set_data(struct packet_sock *po, char __user *data, in fanout_set_data() argument
1577 switch (po->fanout->type) { in fanout_set_data()
1579 return fanout_set_data_cbpf(po, data, len); in fanout_set_data()
1581 return fanout_set_data_ebpf(po, data, len); in fanout_set_data()
1629 struct packet_sock *po = pkt_sk(sk); in fanout_add() local
1654 if (po->fanout) in fanout_add()
1705 match->prot_hook.type = po->prot_hook.type; in fanout_add()
1706 match->prot_hook.dev = po->prot_hook.dev; in fanout_add()
1714 spin_lock(&po->bind_lock); in fanout_add()
1715 if (po->running && in fanout_add()
1717 match->prot_hook.type == po->prot_hook.type && in fanout_add()
1718 match->prot_hook.dev == po->prot_hook.dev) { in fanout_add()
1721 __dev_remove_pack(&po->prot_hook); in fanout_add()
1722 po->fanout = match; in fanout_add()
1723 po->rollover = rollover; in fanout_add()
1726 __fanout_link(sk, po); in fanout_add()
1730 spin_unlock(&po->bind_lock); in fanout_add()
1750 struct packet_sock *po = pkt_sk(sk); in fanout_release() local
1754 f = po->fanout; in fanout_release()
1756 po->fanout = NULL; in fanout_release()
2042 struct packet_sock *po; in packet_rcv() local
2052 po = pkt_sk(sk); in packet_rcv()
2104 if (unlikely(po->origdev)) in packet_rcv()
2127 po->stats.stats1.tp_packets++; in packet_rcv()
2136 atomic_inc(&po->tp_drops); in packet_rcv()
2156 struct packet_sock *po; in tpacket_rcv() local
2181 po = pkt_sk(sk); in tpacket_rcv()
2202 if (__packet_rcv_has_room(po, skb) == ROOM_NONE) { in tpacket_rcv()
2203 atomic_inc(&po->tp_drops); in tpacket_rcv()
2218 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + in tpacket_rcv()
2219 po->tp_reserve; in tpacket_rcv()
2222 netoff = TPACKET_ALIGN(po->tp_hdrlen + in tpacket_rcv()
2224 po->tp_reserve; in tpacket_rcv()
2225 if (po->has_vnet_hdr) { in tpacket_rcv()
2231 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2232 if (macoff + snaplen > po->rx_ring.frame_size) { in tpacket_rcv()
2233 if (po->copy_thresh && in tpacket_rcv()
2244 snaplen = po->rx_ring.frame_size - macoff; in tpacket_rcv()
2251 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) { in tpacket_rcv()
2254 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff; in tpacket_rcv()
2260 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len; in tpacket_rcv()
2265 h.raw = packet_current_rx_frame(po, skb, in tpacket_rcv()
2269 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2270 packet_increment_rx_head(po, &po->rx_ring); in tpacket_rcv()
2277 if (atomic_read(&po->tp_drops)) in tpacket_rcv()
2287 po->stats.stats1.tp_packets++; in tpacket_rcv()
2296 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in tpacket_rcv()
2301 switch (po->tp_version) { in tpacket_rcv()
2353 if (unlikely(po->origdev)) in tpacket_rcv()
2361 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2373 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2374 __packet_set_status(po, h.raw, status); in tpacket_rcv()
2377 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
2394 atomic_inc(&po->tp_drops); in tpacket_rcv()
2404 struct packet_sock *po = pkt_sk(skb->sk); in tpacket_destruct_skb() local
2406 if (likely(po->tx_ring.pg_vec)) { in tpacket_destruct_skb()
2411 packet_dec_pending(&po->tx_ring); in tpacket_destruct_skb()
2413 ts = __packet_set_timestamp(po, ph, skb); in tpacket_destruct_skb()
2414 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); in tpacket_destruct_skb()
2416 if (!packet_read_pending(&po->tx_ring)) in tpacket_destruct_skb()
2417 complete(&po->skb_completion); in tpacket_destruct_skb()
2452 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, in tpacket_fill_skb() argument
2459 struct socket *sock = po->sk.sk_socket; in tpacket_fill_skb()
2467 skb->priority = po->sk.sk_priority; in tpacket_fill_skb()
2468 skb->mark = po->sk.sk_mark; in tpacket_fill_skb()
2505 refcount_add(to_write, &po->sk.sk_wmem_alloc); in tpacket_fill_skb()
2532 static int tpacket_parse_header(struct packet_sock *po, void *frame, in tpacket_parse_header() argument
2540 switch (po->tp_version) { in tpacket_parse_header()
2560 if (unlikely(po->tp_tx_has_off)) { in tpacket_parse_header()
2563 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2564 off_max = po->tx_ring.frame_size - tp_len; in tpacket_parse_header()
2565 if (po->sk.sk_type == SOCK_DGRAM) { in tpacket_parse_header()
2566 switch (po->tp_version) { in tpacket_parse_header()
2578 switch (po->tp_version) { in tpacket_parse_header()
2593 off = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_parse_header()
2600 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) in tpacket_snd() argument
2619 mutex_lock(&po->pg_vec_lock); in tpacket_snd()
2624 if (unlikely(!po->tx_ring.pg_vec)) { in tpacket_snd()
2629 dev = packet_cached_dev_get(po); in tpacket_snd()
2630 proto = po->num; in tpacket_snd()
2640 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); in tpacket_snd()
2641 if (po->sk.sk_socket->type == SOCK_DGRAM) { in tpacket_snd()
2656 sockcm_init(&sockc, &po->sk); in tpacket_snd()
2658 err = sock_cmsg_send(&po->sk, msg, &sockc); in tpacket_snd()
2663 if (po->sk.sk_socket->type == SOCK_RAW) in tpacket_snd()
2665 size_max = po->tx_ring.frame_size in tpacket_snd()
2666 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); in tpacket_snd()
2668 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr) in tpacket_snd()
2671 reinit_completion(&po->skb_completion); in tpacket_snd()
2674 ph = packet_current_frame(po, &po->tx_ring, in tpacket_snd()
2678 timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT); in tpacket_snd()
2679 timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo); in tpacket_snd()
2690 tp_len = tpacket_parse_header(po, ph, size_max, &data); in tpacket_snd()
2697 if (po->has_vnet_hdr) { in tpacket_snd()
2710 skb = sock_alloc_send_skb(&po->sk, in tpacket_snd()
2721 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, in tpacket_snd()
2725 !po->has_vnet_hdr && in tpacket_snd()
2731 if (po->tp_loss) { in tpacket_snd()
2732 __packet_set_status(po, ph, in tpacket_snd()
2734 packet_increment_head(&po->tx_ring); in tpacket_snd()
2744 if (po->has_vnet_hdr) { in tpacket_snd()
2753 __packet_set_status(po, ph, TP_STATUS_SENDING); in tpacket_snd()
2754 packet_inc_pending(&po->tx_ring); in tpacket_snd()
2757 err = po->xmit(skb); in tpacket_snd()
2760 if (err && __packet_get_status(po, ph) == in tpacket_snd()
2772 packet_increment_head(&po->tx_ring); in tpacket_snd()
2781 (need_wait && packet_read_pending(&po->tx_ring)))); in tpacket_snd()
2787 __packet_set_status(po, ph, status); in tpacket_snd()
2792 mutex_unlock(&po->pg_vec_lock); in tpacket_snd()
2832 struct packet_sock *po = pkt_sk(sk); in packet_snd() local
2842 dev = packet_cached_dev_get(po); in packet_snd()
2843 proto = po->num; in packet_snd()
2877 if (po->has_vnet_hdr) { in packet_snd()
2959 err = po->xmit(skb); in packet_snd()
2979 struct packet_sock *po = pkt_sk(sk); in packet_sendmsg() local
2981 if (po->tx_ring.pg_vec) in packet_sendmsg()
2982 return tpacket_snd(po, msg); in packet_sendmsg()
2995 struct packet_sock *po; in packet_release() local
3004 po = pkt_sk(sk); in packet_release()
3014 spin_lock(&po->bind_lock); in packet_release()
3016 packet_cached_dev_reset(po); in packet_release()
3018 if (po->prot_hook.dev) { in packet_release()
3019 dev_put(po->prot_hook.dev); in packet_release()
3020 po->prot_hook.dev = NULL; in packet_release()
3022 spin_unlock(&po->bind_lock); in packet_release()
3027 if (po->rx_ring.pg_vec) { in packet_release()
3032 if (po->tx_ring.pg_vec) { in packet_release()
3042 kfree(po->rollover); in packet_release()
3056 packet_free_pending(po); in packet_release()
3070 struct packet_sock *po = pkt_sk(sk); in packet_do_bind() local
3079 spin_lock(&po->bind_lock); in packet_do_bind()
3082 if (po->fanout) { in packet_do_bind()
3104 proto_curr = po->prot_hook.type; in packet_do_bind()
3105 dev_curr = po->prot_hook.dev; in packet_do_bind()
3110 if (po->running) { in packet_do_bind()
3115 po->num = 0; in packet_do_bind()
3118 dev_curr = po->prot_hook.dev; in packet_do_bind()
3124 BUG_ON(po->running); in packet_do_bind()
3125 po->num = proto; in packet_do_bind()
3126 po->prot_hook.type = proto; in packet_do_bind()
3130 po->prot_hook.dev = NULL; in packet_do_bind()
3131 po->ifindex = -1; in packet_do_bind()
3132 packet_cached_dev_reset(po); in packet_do_bind()
3134 po->prot_hook.dev = dev; in packet_do_bind()
3135 po->ifindex = dev ? dev->ifindex : 0; in packet_do_bind()
3136 packet_cached_dev_assign(po, dev); in packet_do_bind()
3155 spin_unlock(&po->bind_lock); in packet_do_bind()
3217 struct packet_sock *po; in packet_create() local
3240 po = pkt_sk(sk); in packet_create()
3241 init_completion(&po->skb_completion); in packet_create()
3243 po->num = proto; in packet_create()
3244 po->xmit = dev_queue_xmit; in packet_create()
3246 err = packet_alloc_pending(po); in packet_create()
3250 packet_cached_dev_reset(po); in packet_create()
3259 spin_lock_init(&po->bind_lock); in packet_create()
3260 mutex_init(&po->pg_vec_lock); in packet_create()
3261 po->rollover = NULL; in packet_create()
3262 po->prot_hook.func = packet_rcv; in packet_create()
3265 po->prot_hook.func = packet_rcv_spkt; in packet_create()
3267 po->prot_hook.af_packet_priv = sk; in packet_create()
3270 po->prot_hook.type = proto; in packet_create()
3462 struct packet_sock *po = pkt_sk(sk); in packet_getname() local
3469 sll->sll_ifindex = po->ifindex; in packet_getname()
3470 sll->sll_protocol = po->num; in packet_getname()
3473 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex); in packet_getname()
3534 struct packet_sock *po = pkt_sk(sk); in packet_mc_add() local
3556 for (ml = po->mclist; ml; ml = ml->next) { in packet_mc_add()
3574 i->next = po->mclist; in packet_mc_add()
3575 po->mclist = i; in packet_mc_add()
3578 po->mclist = i->next; in packet_mc_add()
3615 struct packet_sock *po = pkt_sk(sk); in packet_flush_mclist() local
3618 if (!po->mclist) in packet_flush_mclist()
3622 while ((ml = po->mclist) != NULL) { in packet_flush_mclist()
3625 po->mclist = ml->next; in packet_flush_mclist()
3638 struct packet_sock *po = pkt_sk(sk); in packet_setsockopt() local
3673 switch (po->tp_version) { in packet_setsockopt()
3724 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3727 po->tp_version = val; in packet_setsockopt()
3744 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3747 po->tp_reserve = val; in packet_setsockopt()
3763 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3766 po->tp_loss = !!val; in packet_setsockopt()
3782 po->auxdata = !!val; in packet_setsockopt()
3796 po->origdev = !!val; in packet_setsockopt()
3812 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3815 po->has_vnet_hdr = !!val; in packet_setsockopt()
3830 po->tp_tstamp = val; in packet_setsockopt()
3846 if (!po->fanout) in packet_setsockopt()
3849 return fanout_set_data(po, optval, optlen); in packet_setsockopt()
3862 po->prot_hook.ignore_outgoing = !!val; in packet_setsockopt()
3875 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { in packet_setsockopt()
3878 po->tp_tx_has_off = !!val; in packet_setsockopt()
3893 po->xmit = val ? packet_direct_xmit : dev_queue_xmit; in packet_setsockopt()
3907 struct packet_sock *po = pkt_sk(sk); in packet_getsockopt() local
3925 memcpy(&st, &po->stats, sizeof(st)); in packet_getsockopt()
3926 memset(&po->stats, 0, sizeof(po->stats)); in packet_getsockopt()
3928 drops = atomic_xchg(&po->tp_drops, 0); in packet_getsockopt()
3930 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
3944 val = po->auxdata; in packet_getsockopt()
3947 val = po->origdev; in packet_getsockopt()
3950 val = po->has_vnet_hdr; in packet_getsockopt()
3953 val = po->tp_version; in packet_getsockopt()
3977 val = po->tp_reserve; in packet_getsockopt()
3980 val = po->tp_loss; in packet_getsockopt()
3983 val = po->tp_tstamp; in packet_getsockopt()
3986 val = (po->fanout ? in packet_getsockopt()
3987 ((u32)po->fanout->id | in packet_getsockopt()
3988 ((u32)po->fanout->type << 16) | in packet_getsockopt()
3989 ((u32)po->fanout->flags << 24)) : in packet_getsockopt()
3993 val = po->prot_hook.ignore_outgoing; in packet_getsockopt()
3996 if (!po->rollover) in packet_getsockopt()
3998 rstats.tp_all = atomic_long_read(&po->rollover->num); in packet_getsockopt()
3999 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge); in packet_getsockopt()
4000 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed); in packet_getsockopt()
4005 val = po->tp_tx_has_off; in packet_getsockopt()
4008 val = packet_use_direct_xmit(po); in packet_getsockopt()
4028 struct packet_sock *po = pkt_sk(sock->sk); in compat_packet_setsockopt() local
4034 po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) { in compat_packet_setsockopt()
4054 struct packet_sock *po = pkt_sk(sk); in packet_notifier() local
4058 if (po->mclist) in packet_notifier()
4059 packet_dev_mclist_delete(dev, &po->mclist); in packet_notifier()
4063 if (dev->ifindex == po->ifindex) { in packet_notifier()
4064 spin_lock(&po->bind_lock); in packet_notifier()
4065 if (po->running) { in packet_notifier()
4072 packet_cached_dev_reset(po); in packet_notifier()
4073 po->ifindex = -1; in packet_notifier()
4074 if (po->prot_hook.dev) in packet_notifier()
4075 dev_put(po->prot_hook.dev); in packet_notifier()
4076 po->prot_hook.dev = NULL; in packet_notifier()
4078 spin_unlock(&po->bind_lock); in packet_notifier()
4082 if (dev->ifindex == po->ifindex) { in packet_notifier()
4083 spin_lock(&po->bind_lock); in packet_notifier()
4084 if (po->num) in packet_notifier()
4086 spin_unlock(&po->bind_lock); in packet_notifier()
4148 struct packet_sock *po = pkt_sk(sk); in packet_poll() local
4152 if (po->rx_ring.pg_vec) { in packet_poll()
4153 if (!packet_previous_rx_frame(po, &po->rx_ring, in packet_poll()
4157 packet_rcv_try_clear_pressure(po); in packet_poll()
4160 if (po->tx_ring.pg_vec) { in packet_poll()
4161 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) in packet_poll()
4270 struct packet_sock *po = pkt_sk(sk); in packet_set_ring() local
4279 rb = tx_ring ? &po->tx_ring : &po->rx_ring; in packet_set_ring()
4284 if (atomic_read(&po->mapped)) in packet_set_ring()
4298 switch (po->tp_version) { in packet_set_ring()
4300 po->tp_hdrlen = TPACKET_HDRLEN; in packet_set_ring()
4303 po->tp_hdrlen = TPACKET2_HDRLEN; in packet_set_ring()
4306 po->tp_hdrlen = TPACKET3_HDRLEN; in packet_set_ring()
4315 min_frame_size = po->tp_hdrlen + po->tp_reserve; in packet_set_ring()
4316 if (po->tp_version >= TPACKET_V3 && in packet_set_ring()
4339 switch (po->tp_version) { in packet_set_ring()
4343 init_prb_bdqc(po, rb, pg_vec, req_u); in packet_set_ring()
4368 spin_lock(&po->bind_lock); in packet_set_ring()
4369 was_running = po->running; in packet_set_ring()
4370 num = po->num; in packet_set_ring()
4372 po->num = 0; in packet_set_ring()
4375 spin_unlock(&po->bind_lock); in packet_set_ring()
4380 mutex_lock(&po->pg_vec_lock); in packet_set_ring()
4381 if (closing || atomic_read(&po->mapped) == 0) { in packet_set_ring()
4394 po->prot_hook.func = (po->rx_ring.pg_vec) ? in packet_set_ring()
4397 if (atomic_read(&po->mapped)) in packet_set_ring()
4399 atomic_read(&po->mapped)); in packet_set_ring()
4401 mutex_unlock(&po->pg_vec_lock); in packet_set_ring()
4403 spin_lock(&po->bind_lock); in packet_set_ring()
4405 po->num = num; in packet_set_ring()
4408 spin_unlock(&po->bind_lock); in packet_set_ring()
4409 if (pg_vec && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
4412 prb_shutdown_retire_blk_timer(po, rb_queue); in packet_set_ring()
4426 struct packet_sock *po = pkt_sk(sk); in packet_mmap() local
4436 mutex_lock(&po->pg_vec_lock); in packet_mmap()
4439 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4455 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4475 atomic_inc(&po->mapped); in packet_mmap()
4480 mutex_unlock(&po->pg_vec_lock); in packet_mmap()
4570 const struct packet_sock *po = pkt_sk(s); in packet_seq_show() local
4577 ntohs(po->num), in packet_seq_show()
4578 po->ifindex, in packet_seq_show()
4579 po->running, in packet_seq_show()