1 /*
2 * Virtual network driver for conversing with remote driver backends.
3 *
4 * Copyright (c) 2002-2005, K A Fraser
5 * Copyright (c) 2005, XenSource Ltd
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version 2
9 * as published by the Free Software Foundation; or, when distributed
10 * separately from the Linux kernel or incorporated into other
11 * software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34 #include <linux/module.h>
35 #include <linux/kernel.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/skbuff.h>
39 #include <linux/ethtool.h>
40 #include <linux/if_ether.h>
41 #include <net/tcp.h>
42 #include <linux/udp.h>
43 #include <linux/moduleparam.h>
44 #include <linux/mm.h>
45 #include <linux/slab.h>
46 #include <net/ip.h>
47 #include <linux/bpf.h>
48 #include <net/page_pool.h>
49 #include <linux/bpf_trace.h>
50
51 #include <xen/xen.h>
52 #include <xen/xenbus.h>
53 #include <xen/events.h>
54 #include <xen/page.h>
55 #include <xen/platform_pci.h>
56 #include <xen/grant_table.h>
57
58 #include <xen/interface/io/netif.h>
59 #include <xen/interface/memory.h>
60 #include <xen/interface/grant_table.h>
61
62 /* Module parameters */
63 #define MAX_QUEUES_DEFAULT 8
64 static unsigned int xennet_max_queues;
65 module_param_named(max_queues, xennet_max_queues, uint, 0644);
66 MODULE_PARM_DESC(max_queues,
67 "Maximum number of queues per virtual interface");
68
69 #define XENNET_TIMEOUT (5 * HZ)
70
71 static const struct ethtool_ops xennet_ethtool_ops;
72
73 struct netfront_cb {
74 int pull_to;
75 };
76
77 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb))
78
79 #define RX_COPY_THRESHOLD 256
80
81 #define GRANT_INVALID_REF 0
82
83 #define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE)
84 #define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE)
85
86 /* Minimum number of Rx slots (includes slot for GSO metadata). */
87 #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1)
88
89 /* Queue name is interface name with "-qNNN" appended */
90 #define QUEUE_NAME_SIZE (IFNAMSIZ + 6)
91
92 /* IRQ name is queue name with "-tx" or "-rx" appended */
93 #define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3)
94
95 static DECLARE_WAIT_QUEUE_HEAD(module_wq);
96
97 struct netfront_stats {
98 u64 packets;
99 u64 bytes;
100 struct u64_stats_sync syncp;
101 };
102
103 struct netfront_info;
104
105 struct netfront_queue {
106 unsigned int id; /* Queue ID, 0-based */
107 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
108 struct netfront_info *info;
109
110 struct bpf_prog __rcu *xdp_prog;
111
112 struct napi_struct napi;
113
114 /* Split event channels support, tx_* == rx_* when using
115 * single event channel.
116 */
117 unsigned int tx_evtchn, rx_evtchn;
118 unsigned int tx_irq, rx_irq;
119 /* Only used when split event channels support is enabled */
120 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
121 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
122
123 spinlock_t tx_lock;
124 struct xen_netif_tx_front_ring tx;
125 int tx_ring_ref;
126
127 /*
128 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
129 * are linked from tx_skb_freelist through tx_link.
130 */
131 struct sk_buff *tx_skbs[NET_TX_RING_SIZE];
132 unsigned short tx_link[NET_TX_RING_SIZE];
133 #define TX_LINK_NONE 0xffff
134 #define TX_PENDING 0xfffe
135 grant_ref_t gref_tx_head;
136 grant_ref_t grant_tx_ref[NET_TX_RING_SIZE];
137 struct page *grant_tx_page[NET_TX_RING_SIZE];
138 unsigned tx_skb_freelist;
139 unsigned int tx_pend_queue;
140
141 spinlock_t rx_lock ____cacheline_aligned_in_smp;
142 struct xen_netif_rx_front_ring rx;
143 int rx_ring_ref;
144
145 struct timer_list rx_refill_timer;
146
147 struct sk_buff *rx_skbs[NET_RX_RING_SIZE];
148 grant_ref_t gref_rx_head;
149 grant_ref_t grant_rx_ref[NET_RX_RING_SIZE];
150
151 struct page_pool *page_pool;
152 struct xdp_rxq_info xdp_rxq;
153 };
154
155 struct netfront_info {
156 struct list_head list;
157 struct net_device *netdev;
158
159 struct xenbus_device *xbdev;
160
161 /* Multi-queue support */
162 struct netfront_queue *queues;
163
164 /* Statistics */
165 struct netfront_stats __percpu *rx_stats;
166 struct netfront_stats __percpu *tx_stats;
167
168 /* XDP state */
169 bool netback_has_xdp_headroom;
170 bool netfront_xdp_enabled;
171
172 /* Is device behaving sane? */
173 bool broken;
174
175 atomic_t rx_gso_checksum_fixup;
176 };
177
178 struct netfront_rx_info {
179 struct xen_netif_rx_response rx;
180 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
181 };
182
183 /*
184 * Access macros for acquiring freeing slots in tx_skbs[].
185 */
186
add_id_to_list(unsigned * head,unsigned short * list,unsigned short id)187 static void add_id_to_list(unsigned *head, unsigned short *list,
188 unsigned short id)
189 {
190 list[id] = *head;
191 *head = id;
192 }
193
get_id_from_list(unsigned * head,unsigned short * list)194 static unsigned short get_id_from_list(unsigned *head, unsigned short *list)
195 {
196 unsigned int id = *head;
197
198 if (id != TX_LINK_NONE) {
199 *head = list[id];
200 list[id] = TX_LINK_NONE;
201 }
202 return id;
203 }
204
xennet_rxidx(RING_IDX idx)205 static int xennet_rxidx(RING_IDX idx)
206 {
207 return idx & (NET_RX_RING_SIZE - 1);
208 }
209
xennet_get_rx_skb(struct netfront_queue * queue,RING_IDX ri)210 static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue,
211 RING_IDX ri)
212 {
213 int i = xennet_rxidx(ri);
214 struct sk_buff *skb = queue->rx_skbs[i];
215 queue->rx_skbs[i] = NULL;
216 return skb;
217 }
218
xennet_get_rx_ref(struct netfront_queue * queue,RING_IDX ri)219 static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue,
220 RING_IDX ri)
221 {
222 int i = xennet_rxidx(ri);
223 grant_ref_t ref = queue->grant_rx_ref[i];
224 queue->grant_rx_ref[i] = GRANT_INVALID_REF;
225 return ref;
226 }
227
228 #ifdef CONFIG_SYSFS
229 static const struct attribute_group xennet_dev_group;
230 #endif
231
xennet_can_sg(struct net_device * dev)232 static bool xennet_can_sg(struct net_device *dev)
233 {
234 return dev->features & NETIF_F_SG;
235 }
236
237
rx_refill_timeout(struct timer_list * t)238 static void rx_refill_timeout(struct timer_list *t)
239 {
240 struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer);
241 napi_schedule(&queue->napi);
242 }
243
netfront_tx_slot_available(struct netfront_queue * queue)244 static int netfront_tx_slot_available(struct netfront_queue *queue)
245 {
246 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) <
247 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1);
248 }
249
xennet_maybe_wake_tx(struct netfront_queue * queue)250 static void xennet_maybe_wake_tx(struct netfront_queue *queue)
251 {
252 struct net_device *dev = queue->info->netdev;
253 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id);
254
255 if (unlikely(netif_tx_queue_stopped(dev_queue)) &&
256 netfront_tx_slot_available(queue) &&
257 likely(netif_running(dev)))
258 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id));
259 }
260
261
xennet_alloc_one_rx_buffer(struct netfront_queue * queue)262 static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)
263 {
264 struct sk_buff *skb;
265 struct page *page;
266
267 skb = __netdev_alloc_skb(queue->info->netdev,
268 RX_COPY_THRESHOLD + NET_IP_ALIGN,
269 GFP_ATOMIC | __GFP_NOWARN);
270 if (unlikely(!skb))
271 return NULL;
272
273 page = page_pool_dev_alloc_pages(queue->page_pool);
274 if (unlikely(!page)) {
275 kfree_skb(skb);
276 return NULL;
277 }
278 skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE);
279
280 /* Align ip header to a 16 bytes boundary */
281 skb_reserve(skb, NET_IP_ALIGN);
282 skb->dev = queue->info->netdev;
283
284 return skb;
285 }
286
287
xennet_alloc_rx_buffers(struct netfront_queue * queue)288 static void xennet_alloc_rx_buffers(struct netfront_queue *queue)
289 {
290 RING_IDX req_prod = queue->rx.req_prod_pvt;
291 int notify;
292 int err = 0;
293
294 if (unlikely(!netif_carrier_ok(queue->info->netdev)))
295 return;
296
297 for (req_prod = queue->rx.req_prod_pvt;
298 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE;
299 req_prod++) {
300 struct sk_buff *skb;
301 unsigned short id;
302 grant_ref_t ref;
303 struct page *page;
304 struct xen_netif_rx_request *req;
305
306 skb = xennet_alloc_one_rx_buffer(queue);
307 if (!skb) {
308 err = -ENOMEM;
309 break;
310 }
311
312 id = xennet_rxidx(req_prod);
313
314 BUG_ON(queue->rx_skbs[id]);
315 queue->rx_skbs[id] = skb;
316
317 ref = gnttab_claim_grant_reference(&queue->gref_rx_head);
318 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
319 queue->grant_rx_ref[id] = ref;
320
321 page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
322
323 req = RING_GET_REQUEST(&queue->rx, req_prod);
324 gnttab_page_grant_foreign_access_ref_one(ref,
325 queue->info->xbdev->otherend_id,
326 page,
327 0);
328 req->id = id;
329 req->gref = ref;
330 }
331
332 queue->rx.req_prod_pvt = req_prod;
333
334 /* Try again later if there are not enough requests or skb allocation
335 * failed.
336 * Enough requests is quantified as the sum of newly created slots and
337 * the unconsumed slots at the backend.
338 */
339 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN ||
340 unlikely(err)) {
341 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10));
342 return;
343 }
344
345 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify);
346 if (notify)
347 notify_remote_via_irq(queue->rx_irq);
348 }
349
xennet_open(struct net_device * dev)350 static int xennet_open(struct net_device *dev)
351 {
352 struct netfront_info *np = netdev_priv(dev);
353 unsigned int num_queues = dev->real_num_tx_queues;
354 unsigned int i = 0;
355 struct netfront_queue *queue = NULL;
356
357 if (!np->queues || np->broken)
358 return -ENODEV;
359
360 for (i = 0; i < num_queues; ++i) {
361 queue = &np->queues[i];
362 napi_enable(&queue->napi);
363
364 spin_lock_bh(&queue->rx_lock);
365 if (netif_carrier_ok(dev)) {
366 xennet_alloc_rx_buffers(queue);
367 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1;
368 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))
369 napi_schedule(&queue->napi);
370 }
371 spin_unlock_bh(&queue->rx_lock);
372 }
373
374 netif_tx_start_all_queues(dev);
375
376 return 0;
377 }
378
xennet_tx_buf_gc(struct netfront_queue * queue)379 static void xennet_tx_buf_gc(struct netfront_queue *queue)
380 {
381 RING_IDX cons, prod;
382 unsigned short id;
383 struct sk_buff *skb;
384 bool more_to_do;
385 const struct device *dev = &queue->info->netdev->dev;
386
387 BUG_ON(!netif_carrier_ok(queue->info->netdev));
388
389 do {
390 prod = queue->tx.sring->rsp_prod;
391 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) {
392 dev_alert(dev, "Illegal number of responses %u\n",
393 prod - queue->tx.rsp_cons);
394 goto err;
395 }
396 rmb(); /* Ensure we see responses up to 'rp'. */
397
398 for (cons = queue->tx.rsp_cons; cons != prod; cons++) {
399 struct xen_netif_tx_response txrsp;
400
401 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp);
402 if (txrsp.status == XEN_NETIF_RSP_NULL)
403 continue;
404
405 id = txrsp.id;
406 if (id >= RING_SIZE(&queue->tx)) {
407 dev_alert(dev,
408 "Response has incorrect id (%u)\n",
409 id);
410 goto err;
411 }
412 if (queue->tx_link[id] != TX_PENDING) {
413 dev_alert(dev,
414 "Response for inactive request\n");
415 goto err;
416 }
417
418 queue->tx_link[id] = TX_LINK_NONE;
419 skb = queue->tx_skbs[id];
420 queue->tx_skbs[id] = NULL;
421 if (unlikely(gnttab_query_foreign_access(
422 queue->grant_tx_ref[id]) != 0)) {
423 dev_alert(dev,
424 "Grant still in use by backend domain\n");
425 goto err;
426 }
427 gnttab_end_foreign_access_ref(
428 queue->grant_tx_ref[id], GNTMAP_readonly);
429 gnttab_release_grant_reference(
430 &queue->gref_tx_head, queue->grant_tx_ref[id]);
431 queue->grant_tx_ref[id] = GRANT_INVALID_REF;
432 queue->grant_tx_page[id] = NULL;
433 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id);
434 dev_kfree_skb_irq(skb);
435 }
436
437 queue->tx.rsp_cons = prod;
438
439 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do);
440 } while (more_to_do);
441
442 xennet_maybe_wake_tx(queue);
443
444 return;
445
446 err:
447 queue->info->broken = true;
448 dev_alert(dev, "Disabled for further use\n");
449 }
450
451 struct xennet_gnttab_make_txreq {
452 struct netfront_queue *queue;
453 struct sk_buff *skb;
454 struct page *page;
455 struct xen_netif_tx_request *tx; /* Last request on ring page */
456 struct xen_netif_tx_request tx_local; /* Last request local copy*/
457 unsigned int size;
458 };
459
xennet_tx_setup_grant(unsigned long gfn,unsigned int offset,unsigned int len,void * data)460 static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset,
461 unsigned int len, void *data)
462 {
463 struct xennet_gnttab_make_txreq *info = data;
464 unsigned int id;
465 struct xen_netif_tx_request *tx;
466 grant_ref_t ref;
467 /* convenient aliases */
468 struct page *page = info->page;
469 struct netfront_queue *queue = info->queue;
470 struct sk_buff *skb = info->skb;
471
472 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link);
473 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
474 ref = gnttab_claim_grant_reference(&queue->gref_tx_head);
475 WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
476
477 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id,
478 gfn, GNTMAP_readonly);
479
480 queue->tx_skbs[id] = skb;
481 queue->grant_tx_page[id] = page;
482 queue->grant_tx_ref[id] = ref;
483
484 info->tx_local.id = id;
485 info->tx_local.gref = ref;
486 info->tx_local.offset = offset;
487 info->tx_local.size = len;
488 info->tx_local.flags = 0;
489
490 *tx = info->tx_local;
491
492 /*
493 * Put the request in the pending queue, it will be set to be pending
494 * when the producer index is about to be raised.
495 */
496 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id);
497
498 info->tx = tx;
499 info->size += info->tx_local.size;
500 }
501
xennet_make_first_txreq(struct xennet_gnttab_make_txreq * info,unsigned int offset,unsigned int len)502 static struct xen_netif_tx_request *xennet_make_first_txreq(
503 struct xennet_gnttab_make_txreq *info,
504 unsigned int offset, unsigned int len)
505 {
506 info->size = 0;
507
508 gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info);
509
510 return info->tx;
511 }
512
xennet_make_one_txreq(unsigned long gfn,unsigned int offset,unsigned int len,void * data)513 static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset,
514 unsigned int len, void *data)
515 {
516 struct xennet_gnttab_make_txreq *info = data;
517
518 info->tx->flags |= XEN_NETTXF_more_data;
519 skb_get(info->skb);
520 xennet_tx_setup_grant(gfn, offset, len, data);
521 }
522
xennet_make_txreqs(struct xennet_gnttab_make_txreq * info,struct page * page,unsigned int offset,unsigned int len)523 static void xennet_make_txreqs(
524 struct xennet_gnttab_make_txreq *info,
525 struct page *page,
526 unsigned int offset, unsigned int len)
527 {
528 /* Skip unused frames from start of page */
529 page += offset >> PAGE_SHIFT;
530 offset &= ~PAGE_MASK;
531
532 while (len) {
533 info->page = page;
534 info->size = 0;
535
536 gnttab_foreach_grant_in_range(page, offset, len,
537 xennet_make_one_txreq,
538 info);
539
540 page++;
541 offset = 0;
542 len -= info->size;
543 }
544 }
545
546 /*
547 * Count how many ring slots are required to send this skb. Each frag
548 * might be a compound page.
549 */
xennet_count_skb_slots(struct sk_buff * skb)550 static int xennet_count_skb_slots(struct sk_buff *skb)
551 {
552 int i, frags = skb_shinfo(skb)->nr_frags;
553 int slots;
554
555 slots = gnttab_count_grant(offset_in_page(skb->data),
556 skb_headlen(skb));
557
558 for (i = 0; i < frags; i++) {
559 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
560 unsigned long size = skb_frag_size(frag);
561 unsigned long offset = skb_frag_off(frag);
562
563 /* Skip unused frames from start of page */
564 offset &= ~PAGE_MASK;
565
566 slots += gnttab_count_grant(offset, size);
567 }
568
569 return slots;
570 }
571
xennet_select_queue(struct net_device * dev,struct sk_buff * skb,struct net_device * sb_dev)572 static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb,
573 struct net_device *sb_dev)
574 {
575 unsigned int num_queues = dev->real_num_tx_queues;
576 u32 hash;
577 u16 queue_idx;
578
579 /* First, check if there is only one queue */
580 if (num_queues == 1) {
581 queue_idx = 0;
582 } else {
583 hash = skb_get_hash(skb);
584 queue_idx = hash % num_queues;
585 }
586
587 return queue_idx;
588 }
589
xennet_mark_tx_pending(struct netfront_queue * queue)590 static void xennet_mark_tx_pending(struct netfront_queue *queue)
591 {
592 unsigned int i;
593
594 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) !=
595 TX_LINK_NONE)
596 queue->tx_link[i] = TX_PENDING;
597 }
598
xennet_xdp_xmit_one(struct net_device * dev,struct netfront_queue * queue,struct xdp_frame * xdpf)599 static int xennet_xdp_xmit_one(struct net_device *dev,
600 struct netfront_queue *queue,
601 struct xdp_frame *xdpf)
602 {
603 struct netfront_info *np = netdev_priv(dev);
604 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
605 struct xennet_gnttab_make_txreq info = {
606 .queue = queue,
607 .skb = NULL,
608 .page = virt_to_page(xdpf->data),
609 };
610 int notify;
611
612 xennet_make_first_txreq(&info,
613 offset_in_page(xdpf->data),
614 xdpf->len);
615
616 xennet_mark_tx_pending(queue);
617
618 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
619 if (notify)
620 notify_remote_via_irq(queue->tx_irq);
621
622 u64_stats_update_begin(&tx_stats->syncp);
623 tx_stats->bytes += xdpf->len;
624 tx_stats->packets++;
625 u64_stats_update_end(&tx_stats->syncp);
626
627 xennet_tx_buf_gc(queue);
628
629 return 0;
630 }
631
xennet_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)632 static int xennet_xdp_xmit(struct net_device *dev, int n,
633 struct xdp_frame **frames, u32 flags)
634 {
635 unsigned int num_queues = dev->real_num_tx_queues;
636 struct netfront_info *np = netdev_priv(dev);
637 struct netfront_queue *queue = NULL;
638 unsigned long irq_flags;
639 int nxmit = 0;
640 int i;
641
642 if (unlikely(np->broken))
643 return -ENODEV;
644 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
645 return -EINVAL;
646
647 queue = &np->queues[smp_processor_id() % num_queues];
648
649 spin_lock_irqsave(&queue->tx_lock, irq_flags);
650 for (i = 0; i < n; i++) {
651 struct xdp_frame *xdpf = frames[i];
652
653 if (!xdpf)
654 continue;
655 if (xennet_xdp_xmit_one(dev, queue, xdpf))
656 break;
657 nxmit++;
658 }
659 spin_unlock_irqrestore(&queue->tx_lock, irq_flags);
660
661 return nxmit;
662 }
663
664
665 #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
666
xennet_start_xmit(struct sk_buff * skb,struct net_device * dev)667 static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
668 {
669 struct netfront_info *np = netdev_priv(dev);
670 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
671 struct xen_netif_tx_request *first_tx;
672 unsigned int i;
673 int notify;
674 int slots;
675 struct page *page;
676 unsigned int offset;
677 unsigned int len;
678 unsigned long flags;
679 struct netfront_queue *queue = NULL;
680 struct xennet_gnttab_make_txreq info = { };
681 unsigned int num_queues = dev->real_num_tx_queues;
682 u16 queue_index;
683 struct sk_buff *nskb;
684
685 /* Drop the packet if no queues are set up */
686 if (num_queues < 1)
687 goto drop;
688 if (unlikely(np->broken))
689 goto drop;
690 /* Determine which queue to transmit this SKB on */
691 queue_index = skb_get_queue_mapping(skb);
692 queue = &np->queues[queue_index];
693
694 /* If skb->len is too big for wire format, drop skb and alert
695 * user about misconfiguration.
696 */
697 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) {
698 net_alert_ratelimited(
699 "xennet: skb->len = %u, too big for wire format\n",
700 skb->len);
701 goto drop;
702 }
703
704 slots = xennet_count_skb_slots(skb);
705 if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) {
706 net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n",
707 slots, skb->len);
708 if (skb_linearize(skb))
709 goto drop;
710 }
711
712 page = virt_to_page(skb->data);
713 offset = offset_in_page(skb->data);
714
715 /* The first req should be at least ETH_HLEN size or the packet will be
716 * dropped by netback.
717 */
718 if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
719 nskb = skb_copy(skb, GFP_ATOMIC);
720 if (!nskb)
721 goto drop;
722 dev_consume_skb_any(skb);
723 skb = nskb;
724 page = virt_to_page(skb->data);
725 offset = offset_in_page(skb->data);
726 }
727
728 len = skb_headlen(skb);
729
730 spin_lock_irqsave(&queue->tx_lock, flags);
731
732 if (unlikely(!netif_carrier_ok(dev) ||
733 (slots > 1 && !xennet_can_sg(dev)) ||
734 netif_needs_gso(skb, netif_skb_features(skb)))) {
735 spin_unlock_irqrestore(&queue->tx_lock, flags);
736 goto drop;
737 }
738
739 /* First request for the linear area. */
740 info.queue = queue;
741 info.skb = skb;
742 info.page = page;
743 first_tx = xennet_make_first_txreq(&info, offset, len);
744 offset += info.tx_local.size;
745 if (offset == PAGE_SIZE) {
746 page++;
747 offset = 0;
748 }
749 len -= info.tx_local.size;
750
751 if (skb->ip_summed == CHECKSUM_PARTIAL)
752 /* local packet? */
753 first_tx->flags |= XEN_NETTXF_csum_blank |
754 XEN_NETTXF_data_validated;
755 else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
756 /* remote but checksummed. */
757 first_tx->flags |= XEN_NETTXF_data_validated;
758
759 /* Optional extra info after the first request. */
760 if (skb_shinfo(skb)->gso_size) {
761 struct xen_netif_extra_info *gso;
762
763 gso = (struct xen_netif_extra_info *)
764 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
765
766 first_tx->flags |= XEN_NETTXF_extra_info;
767
768 gso->u.gso.size = skb_shinfo(skb)->gso_size;
769 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ?
770 XEN_NETIF_GSO_TYPE_TCPV6 :
771 XEN_NETIF_GSO_TYPE_TCPV4;
772 gso->u.gso.pad = 0;
773 gso->u.gso.features = 0;
774
775 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
776 gso->flags = 0;
777 }
778
779 /* Requests for the rest of the linear area. */
780 xennet_make_txreqs(&info, page, offset, len);
781
782 /* Requests for all the frags. */
783 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
784 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
785 xennet_make_txreqs(&info, skb_frag_page(frag),
786 skb_frag_off(frag),
787 skb_frag_size(frag));
788 }
789
790 /* First request has the packet length. */
791 first_tx->size = skb->len;
792
793 /* timestamp packet in software */
794 skb_tx_timestamp(skb);
795
796 xennet_mark_tx_pending(queue);
797
798 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
799 if (notify)
800 notify_remote_via_irq(queue->tx_irq);
801
802 u64_stats_update_begin(&tx_stats->syncp);
803 tx_stats->bytes += skb->len;
804 tx_stats->packets++;
805 u64_stats_update_end(&tx_stats->syncp);
806
807 /* Note: It is not safe to access skb after xennet_tx_buf_gc()! */
808 xennet_tx_buf_gc(queue);
809
810 if (!netfront_tx_slot_available(queue))
811 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
812
813 spin_unlock_irqrestore(&queue->tx_lock, flags);
814
815 return NETDEV_TX_OK;
816
817 drop:
818 dev->stats.tx_dropped++;
819 dev_kfree_skb_any(skb);
820 return NETDEV_TX_OK;
821 }
822
xennet_close(struct net_device * dev)823 static int xennet_close(struct net_device *dev)
824 {
825 struct netfront_info *np = netdev_priv(dev);
826 unsigned int num_queues = dev->real_num_tx_queues;
827 unsigned int i;
828 struct netfront_queue *queue;
829 netif_tx_stop_all_queues(np->netdev);
830 for (i = 0; i < num_queues; ++i) {
831 queue = &np->queues[i];
832 napi_disable(&queue->napi);
833 }
834 return 0;
835 }
836
xennet_move_rx_slot(struct netfront_queue * queue,struct sk_buff * skb,grant_ref_t ref)837 static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb,
838 grant_ref_t ref)
839 {
840 int new = xennet_rxidx(queue->rx.req_prod_pvt);
841
842 BUG_ON(queue->rx_skbs[new]);
843 queue->rx_skbs[new] = skb;
844 queue->grant_rx_ref[new] = ref;
845 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new;
846 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref;
847 queue->rx.req_prod_pvt++;
848 }
849
xennet_get_extras(struct netfront_queue * queue,struct xen_netif_extra_info * extras,RING_IDX rp)850 static int xennet_get_extras(struct netfront_queue *queue,
851 struct xen_netif_extra_info *extras,
852 RING_IDX rp)
853
854 {
855 struct xen_netif_extra_info extra;
856 struct device *dev = &queue->info->netdev->dev;
857 RING_IDX cons = queue->rx.rsp_cons;
858 int err = 0;
859
860 do {
861 struct sk_buff *skb;
862 grant_ref_t ref;
863
864 if (unlikely(cons + 1 == rp)) {
865 if (net_ratelimit())
866 dev_warn(dev, "Missing extra info\n");
867 err = -EBADR;
868 break;
869 }
870
871 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra);
872
873 if (unlikely(!extra.type ||
874 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
875 if (net_ratelimit())
876 dev_warn(dev, "Invalid extra type: %d\n",
877 extra.type);
878 err = -EINVAL;
879 } else {
880 extras[extra.type - 1] = extra;
881 }
882
883 skb = xennet_get_rx_skb(queue, cons);
884 ref = xennet_get_rx_ref(queue, cons);
885 xennet_move_rx_slot(queue, skb, ref);
886 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
887
888 queue->rx.rsp_cons = cons;
889 return err;
890 }
891
xennet_run_xdp(struct netfront_queue * queue,struct page * pdata,struct xen_netif_rx_response * rx,struct bpf_prog * prog,struct xdp_buff * xdp,bool * need_xdp_flush)892 static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata,
893 struct xen_netif_rx_response *rx, struct bpf_prog *prog,
894 struct xdp_buff *xdp, bool *need_xdp_flush)
895 {
896 struct xdp_frame *xdpf;
897 u32 len = rx->status;
898 u32 act;
899 int err;
900
901 xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
902 &queue->xdp_rxq);
903 xdp_prepare_buff(xdp, page_address(pdata), XDP_PACKET_HEADROOM,
904 len, false);
905
906 act = bpf_prog_run_xdp(prog, xdp);
907 switch (act) {
908 case XDP_TX:
909 get_page(pdata);
910 xdpf = xdp_convert_buff_to_frame(xdp);
911 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0);
912 if (unlikely(!err))
913 xdp_return_frame_rx_napi(xdpf);
914 else if (unlikely(err < 0))
915 trace_xdp_exception(queue->info->netdev, prog, act);
916 break;
917 case XDP_REDIRECT:
918 get_page(pdata);
919 err = xdp_do_redirect(queue->info->netdev, xdp, prog);
920 *need_xdp_flush = true;
921 if (unlikely(err))
922 trace_xdp_exception(queue->info->netdev, prog, act);
923 break;
924 case XDP_PASS:
925 case XDP_DROP:
926 break;
927
928 case XDP_ABORTED:
929 trace_xdp_exception(queue->info->netdev, prog, act);
930 break;
931
932 default:
933 bpf_warn_invalid_xdp_action(act);
934 }
935
936 return act;
937 }
938
xennet_get_responses(struct netfront_queue * queue,struct netfront_rx_info * rinfo,RING_IDX rp,struct sk_buff_head * list,bool * need_xdp_flush)939 static int xennet_get_responses(struct netfront_queue *queue,
940 struct netfront_rx_info *rinfo, RING_IDX rp,
941 struct sk_buff_head *list,
942 bool *need_xdp_flush)
943 {
944 struct xen_netif_rx_response *rx = &rinfo->rx, rx_local;
945 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD);
946 RING_IDX cons = queue->rx.rsp_cons;
947 struct sk_buff *skb = xennet_get_rx_skb(queue, cons);
948 struct xen_netif_extra_info *extras = rinfo->extras;
949 grant_ref_t ref = xennet_get_rx_ref(queue, cons);
950 struct device *dev = &queue->info->netdev->dev;
951 struct bpf_prog *xdp_prog;
952 struct xdp_buff xdp;
953 unsigned long ret;
954 int slots = 1;
955 int err = 0;
956 u32 verdict;
957
958 if (rx->flags & XEN_NETRXF_extra_info) {
959 err = xennet_get_extras(queue, extras, rp);
960 if (!err) {
961 if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) {
962 struct xen_netif_extra_info *xdp;
963
964 xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1];
965 rx->offset = xdp->u.xdp.headroom;
966 }
967 }
968 cons = queue->rx.rsp_cons;
969 }
970
971 for (;;) {
972 if (unlikely(rx->status < 0 ||
973 rx->offset + rx->status > XEN_PAGE_SIZE)) {
974 if (net_ratelimit())
975 dev_warn(dev, "rx->offset: %u, size: %d\n",
976 rx->offset, rx->status);
977 xennet_move_rx_slot(queue, skb, ref);
978 err = -EINVAL;
979 goto next;
980 }
981
982 /*
983 * This definitely indicates a bug, either in this driver or in
984 * the backend driver. In future this should flag the bad
985 * situation to the system controller to reboot the backend.
986 */
987 if (ref == GRANT_INVALID_REF) {
988 if (net_ratelimit())
989 dev_warn(dev, "Bad rx response id %d.\n",
990 rx->id);
991 err = -EINVAL;
992 goto next;
993 }
994
995 ret = gnttab_end_foreign_access_ref(ref, 0);
996 BUG_ON(!ret);
997
998 gnttab_release_grant_reference(&queue->gref_rx_head, ref);
999
1000 rcu_read_lock();
1001 xdp_prog = rcu_dereference(queue->xdp_prog);
1002 if (xdp_prog) {
1003 if (!(rx->flags & XEN_NETRXF_more_data)) {
1004 /* currently only a single page contains data */
1005 verdict = xennet_run_xdp(queue,
1006 skb_frag_page(&skb_shinfo(skb)->frags[0]),
1007 rx, xdp_prog, &xdp, need_xdp_flush);
1008 if (verdict != XDP_PASS)
1009 err = -EINVAL;
1010 } else {
1011 /* drop the frame */
1012 err = -EINVAL;
1013 }
1014 }
1015 rcu_read_unlock();
1016 next:
1017 __skb_queue_tail(list, skb);
1018 if (!(rx->flags & XEN_NETRXF_more_data))
1019 break;
1020
1021 if (cons + slots == rp) {
1022 if (net_ratelimit())
1023 dev_warn(dev, "Need more slots\n");
1024 err = -ENOENT;
1025 break;
1026 }
1027
1028 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local);
1029 rx = &rx_local;
1030 skb = xennet_get_rx_skb(queue, cons + slots);
1031 ref = xennet_get_rx_ref(queue, cons + slots);
1032 slots++;
1033 }
1034
1035 if (unlikely(slots > max)) {
1036 if (net_ratelimit())
1037 dev_warn(dev, "Too many slots\n");
1038 err = -E2BIG;
1039 }
1040
1041 if (unlikely(err))
1042 queue->rx.rsp_cons = cons + slots;
1043
1044 return err;
1045 }
1046
xennet_set_skb_gso(struct sk_buff * skb,struct xen_netif_extra_info * gso)1047 static int xennet_set_skb_gso(struct sk_buff *skb,
1048 struct xen_netif_extra_info *gso)
1049 {
1050 if (!gso->u.gso.size) {
1051 if (net_ratelimit())
1052 pr_warn("GSO size must not be zero\n");
1053 return -EINVAL;
1054 }
1055
1056 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 &&
1057 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) {
1058 if (net_ratelimit())
1059 pr_warn("Bad GSO type %d\n", gso->u.gso.type);
1060 return -EINVAL;
1061 }
1062
1063 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1064 skb_shinfo(skb)->gso_type =
1065 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ?
1066 SKB_GSO_TCPV4 :
1067 SKB_GSO_TCPV6;
1068
1069 /* Header must be checked, and gso_segs computed. */
1070 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1071 skb_shinfo(skb)->gso_segs = 0;
1072
1073 return 0;
1074 }
1075
xennet_fill_frags(struct netfront_queue * queue,struct sk_buff * skb,struct sk_buff_head * list)1076 static int xennet_fill_frags(struct netfront_queue *queue,
1077 struct sk_buff *skb,
1078 struct sk_buff_head *list)
1079 {
1080 RING_IDX cons = queue->rx.rsp_cons;
1081 struct sk_buff *nskb;
1082
1083 while ((nskb = __skb_dequeue(list))) {
1084 struct xen_netif_rx_response rx;
1085 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0];
1086
1087 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx);
1088
1089 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) {
1090 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1091
1092 BUG_ON(pull_to < skb_headlen(skb));
1093 __pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1094 }
1095 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) {
1096 queue->rx.rsp_cons = ++cons + skb_queue_len(list);
1097 kfree_skb(nskb);
1098 return -ENOENT;
1099 }
1100
1101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1102 skb_frag_page(nfrag),
1103 rx.offset, rx.status, PAGE_SIZE);
1104
1105 skb_shinfo(nskb)->nr_frags = 0;
1106 kfree_skb(nskb);
1107 }
1108
1109 queue->rx.rsp_cons = cons;
1110
1111 return 0;
1112 }
1113
checksum_setup(struct net_device * dev,struct sk_buff * skb)1114 static int checksum_setup(struct net_device *dev, struct sk_buff *skb)
1115 {
1116 bool recalculate_partial_csum = false;
1117
1118 /*
1119 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1120 * peers can fail to set NETRXF_csum_blank when sending a GSO
1121 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1122 * recalculate the partial checksum.
1123 */
1124 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1125 struct netfront_info *np = netdev_priv(dev);
1126 atomic_inc(&np->rx_gso_checksum_fixup);
1127 skb->ip_summed = CHECKSUM_PARTIAL;
1128 recalculate_partial_csum = true;
1129 }
1130
1131 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1132 if (skb->ip_summed != CHECKSUM_PARTIAL)
1133 return 0;
1134
1135 return skb_checksum_setup(skb, recalculate_partial_csum);
1136 }
1137
handle_incoming_queue(struct netfront_queue * queue,struct sk_buff_head * rxq)1138 static int handle_incoming_queue(struct netfront_queue *queue,
1139 struct sk_buff_head *rxq)
1140 {
1141 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats);
1142 int packets_dropped = 0;
1143 struct sk_buff *skb;
1144
1145 while ((skb = __skb_dequeue(rxq)) != NULL) {
1146 int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1147
1148 if (pull_to > skb_headlen(skb))
1149 __pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1150
1151 /* Ethernet work: Delayed to here as it peeks the header. */
1152 skb->protocol = eth_type_trans(skb, queue->info->netdev);
1153 skb_reset_network_header(skb);
1154
1155 if (checksum_setup(queue->info->netdev, skb)) {
1156 kfree_skb(skb);
1157 packets_dropped++;
1158 queue->info->netdev->stats.rx_errors++;
1159 continue;
1160 }
1161
1162 u64_stats_update_begin(&rx_stats->syncp);
1163 rx_stats->packets++;
1164 rx_stats->bytes += skb->len;
1165 u64_stats_update_end(&rx_stats->syncp);
1166
1167 /* Pass it up. */
1168 napi_gro_receive(&queue->napi, skb);
1169 }
1170
1171 return packets_dropped;
1172 }
1173
xennet_poll(struct napi_struct * napi,int budget)1174 static int xennet_poll(struct napi_struct *napi, int budget)
1175 {
1176 struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi);
1177 struct net_device *dev = queue->info->netdev;
1178 struct sk_buff *skb;
1179 struct netfront_rx_info rinfo;
1180 struct xen_netif_rx_response *rx = &rinfo.rx;
1181 struct xen_netif_extra_info *extras = rinfo.extras;
1182 RING_IDX i, rp;
1183 int work_done;
1184 struct sk_buff_head rxq;
1185 struct sk_buff_head errq;
1186 struct sk_buff_head tmpq;
1187 int err;
1188 bool need_xdp_flush = false;
1189
1190 spin_lock(&queue->rx_lock);
1191
1192 skb_queue_head_init(&rxq);
1193 skb_queue_head_init(&errq);
1194 skb_queue_head_init(&tmpq);
1195
1196 rp = queue->rx.sring->rsp_prod;
1197 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) {
1198 dev_alert(&dev->dev, "Illegal number of responses %u\n",
1199 rp - queue->rx.rsp_cons);
1200 queue->info->broken = true;
1201 spin_unlock(&queue->rx_lock);
1202 return 0;
1203 }
1204 rmb(); /* Ensure we see queued responses up to 'rp'. */
1205
1206 i = queue->rx.rsp_cons;
1207 work_done = 0;
1208 while ((i != rp) && (work_done < budget)) {
1209 RING_COPY_RESPONSE(&queue->rx, i, rx);
1210 memset(extras, 0, sizeof(rinfo.extras));
1211
1212 err = xennet_get_responses(queue, &rinfo, rp, &tmpq,
1213 &need_xdp_flush);
1214
1215 if (unlikely(err)) {
1216 err:
1217 while ((skb = __skb_dequeue(&tmpq)))
1218 __skb_queue_tail(&errq, skb);
1219 dev->stats.rx_errors++;
1220 i = queue->rx.rsp_cons;
1221 continue;
1222 }
1223
1224 skb = __skb_dequeue(&tmpq);
1225
1226 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1227 struct xen_netif_extra_info *gso;
1228 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1229
1230 if (unlikely(xennet_set_skb_gso(skb, gso))) {
1231 __skb_queue_head(&tmpq, skb);
1232 queue->rx.rsp_cons += skb_queue_len(&tmpq);
1233 goto err;
1234 }
1235 }
1236
1237 NETFRONT_SKB_CB(skb)->pull_to = rx->status;
1238 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD)
1239 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD;
1240
1241 skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset);
1242 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status);
1243 skb->data_len = rx->status;
1244 skb->len += rx->status;
1245
1246 if (unlikely(xennet_fill_frags(queue, skb, &tmpq)))
1247 goto err;
1248
1249 if (rx->flags & XEN_NETRXF_csum_blank)
1250 skb->ip_summed = CHECKSUM_PARTIAL;
1251 else if (rx->flags & XEN_NETRXF_data_validated)
1252 skb->ip_summed = CHECKSUM_UNNECESSARY;
1253
1254 __skb_queue_tail(&rxq, skb);
1255
1256 i = ++queue->rx.rsp_cons;
1257 work_done++;
1258 }
1259 if (need_xdp_flush)
1260 xdp_do_flush();
1261
1262 __skb_queue_purge(&errq);
1263
1264 work_done -= handle_incoming_queue(queue, &rxq);
1265
1266 xennet_alloc_rx_buffers(queue);
1267
1268 if (work_done < budget) {
1269 int more_to_do = 0;
1270
1271 napi_complete_done(napi, work_done);
1272
1273 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do);
1274 if (more_to_do)
1275 napi_schedule(napi);
1276 }
1277
1278 spin_unlock(&queue->rx_lock);
1279
1280 return work_done;
1281 }
1282
xennet_change_mtu(struct net_device * dev,int mtu)1283 static int xennet_change_mtu(struct net_device *dev, int mtu)
1284 {
1285 int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN;
1286
1287 if (mtu > max)
1288 return -EINVAL;
1289 dev->mtu = mtu;
1290 return 0;
1291 }
1292
xennet_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * tot)1293 static void xennet_get_stats64(struct net_device *dev,
1294 struct rtnl_link_stats64 *tot)
1295 {
1296 struct netfront_info *np = netdev_priv(dev);
1297 int cpu;
1298
1299 for_each_possible_cpu(cpu) {
1300 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu);
1301 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu);
1302 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1303 unsigned int start;
1304
1305 do {
1306 start = u64_stats_fetch_begin_irq(&tx_stats->syncp);
1307 tx_packets = tx_stats->packets;
1308 tx_bytes = tx_stats->bytes;
1309 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start));
1310
1311 do {
1312 start = u64_stats_fetch_begin_irq(&rx_stats->syncp);
1313 rx_packets = rx_stats->packets;
1314 rx_bytes = rx_stats->bytes;
1315 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start));
1316
1317 tot->rx_packets += rx_packets;
1318 tot->tx_packets += tx_packets;
1319 tot->rx_bytes += rx_bytes;
1320 tot->tx_bytes += tx_bytes;
1321 }
1322
1323 tot->rx_errors = dev->stats.rx_errors;
1324 tot->tx_dropped = dev->stats.tx_dropped;
1325 }
1326
xennet_release_tx_bufs(struct netfront_queue * queue)1327 static void xennet_release_tx_bufs(struct netfront_queue *queue)
1328 {
1329 struct sk_buff *skb;
1330 int i;
1331
1332 for (i = 0; i < NET_TX_RING_SIZE; i++) {
1333 /* Skip over entries which are actually freelist references */
1334 if (!queue->tx_skbs[i])
1335 continue;
1336
1337 skb = queue->tx_skbs[i];
1338 queue->tx_skbs[i] = NULL;
1339 get_page(queue->grant_tx_page[i]);
1340 gnttab_end_foreign_access(queue->grant_tx_ref[i],
1341 GNTMAP_readonly,
1342 (unsigned long)page_address(queue->grant_tx_page[i]));
1343 queue->grant_tx_page[i] = NULL;
1344 queue->grant_tx_ref[i] = GRANT_INVALID_REF;
1345 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i);
1346 dev_kfree_skb_irq(skb);
1347 }
1348 }
1349
xennet_release_rx_bufs(struct netfront_queue * queue)1350 static void xennet_release_rx_bufs(struct netfront_queue *queue)
1351 {
1352 int id, ref;
1353
1354 spin_lock_bh(&queue->rx_lock);
1355
1356 for (id = 0; id < NET_RX_RING_SIZE; id++) {
1357 struct sk_buff *skb;
1358 struct page *page;
1359
1360 skb = queue->rx_skbs[id];
1361 if (!skb)
1362 continue;
1363
1364 ref = queue->grant_rx_ref[id];
1365 if (ref == GRANT_INVALID_REF)
1366 continue;
1367
1368 page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
1369
1370 /* gnttab_end_foreign_access() needs a page ref until
1371 * foreign access is ended (which may be deferred).
1372 */
1373 get_page(page);
1374 gnttab_end_foreign_access(ref, 0,
1375 (unsigned long)page_address(page));
1376 queue->grant_rx_ref[id] = GRANT_INVALID_REF;
1377
1378 kfree_skb(skb);
1379 }
1380
1381 spin_unlock_bh(&queue->rx_lock);
1382 }
1383
xennet_fix_features(struct net_device * dev,netdev_features_t features)1384 static netdev_features_t xennet_fix_features(struct net_device *dev,
1385 netdev_features_t features)
1386 {
1387 struct netfront_info *np = netdev_priv(dev);
1388
1389 if (features & NETIF_F_SG &&
1390 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0))
1391 features &= ~NETIF_F_SG;
1392
1393 if (features & NETIF_F_IPV6_CSUM &&
1394 !xenbus_read_unsigned(np->xbdev->otherend,
1395 "feature-ipv6-csum-offload", 0))
1396 features &= ~NETIF_F_IPV6_CSUM;
1397
1398 if (features & NETIF_F_TSO &&
1399 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0))
1400 features &= ~NETIF_F_TSO;
1401
1402 if (features & NETIF_F_TSO6 &&
1403 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0))
1404 features &= ~NETIF_F_TSO6;
1405
1406 return features;
1407 }
1408
xennet_set_features(struct net_device * dev,netdev_features_t features)1409 static int xennet_set_features(struct net_device *dev,
1410 netdev_features_t features)
1411 {
1412 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) {
1413 netdev_info(dev, "Reducing MTU because no SG offload");
1414 dev->mtu = ETH_DATA_LEN;
1415 }
1416
1417 return 0;
1418 }
1419
xennet_tx_interrupt(int irq,void * dev_id)1420 static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
1421 {
1422 struct netfront_queue *queue = dev_id;
1423 unsigned long flags;
1424
1425 if (queue->info->broken)
1426 return IRQ_HANDLED;
1427
1428 spin_lock_irqsave(&queue->tx_lock, flags);
1429 xennet_tx_buf_gc(queue);
1430 spin_unlock_irqrestore(&queue->tx_lock, flags);
1431
1432 return IRQ_HANDLED;
1433 }
1434
xennet_rx_interrupt(int irq,void * dev_id)1435 static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
1436 {
1437 struct netfront_queue *queue = dev_id;
1438 struct net_device *dev = queue->info->netdev;
1439
1440 if (queue->info->broken)
1441 return IRQ_HANDLED;
1442
1443 if (likely(netif_carrier_ok(dev) &&
1444 RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)))
1445 napi_schedule(&queue->napi);
1446
1447 return IRQ_HANDLED;
1448 }
1449
xennet_interrupt(int irq,void * dev_id)1450 static irqreturn_t xennet_interrupt(int irq, void *dev_id)
1451 {
1452 xennet_tx_interrupt(irq, dev_id);
1453 xennet_rx_interrupt(irq, dev_id);
1454 return IRQ_HANDLED;
1455 }
1456
1457 #ifdef CONFIG_NET_POLL_CONTROLLER
xennet_poll_controller(struct net_device * dev)1458 static void xennet_poll_controller(struct net_device *dev)
1459 {
1460 /* Poll each queue */
1461 struct netfront_info *info = netdev_priv(dev);
1462 unsigned int num_queues = dev->real_num_tx_queues;
1463 unsigned int i;
1464
1465 if (info->broken)
1466 return;
1467
1468 for (i = 0; i < num_queues; ++i)
1469 xennet_interrupt(0, &info->queues[i]);
1470 }
1471 #endif
1472
1473 #define NETBACK_XDP_HEADROOM_DISABLE 0
1474 #define NETBACK_XDP_HEADROOM_ENABLE 1
1475
talk_to_netback_xdp(struct netfront_info * np,int xdp)1476 static int talk_to_netback_xdp(struct netfront_info *np, int xdp)
1477 {
1478 int err;
1479 unsigned short headroom;
1480
1481 headroom = xdp ? XDP_PACKET_HEADROOM : 0;
1482 err = xenbus_printf(XBT_NIL, np->xbdev->nodename,
1483 "xdp-headroom", "%hu",
1484 headroom);
1485 if (err)
1486 pr_warn("Error writing xdp-headroom\n");
1487
1488 return err;
1489 }
1490
xennet_xdp_set(struct net_device * dev,struct bpf_prog * prog,struct netlink_ext_ack * extack)1491 static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1492 struct netlink_ext_ack *extack)
1493 {
1494 unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM;
1495 struct netfront_info *np = netdev_priv(dev);
1496 struct bpf_prog *old_prog;
1497 unsigned int i, err;
1498
1499 if (dev->mtu > max_mtu) {
1500 netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu);
1501 return -EINVAL;
1502 }
1503
1504 if (!np->netback_has_xdp_headroom)
1505 return 0;
1506
1507 xenbus_switch_state(np->xbdev, XenbusStateReconfiguring);
1508
1509 err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE :
1510 NETBACK_XDP_HEADROOM_DISABLE);
1511 if (err)
1512 return err;
1513
1514 /* avoid the race with XDP headroom adjustment */
1515 wait_event(module_wq,
1516 xenbus_read_driver_state(np->xbdev->otherend) ==
1517 XenbusStateReconfigured);
1518 np->netfront_xdp_enabled = true;
1519
1520 old_prog = rtnl_dereference(np->queues[0].xdp_prog);
1521
1522 if (prog)
1523 bpf_prog_add(prog, dev->real_num_tx_queues);
1524
1525 for (i = 0; i < dev->real_num_tx_queues; ++i)
1526 rcu_assign_pointer(np->queues[i].xdp_prog, prog);
1527
1528 if (old_prog)
1529 for (i = 0; i < dev->real_num_tx_queues; ++i)
1530 bpf_prog_put(old_prog);
1531
1532 xenbus_switch_state(np->xbdev, XenbusStateConnected);
1533
1534 return 0;
1535 }
1536
xennet_xdp(struct net_device * dev,struct netdev_bpf * xdp)1537 static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1538 {
1539 struct netfront_info *np = netdev_priv(dev);
1540
1541 if (np->broken)
1542 return -ENODEV;
1543
1544 switch (xdp->command) {
1545 case XDP_SETUP_PROG:
1546 return xennet_xdp_set(dev, xdp->prog, xdp->extack);
1547 default:
1548 return -EINVAL;
1549 }
1550 }
1551
1552 static const struct net_device_ops xennet_netdev_ops = {
1553 .ndo_open = xennet_open,
1554 .ndo_stop = xennet_close,
1555 .ndo_start_xmit = xennet_start_xmit,
1556 .ndo_change_mtu = xennet_change_mtu,
1557 .ndo_get_stats64 = xennet_get_stats64,
1558 .ndo_set_mac_address = eth_mac_addr,
1559 .ndo_validate_addr = eth_validate_addr,
1560 .ndo_fix_features = xennet_fix_features,
1561 .ndo_set_features = xennet_set_features,
1562 .ndo_select_queue = xennet_select_queue,
1563 .ndo_bpf = xennet_xdp,
1564 .ndo_xdp_xmit = xennet_xdp_xmit,
1565 #ifdef CONFIG_NET_POLL_CONTROLLER
1566 .ndo_poll_controller = xennet_poll_controller,
1567 #endif
1568 };
1569
xennet_free_netdev(struct net_device * netdev)1570 static void xennet_free_netdev(struct net_device *netdev)
1571 {
1572 struct netfront_info *np = netdev_priv(netdev);
1573
1574 free_percpu(np->rx_stats);
1575 free_percpu(np->tx_stats);
1576 free_netdev(netdev);
1577 }
1578
xennet_create_dev(struct xenbus_device * dev)1579 static struct net_device *xennet_create_dev(struct xenbus_device *dev)
1580 {
1581 int err;
1582 struct net_device *netdev;
1583 struct netfront_info *np;
1584
1585 netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues);
1586 if (!netdev)
1587 return ERR_PTR(-ENOMEM);
1588
1589 np = netdev_priv(netdev);
1590 np->xbdev = dev;
1591
1592 np->queues = NULL;
1593
1594 err = -ENOMEM;
1595 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1596 if (np->rx_stats == NULL)
1597 goto exit;
1598 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1599 if (np->tx_stats == NULL)
1600 goto exit;
1601
1602 netdev->netdev_ops = &xennet_netdev_ops;
1603
1604 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
1605 NETIF_F_GSO_ROBUST;
1606 netdev->hw_features = NETIF_F_SG |
1607 NETIF_F_IPV6_CSUM |
1608 NETIF_F_TSO | NETIF_F_TSO6;
1609
1610 /*
1611 * Assume that all hw features are available for now. This set
1612 * will be adjusted by the call to netdev_update_features() in
1613 * xennet_connect() which is the earliest point where we can
1614 * negotiate with the backend regarding supported features.
1615 */
1616 netdev->features |= netdev->hw_features;
1617
1618 netdev->ethtool_ops = &xennet_ethtool_ops;
1619 netdev->min_mtu = ETH_MIN_MTU;
1620 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE;
1621 SET_NETDEV_DEV(netdev, &dev->dev);
1622
1623 np->netdev = netdev;
1624 np->netfront_xdp_enabled = false;
1625
1626 netif_carrier_off(netdev);
1627
1628 do {
1629 xenbus_switch_state(dev, XenbusStateInitialising);
1630 err = wait_event_timeout(module_wq,
1631 xenbus_read_driver_state(dev->otherend) !=
1632 XenbusStateClosed &&
1633 xenbus_read_driver_state(dev->otherend) !=
1634 XenbusStateUnknown, XENNET_TIMEOUT);
1635 } while (!err);
1636
1637 return netdev;
1638
1639 exit:
1640 xennet_free_netdev(netdev);
1641 return ERR_PTR(err);
1642 }
1643
1644 /*
1645 * Entry point to this code when a new device is created. Allocate the basic
1646 * structures and the ring buffers for communication with the backend, and
1647 * inform the backend of the appropriate details for those.
1648 */
netfront_probe(struct xenbus_device * dev,const struct xenbus_device_id * id)1649 static int netfront_probe(struct xenbus_device *dev,
1650 const struct xenbus_device_id *id)
1651 {
1652 int err;
1653 struct net_device *netdev;
1654 struct netfront_info *info;
1655
1656 netdev = xennet_create_dev(dev);
1657 if (IS_ERR(netdev)) {
1658 err = PTR_ERR(netdev);
1659 xenbus_dev_fatal(dev, err, "creating netdev");
1660 return err;
1661 }
1662
1663 info = netdev_priv(netdev);
1664 dev_set_drvdata(&dev->dev, info);
1665 #ifdef CONFIG_SYSFS
1666 info->netdev->sysfs_groups[0] = &xennet_dev_group;
1667 #endif
1668
1669 return 0;
1670 }
1671
xennet_end_access(int ref,void * page)1672 static void xennet_end_access(int ref, void *page)
1673 {
1674 /* This frees the page as a side-effect */
1675 if (ref != GRANT_INVALID_REF)
1676 gnttab_end_foreign_access(ref, 0, (unsigned long)page);
1677 }
1678
xennet_disconnect_backend(struct netfront_info * info)1679 static void xennet_disconnect_backend(struct netfront_info *info)
1680 {
1681 unsigned int i = 0;
1682 unsigned int num_queues = info->netdev->real_num_tx_queues;
1683
1684 netif_carrier_off(info->netdev);
1685
1686 for (i = 0; i < num_queues && info->queues; ++i) {
1687 struct netfront_queue *queue = &info->queues[i];
1688
1689 del_timer_sync(&queue->rx_refill_timer);
1690
1691 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq))
1692 unbind_from_irqhandler(queue->tx_irq, queue);
1693 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) {
1694 unbind_from_irqhandler(queue->tx_irq, queue);
1695 unbind_from_irqhandler(queue->rx_irq, queue);
1696 }
1697 queue->tx_evtchn = queue->rx_evtchn = 0;
1698 queue->tx_irq = queue->rx_irq = 0;
1699
1700 if (netif_running(info->netdev))
1701 napi_synchronize(&queue->napi);
1702
1703 xennet_release_tx_bufs(queue);
1704 xennet_release_rx_bufs(queue);
1705 gnttab_free_grant_references(queue->gref_tx_head);
1706 gnttab_free_grant_references(queue->gref_rx_head);
1707
1708 /* End access and free the pages */
1709 xennet_end_access(queue->tx_ring_ref, queue->tx.sring);
1710 xennet_end_access(queue->rx_ring_ref, queue->rx.sring);
1711
1712 queue->tx_ring_ref = GRANT_INVALID_REF;
1713 queue->rx_ring_ref = GRANT_INVALID_REF;
1714 queue->tx.sring = NULL;
1715 queue->rx.sring = NULL;
1716
1717 page_pool_destroy(queue->page_pool);
1718 }
1719 }
1720
1721 /*
1722 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1723 * driver restart. We tear down our netif structure and recreate it, but
1724 * leave the device-layer structures intact so that this is transparent to the
1725 * rest of the kernel.
1726 */
netfront_resume(struct xenbus_device * dev)1727 static int netfront_resume(struct xenbus_device *dev)
1728 {
1729 struct netfront_info *info = dev_get_drvdata(&dev->dev);
1730
1731 dev_dbg(&dev->dev, "%s\n", dev->nodename);
1732
1733 netif_tx_lock_bh(info->netdev);
1734 netif_device_detach(info->netdev);
1735 netif_tx_unlock_bh(info->netdev);
1736
1737 xennet_disconnect_backend(info);
1738 return 0;
1739 }
1740
xen_net_read_mac(struct xenbus_device * dev,u8 mac[])1741 static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[])
1742 {
1743 char *s, *e, *macstr;
1744 int i;
1745
1746 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL);
1747 if (IS_ERR(macstr))
1748 return PTR_ERR(macstr);
1749
1750 for (i = 0; i < ETH_ALEN; i++) {
1751 mac[i] = simple_strtoul(s, &e, 16);
1752 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) {
1753 kfree(macstr);
1754 return -ENOENT;
1755 }
1756 s = e+1;
1757 }
1758
1759 kfree(macstr);
1760 return 0;
1761 }
1762
setup_netfront_single(struct netfront_queue * queue)1763 static int setup_netfront_single(struct netfront_queue *queue)
1764 {
1765 int err;
1766
1767 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1768 if (err < 0)
1769 goto fail;
1770
1771 err = bind_evtchn_to_irqhandler(queue->tx_evtchn,
1772 xennet_interrupt,
1773 0, queue->info->netdev->name, queue);
1774 if (err < 0)
1775 goto bind_fail;
1776 queue->rx_evtchn = queue->tx_evtchn;
1777 queue->rx_irq = queue->tx_irq = err;
1778
1779 return 0;
1780
1781 bind_fail:
1782 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1783 queue->tx_evtchn = 0;
1784 fail:
1785 return err;
1786 }
1787
setup_netfront_split(struct netfront_queue * queue)1788 static int setup_netfront_split(struct netfront_queue *queue)
1789 {
1790 int err;
1791
1792 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1793 if (err < 0)
1794 goto fail;
1795 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn);
1796 if (err < 0)
1797 goto alloc_rx_evtchn_fail;
1798
1799 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
1800 "%s-tx", queue->name);
1801 err = bind_evtchn_to_irqhandler(queue->tx_evtchn,
1802 xennet_tx_interrupt,
1803 0, queue->tx_irq_name, queue);
1804 if (err < 0)
1805 goto bind_tx_fail;
1806 queue->tx_irq = err;
1807
1808 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
1809 "%s-rx", queue->name);
1810 err = bind_evtchn_to_irqhandler(queue->rx_evtchn,
1811 xennet_rx_interrupt,
1812 0, queue->rx_irq_name, queue);
1813 if (err < 0)
1814 goto bind_rx_fail;
1815 queue->rx_irq = err;
1816
1817 return 0;
1818
1819 bind_rx_fail:
1820 unbind_from_irqhandler(queue->tx_irq, queue);
1821 queue->tx_irq = 0;
1822 bind_tx_fail:
1823 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn);
1824 queue->rx_evtchn = 0;
1825 alloc_rx_evtchn_fail:
1826 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1827 queue->tx_evtchn = 0;
1828 fail:
1829 return err;
1830 }
1831
setup_netfront(struct xenbus_device * dev,struct netfront_queue * queue,unsigned int feature_split_evtchn)1832 static int setup_netfront(struct xenbus_device *dev,
1833 struct netfront_queue *queue, unsigned int feature_split_evtchn)
1834 {
1835 struct xen_netif_tx_sring *txs;
1836 struct xen_netif_rx_sring *rxs;
1837 grant_ref_t gref;
1838 int err;
1839
1840 queue->tx_ring_ref = GRANT_INVALID_REF;
1841 queue->rx_ring_ref = GRANT_INVALID_REF;
1842 queue->rx.sring = NULL;
1843 queue->tx.sring = NULL;
1844
1845 txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH);
1846 if (!txs) {
1847 err = -ENOMEM;
1848 xenbus_dev_fatal(dev, err, "allocating tx ring page");
1849 goto fail;
1850 }
1851 SHARED_RING_INIT(txs);
1852 FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE);
1853
1854 err = xenbus_grant_ring(dev, txs, 1, &gref);
1855 if (err < 0)
1856 goto grant_tx_ring_fail;
1857 queue->tx_ring_ref = gref;
1858
1859 rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH);
1860 if (!rxs) {
1861 err = -ENOMEM;
1862 xenbus_dev_fatal(dev, err, "allocating rx ring page");
1863 goto alloc_rx_ring_fail;
1864 }
1865 SHARED_RING_INIT(rxs);
1866 FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE);
1867
1868 err = xenbus_grant_ring(dev, rxs, 1, &gref);
1869 if (err < 0)
1870 goto grant_rx_ring_fail;
1871 queue->rx_ring_ref = gref;
1872
1873 if (feature_split_evtchn)
1874 err = setup_netfront_split(queue);
1875 /* setup single event channel if
1876 * a) feature-split-event-channels == 0
1877 * b) feature-split-event-channels == 1 but failed to setup
1878 */
1879 if (!feature_split_evtchn || err)
1880 err = setup_netfront_single(queue);
1881
1882 if (err)
1883 goto alloc_evtchn_fail;
1884
1885 return 0;
1886
1887 /* If we fail to setup netfront, it is safe to just revoke access to
1888 * granted pages because backend is not accessing it at this point.
1889 */
1890 alloc_evtchn_fail:
1891 gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0);
1892 grant_rx_ring_fail:
1893 free_page((unsigned long)rxs);
1894 alloc_rx_ring_fail:
1895 gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0);
1896 grant_tx_ring_fail:
1897 free_page((unsigned long)txs);
1898 fail:
1899 return err;
1900 }
1901
1902 /* Queue-specific initialisation
1903 * This used to be done in xennet_create_dev() but must now
1904 * be run per-queue.
1905 */
xennet_init_queue(struct netfront_queue * queue)1906 static int xennet_init_queue(struct netfront_queue *queue)
1907 {
1908 unsigned short i;
1909 int err = 0;
1910 char *devid;
1911
1912 spin_lock_init(&queue->tx_lock);
1913 spin_lock_init(&queue->rx_lock);
1914
1915 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0);
1916
1917 devid = strrchr(queue->info->xbdev->nodename, '/') + 1;
1918 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u",
1919 devid, queue->id);
1920
1921 /* Initialise tx_skb_freelist as a free chain containing every entry. */
1922 queue->tx_skb_freelist = 0;
1923 queue->tx_pend_queue = TX_LINK_NONE;
1924 for (i = 0; i < NET_TX_RING_SIZE; i++) {
1925 queue->tx_link[i] = i + 1;
1926 queue->grant_tx_ref[i] = GRANT_INVALID_REF;
1927 queue->grant_tx_page[i] = NULL;
1928 }
1929 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE;
1930
1931 /* Clear out rx_skbs */
1932 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1933 queue->rx_skbs[i] = NULL;
1934 queue->grant_rx_ref[i] = GRANT_INVALID_REF;
1935 }
1936
1937 /* A grant for every tx ring slot */
1938 if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
1939 &queue->gref_tx_head) < 0) {
1940 pr_alert("can't alloc tx grant refs\n");
1941 err = -ENOMEM;
1942 goto exit;
1943 }
1944
1945 /* A grant for every rx ring slot */
1946 if (gnttab_alloc_grant_references(NET_RX_RING_SIZE,
1947 &queue->gref_rx_head) < 0) {
1948 pr_alert("can't alloc rx grant refs\n");
1949 err = -ENOMEM;
1950 goto exit_free_tx;
1951 }
1952
1953 return 0;
1954
1955 exit_free_tx:
1956 gnttab_free_grant_references(queue->gref_tx_head);
1957 exit:
1958 return err;
1959 }
1960
write_queue_xenstore_keys(struct netfront_queue * queue,struct xenbus_transaction * xbt,int write_hierarchical)1961 static int write_queue_xenstore_keys(struct netfront_queue *queue,
1962 struct xenbus_transaction *xbt, int write_hierarchical)
1963 {
1964 /* Write the queue-specific keys into XenStore in the traditional
1965 * way for a single queue, or in a queue subkeys for multiple
1966 * queues.
1967 */
1968 struct xenbus_device *dev = queue->info->xbdev;
1969 int err;
1970 const char *message;
1971 char *path;
1972 size_t pathsize;
1973
1974 /* Choose the correct place to write the keys */
1975 if (write_hierarchical) {
1976 pathsize = strlen(dev->nodename) + 10;
1977 path = kzalloc(pathsize, GFP_KERNEL);
1978 if (!path) {
1979 err = -ENOMEM;
1980 message = "out of memory while writing ring references";
1981 goto error;
1982 }
1983 snprintf(path, pathsize, "%s/queue-%u",
1984 dev->nodename, queue->id);
1985 } else {
1986 path = (char *)dev->nodename;
1987 }
1988
1989 /* Write ring references */
1990 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u",
1991 queue->tx_ring_ref);
1992 if (err) {
1993 message = "writing tx-ring-ref";
1994 goto error;
1995 }
1996
1997 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u",
1998 queue->rx_ring_ref);
1999 if (err) {
2000 message = "writing rx-ring-ref";
2001 goto error;
2002 }
2003
2004 /* Write event channels; taking into account both shared
2005 * and split event channel scenarios.
2006 */
2007 if (queue->tx_evtchn == queue->rx_evtchn) {
2008 /* Shared event channel */
2009 err = xenbus_printf(*xbt, path,
2010 "event-channel", "%u", queue->tx_evtchn);
2011 if (err) {
2012 message = "writing event-channel";
2013 goto error;
2014 }
2015 } else {
2016 /* Split event channels */
2017 err = xenbus_printf(*xbt, path,
2018 "event-channel-tx", "%u", queue->tx_evtchn);
2019 if (err) {
2020 message = "writing event-channel-tx";
2021 goto error;
2022 }
2023
2024 err = xenbus_printf(*xbt, path,
2025 "event-channel-rx", "%u", queue->rx_evtchn);
2026 if (err) {
2027 message = "writing event-channel-rx";
2028 goto error;
2029 }
2030 }
2031
2032 if (write_hierarchical)
2033 kfree(path);
2034 return 0;
2035
2036 error:
2037 if (write_hierarchical)
2038 kfree(path);
2039 xenbus_dev_fatal(dev, err, "%s", message);
2040 return err;
2041 }
2042
xennet_destroy_queues(struct netfront_info * info)2043 static void xennet_destroy_queues(struct netfront_info *info)
2044 {
2045 unsigned int i;
2046
2047 for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
2048 struct netfront_queue *queue = &info->queues[i];
2049
2050 if (netif_running(info->netdev))
2051 napi_disable(&queue->napi);
2052 netif_napi_del(&queue->napi);
2053 }
2054
2055 kfree(info->queues);
2056 info->queues = NULL;
2057 }
2058
2059
2060
xennet_create_page_pool(struct netfront_queue * queue)2061 static int xennet_create_page_pool(struct netfront_queue *queue)
2062 {
2063 int err;
2064 struct page_pool_params pp_params = {
2065 .order = 0,
2066 .flags = 0,
2067 .pool_size = NET_RX_RING_SIZE,
2068 .nid = NUMA_NO_NODE,
2069 .dev = &queue->info->netdev->dev,
2070 .offset = XDP_PACKET_HEADROOM,
2071 .max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
2072 };
2073
2074 queue->page_pool = page_pool_create(&pp_params);
2075 if (IS_ERR(queue->page_pool)) {
2076 err = PTR_ERR(queue->page_pool);
2077 queue->page_pool = NULL;
2078 return err;
2079 }
2080
2081 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev,
2082 queue->id, 0);
2083 if (err) {
2084 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n");
2085 goto err_free_pp;
2086 }
2087
2088 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq,
2089 MEM_TYPE_PAGE_POOL, queue->page_pool);
2090 if (err) {
2091 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n");
2092 goto err_unregister_rxq;
2093 }
2094 return 0;
2095
2096 err_unregister_rxq:
2097 xdp_rxq_info_unreg(&queue->xdp_rxq);
2098 err_free_pp:
2099 page_pool_destroy(queue->page_pool);
2100 queue->page_pool = NULL;
2101 return err;
2102 }
2103
xennet_create_queues(struct netfront_info * info,unsigned int * num_queues)2104 static int xennet_create_queues(struct netfront_info *info,
2105 unsigned int *num_queues)
2106 {
2107 unsigned int i;
2108 int ret;
2109
2110 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue),
2111 GFP_KERNEL);
2112 if (!info->queues)
2113 return -ENOMEM;
2114
2115 for (i = 0; i < *num_queues; i++) {
2116 struct netfront_queue *queue = &info->queues[i];
2117
2118 queue->id = i;
2119 queue->info = info;
2120
2121 ret = xennet_init_queue(queue);
2122 if (ret < 0) {
2123 dev_warn(&info->xbdev->dev,
2124 "only created %d queues\n", i);
2125 *num_queues = i;
2126 break;
2127 }
2128
2129 /* use page pool recycling instead of buddy allocator */
2130 ret = xennet_create_page_pool(queue);
2131 if (ret < 0) {
2132 dev_err(&info->xbdev->dev, "can't allocate page pool\n");
2133 *num_queues = i;
2134 return ret;
2135 }
2136
2137 netif_napi_add(queue->info->netdev, &queue->napi,
2138 xennet_poll, 64);
2139 if (netif_running(info->netdev))
2140 napi_enable(&queue->napi);
2141 }
2142
2143 netif_set_real_num_tx_queues(info->netdev, *num_queues);
2144
2145 if (*num_queues == 0) {
2146 dev_err(&info->xbdev->dev, "no queues\n");
2147 return -EINVAL;
2148 }
2149 return 0;
2150 }
2151
2152 /* Common code used when first setting up, and when resuming. */
talk_to_netback(struct xenbus_device * dev,struct netfront_info * info)2153 static int talk_to_netback(struct xenbus_device *dev,
2154 struct netfront_info *info)
2155 {
2156 const char *message;
2157 struct xenbus_transaction xbt;
2158 int err;
2159 unsigned int feature_split_evtchn;
2160 unsigned int i = 0;
2161 unsigned int max_queues = 0;
2162 struct netfront_queue *queue = NULL;
2163 unsigned int num_queues = 1;
2164
2165 info->netdev->irq = 0;
2166
2167 /* Check if backend supports multiple queues */
2168 max_queues = xenbus_read_unsigned(info->xbdev->otherend,
2169 "multi-queue-max-queues", 1);
2170 num_queues = min(max_queues, xennet_max_queues);
2171
2172 /* Check feature-split-event-channels */
2173 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend,
2174 "feature-split-event-channels", 0);
2175
2176 /* Read mac addr. */
2177 err = xen_net_read_mac(dev, info->netdev->dev_addr);
2178 if (err) {
2179 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename);
2180 goto out_unlocked;
2181 }
2182
2183 info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend,
2184 "feature-xdp-headroom", 0);
2185 if (info->netback_has_xdp_headroom) {
2186 /* set the current xen-netfront xdp state */
2187 err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ?
2188 NETBACK_XDP_HEADROOM_ENABLE :
2189 NETBACK_XDP_HEADROOM_DISABLE);
2190 if (err)
2191 goto out_unlocked;
2192 }
2193
2194 rtnl_lock();
2195 if (info->queues)
2196 xennet_destroy_queues(info);
2197
2198 /* For the case of a reconnect reset the "broken" indicator. */
2199 info->broken = false;
2200
2201 err = xennet_create_queues(info, &num_queues);
2202 if (err < 0) {
2203 xenbus_dev_fatal(dev, err, "creating queues");
2204 kfree(info->queues);
2205 info->queues = NULL;
2206 goto out;
2207 }
2208 rtnl_unlock();
2209
2210 /* Create shared ring, alloc event channel -- for each queue */
2211 for (i = 0; i < num_queues; ++i) {
2212 queue = &info->queues[i];
2213 err = setup_netfront(dev, queue, feature_split_evtchn);
2214 if (err)
2215 goto destroy_ring;
2216 }
2217
2218 again:
2219 err = xenbus_transaction_start(&xbt);
2220 if (err) {
2221 xenbus_dev_fatal(dev, err, "starting transaction");
2222 goto destroy_ring;
2223 }
2224
2225 if (xenbus_exists(XBT_NIL,
2226 info->xbdev->otherend, "multi-queue-max-queues")) {
2227 /* Write the number of queues */
2228 err = xenbus_printf(xbt, dev->nodename,
2229 "multi-queue-num-queues", "%u", num_queues);
2230 if (err) {
2231 message = "writing multi-queue-num-queues";
2232 goto abort_transaction_no_dev_fatal;
2233 }
2234 }
2235
2236 if (num_queues == 1) {
2237 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */
2238 if (err)
2239 goto abort_transaction_no_dev_fatal;
2240 } else {
2241 /* Write the keys for each queue */
2242 for (i = 0; i < num_queues; ++i) {
2243 queue = &info->queues[i];
2244 err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */
2245 if (err)
2246 goto abort_transaction_no_dev_fatal;
2247 }
2248 }
2249
2250 /* The remaining keys are not queue-specific */
2251 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u",
2252 1);
2253 if (err) {
2254 message = "writing request-rx-copy";
2255 goto abort_transaction;
2256 }
2257
2258 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1);
2259 if (err) {
2260 message = "writing feature-rx-notify";
2261 goto abort_transaction;
2262 }
2263
2264 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
2265 if (err) {
2266 message = "writing feature-sg";
2267 goto abort_transaction;
2268 }
2269
2270 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1);
2271 if (err) {
2272 message = "writing feature-gso-tcpv4";
2273 goto abort_transaction;
2274 }
2275
2276 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1");
2277 if (err) {
2278 message = "writing feature-gso-tcpv6";
2279 goto abort_transaction;
2280 }
2281
2282 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload",
2283 "1");
2284 if (err) {
2285 message = "writing feature-ipv6-csum-offload";
2286 goto abort_transaction;
2287 }
2288
2289 err = xenbus_transaction_end(xbt, 0);
2290 if (err) {
2291 if (err == -EAGAIN)
2292 goto again;
2293 xenbus_dev_fatal(dev, err, "completing transaction");
2294 goto destroy_ring;
2295 }
2296
2297 return 0;
2298
2299 abort_transaction:
2300 xenbus_dev_fatal(dev, err, "%s", message);
2301 abort_transaction_no_dev_fatal:
2302 xenbus_transaction_end(xbt, 1);
2303 destroy_ring:
2304 xennet_disconnect_backend(info);
2305 rtnl_lock();
2306 xennet_destroy_queues(info);
2307 out:
2308 rtnl_unlock();
2309 out_unlocked:
2310 device_unregister(&dev->dev);
2311 return err;
2312 }
2313
xennet_connect(struct net_device * dev)2314 static int xennet_connect(struct net_device *dev)
2315 {
2316 struct netfront_info *np = netdev_priv(dev);
2317 unsigned int num_queues = 0;
2318 int err;
2319 unsigned int j = 0;
2320 struct netfront_queue *queue = NULL;
2321
2322 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) {
2323 dev_info(&dev->dev,
2324 "backend does not support copying receive path\n");
2325 return -ENODEV;
2326 }
2327
2328 err = talk_to_netback(np->xbdev, np);
2329 if (err)
2330 return err;
2331 if (np->netback_has_xdp_headroom)
2332 pr_info("backend supports XDP headroom\n");
2333
2334 /* talk_to_netback() sets the correct number of queues */
2335 num_queues = dev->real_num_tx_queues;
2336
2337 if (dev->reg_state == NETREG_UNINITIALIZED) {
2338 err = register_netdev(dev);
2339 if (err) {
2340 pr_warn("%s: register_netdev err=%d\n", __func__, err);
2341 device_unregister(&np->xbdev->dev);
2342 return err;
2343 }
2344 }
2345
2346 rtnl_lock();
2347 netdev_update_features(dev);
2348 rtnl_unlock();
2349
2350 /*
2351 * All public and private state should now be sane. Get
2352 * ready to start sending and receiving packets and give the driver
2353 * domain a kick because we've probably just requeued some
2354 * packets.
2355 */
2356 netif_tx_lock_bh(np->netdev);
2357 netif_device_attach(np->netdev);
2358 netif_tx_unlock_bh(np->netdev);
2359
2360 netif_carrier_on(np->netdev);
2361 for (j = 0; j < num_queues; ++j) {
2362 queue = &np->queues[j];
2363
2364 notify_remote_via_irq(queue->tx_irq);
2365 if (queue->tx_irq != queue->rx_irq)
2366 notify_remote_via_irq(queue->rx_irq);
2367
2368 spin_lock_irq(&queue->tx_lock);
2369 xennet_tx_buf_gc(queue);
2370 spin_unlock_irq(&queue->tx_lock);
2371
2372 spin_lock_bh(&queue->rx_lock);
2373 xennet_alloc_rx_buffers(queue);
2374 spin_unlock_bh(&queue->rx_lock);
2375 }
2376
2377 return 0;
2378 }
2379
2380 /*
2381 * Callback received when the backend's state changes.
2382 */
netback_changed(struct xenbus_device * dev,enum xenbus_state backend_state)2383 static void netback_changed(struct xenbus_device *dev,
2384 enum xenbus_state backend_state)
2385 {
2386 struct netfront_info *np = dev_get_drvdata(&dev->dev);
2387 struct net_device *netdev = np->netdev;
2388
2389 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state));
2390
2391 wake_up_all(&module_wq);
2392
2393 switch (backend_state) {
2394 case XenbusStateInitialising:
2395 case XenbusStateInitialised:
2396 case XenbusStateReconfiguring:
2397 case XenbusStateReconfigured:
2398 case XenbusStateUnknown:
2399 break;
2400
2401 case XenbusStateInitWait:
2402 if (dev->state != XenbusStateInitialising)
2403 break;
2404 if (xennet_connect(netdev) != 0)
2405 break;
2406 xenbus_switch_state(dev, XenbusStateConnected);
2407 break;
2408
2409 case XenbusStateConnected:
2410 netdev_notify_peers(netdev);
2411 break;
2412
2413 case XenbusStateClosed:
2414 if (dev->state == XenbusStateClosed)
2415 break;
2416 fallthrough; /* Missed the backend's CLOSING state */
2417 case XenbusStateClosing:
2418 xenbus_frontend_closed(dev);
2419 break;
2420 }
2421 }
2422
2423 static const struct xennet_stat {
2424 char name[ETH_GSTRING_LEN];
2425 u16 offset;
2426 } xennet_stats[] = {
2427 {
2428 "rx_gso_checksum_fixup",
2429 offsetof(struct netfront_info, rx_gso_checksum_fixup)
2430 },
2431 };
2432
xennet_get_sset_count(struct net_device * dev,int string_set)2433 static int xennet_get_sset_count(struct net_device *dev, int string_set)
2434 {
2435 switch (string_set) {
2436 case ETH_SS_STATS:
2437 return ARRAY_SIZE(xennet_stats);
2438 default:
2439 return -EINVAL;
2440 }
2441 }
2442
xennet_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)2443 static void xennet_get_ethtool_stats(struct net_device *dev,
2444 struct ethtool_stats *stats, u64 * data)
2445 {
2446 void *np = netdev_priv(dev);
2447 int i;
2448
2449 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2450 data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset));
2451 }
2452
xennet_get_strings(struct net_device * dev,u32 stringset,u8 * data)2453 static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data)
2454 {
2455 int i;
2456
2457 switch (stringset) {
2458 case ETH_SS_STATS:
2459 for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2460 memcpy(data + i * ETH_GSTRING_LEN,
2461 xennet_stats[i].name, ETH_GSTRING_LEN);
2462 break;
2463 }
2464 }
2465
2466 static const struct ethtool_ops xennet_ethtool_ops =
2467 {
2468 .get_link = ethtool_op_get_link,
2469
2470 .get_sset_count = xennet_get_sset_count,
2471 .get_ethtool_stats = xennet_get_ethtool_stats,
2472 .get_strings = xennet_get_strings,
2473 .get_ts_info = ethtool_op_get_ts_info,
2474 };
2475
2476 #ifdef CONFIG_SYSFS
show_rxbuf(struct device * dev,struct device_attribute * attr,char * buf)2477 static ssize_t show_rxbuf(struct device *dev,
2478 struct device_attribute *attr, char *buf)
2479 {
2480 return sprintf(buf, "%lu\n", NET_RX_RING_SIZE);
2481 }
2482
store_rxbuf(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2483 static ssize_t store_rxbuf(struct device *dev,
2484 struct device_attribute *attr,
2485 const char *buf, size_t len)
2486 {
2487 char *endp;
2488
2489 if (!capable(CAP_NET_ADMIN))
2490 return -EPERM;
2491
2492 simple_strtoul(buf, &endp, 0);
2493 if (endp == buf)
2494 return -EBADMSG;
2495
2496 /* rxbuf_min and rxbuf_max are no longer configurable. */
2497
2498 return len;
2499 }
2500
2501 static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf);
2502 static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf);
2503 static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL);
2504
2505 static struct attribute *xennet_dev_attrs[] = {
2506 &dev_attr_rxbuf_min.attr,
2507 &dev_attr_rxbuf_max.attr,
2508 &dev_attr_rxbuf_cur.attr,
2509 NULL
2510 };
2511
2512 static const struct attribute_group xennet_dev_group = {
2513 .attrs = xennet_dev_attrs
2514 };
2515 #endif /* CONFIG_SYSFS */
2516
xennet_bus_close(struct xenbus_device * dev)2517 static void xennet_bus_close(struct xenbus_device *dev)
2518 {
2519 int ret;
2520
2521 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2522 return;
2523 do {
2524 xenbus_switch_state(dev, XenbusStateClosing);
2525 ret = wait_event_timeout(module_wq,
2526 xenbus_read_driver_state(dev->otherend) ==
2527 XenbusStateClosing ||
2528 xenbus_read_driver_state(dev->otherend) ==
2529 XenbusStateClosed ||
2530 xenbus_read_driver_state(dev->otherend) ==
2531 XenbusStateUnknown,
2532 XENNET_TIMEOUT);
2533 } while (!ret);
2534
2535 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2536 return;
2537
2538 do {
2539 xenbus_switch_state(dev, XenbusStateClosed);
2540 ret = wait_event_timeout(module_wq,
2541 xenbus_read_driver_state(dev->otherend) ==
2542 XenbusStateClosed ||
2543 xenbus_read_driver_state(dev->otherend) ==
2544 XenbusStateUnknown,
2545 XENNET_TIMEOUT);
2546 } while (!ret);
2547 }
2548
xennet_remove(struct xenbus_device * dev)2549 static int xennet_remove(struct xenbus_device *dev)
2550 {
2551 struct netfront_info *info = dev_get_drvdata(&dev->dev);
2552
2553 xennet_bus_close(dev);
2554 xennet_disconnect_backend(info);
2555
2556 if (info->netdev->reg_state == NETREG_REGISTERED)
2557 unregister_netdev(info->netdev);
2558
2559 if (info->queues) {
2560 rtnl_lock();
2561 xennet_destroy_queues(info);
2562 rtnl_unlock();
2563 }
2564 xennet_free_netdev(info->netdev);
2565
2566 return 0;
2567 }
2568
2569 static const struct xenbus_device_id netfront_ids[] = {
2570 { "vif" },
2571 { "" }
2572 };
2573
2574 static struct xenbus_driver netfront_driver = {
2575 .ids = netfront_ids,
2576 .probe = netfront_probe,
2577 .remove = xennet_remove,
2578 .resume = netfront_resume,
2579 .otherend_changed = netback_changed,
2580 };
2581
netif_init(void)2582 static int __init netif_init(void)
2583 {
2584 if (!xen_domain())
2585 return -ENODEV;
2586
2587 if (!xen_has_pv_nic_devices())
2588 return -ENODEV;
2589
2590 pr_info("Initialising Xen virtual ethernet driver\n");
2591
2592 /* Allow as many queues as there are CPUs inut max. 8 if user has not
2593 * specified a value.
2594 */
2595 if (xennet_max_queues == 0)
2596 xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT,
2597 num_online_cpus());
2598
2599 return xenbus_register_frontend(&netfront_driver);
2600 }
2601 module_init(netif_init);
2602
2603
netif_exit(void)2604 static void __exit netif_exit(void)
2605 {
2606 xenbus_unregister_driver(&netfront_driver);
2607 }
2608 module_exit(netif_exit);
2609
2610 MODULE_DESCRIPTION("Xen virtual network device frontend");
2611 MODULE_LICENSE("GPL");
2612 MODULE_ALIAS("xen:vif");
2613 MODULE_ALIAS("xennet");
2614
