1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 * Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2019 Google, Inc.
5 */
6
7 #ifndef _GVE_H_
8 #define _GVE_H_
9
10 #include <linux/dma-mapping.h>
11 #include <linux/netdevice.h>
12 #include <linux/pci.h>
13 #include <linux/u64_stats_sync.h>
14 #include "gve_desc.h"
15
16 #ifndef PCI_VENDOR_ID_GOOGLE
17 #define PCI_VENDOR_ID_GOOGLE 0x1ae0
18 #endif
19
20 #define PCI_DEV_ID_GVNIC 0x0042
21
22 #define GVE_REGISTER_BAR 0
23 #define GVE_DOORBELL_BAR 2
24
25 /* Driver can alloc up to 2 segments for the header and 2 for the payload. */
26 #define GVE_TX_MAX_IOVEC 4
27 /* 1 for management, 1 for rx, 1 for tx */
28 #define GVE_MIN_MSIX 3
29
30 /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
31 struct gve_rx_desc_queue {
32 struct gve_rx_desc *desc_ring; /* the descriptor ring */
33 dma_addr_t bus; /* the bus for the desc_ring */
34 u8 seqno; /* the next expected seqno for this desc*/
35 };
36
37 /* The page info for a single slot in the RX data queue */
38 struct gve_rx_slot_page_info {
39 struct page *page;
40 void *page_address;
41 u32 page_offset; /* offset to write to in page */
42 };
43
44 /* A list of pages registered with the device during setup and used by a queue
45 * as buffers
46 */
47 struct gve_queue_page_list {
48 u32 id; /* unique id */
49 u32 num_entries;
50 struct page **pages; /* list of num_entries pages */
51 dma_addr_t *page_buses; /* the dma addrs of the pages */
52 };
53
54 /* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */
55 struct gve_rx_data_queue {
56 struct gve_rx_data_slot *data_ring; /* read by NIC */
57 dma_addr_t data_bus; /* dma mapping of the slots */
58 struct gve_rx_slot_page_info *page_info; /* page info of the buffers */
59 struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
60 };
61
62 struct gve_priv;
63
64 /* An RX ring that contains a power-of-two sized desc and data ring. */
65 struct gve_rx_ring {
66 struct gve_priv *gve;
67 struct gve_rx_desc_queue desc;
68 struct gve_rx_data_queue data;
69 u64 rbytes; /* free-running bytes received */
70 u64 rpackets; /* free-running packets received */
71 u32 cnt; /* free-running total number of completed packets */
72 u32 fill_cnt; /* free-running total number of descs and buffs posted */
73 u32 mask; /* masks the cnt and fill_cnt to the size of the ring */
74 u32 q_num; /* queue index */
75 u32 ntfy_id; /* notification block index */
76 struct gve_queue_resources *q_resources; /* head and tail pointer idx */
77 dma_addr_t q_resources_bus; /* dma address for the queue resources */
78 struct u64_stats_sync statss; /* sync stats for 32bit archs */
79 };
80
81 /* A TX desc ring entry */
82 union gve_tx_desc {
83 struct gve_tx_pkt_desc pkt; /* first desc for a packet */
84 struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
85 };
86
87 /* Tracks the memory in the fifo occupied by a segment of a packet */
88 struct gve_tx_iovec {
89 u32 iov_offset; /* offset into this segment */
90 u32 iov_len; /* length */
91 u32 iov_padding; /* padding associated with this segment */
92 };
93
94 /* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc
95 * ring entry but only used for a pkt_desc not a seg_desc
96 */
97 struct gve_tx_buffer_state {
98 struct sk_buff *skb; /* skb for this pkt */
99 struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
100 };
101
102 /* A TX buffer - each queue has one */
103 struct gve_tx_fifo {
104 void *base; /* address of base of FIFO */
105 u32 size; /* total size */
106 atomic_t available; /* how much space is still available */
107 u32 head; /* offset to write at */
108 struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */
109 };
110
111 /* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */
112 struct gve_tx_ring {
113 /* Cacheline 0 -- Accessed & dirtied during transmit */
114 struct gve_tx_fifo tx_fifo;
115 u32 req; /* driver tracked head pointer */
116 u32 done; /* driver tracked tail pointer */
117
118 /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
119 __be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */
120 u64 pkt_done; /* free-running - total packets completed */
121 u64 bytes_done; /* free-running - total bytes completed */
122
123 /* Cacheline 2 -- Read-mostly fields */
124 union gve_tx_desc *desc ____cacheline_aligned;
125 struct gve_tx_buffer_state *info; /* Maps 1:1 to a desc */
126 struct netdev_queue *netdev_txq;
127 struct gve_queue_resources *q_resources; /* head and tail pointer idx */
128 u32 mask; /* masks req and done down to queue size */
129
130 /* Slow-path fields */
131 u32 q_num ____cacheline_aligned; /* queue idx */
132 u32 stop_queue; /* count of queue stops */
133 u32 wake_queue; /* count of queue wakes */
134 u32 ntfy_id; /* notification block index */
135 dma_addr_t bus; /* dma address of the descr ring */
136 dma_addr_t q_resources_bus; /* dma address of the queue resources */
137 struct u64_stats_sync statss; /* sync stats for 32bit archs */
138 } ____cacheline_aligned;
139
140 /* Wraps the info for one irq including the napi struct and the queues
141 * associated with that irq.
142 */
143 struct gve_notify_block {
144 __be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
145 char name[IFNAMSIZ + 16]; /* name registered with the kernel */
146 struct napi_struct napi; /* kernel napi struct for this block */
147 struct gve_priv *priv;
148 struct gve_tx_ring *tx; /* tx rings on this block */
149 struct gve_rx_ring *rx; /* rx rings on this block */
150 } ____cacheline_aligned;
151
152 /* Tracks allowed and current queue settings */
153 struct gve_queue_config {
154 u16 max_queues;
155 u16 num_queues; /* current */
156 };
157
158 /* Tracks the available and used qpl IDs */
159 struct gve_qpl_config {
160 u32 qpl_map_size; /* map memory size */
161 unsigned long *qpl_id_map; /* bitmap of used qpl ids */
162 };
163
164 struct gve_priv {
165 struct net_device *dev;
166 struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
167 struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */
168 struct gve_queue_page_list *qpls; /* array of num qpls */
169 struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */
170 dma_addr_t ntfy_block_bus;
171 struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */
172 char mgmt_msix_name[IFNAMSIZ + 16];
173 u32 mgmt_msix_idx;
174 __be32 *counter_array; /* array of num_event_counters */
175 dma_addr_t counter_array_bus;
176
177 u16 num_event_counters;
178 u16 tx_desc_cnt; /* num desc per ring */
179 u16 rx_desc_cnt; /* num desc per ring */
180 u16 tx_pages_per_qpl; /* tx buffer length */
181 u16 rx_pages_per_qpl; /* rx buffer length */
182 u64 max_registered_pages;
183 u64 num_registered_pages; /* num pages registered with NIC */
184 u32 rx_copybreak; /* copy packets smaller than this */
185 u16 default_num_queues; /* default num queues to set up */
186
187 struct gve_queue_config tx_cfg;
188 struct gve_queue_config rx_cfg;
189 struct gve_qpl_config qpl_cfg; /* map used QPL ids */
190 u32 num_ntfy_blks; /* spilt between TX and RX so must be even */
191
192 struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
193 __be32 __iomem *db_bar2; /* "array" of doorbells */
194 u32 msg_enable; /* level for netif* netdev print macros */
195 struct pci_dev *pdev;
196
197 /* metrics */
198 u32 tx_timeo_cnt;
199
200 /* Admin queue - see gve_adminq.h*/
201 union gve_adminq_command *adminq;
202 dma_addr_t adminq_bus_addr;
203 u32 adminq_mask; /* masks prod_cnt to adminq size */
204 u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
205
206 struct workqueue_struct *gve_wq;
207 struct work_struct service_task;
208 unsigned long service_task_flags;
209 unsigned long state_flags;
210 };
211
212 enum gve_service_task_flags {
213 GVE_PRIV_FLAGS_DO_RESET = BIT(1),
214 GVE_PRIV_FLAGS_RESET_IN_PROGRESS = BIT(2),
215 GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = BIT(3),
216 };
217
218 enum gve_state_flags {
219 GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = BIT(1),
220 GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = BIT(2),
221 GVE_PRIV_FLAGS_DEVICE_RINGS_OK = BIT(3),
222 GVE_PRIV_FLAGS_NAPI_ENABLED = BIT(4),
223 };
224
gve_get_do_reset(struct gve_priv * priv)225 static inline bool gve_get_do_reset(struct gve_priv *priv)
226 {
227 return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
228 }
229
gve_set_do_reset(struct gve_priv * priv)230 static inline void gve_set_do_reset(struct gve_priv *priv)
231 {
232 set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
233 }
234
gve_clear_do_reset(struct gve_priv * priv)235 static inline void gve_clear_do_reset(struct gve_priv *priv)
236 {
237 clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
238 }
239
gve_get_reset_in_progress(struct gve_priv * priv)240 static inline bool gve_get_reset_in_progress(struct gve_priv *priv)
241 {
242 return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS,
243 &priv->service_task_flags);
244 }
245
gve_set_reset_in_progress(struct gve_priv * priv)246 static inline void gve_set_reset_in_progress(struct gve_priv *priv)
247 {
248 set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
249 }
250
gve_clear_reset_in_progress(struct gve_priv * priv)251 static inline void gve_clear_reset_in_progress(struct gve_priv *priv)
252 {
253 clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
254 }
255
gve_get_probe_in_progress(struct gve_priv * priv)256 static inline bool gve_get_probe_in_progress(struct gve_priv *priv)
257 {
258 return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS,
259 &priv->service_task_flags);
260 }
261
gve_set_probe_in_progress(struct gve_priv * priv)262 static inline void gve_set_probe_in_progress(struct gve_priv *priv)
263 {
264 set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
265 }
266
gve_clear_probe_in_progress(struct gve_priv * priv)267 static inline void gve_clear_probe_in_progress(struct gve_priv *priv)
268 {
269 clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
270 }
271
gve_get_admin_queue_ok(struct gve_priv * priv)272 static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
273 {
274 return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
275 }
276
gve_set_admin_queue_ok(struct gve_priv * priv)277 static inline void gve_set_admin_queue_ok(struct gve_priv *priv)
278 {
279 set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
280 }
281
gve_clear_admin_queue_ok(struct gve_priv * priv)282 static inline void gve_clear_admin_queue_ok(struct gve_priv *priv)
283 {
284 clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
285 }
286
gve_get_device_resources_ok(struct gve_priv * priv)287 static inline bool gve_get_device_resources_ok(struct gve_priv *priv)
288 {
289 return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
290 }
291
gve_set_device_resources_ok(struct gve_priv * priv)292 static inline void gve_set_device_resources_ok(struct gve_priv *priv)
293 {
294 set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
295 }
296
gve_clear_device_resources_ok(struct gve_priv * priv)297 static inline void gve_clear_device_resources_ok(struct gve_priv *priv)
298 {
299 clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
300 }
301
gve_get_device_rings_ok(struct gve_priv * priv)302 static inline bool gve_get_device_rings_ok(struct gve_priv *priv)
303 {
304 return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
305 }
306
gve_set_device_rings_ok(struct gve_priv * priv)307 static inline void gve_set_device_rings_ok(struct gve_priv *priv)
308 {
309 set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
310 }
311
gve_clear_device_rings_ok(struct gve_priv * priv)312 static inline void gve_clear_device_rings_ok(struct gve_priv *priv)
313 {
314 clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
315 }
316
gve_get_napi_enabled(struct gve_priv * priv)317 static inline bool gve_get_napi_enabled(struct gve_priv *priv)
318 {
319 return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
320 }
321
gve_set_napi_enabled(struct gve_priv * priv)322 static inline void gve_set_napi_enabled(struct gve_priv *priv)
323 {
324 set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
325 }
326
gve_clear_napi_enabled(struct gve_priv * priv)327 static inline void gve_clear_napi_enabled(struct gve_priv *priv)
328 {
329 clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
330 }
331
332 /* Returns the address of the ntfy_blocks irq doorbell
333 */
gve_irq_doorbell(struct gve_priv * priv,struct gve_notify_block * block)334 static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv,
335 struct gve_notify_block *block)
336 {
337 return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
338 }
339
340 /* Returns the index into ntfy_blocks of the given tx ring's block
341 */
gve_tx_idx_to_ntfy(struct gve_priv * priv,u32 queue_idx)342 static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
343 {
344 return queue_idx;
345 }
346
347 /* Returns the index into ntfy_blocks of the given rx ring's block
348 */
gve_rx_idx_to_ntfy(struct gve_priv * priv,u32 queue_idx)349 static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
350 {
351 return (priv->num_ntfy_blks / 2) + queue_idx;
352 }
353
354 /* Returns the number of tx queue page lists
355 */
gve_num_tx_qpls(struct gve_priv * priv)356 static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
357 {
358 return priv->tx_cfg.num_queues;
359 }
360
361 /* Returns the number of rx queue page lists
362 */
gve_num_rx_qpls(struct gve_priv * priv)363 static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
364 {
365 return priv->rx_cfg.num_queues;
366 }
367
368 /* Returns a pointer to the next available tx qpl in the list of qpls
369 */
370 static inline
gve_assign_tx_qpl(struct gve_priv * priv)371 struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv)
372 {
373 int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,
374 priv->qpl_cfg.qpl_map_size);
375
376 /* we are out of tx qpls */
377 if (id >= gve_num_tx_qpls(priv))
378 return NULL;
379
380 set_bit(id, priv->qpl_cfg.qpl_id_map);
381 return &priv->qpls[id];
382 }
383
384 /* Returns a pointer to the next available rx qpl in the list of qpls
385 */
386 static inline
gve_assign_rx_qpl(struct gve_priv * priv)387 struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv)
388 {
389 int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,
390 priv->qpl_cfg.qpl_map_size,
391 gve_num_tx_qpls(priv));
392
393 /* we are out of rx qpls */
394 if (id == priv->qpl_cfg.qpl_map_size)
395 return NULL;
396
397 set_bit(id, priv->qpl_cfg.qpl_id_map);
398 return &priv->qpls[id];
399 }
400
401 /* Unassigns the qpl with the given id
402 */
gve_unassign_qpl(struct gve_priv * priv,int id)403 static inline void gve_unassign_qpl(struct gve_priv *priv, int id)
404 {
405 clear_bit(id, priv->qpl_cfg.qpl_id_map);
406 }
407
408 /* Returns the correct dma direction for tx and rx qpls
409 */
gve_qpl_dma_dir(struct gve_priv * priv,int id)410 static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
411 int id)
412 {
413 if (id < gve_num_tx_qpls(priv))
414 return DMA_TO_DEVICE;
415 else
416 return DMA_FROM_DEVICE;
417 }
418
419 /* Returns true if the max mtu allows page recycling */
gve_can_recycle_pages(struct net_device * dev)420 static inline bool gve_can_recycle_pages(struct net_device *dev)
421 {
422 /* We can't recycle the pages if we can't fit a packet into half a
423 * page.
424 */
425 return dev->max_mtu <= PAGE_SIZE / 2;
426 }
427
428 /* buffers */
429 int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
430 enum dma_data_direction);
431 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
432 enum dma_data_direction);
433 /* tx handling */
434 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev);
435 bool gve_tx_poll(struct gve_notify_block *block, int budget);
436 int gve_tx_alloc_rings(struct gve_priv *priv);
437 void gve_tx_free_rings(struct gve_priv *priv);
438 __be32 gve_tx_load_event_counter(struct gve_priv *priv,
439 struct gve_tx_ring *tx);
440 /* rx handling */
441 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
442 bool gve_rx_poll(struct gve_notify_block *block, int budget);
443 int gve_rx_alloc_rings(struct gve_priv *priv);
444 void gve_rx_free_rings(struct gve_priv *priv);
445 bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
446 netdev_features_t feat);
447 /* Reset */
448 void gve_schedule_reset(struct gve_priv *priv);
449 int gve_reset(struct gve_priv *priv, bool attempt_teardown);
450 int gve_adjust_queues(struct gve_priv *priv,
451 struct gve_queue_config new_rx_config,
452 struct gve_queue_config new_tx_config);
453 /* exported by ethtool.c */
454 extern const struct ethtool_ops gve_ethtool_ops;
455 /* needed by ethtool */
456 extern const char gve_version_str[];
457 #endif /* _GVE_H_ */
458
