1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT)
4  * Copyright (C) 2015-2021 Google, Inc.
10 #include <linux/dma-mapping.h>
29 /* 1 for management, 1 for rx, 1 for tx */
32 /* Numbers of gve tx/rx stats in stats report. */
39 /* Numbers of NIC tx/rx stats in stats report. */
43 #define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1))
57 /* The page info for a single slot in the RX data queue */
66 /* A list of pages registered with the device during setup and used by a queue
81 	struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
87 /* RX buffer queue for posting buffers to HW.
88  * Each RX (completion) queue has a corresponding buffer queue.
98 /* RX completion queue to receive packets from HW. */
104 	 * post more buffers than the queue size to avoid HW overrunning the
105 	 * queue.
135 	/* Linked list index to next element in the list, or -1 if none */
139 /* `head` and `tail` are indices into an array, or -1 if empty. */
145 /* Contains datapath state used to represent an RX queue. */
167 			 * buf_states, or -1 if empty.
172 			 * buf_states, or -1 if empty.
184 			 * buf_states, or -1 if empty.
193 	u64 rbytes; /* free-running bytes received */
194 	u64 rpackets; /* free-running packets received */
195 	u32 cnt; /* free-running total number of completed packets */
196 	u32 fill_cnt; /* free-running total number of descs and buffs posted */
198 	u64 rx_copybreak_pkt; /* free-running count of copybreak packets */
199 	u64 rx_copied_pkt; /* free-running total number of copied packets */
200 	u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */
201 	u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */
202 	u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */
203 	u32 q_num; /* queue index */
206 	dma_addr_t q_resources_bus; /* dma address for the queue resources */
243 /* A TX buffer - each queue has one */
267 	 * re-injection completion.
286 	/* Linked list index to next element in the list, or -1 if none */
289 	/* Linked list index to prev element in the list, or -1 if none.
301 	 * freed if the corresponding re-injection completion is not received
307 /* Contains datapath state used to represent a TX queue. */
309 	/* Cacheline 0 -- Accessed & dirtied during transmit */
321 			 * pending_packets, or -1 if empty.
341 	/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
357 			 * pending_packets, or -1 if empty.
381 	u64 pkt_done; /* free-running - total packets completed */
382 	u64 bytes_done; /* free-running - total bytes completed */
383 	u64 dropped_pkt; /* free-running - total packets dropped */
386 	/* Cacheline 2 -- Read-mostly fields */
410 	u32 mask; /* masks req and done down to queue size */
413 	/* Slow-path fields */
414 	u32 q_num ____cacheline_aligned; /* queue idx */
415 	u32 stop_queue; /* count of queue stops */
416 	u32 wake_queue; /* count of queue wakes */
419 	dma_addr_t q_resources_bus; /* dma address of the queue resources */
428 	__be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
433 	struct gve_rx_ring *rx; /* rx rings on this block */  member
436 /* Tracks allowed and current queue settings */
476 	struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */  member
490 	u16 rx_data_slot_cnt; /* rx buffer length */
499 	u32 num_ntfy_blks; /* spilt between TX and RX so must be even */
509 	/* Admin queue - see gve_adminq.h*/
513 	u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
514 	u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */
515 	u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */
516 	/* free-running count of per AQ cmd executed */
537 	u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */
584 	return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_get_do_reset()
589 	set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_set_do_reset()
594 	clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_clear_do_reset()
600 			&priv->service_task_flags);  in gve_get_reset_in_progress()
605 	set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_set_reset_in_progress()
610 	clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_reset_in_progress()
616 			&priv->service_task_flags);  in gve_get_probe_in_progress()
621 	set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_set_probe_in_progress()
626 	clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_probe_in_progress()
632 			&priv->service_task_flags);  in gve_get_do_report_stats()
637 	set_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_set_do_report_stats()
642 	clear_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_clear_do_report_stats()
647 	return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_get_admin_queue_ok()
652 	set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_set_admin_queue_ok()
657 	clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_clear_admin_queue_ok()
662 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_get_device_resources_ok()
667 	set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_set_device_resources_ok()
672 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_clear_device_resources_ok()
677 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_get_device_rings_ok()
682 	set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_set_device_rings_ok()
687 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_clear_device_rings_ok()
692 	return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_get_napi_enabled()
697 	set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_set_napi_enabled()
702 	clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_clear_napi_enabled()
707 	return test_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_get_report_stats()
712 	clear_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_clear_report_stats()
720 	return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];  in gve_irq_doorbell()
730 /* Returns the index into ntfy_blocks of the given rx ring's block
734 	return (priv->num_ntfy_blks / 2) + queue_idx;  in gve_rx_idx_to_ntfy()
737 /* Returns the number of tx queue page lists
741 	if (priv->queue_format != GVE_GQI_QPL_FORMAT)  in gve_num_tx_qpls()
744 	return priv->tx_cfg.num_queues;  in gve_num_tx_qpls()
747 /* Returns the number of rx queue page lists
751 	if (priv->queue_format != GVE_GQI_QPL_FORMAT)  in gve_num_rx_qpls()
754 	return priv->rx_cfg.num_queues;  in gve_num_rx_qpls()
762 	int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,  in gve_assign_tx_qpl()
763 				     priv->qpl_cfg.qpl_map_size);  in gve_assign_tx_qpl()
769 	set_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_assign_tx_qpl()
770 	return &priv->qpls[id];  in gve_assign_tx_qpl()
773 /* Returns a pointer to the next available rx qpl in the list of qpls
778 	int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,  in gve_assign_rx_qpl()
779 				    priv->qpl_cfg.qpl_map_size,  in gve_assign_rx_qpl()
782 	/* we are out of rx qpls */  in gve_assign_rx_qpl()
786 	set_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_assign_rx_qpl()
787 	return &priv->qpls[id];  in gve_assign_rx_qpl()
794 	clear_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_unassign_qpl()
797 /* Returns the correct dma direction for tx and rx qpls
810 	return priv->queue_format == GVE_GQI_RDA_FORMAT ||  in gve_is_gqi()
811 		priv->queue_format == GVE_GQI_QPL_FORMAT;  in gve_is_gqi()
827 /* rx handling */
828 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
832 bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,