1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT)
4  * Copyright (C) 2015-2021 Google, Inc.
10 #include <linux/dma-mapping.h>
43 #define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1))
131 	 * which point every other offset is free to be reused.
135 	/* Linked list index to next element in the list, or -1 if none */
139 /* `head` and `tail` are indices into an array, or -1 if empty. */
181 			 * buf_states, or -1 if empty.
186 			 * buf_states, or -1 if empty.
198 			 * buf_states, or -1 if empty.
207 	u64 rbytes; /* free-running bytes received */
208 	u64 rpackets; /* free-running packets received */
209 	u32 cnt; /* free-running total number of completed packets */
210 	u32 fill_cnt; /* free-running total number of descs and buffs posted */
212 	u64 rx_copybreak_pkt; /* free-running count of copybreak packets */
213 	u64 rx_copied_pkt; /* free-running total number of copied packets */
214 	u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */
215 	u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */
216 	u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */
217 	u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */
218 	u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */
219 	u64 rx_frag_copy_cnt; /* free-running count of rx segments copied into skb linear portion */
257 /* A TX buffer - each queue has one */
274 	/* Packet is in free list, available to be allocated.
281 	 * re-injection completion.
301 	/* Linked list index to next element in the list, or -1 if none */
304 	/* Linked list index to prev element in the list, or -1 if none.
316 	 * freed if the corresponding re-injection completion is not received
324 	/* Cacheline 0 -- Accessed & dirtied during transmit */
336 			 * pending_packets, or -1 if empty.
356 	/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
372 			 * pending_packets, or -1 if empty.
396 	u64 pkt_done; /* free-running - total packets completed */
397 	u64 bytes_done; /* free-running - total bytes completed */
398 	u64 dropped_pkt; /* free-running - total packets dropped */
401 	/* Cacheline 2 -- Read-mostly fields */
428 	/* Slow-path fields */
531 	/* Admin queue - see gve_adminq.h*/
535 	u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
536 	u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */
537 	u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */
538 	/* free-running count of per AQ cmd executed */
559 	u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */
613 	return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_get_do_reset()
618 	set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_set_do_reset()
623 	clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);  in gve_clear_do_reset()
629 			&priv->service_task_flags);  in gve_get_reset_in_progress()
634 	set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_set_reset_in_progress()
639 	clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_reset_in_progress()
645 			&priv->service_task_flags);  in gve_get_probe_in_progress()
650 	set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_set_probe_in_progress()
655 	clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);  in gve_clear_probe_in_progress()
661 			&priv->service_task_flags);  in gve_get_do_report_stats()
666 	set_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_set_do_report_stats()
671 	clear_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags);  in gve_clear_do_report_stats()
676 	return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_get_admin_queue_ok()
681 	set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_set_admin_queue_ok()
686 	clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);  in gve_clear_admin_queue_ok()
691 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_get_device_resources_ok()
696 	set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_set_device_resources_ok()
701 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);  in gve_clear_device_resources_ok()
706 	return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_get_device_rings_ok()
711 	set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_set_device_rings_ok()
716 	clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);  in gve_clear_device_rings_ok()
721 	return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_get_napi_enabled()
726 	set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_set_napi_enabled()
731 	clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);  in gve_clear_napi_enabled()
736 	return test_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_get_report_stats()
741 	clear_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags);  in gve_clear_report_stats()
749 	return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)];  in gve_irq_doorbell()
763 	return (priv->num_ntfy_blks / 2) + queue_idx;  in gve_rx_idx_to_ntfy()
770 	if (priv->queue_format != GVE_GQI_QPL_FORMAT)  in gve_num_tx_qpls()
773 	return priv->tx_cfg.num_queues;  in gve_num_tx_qpls()
780 	if (priv->queue_format != GVE_GQI_QPL_FORMAT)  in gve_num_rx_qpls()
783 	return priv->rx_cfg.num_queues;  in gve_num_rx_qpls()
791 	int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,  in gve_assign_tx_qpl()
792 				     priv->qpl_cfg.qpl_map_size);  in gve_assign_tx_qpl()
798 	set_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_assign_tx_qpl()
799 	return &priv->qpls[id];  in gve_assign_tx_qpl()
807 	int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,  in gve_assign_rx_qpl()
808 				    priv->qpl_cfg.qpl_map_size,  in gve_assign_rx_qpl()
815 	set_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_assign_rx_qpl()
816 	return &priv->qpls[id];  in gve_assign_rx_qpl()
823 	clear_bit(id, priv->qpl_cfg.qpl_id_map);  in gve_unassign_qpl()
839 	return priv->queue_format == GVE_GQI_RDA_FORMAT ||  in gve_is_gqi()
840 		priv->queue_format == GVE_GQI_QPL_FORMAT;  in gve_is_gqi()