319 static int rocker_desc_err(const struct rocker_desc_info *desc_info)  in rocker_desc_err()  argument
321 	int err = desc_info->desc->comp_err & ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_err()
347 static void rocker_desc_gen_clear(const struct rocker_desc_info *desc_info)  in rocker_desc_gen_clear()  argument
349 	desc_info->desc->comp_err &= ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_gen_clear()
352 static bool rocker_desc_gen(const struct rocker_desc_info *desc_info)  in rocker_desc_gen()  argument
354 	u32 comp_err = desc_info->desc->comp_err;  in rocker_desc_gen()
360 rocker_desc_cookie_ptr_get(const struct rocker_desc_info *desc_info)  in rocker_desc_cookie_ptr_get()  argument
362 	return (void *)(uintptr_t)desc_info->desc->cookie;  in rocker_desc_cookie_ptr_get()
365 static void rocker_desc_cookie_ptr_set(const struct rocker_desc_info *desc_info,  in rocker_desc_cookie_ptr_set()  argument
368 	desc_info->desc->cookie = (uintptr_t) ptr;  in rocker_desc_cookie_ptr_set()
374 	static struct rocker_desc_info *desc_info;  in rocker_desc_head_get()  local
377 	desc_info = &info->desc_info[info->head];  in rocker_desc_head_get()
380 	desc_info->tlv_size = 0;  in rocker_desc_head_get()
381 	return desc_info;  in rocker_desc_head_get()
384 static void rocker_desc_commit(const struct rocker_desc_info *desc_info)  in rocker_desc_commit()  argument
386 	desc_info->desc->buf_size = desc_info->data_size;  in rocker_desc_commit()
387 	desc_info->desc->tlv_size = desc_info->tlv_size;  in rocker_desc_commit()
392 				 const struct rocker_desc_info *desc_info)  in rocker_desc_head_set()  argument
397 	rocker_desc_commit(desc_info);  in rocker_desc_head_set()
405 	static struct rocker_desc_info *desc_info;  in rocker_desc_tail_get()  local
409 	desc_info = &info->desc_info[info->tail];  in rocker_desc_tail_get()
410 	if (!rocker_desc_gen(desc_info))  in rocker_desc_tail_get()
413 	desc_info->tlv_size = desc_info->desc->tlv_size;  in rocker_desc_tail_get()
414 	return desc_info;  in rocker_desc_tail_get()
443 	info->desc_info = kcalloc(info->size, sizeof(*info->desc_info),  in rocker_dma_ring_create()
445 	if (!info->desc_info)  in rocker_dma_ring_create()
452 		kfree(info->desc_info);  in rocker_dma_ring_create()
457 		info->desc_info[i].desc = &info->desc[i];  in rocker_dma_ring_create()
475 	kfree(info->desc_info);  in rocker_dma_ring_destroy()
489 		rocker_desc_head_set(rocker, info, &info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
490 	rocker_desc_commit(&info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
502 		struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
520 		desc_info->data = buf;  in rocker_dma_ring_bufs_alloc()
521 		desc_info->data_size = buf_size;  in rocker_dma_ring_bufs_alloc()
522 		dma_unmap_addr_set(desc_info, mapaddr, dma_handle);  in rocker_dma_ring_bufs_alloc()
531 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
533 		pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_alloc()
534 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_alloc()
535 		kfree(desc_info->data);  in rocker_dma_ring_bufs_alloc()
548 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_free()  local
553 		pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_free()
554 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_free()
555 		kfree(desc_info->data);  in rocker_dma_ring_bufs_free()
559 static int rocker_dma_cmd_ring_wait_alloc(struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_alloc()  argument
566 	rocker_desc_cookie_ptr_set(desc_info, wait);  in rocker_dma_cmd_ring_wait_alloc()
571 rocker_dma_cmd_ring_wait_free(const struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_free()  argument
573 	struct rocker_wait *wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_cmd_ring_wait_free()
585 		err = rocker_dma_cmd_ring_wait_alloc(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
593 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
603 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_free()
675 				      struct rocker_desc_info *desc_info,  in rocker_dma_rx_ring_skb_map()  argument
686 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_RX_FRAG_ADDR, dma_handle))  in rocker_dma_rx_ring_skb_map()
688 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_RX_FRAG_MAX_LEN, buf_len))  in rocker_dma_rx_ring_skb_map()
694 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_map()
704 					struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_alloc()  argument
714 	rocker_desc_cookie_ptr_set(desc_info, NULL);  in rocker_dma_rx_ring_skb_alloc()
715 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_alloc()
720 	err = rocker_dma_rx_ring_skb_map(rocker_port, desc_info, skb, buf_len);  in rocker_dma_rx_ring_skb_alloc()
725 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_dma_rx_ring_skb_alloc()
745 					const struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_free()  argument
748 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_rx_ring_skb_free()
752 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_dma_rx_ring_skb_free()
766 						   &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
774 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
785 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_free()
881 	const struct rocker_desc_info *desc_info;  in rocker_cmd_irq_handler()  local
886 	while ((desc_info = rocker_desc_tail_get(&rocker->cmd_ring))) {  in rocker_cmd_irq_handler()
887 		wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_irq_handler()
889 			rocker_desc_gen_clear(desc_info);  in rocker_cmd_irq_handler()
974 				const struct rocker_desc_info *desc_info)  in rocker_event_process()  argument
980 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_EVENT_MAX, desc_info);  in rocker_event_process()
1002 	const struct rocker_desc_info *desc_info;  in rocker_event_irq_handler()  local
1006 	while ((desc_info = rocker_desc_tail_get(&rocker->event_ring))) {  in rocker_event_irq_handler()
1007 		err = rocker_desc_err(desc_info);  in rocker_event_irq_handler()
1012 			err = rocker_event_process(rocker, desc_info);  in rocker_event_irq_handler()
1017 		rocker_desc_gen_clear(desc_info);  in rocker_event_irq_handler()
1018 		rocker_desc_head_set(rocker, &rocker->event_ring, desc_info);  in rocker_event_irq_handler()
1051 	struct rocker_desc_info *desc_info;  in rocker_cmd_exec()  local
1058 	desc_info = rocker_desc_head_get(&rocker->cmd_ring);  in rocker_cmd_exec()
1059 	if (!desc_info) {  in rocker_cmd_exec()
1064 	wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_exec()
1068 	err = prepare(rocker_port, desc_info, prepare_priv);  in rocker_cmd_exec()
1074 	rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);  in rocker_cmd_exec()
1084 	err = rocker_desc_err(desc_info);  in rocker_cmd_exec()
1089 		err = process(rocker_port, desc_info, process_priv);  in rocker_cmd_exec()
1091 	rocker_desc_gen_clear(desc_info);  in rocker_cmd_exec()
1097 				  struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_prep()  argument
1102 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_settings_prep()
1105 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_settings_prep()
1108 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_get_port_settings_prep()
1111 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_get_port_settings_prep()
1117 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_ethtool_proc()  argument
1127 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_ethtool_proc()
1156 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_macaddr_proc()  argument
1164 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_macaddr_proc()
1183 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_mode_proc()  argument
1191 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_mode_proc()
1212 					    const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_phys_name_proc()  argument
1222 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_phys_name_proc()
1253 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_ethtool_prep()  argument
1259 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_ethtool_prep()
1262 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_ethtool_prep()
1265 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_ethtool_prep()
1268 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,  in rocker_cmd_set_port_settings_ethtool_prep()
1271 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,  in rocker_cmd_set_port_settings_ethtool_prep()
1274 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,  in rocker_cmd_set_port_settings_ethtool_prep()
1277 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_ethtool_prep()
1283 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_macaddr_prep()  argument
1289 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_macaddr_prep()
1292 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_macaddr_prep()
1295 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_macaddr_prep()
1298 	if (rocker_tlv_put(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,  in rocker_cmd_set_port_settings_macaddr_prep()
1301 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_macaddr_prep()
1307 				      struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_mtu_prep()  argument
1313 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_mtu_prep()
1316 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_mtu_prep()
1319 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_mtu_prep()
1322 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MTU,  in rocker_cmd_set_port_settings_mtu_prep()
1325 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_mtu_prep()
1331 				  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_learning_prep()  argument
1337 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_learning_prep()
1340 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_learning_prep()
1343 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_learning_prep()
1346 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,  in rocker_cmd_set_port_learning_prep()
1349 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_learning_prep()
1831 				       const struct rocker_desc_info *desc_info)  in rocker_tx_desc_frags_unmap()  argument
1839 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_TX_MAX, desc_info);  in rocker_tx_desc_frags_unmap()
1861 				       struct rocker_desc_info *desc_info,  in rocker_tx_desc_frag_map_put()  argument
1875 	frag = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAG);  in rocker_tx_desc_frag_map_put()
1878 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_TX_FRAG_ATTR_ADDR,  in rocker_tx_desc_frag_map_put()
1881 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_TX_FRAG_ATTR_LEN,  in rocker_tx_desc_frag_map_put()
1884 	rocker_tlv_nest_end(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1888 	rocker_tlv_nest_cancel(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1898 	struct rocker_desc_info *desc_info;  in rocker_port_xmit()  local
1903 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1904 	if (unlikely(!desc_info)) {  in rocker_port_xmit()
1910 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_port_xmit()
1912 	frags = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAGS);  in rocker_port_xmit()
1915 	err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1928 		err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1934 	rocker_tlv_nest_end(desc_info, frags);  in rocker_port_xmit()
1936 	rocker_desc_gen_clear(desc_info);  in rocker_port_xmit()
1937 	rocker_desc_head_set(rocker, &rocker_port->tx_ring, desc_info);  in rocker_port_xmit()
1939 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1940 	if (!desc_info)  in rocker_port_xmit()
1946 	rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_xmit()
1948 	rocker_tlv_nest_cancel(desc_info, frags);  in rocker_port_xmit()
2299 			       struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_prep()  argument
2304 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_stats_prep()
2308 	cmd_stats = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_stats_prep()
2312 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_STATS_PPORT,  in rocker_cmd_get_port_stats_prep()
2316 	rocker_tlv_nest_end(desc_info, cmd_stats);  in rocker_cmd_get_port_stats_prep()
2323 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_ethtool_proc()  argument
2333 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_stats_ethtool_proc()
2414 	const struct rocker_desc_info *desc_info;  in rocker_port_poll_tx()  local
2419 	while ((desc_info = rocker_desc_tail_get(&rocker_port->tx_ring))) {  in rocker_port_poll_tx()
2422 		err = rocker_desc_err(desc_info);  in rocker_port_poll_tx()
2426 		rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_poll_tx()
2428 		skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_poll_tx()
2451 			       struct rocker_desc_info *desc_info)  in rocker_port_rx_proc()  argument
2454 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_rx_proc()
2461 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_port_rx_proc()
2481 	return rocker_dma_rx_ring_skb_alloc(rocker_port, desc_info);  in rocker_port_rx_proc()
2493 	struct rocker_desc_info *desc_info;  in rocker_port_poll_rx()  local
2499 	       (desc_info = rocker_desc_tail_get(&rocker_port->rx_ring))) {  in rocker_port_poll_rx()
2500 		err = rocker_desc_err(desc_info);  in rocker_port_poll_rx()
2507 						  desc_info);  in rocker_port_poll_rx()
2515 		rocker_desc_gen_clear(desc_info);  in rocker_port_poll_rx()
2516 		rocker_desc_head_set(rocker, &rocker_port->rx_ring, desc_info);  in rocker_port_poll_rx()