33 void netvsc_switch_datapath(struct net_device *ndev, bool vf)  in netvsc_switch_datapath()
94 	struct netvsc_device *net_device;  in alloc_net_device()  local
96 	net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL);  in alloc_net_device()
97 	if (!net_device)  in alloc_net_device()
100 	init_waitqueue_head(&net_device->wait_drain);  in alloc_net_device()
101 	net_device->destroy = false;  in alloc_net_device()
102 	net_device->tx_disable = false;  in alloc_net_device()
104 	net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;  in alloc_net_device()
105 	net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;  in alloc_net_device()
107 	init_completion(&net_device->channel_init_wait);  in alloc_net_device()
108 	init_waitqueue_head(&net_device->subchan_open);  in alloc_net_device()
109 	INIT_WORK(&net_device->subchan_work, netvsc_subchan_work);  in alloc_net_device()
111 	return net_device;  in alloc_net_device()
137 				   struct netvsc_device *net_device,  in netvsc_revoke_recv_buf()  argument
138 				   struct net_device *ndev)  in netvsc_revoke_recv_buf()
149 	if (net_device->recv_section_cnt) {  in netvsc_revoke_recv_buf()
151 		revoke_packet = &net_device->revoke_packet;  in netvsc_revoke_recv_buf()
182 		net_device->recv_section_cnt = 0;  in netvsc_revoke_recv_buf()
187 				   struct netvsc_device *net_device,  in netvsc_revoke_send_buf()  argument
188 				   struct net_device *ndev)  in netvsc_revoke_send_buf()
199 	if (net_device->send_section_cnt) {  in netvsc_revoke_send_buf()
201 		revoke_packet = &net_device->revoke_packet;  in netvsc_revoke_send_buf()
233 		net_device->send_section_cnt = 0;  in netvsc_revoke_send_buf()
238 				       struct netvsc_device *net_device,  in netvsc_teardown_recv_gpadl()  argument
239 				       struct net_device *ndev)  in netvsc_teardown_recv_gpadl()
243 	if (net_device->recv_buf_gpadl_handle) {  in netvsc_teardown_recv_gpadl()
245 					   net_device->recv_buf_gpadl_handle);  in netvsc_teardown_recv_gpadl()
255 		net_device->recv_buf_gpadl_handle = 0;  in netvsc_teardown_recv_gpadl()
260 				       struct netvsc_device *net_device,  in netvsc_teardown_send_gpadl()  argument
261 				       struct net_device *ndev)  in netvsc_teardown_send_gpadl()
265 	if (net_device->send_buf_gpadl_handle) {  in netvsc_teardown_send_gpadl()
267 					   net_device->send_buf_gpadl_handle);  in netvsc_teardown_send_gpadl()
277 		net_device->send_buf_gpadl_handle = 0;  in netvsc_teardown_send_gpadl()
281 int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)  in netvsc_alloc_recv_comp_ring()  argument
283 	struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];  in netvsc_alloc_recv_comp_ring()
287 	size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data);  in netvsc_alloc_recv_comp_ring()
296 			   struct netvsc_device *net_device,  in netvsc_init_buf()  argument
300 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_init_buf()
311 	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)  in netvsc_init_buf()
315 	net_device->recv_buf = vzalloc(buf_size);  in netvsc_init_buf()
316 	if (!net_device->recv_buf) {  in netvsc_init_buf()
324 	net_device->recv_buf_size = buf_size;  in netvsc_init_buf()
331 	ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf,  in netvsc_init_buf()
333 				    &net_device->recv_buf_gpadl_handle);  in netvsc_init_buf()
341 	init_packet = &net_device->channel_init_pkt;  in netvsc_init_buf()
345 		gpadl_handle = net_device->recv_buf_gpadl_handle;  in netvsc_init_buf()
363 	wait_for_completion(&net_device->channel_init_wait);  in netvsc_init_buf()
386 	net_device->recv_section_size = resp->sections[0].sub_alloc_size;  in netvsc_init_buf()
387 	net_device->recv_section_cnt = resp->sections[0].num_sub_allocs;  in netvsc_init_buf()
390 	net_device->recv_completion_cnt  in netvsc_init_buf()
391 		= round_up(net_device->recv_section_cnt + 1,  in netvsc_init_buf()
393 	ret = netvsc_alloc_recv_comp_ring(net_device, 0);  in netvsc_init_buf()
401 	net_device->send_buf = vzalloc(buf_size);  in netvsc_init_buf()
402 	if (!net_device->send_buf) {  in netvsc_init_buf()
413 	ret = vmbus_establish_gpadl(device->channel, net_device->send_buf,  in netvsc_init_buf()
415 				    &net_device->send_buf_gpadl_handle);  in netvsc_init_buf()
423 	init_packet = &net_device->channel_init_pkt;  in netvsc_init_buf()
427 		net_device->send_buf_gpadl_handle;  in netvsc_init_buf()
444 	wait_for_completion(&net_device->channel_init_wait);  in netvsc_init_buf()
458 	net_device->send_section_size = init_packet->msg.  in netvsc_init_buf()
462 	net_device->send_section_cnt = buf_size / net_device->send_section_size;  in netvsc_init_buf()
465 		   net_device->send_section_size, net_device->send_section_cnt);  in netvsc_init_buf()
468 	map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG);  in netvsc_init_buf()
470 	net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL);  in netvsc_init_buf()
471 	if (net_device->send_section_map == NULL) {  in netvsc_init_buf()
479 	netvsc_revoke_recv_buf(device, net_device, ndev);  in netvsc_init_buf()
480 	netvsc_revoke_send_buf(device, net_device, ndev);  in netvsc_init_buf()
481 	netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_init_buf()
482 	netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_init_buf()
490 			      struct netvsc_device *net_device,  in negotiate_nvsp_ver()  argument
494 	struct net_device *ndev = hv_get_drvdata(device);  in negotiate_nvsp_ver()
513 	wait_for_completion(&net_device->channel_init_wait);  in negotiate_nvsp_ver()
549 			      struct netvsc_device *net_device,  in netvsc_connect_vsp()  argument
552 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_connect_vsp()
561 	init_packet = &net_device->channel_init_pkt;  in netvsc_connect_vsp()
565 		if (negotiate_nvsp_ver(device, net_device, init_packet,  in netvsc_connect_vsp()
567 			net_device->nvsp_version = ver_list[i];  in netvsc_connect_vsp()
576 	pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version);  in netvsc_connect_vsp()
581 	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4)  in netvsc_connect_vsp()
605 	ret = netvsc_init_buf(device, net_device, device_info);  in netvsc_connect_vsp()
616 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_device_remove()
618 	struct netvsc_device *net_device  in netvsc_device_remove()  local
626 	netvsc_revoke_recv_buf(device, net_device, ndev);  in netvsc_device_remove()
628 		netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_device_remove()
630 	netvsc_revoke_send_buf(device, net_device, ndev);  in netvsc_device_remove()
632 		netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_device_remove()
637 	for (i = 0; i < net_device->num_chn; i++)  in netvsc_device_remove()
638 		netif_napi_del(&net_device->chan_table[i].napi);  in netvsc_device_remove()
654 		netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_device_remove()
655 		netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_device_remove()
659 	free_netvsc_device_rcu(net_device);  in netvsc_device_remove()
665 static inline void netvsc_free_send_slot(struct netvsc_device *net_device,  in netvsc_free_send_slot()  argument
668 	sync_change_bit(index, net_device->send_section_map);  in netvsc_free_send_slot()
671 static void netvsc_send_tx_complete(struct net_device *ndev,  in netvsc_send_tx_complete()
672 				    struct netvsc_device *net_device,  in netvsc_send_tx_complete()  argument
690 			netvsc_free_send_slot(net_device, send_index);  in netvsc_send_tx_complete()
693 		tx_stats = &net_device->chan_table[q_idx].tx_stats;  in netvsc_send_tx_complete()
704 		atomic_dec_return(&net_device->chan_table[q_idx].queue_sends);  in netvsc_send_tx_complete()
706 	if (unlikely(net_device->destroy)) {  in netvsc_send_tx_complete()
708 			wake_up(&net_device->wait_drain);  in netvsc_send_tx_complete()
712 		if (netif_tx_queue_stopped(txq) && !net_device->tx_disable &&  in netvsc_send_tx_complete()
721 static void netvsc_send_completion(struct net_device *ndev,  in netvsc_send_completion()
722 				   struct netvsc_device *net_device,  in netvsc_send_completion()  argument
735 		memcpy(&net_device->channel_init_pkt, nvsp_packet,  in netvsc_send_completion()
737 		complete(&net_device->channel_init_wait);  in netvsc_send_completion()
741 		netvsc_send_tx_complete(ndev, net_device, incoming_channel,  in netvsc_send_completion()
752 static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)  in netvsc_get_next_send_section()  argument
754 	unsigned long *map_addr = net_device->send_section_map;  in netvsc_get_next_send_section()
757 	for_each_clear_bit(i, map_addr, net_device->send_section_cnt) {  in netvsc_get_next_send_section()
765 static void netvsc_copy_to_send_buf(struct netvsc_device *net_device,  in netvsc_copy_to_send_buf()  argument
773 	char *start = net_device->send_buf;  in netvsc_copy_to_send_buf()
774 	char *dest = start + (section_index * net_device->send_section_size)  in netvsc_copy_to_send_buf()
783 	remain = packet->total_data_buflen & (net_device->pkt_align - 1);  in netvsc_copy_to_send_buf()
785 		padding = net_device->pkt_align - remain;  in netvsc_copy_to_send_buf()
806 	struct netvsc_device *net_device,  in netvsc_send_pkt()  argument
814 		&net_device->chan_table[packet->q_idx];  in netvsc_send_pkt()
816 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_send_pkt()
876 	    !net_device->tx_disable) {  in netvsc_send_pkt()
899 int netvsc_send(struct net_device *ndev,  in netvsc_send()
906 	struct netvsc_device *net_device  in netvsc_send()  local
919 	if (unlikely(!net_device || net_device->destroy))  in netvsc_send()
922 	nvchan = &net_device->chan_table[packet->q_idx];  in netvsc_send()
930 		return netvsc_send_pkt(device, packet, net_device, pb, skb);  in netvsc_send()
937 	try_batch =  msd_len > 0 && msdp->count < net_device->max_pkt;  in netvsc_send()
938 	if (try_batch && msd_len + pktlen + net_device->pkt_align <  in netvsc_send()
939 	    net_device->send_section_size) {  in netvsc_send()
943 		   net_device->send_section_size) {  in netvsc_send()
947 	} else if (pktlen + net_device->pkt_align <  in netvsc_send()
948 		   net_device->send_section_size) {  in netvsc_send()
949 		section_index = netvsc_get_next_send_section(net_device);  in netvsc_send()
966 		netvsc_copy_to_send_buf(net_device,  in netvsc_send()
1004 		int m_ret = netvsc_send_pkt(device, msd_send, net_device,  in netvsc_send()
1008 			netvsc_free_send_slot(net_device,  in netvsc_send()
1015 		ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb);  in netvsc_send()
1018 		netvsc_free_send_slot(net_device, section_index);  in netvsc_send()
1024 static int send_recv_completions(struct net_device *ndev,  in send_recv_completions()
1079 static void enq_receive_complete(struct net_device *ndev,  in enq_receive_complete()
1109 static int netvsc_receive(struct net_device *ndev,  in netvsc_receive()
1110 			  struct netvsc_device *net_device,  in netvsc_receive()  argument
1120 	char *recv_buf = net_device->recv_buf;  in netvsc_receive()
1150 		if (unlikely(offset + buflen > net_device->recv_buf_size)) {  in netvsc_receive()
1167 		ret = rndis_filter_receive(ndev, net_device,  in netvsc_receive()
1174 	enq_receive_complete(ndev, net_device, q_idx,  in netvsc_receive()
1180 static void netvsc_send_table(struct net_device *ndev,  in netvsc_send_table()
1219 static void netvsc_send_vf(struct net_device *ndev,  in netvsc_send_vf()
1231 static void netvsc_receive_inband(struct net_device *ndev,  in netvsc_receive_inband()
1249 				  struct netvsc_device *net_device,  in netvsc_process_raw_pkt()  argument
1250 				  struct net_device *ndev,  in netvsc_process_raw_pkt()
1262 		netvsc_send_completion(ndev, net_device, channel,  in netvsc_process_raw_pkt()
1267 		return netvsc_receive(ndev, net_device, nvchan,  in netvsc_process_raw_pkt()
1272 		netvsc_receive_inband(ndev, net_device, nvmsg, msglen);  in netvsc_process_raw_pkt()
1299 	struct netvsc_device *net_device = nvchan->net_device;  in netvsc_poll()  local
1302 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_poll()
1311 		work_done += netvsc_process_raw_pkt(device, nvchan, net_device,  in netvsc_poll()
1317 	ret = send_recv_completions(ndev, net_device, nvchan);  in netvsc_poll()
1365 	struct netvsc_device *net_device;  in netvsc_device_add()  local
1366 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_device_add()
1369 	net_device = alloc_net_device();  in netvsc_device_add()
1370 	if (!net_device)  in netvsc_device_add()
1389 		struct netvsc_channel *nvchan = &net_device->chan_table[i];  in netvsc_device_add()
1392 		nvchan->net_device = net_device;  in netvsc_device_add()
1398 	netif_napi_add(ndev, &net_device->chan_table[0].napi,  in netvsc_device_add()
1404 			 netvsc_channel_cb, net_device->chan_table);  in netvsc_device_add()
1414 	napi_enable(&net_device->chan_table[0].napi);  in netvsc_device_add()
1417 	ret = netvsc_connect_vsp(device, net_device, device_info);  in netvsc_device_add()
1427 	rcu_assign_pointer(net_device_ctx->nvdev, net_device);  in netvsc_device_add()
1429 	return net_device;  in netvsc_device_add()
1433 	napi_disable(&net_device->chan_table[0].napi);  in netvsc_device_add()
1439 	netif_napi_del(&net_device->chan_table[0].napi);  in netvsc_device_add()
1440 	free_netvsc_device(&net_device->rcu);  in netvsc_device_add()