44 void netvsc_switch_datapath(struct net_device *ndev, bool vf)  in netvsc_switch_datapath()
105 	struct netvsc_device *net_device;  in alloc_net_device()  local
107 	net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL);  in alloc_net_device()
108 	if (!net_device)  in alloc_net_device()
111 	init_waitqueue_head(&net_device->wait_drain);  in alloc_net_device()
112 	net_device->destroy = false;  in alloc_net_device()
114 	net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;  in alloc_net_device()
115 	net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;  in alloc_net_device()
117 	init_completion(&net_device->channel_init_wait);  in alloc_net_device()
118 	init_waitqueue_head(&net_device->subchan_open);  in alloc_net_device()
119 	INIT_WORK(&net_device->subchan_work, netvsc_subchan_work);  in alloc_net_device()
121 	return net_device;  in alloc_net_device()
147 				   struct netvsc_device *net_device,  in netvsc_revoke_recv_buf()  argument
148 				   struct net_device *ndev)  in netvsc_revoke_recv_buf()
159 	if (net_device->recv_section_cnt) {  in netvsc_revoke_recv_buf()
161 		revoke_packet = &net_device->revoke_packet;  in netvsc_revoke_recv_buf()
192 		net_device->recv_section_cnt = 0;  in netvsc_revoke_recv_buf()
197 				   struct netvsc_device *net_device,  in netvsc_revoke_send_buf()  argument
198 				   struct net_device *ndev)  in netvsc_revoke_send_buf()
209 	if (net_device->send_section_cnt) {  in netvsc_revoke_send_buf()
211 		revoke_packet = &net_device->revoke_packet;  in netvsc_revoke_send_buf()
243 		net_device->send_section_cnt = 0;  in netvsc_revoke_send_buf()
248 				       struct netvsc_device *net_device,  in netvsc_teardown_recv_gpadl()  argument
249 				       struct net_device *ndev)  in netvsc_teardown_recv_gpadl()
253 	if (net_device->recv_buf_gpadl_handle) {  in netvsc_teardown_recv_gpadl()
255 					   net_device->recv_buf_gpadl_handle);  in netvsc_teardown_recv_gpadl()
265 		net_device->recv_buf_gpadl_handle = 0;  in netvsc_teardown_recv_gpadl()
270 				       struct netvsc_device *net_device,  in netvsc_teardown_send_gpadl()  argument
271 				       struct net_device *ndev)  in netvsc_teardown_send_gpadl()
275 	if (net_device->send_buf_gpadl_handle) {  in netvsc_teardown_send_gpadl()
277 					   net_device->send_buf_gpadl_handle);  in netvsc_teardown_send_gpadl()
287 		net_device->send_buf_gpadl_handle = 0;  in netvsc_teardown_send_gpadl()
291 int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)  in netvsc_alloc_recv_comp_ring()  argument
293 	struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];  in netvsc_alloc_recv_comp_ring()
297 	size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data);  in netvsc_alloc_recv_comp_ring()
306 			   struct netvsc_device *net_device,  in netvsc_init_buf()  argument
310 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_init_buf()
321 	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)  in netvsc_init_buf()
325 	net_device->recv_buf = vzalloc(buf_size);  in netvsc_init_buf()
326 	if (!net_device->recv_buf) {  in netvsc_init_buf()
334 	net_device->recv_buf_size = buf_size;  in netvsc_init_buf()
341 	ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf,  in netvsc_init_buf()
343 				    &net_device->recv_buf_gpadl_handle);  in netvsc_init_buf()
351 	init_packet = &net_device->channel_init_pkt;  in netvsc_init_buf()
355 		gpadl_handle = net_device->recv_buf_gpadl_handle;  in netvsc_init_buf()
373 	wait_for_completion(&net_device->channel_init_wait);  in netvsc_init_buf()
396 	net_device->recv_section_size = resp->sections[0].sub_alloc_size;  in netvsc_init_buf()
397 	net_device->recv_section_cnt = resp->sections[0].num_sub_allocs;  in netvsc_init_buf()
400 	net_device->recv_completion_cnt  in netvsc_init_buf()
401 		= round_up(net_device->recv_section_cnt + 1,  in netvsc_init_buf()
403 	ret = netvsc_alloc_recv_comp_ring(net_device, 0);  in netvsc_init_buf()
411 	net_device->send_buf = vzalloc(buf_size);  in netvsc_init_buf()
412 	if (!net_device->send_buf) {  in netvsc_init_buf()
423 	ret = vmbus_establish_gpadl(device->channel, net_device->send_buf,  in netvsc_init_buf()
425 				    &net_device->send_buf_gpadl_handle);  in netvsc_init_buf()
433 	init_packet = &net_device->channel_init_pkt;  in netvsc_init_buf()
437 		net_device->send_buf_gpadl_handle;  in netvsc_init_buf()
454 	wait_for_completion(&net_device->channel_init_wait);  in netvsc_init_buf()
468 	net_device->send_section_size = init_packet->msg.  in netvsc_init_buf()
472 	net_device->send_section_cnt = buf_size / net_device->send_section_size;  in netvsc_init_buf()
475 		   net_device->send_section_size, net_device->send_section_cnt);  in netvsc_init_buf()
478 	map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG);  in netvsc_init_buf()
480 	net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL);  in netvsc_init_buf()
481 	if (net_device->send_section_map == NULL) {  in netvsc_init_buf()
489 	netvsc_revoke_recv_buf(device, net_device, ndev);  in netvsc_init_buf()
490 	netvsc_revoke_send_buf(device, net_device, ndev);  in netvsc_init_buf()
491 	netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_init_buf()
492 	netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_init_buf()
500 			      struct netvsc_device *net_device,  in negotiate_nvsp_ver()  argument
504 	struct net_device *ndev = hv_get_drvdata(device);  in negotiate_nvsp_ver()
523 	wait_for_completion(&net_device->channel_init_wait);  in negotiate_nvsp_ver()
556 			      struct netvsc_device *net_device,  in netvsc_connect_vsp()  argument
559 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_connect_vsp()
568 	init_packet = &net_device->channel_init_pkt;  in netvsc_connect_vsp()
572 		if (negotiate_nvsp_ver(device, net_device, init_packet,  in netvsc_connect_vsp()
574 			net_device->nvsp_version = ver_list[i];  in netvsc_connect_vsp()
583 	pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version);  in netvsc_connect_vsp()
588 	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4)  in netvsc_connect_vsp()
612 	ret = netvsc_init_buf(device, net_device, device_info);  in netvsc_connect_vsp()
623 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_device_remove()
625 	struct netvsc_device *net_device  in netvsc_device_remove()  local
633 	netvsc_revoke_recv_buf(device, net_device, ndev);  in netvsc_device_remove()
635 		netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_device_remove()
637 	netvsc_revoke_send_buf(device, net_device, ndev);  in netvsc_device_remove()
639 		netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_device_remove()
644 	for (i = 0; i < net_device->num_chn; i++)  in netvsc_device_remove()
645 		netif_napi_del(&net_device->chan_table[i].napi);  in netvsc_device_remove()
661 		netvsc_teardown_recv_gpadl(device, net_device, ndev);  in netvsc_device_remove()
662 		netvsc_teardown_send_gpadl(device, net_device, ndev);  in netvsc_device_remove()
666 	free_netvsc_device_rcu(net_device);  in netvsc_device_remove()
672 static inline void netvsc_free_send_slot(struct netvsc_device *net_device,  in netvsc_free_send_slot()  argument
675 	sync_change_bit(index, net_device->send_section_map);  in netvsc_free_send_slot()
678 static void netvsc_send_tx_complete(struct net_device *ndev,  in netvsc_send_tx_complete()
679 				    struct netvsc_device *net_device,  in netvsc_send_tx_complete()  argument
697 			netvsc_free_send_slot(net_device, send_index);  in netvsc_send_tx_complete()
700 		tx_stats = &net_device->chan_table[q_idx].tx_stats;  in netvsc_send_tx_complete()
711 		atomic_dec_return(&net_device->chan_table[q_idx].queue_sends);  in netvsc_send_tx_complete()
713 	if (unlikely(net_device->destroy)) {  in netvsc_send_tx_complete()
715 			wake_up(&net_device->wait_drain);  in netvsc_send_tx_complete()
728 static void netvsc_send_completion(struct net_device *ndev,  in netvsc_send_completion()
729 				   struct netvsc_device *net_device,  in netvsc_send_completion()  argument
742 		memcpy(&net_device->channel_init_pkt, nvsp_packet,  in netvsc_send_completion()
744 		complete(&net_device->channel_init_wait);  in netvsc_send_completion()
748 		netvsc_send_tx_complete(ndev, net_device, incoming_channel,  in netvsc_send_completion()
759 static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)  in netvsc_get_next_send_section()  argument
761 	unsigned long *map_addr = net_device->send_section_map;  in netvsc_get_next_send_section()
764 	for_each_clear_bit(i, map_addr, net_device->send_section_cnt) {  in netvsc_get_next_send_section()
772 static void netvsc_copy_to_send_buf(struct netvsc_device *net_device,  in netvsc_copy_to_send_buf()  argument
780 	char *start = net_device->send_buf;  in netvsc_copy_to_send_buf()
781 	char *dest = start + (section_index * net_device->send_section_size)  in netvsc_copy_to_send_buf()
790 	remain = packet->total_data_buflen & (net_device->pkt_align - 1);  in netvsc_copy_to_send_buf()
792 		padding = net_device->pkt_align - remain;  in netvsc_copy_to_send_buf()
813 	struct netvsc_device *net_device,  in netvsc_send_pkt()  argument
821 		&net_device->chan_table[packet->q_idx];  in netvsc_send_pkt()
823 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_send_pkt()
902 int netvsc_send(struct net_device *ndev,  in netvsc_send()
909 	struct netvsc_device *net_device  in netvsc_send()  local
922 	if (unlikely(!net_device || net_device->destroy))  in netvsc_send()
925 	nvchan = &net_device->chan_table[packet->q_idx];  in netvsc_send()
933 		return netvsc_send_pkt(device, packet, net_device, pb, skb);  in netvsc_send()
940 	try_batch =  msd_len > 0 && msdp->count < net_device->max_pkt;  in netvsc_send()
941 	if (try_batch && msd_len + pktlen + net_device->pkt_align <  in netvsc_send()
942 	    net_device->send_section_size) {  in netvsc_send()
946 		   net_device->send_section_size) {  in netvsc_send()
950 	} else if (pktlen + net_device->pkt_align <  in netvsc_send()
951 		   net_device->send_section_size) {  in netvsc_send()
952 		section_index = netvsc_get_next_send_section(net_device);  in netvsc_send()
969 		netvsc_copy_to_send_buf(net_device,  in netvsc_send()
1007 		int m_ret = netvsc_send_pkt(device, msd_send, net_device,  in netvsc_send()
1011 			netvsc_free_send_slot(net_device,  in netvsc_send()
1018 		ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb);  in netvsc_send()
1021 		netvsc_free_send_slot(net_device, section_index);  in netvsc_send()
1027 static int send_recv_completions(struct net_device *ndev,  in send_recv_completions()
1082 static void enq_receive_complete(struct net_device *ndev,  in enq_receive_complete()
1112 static int netvsc_receive(struct net_device *ndev,  in netvsc_receive()
1113 			  struct netvsc_device *net_device,  in netvsc_receive()  argument
1122 	char *recv_buf = net_device->recv_buf;  in netvsc_receive()
1152 		if (unlikely(offset + buflen > net_device->recv_buf_size)) {  in netvsc_receive()
1166 		ret = rndis_filter_receive(ndev, net_device,  in netvsc_receive()
1173 	enq_receive_complete(ndev, net_device, q_idx,  in netvsc_receive()
1179 static void netvsc_send_table(struct net_device *ndev,  in netvsc_send_table()
1199 static void netvsc_send_vf(struct net_device *ndev,  in netvsc_send_vf()
1211 static  void netvsc_receive_inband(struct net_device *ndev,  in netvsc_receive_inband()
1227 				  struct netvsc_device *net_device,  in netvsc_process_raw_pkt()  argument
1228 				  struct net_device *ndev,  in netvsc_process_raw_pkt()
1238 		netvsc_send_completion(ndev, net_device, channel,  in netvsc_process_raw_pkt()
1243 		return netvsc_receive(ndev, net_device, channel,  in netvsc_process_raw_pkt()
1275 	struct netvsc_device *net_device = nvchan->net_device;  in netvsc_poll()  local
1278 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_poll()
1287 		work_done += netvsc_process_raw_pkt(device, channel, net_device,  in netvsc_poll()
1293 	ret = send_recv_completions(ndev, net_device, nvchan);  in netvsc_poll()
1341 	struct netvsc_device *net_device;  in netvsc_device_add()  local
1342 	struct net_device *ndev = hv_get_drvdata(device);  in netvsc_device_add()
1345 	net_device = alloc_net_device();  in netvsc_device_add()
1346 	if (!net_device)  in netvsc_device_add()
1365 		struct netvsc_channel *nvchan = &net_device->chan_table[i];  in netvsc_device_add()
1368 		nvchan->net_device = net_device;  in netvsc_device_add()
1374 	netif_napi_add(ndev, &net_device->chan_table[0].napi,  in netvsc_device_add()
1380 			 netvsc_channel_cb, net_device->chan_table);  in netvsc_device_add()
1390 	napi_enable(&net_device->chan_table[0].napi);  in netvsc_device_add()
1393 	ret = netvsc_connect_vsp(device, net_device, device_info);  in netvsc_device_add()
1403 	rcu_assign_pointer(net_device_ctx->nvdev, net_device);  in netvsc_device_add()
1405 	return net_device;  in netvsc_device_add()
1409 	napi_disable(&net_device->chan_table[0].napi);  in netvsc_device_add()
1415 	netif_napi_del(&net_device->chan_table[0].napi);  in netvsc_device_add()
1416 	free_netvsc_device(&net_device->rcu);  in netvsc_device_add()