155 	struct vmci_queue *produce_q;  member
211 	struct vmci_queue *produce_q;  member
224 	void *produce_q;  member
440 	struct vmci_queue *produce_q = prod_q;  in qp_alloc_ppn_set()  local
444 	if (!produce_q || !num_produce_pages || !consume_q ||  in qp_alloc_ppn_set()
469 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;  in qp_alloc_ppn_set()
588 static void qp_init_queue_mutex(struct vmci_queue *produce_q,  in qp_init_queue_mutex()  argument
596 	if (produce_q->kernel_if->host) {  in qp_init_queue_mutex()
597 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
598 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
599 		mutex_init(produce_q->kernel_if->mutex);  in qp_init_queue_mutex()
606 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,  in qp_cleanup_queue_mutex()  argument
609 	if (produce_q->kernel_if->host) {  in qp_cleanup_queue_mutex()
610 		produce_q->kernel_if->mutex = NULL;  in qp_cleanup_queue_mutex()
662 				   struct vmci_queue *produce_q,  in qp_host_get_user_memory()  argument
669 				     produce_q->kernel_if->num_pages, 1,  in qp_host_get_user_memory()
670 				     produce_q->kernel_if->u.h.header_page);  in qp_host_get_user_memory()
671 	if (retval < (int)produce_q->kernel_if->num_pages) {  in qp_host_get_user_memory()
674 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
688 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
689 				 produce_q->kernel_if->num_pages, false);  in qp_host_get_user_memory()
703 					struct vmci_queue *produce_q,  in qp_host_register_user_memory()  argument
716 	    produce_q->kernel_if->num_pages * PAGE_SIZE;  in qp_host_register_user_memory()
717 	return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,  in qp_host_register_user_memory()
726 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,  in qp_host_unregister_user_memory()  argument
729 	qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_unregister_user_memory()
730 			 produce_q->kernel_if->num_pages, true);  in qp_host_unregister_user_memory()
731 	memset(produce_q->kernel_if->u.h.header_page, 0,  in qp_host_unregister_user_memory()
732 	       sizeof(*produce_q->kernel_if->u.h.header_page) *  in qp_host_unregister_user_memory()
733 	       produce_q->kernel_if->num_pages);  in qp_host_unregister_user_memory()
749 static int qp_host_map_queues(struct vmci_queue *produce_q,  in qp_host_map_queues()  argument
754 	if (!produce_q->q_header || !consume_q->q_header) {  in qp_host_map_queues()
757 		if (produce_q->q_header != consume_q->q_header)  in qp_host_map_queues()
760 		if (produce_q->kernel_if->u.h.header_page == NULL ||  in qp_host_map_queues()
761 		    *produce_q->kernel_if->u.h.header_page == NULL)  in qp_host_map_queues()
764 		headers[0] = *produce_q->kernel_if->u.h.header_page;  in qp_host_map_queues()
767 		produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);  in qp_host_map_queues()
768 		if (produce_q->q_header != NULL) {  in qp_host_map_queues()
771 							 produce_q->q_header +  in qp_host_map_queues()
790 				struct vmci_queue *produce_q,  in qp_host_unmap_queues()  argument
793 	if (produce_q->q_header) {  in qp_host_unmap_queues()
794 		if (produce_q->q_header < consume_q->q_header)  in qp_host_unmap_queues()
795 			vunmap(produce_q->q_header);  in qp_host_unmap_queues()
799 		produce_q->q_header = NULL;  in qp_host_unmap_queues()
888 			 void *produce_q,  in qp_guest_endpoint_create()  argument
911 		entry->produce_q = produce_q;  in qp_guest_endpoint_create()
937 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in qp_guest_endpoint_destroy()
938 	qp_free_queue(entry->produce_q, entry->qp.produce_size);  in qp_guest_endpoint_destroy()
1097 			       struct vmci_queue **produce_q,  in qp_alloc_guest_work()  argument
1149 			my_consume_q = queue_pair_entry->produce_q;  in qp_alloc_guest_work()
1232 	*produce_q = (struct vmci_queue *)my_produce_q;  in qp_alloc_guest_work()
1242 		vmci_q_header_init((*produce_q)->q_header, *handle);  in qp_alloc_guest_work()
1360 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);  in qp_broker_create()
1361 	if (entry->produce_q == NULL) {  in qp_broker_create()
1371 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);  in qp_broker_create()
1385 		entry->produce_q->q_header = entry->local_mem;  in qp_broker_create()
1395 						      entry->produce_q,  in qp_broker_create()
1428 		vmci_q_header_init(entry->produce_q->q_header,  in qp_broker_create()
1440 		qp_host_free_queue(entry->produce_q, guest_produce_size);  in qp_broker_create()
1641 							      entry->produce_q,  in qp_broker_attach()
1773 			      struct vmci_queue **produce_q,  in qp_alloc_host_work()  argument
1809 			*produce_q = entry->consume_q;  in qp_alloc_host_work()
1810 			*consume_q = entry->produce_q;  in qp_alloc_host_work()
1812 			*produce_q = entry->produce_q;  in qp_alloc_host_work()
1832 		  struct vmci_queue **produce_q,  in vmci_qp_alloc()  argument
1843 	if (!handle || !produce_q || !consume_q ||  in vmci_qp_alloc()
1848 		return qp_alloc_guest_work(handle, produce_q,  in vmci_qp_alloc()
1853 		return qp_alloc_host_work(handle, produce_q,  in vmci_qp_alloc()
2018 					 entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2022 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in vmci_qp_broker_set_page_store()
2024 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_set_page_store()
2059 	entry->produce_q->saved_header = NULL;  in qp_reset_saved_headers()
2138 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2139 		headers_mapped = entry->produce_q->q_header ||  in vmci_qp_broker_detach()
2144 						 entry->produce_q,  in vmci_qp_broker_detach()
2151 			qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_detach()
2159 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_detach()
2177 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);  in vmci_qp_broker_detach()
2178 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);  in vmci_qp_broker_detach()
2253 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2257 						 entry->produce_q,  in vmci_qp_broker_map()
2259 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_map()
2286 	if (entry->produce_q->saved_header != NULL &&  in qp_save_headers()
2297 	if (NULL == entry->produce_q->q_header ||  in qp_save_headers()
2299 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);  in qp_save_headers()
2304 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,  in qp_save_headers()
2306 	entry->produce_q->saved_header = &entry->saved_produce_q;  in qp_save_headers()
2355 		qp_acquire_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2361 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);  in vmci_qp_broker_unmap()
2370 		qp_host_unregister_user_memory(entry->produce_q,  in vmci_qp_broker_unmap()
2378 		qp_release_queue_mutex(entry->produce_q);  in vmci_qp_broker_unmap()
2428 	qp_acquire_queue_mutex(qpair->produce_q);  in qp_lock()
2437 	qp_release_queue_mutex(qpair->produce_q);  in qp_unlock()
2444 static int qp_map_queue_headers(struct vmci_queue *produce_q,  in qp_map_queue_headers()  argument
2449 	if (NULL == produce_q->q_header || NULL == consume_q->q_header) {  in qp_map_queue_headers()
2450 		result = qp_host_map_queues(produce_q, consume_q);  in qp_map_queue_headers()
2452 			return (produce_q->saved_header &&  in qp_map_queue_headers()
2473 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);  in qp_get_queue_headers()
2475 		*produce_q_header = qpair->produce_q->q_header;  in qp_get_queue_headers()
2477 	} else if (qpair->produce_q->saved_header &&  in qp_get_queue_headers()
2479 		*produce_q_header = qpair->produce_q->saved_header;  in qp_get_queue_headers()
2537 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,  in qp_enqueue_locked()  argument
2548 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_enqueue_locked()
2552 	free_space = vmci_q_header_free_space(produce_q->q_header,  in qp_enqueue_locked()
2562 	tail = vmci_q_header_producer_tail(produce_q->q_header);  in qp_enqueue_locked()
2564 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);  in qp_enqueue_locked()
2570 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);  in qp_enqueue_locked()
2572 			result = qp_memcpy_to_queue_iter(produce_q, 0, from,  in qp_enqueue_locked()
2579 	vmci_q_header_add_producer_tail(produce_q->q_header, written,  in qp_enqueue_locked()
2597 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,  in qp_dequeue_locked()  argument
2609 	result = qp_map_queue_headers(produce_q, consume_q);  in qp_dequeue_locked()
2614 					    produce_q->q_header,  in qp_dequeue_locked()
2623 	head = vmci_q_header_consumer_head(produce_q->q_header);  in qp_dequeue_locked()
2642 		vmci_q_header_add_consumer_head(produce_q->q_header,  in qp_dequeue_locked()
2738 			       &my_qpair->produce_q,  in vmci_qpair_alloc()
3038 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enqueue()
3082 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequeue()
3127 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peek()
3168 		result = qp_enqueue_locked(qpair->produce_q,  in vmci_qpair_enquev()
3209 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_dequev()
3251 		result = qp_dequeue_locked(qpair->produce_q,  in vmci_qpair_peekv()