1 // SPDX-License-Identifier: GPL-2.0-only
36  *	   once the ring buffer is empty, it will clear the
37  *	   interrupt_mask and re-check to see if new data has
44  * transitions from empty to non-empty (or full to non full on the guest
52 	struct hv_ring_buffer_info *rbi = &channel->outbound;  in hv_signal_on_write()
55 	if (READ_ONCE(rbi->ring_buffer->interrupt_mask))  in hv_signal_on_write()
62 	 * ring transitions from being empty to non-empty.  in hv_signal_on_write()
64 	if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) {  in hv_signal_on_write()
65 		++channel->intr_out_empty;  in hv_signal_on_write()
74 	u32 next = ring_info->ring_buffer->write_index;  in hv_get_next_write_location()
84 	ring_info->ring_buffer->write_index = next_write_location;  in hv_set_next_write_location()
91 	return ring_info->ring_datasize;  in hv_get_ring_buffersize()
98 	return (u64)ring_info->ring_buffer->write_index << 32;  in hv_get_ring_bufferindices()
103  * Assume there is enough room. Handles wrap-around in dest case only!!
118 		start_write_offset -= ring_buffer_size;  in hv_copyto_ringbuffer()
137 	read_loc = READ_ONCE(rbi->ring_buffer->read_index);  in hv_get_ringbuffer_availbytes()
138 	write_loc = READ_ONCE(rbi->ring_buffer->write_index);  in hv_get_ringbuffer_availbytes()
139 	dsize = rbi->ring_datasize;  in hv_get_ringbuffer_availbytes()
141 	*write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :  in hv_get_ringbuffer_availbytes()
142 		read_loc - write_loc;  in hv_get_ringbuffer_availbytes()
143 	*read = dsize - *write;  in hv_get_ringbuffer_availbytes()
153 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
155 	if (!ring_info->ring_buffer) {  in hv_ringbuffer_get_debuginfo()
156 		mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
157 		return -EINVAL;  in hv_ringbuffer_get_debuginfo()
163 	debug_info->bytes_avail_toread = bytes_avail_toread;  in hv_ringbuffer_get_debuginfo()
164 	debug_info->bytes_avail_towrite = bytes_avail_towrite;  in hv_ringbuffer_get_debuginfo()
165 	debug_info->current_read_index = ring_info->ring_buffer->read_index;  in hv_ringbuffer_get_debuginfo()
166 	debug_info->current_write_index = ring_info->ring_buffer->write_index;  in hv_ringbuffer_get_debuginfo()
167 	debug_info->current_interrupt_mask  in hv_ringbuffer_get_debuginfo()
168 		= ring_info->ring_buffer->interrupt_mask;  in hv_ringbuffer_get_debuginfo()
169 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
178 	mutex_init(&channel->inbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
179 	mutex_init(&channel->outbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
195 	pages_wraparound = kcalloc(page_cnt * 2 - 1, sizeof(struct page *),  in hv_ringbuffer_init()
198 		return -ENOMEM;  in hv_ringbuffer_init()
201 	for (i = 0; i < 2 * (page_cnt - 1); i++)  in hv_ringbuffer_init()
202 		pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];  in hv_ringbuffer_init()
204 	ring_info->ring_buffer = (struct hv_ring_buffer *)  in hv_ringbuffer_init()
205 		vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);  in hv_ringbuffer_init()
210 	if (!ring_info->ring_buffer)  in hv_ringbuffer_init()
211 		return -ENOMEM;  in hv_ringbuffer_init()
213 	ring_info->ring_buffer->read_index =  in hv_ringbuffer_init()
214 		ring_info->ring_buffer->write_index = 0;  in hv_ringbuffer_init()
217 	ring_info->ring_buffer->feature_bits.value = 1;  in hv_ringbuffer_init()
219 	ring_info->ring_size = page_cnt << PAGE_SHIFT;  in hv_ringbuffer_init()
220 	ring_info->ring_size_div10_reciprocal =  in hv_ringbuffer_init()
221 		reciprocal_value(ring_info->ring_size / 10);  in hv_ringbuffer_init()
222 	ring_info->ring_datasize = ring_info->ring_size -  in hv_ringbuffer_init()
224 	ring_info->priv_read_index = 0;  in hv_ringbuffer_init()
228 		ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL);  in hv_ringbuffer_init()
229 		if (!ring_info->pkt_buffer)  in hv_ringbuffer_init()
230 			return -ENOMEM;  in hv_ringbuffer_init()
231 		ring_info->pkt_buffer_size = max_pkt_size;  in hv_ringbuffer_init()
234 	spin_lock_init(&ring_info->ring_lock);  in hv_ringbuffer_init()
242 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
243 	vunmap(ring_info->ring_buffer);  in hv_ringbuffer_cleanup()
244 	ring_info->ring_buffer = NULL;  in hv_ringbuffer_cleanup()
245 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
247 	kfree(ring_info->pkt_buffer);  in hv_ringbuffer_cleanup()
248 	ring_info->pkt_buffer = NULL;  in hv_ringbuffer_cleanup()
249 	ring_info->pkt_buffer_size = 0;  in hv_ringbuffer_cleanup()
264 	struct hv_ring_buffer_info *outring_info = &channel->outbound;  in hv_ringbuffer_write()
268 	if (channel->rescind)  in hv_ringbuffer_write()
269 		return -ENODEV;  in hv_ringbuffer_write()
274 	spin_lock_irqsave(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
281 	 * is empty since the read index == write index.  in hv_ringbuffer_write()
284 		++channel->out_full_total;  in hv_ringbuffer_write()
286 		if (!channel->out_full_flag) {  in hv_ringbuffer_write()
287 			++channel->out_full_first;  in hv_ringbuffer_write()
288 			channel->out_full_flag = true;  in hv_ringbuffer_write()
291 		spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
292 		return -EAGAIN;  in hv_ringbuffer_write()
295 	channel->out_full_flag = false;  in hv_ringbuffer_write()
315 	if (desc->flags == VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED) {  in hv_ringbuffer_write()
316 		if (channel->next_request_id_callback != NULL) {  in hv_ringbuffer_write()
317 			rqst_id = channel->next_request_id_callback(channel, requestid);  in hv_ringbuffer_write()
319 				spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
320 				return -EAGAIN;  in hv_ringbuffer_write()
325 	desc->trans_id = (rqst_id == VMBUS_NO_RQSTOR) ? requestid : rqst_id;  in hv_ringbuffer_write()
342 	spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
346 	if (channel->rescind) {  in hv_ringbuffer_write()
349 			if (channel->request_addr_callback != NULL)  in hv_ringbuffer_write()
350 				channel->request_addr_callback(channel, rqst_id);  in hv_ringbuffer_write()
352 		return -ENODEV;  in hv_ringbuffer_write()
366 		return -EINVAL;  in hv_ringbuffer_read()
381 	offset = raw ? 0 : (desc->offset8 << 3);  in hv_ringbuffer_read()
382 	packetlen = (desc->len8 << 3) - offset;  in hv_ringbuffer_read()
384 	*requestid = desc->trans_id;  in hv_ringbuffer_read()
387 		return -ENOBUFS;  in hv_ringbuffer_read()
410 	u32 priv_read_loc = rbi->priv_read_index;  in hv_pkt_iter_avail()
411 	u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index);  in hv_pkt_iter_avail()
414 		return write_loc - priv_read_loc;  in hv_pkt_iter_avail()
416 		return (rbi->ring_datasize - priv_read_loc) + write_loc;  in hv_pkt_iter_avail()
424 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_first_raw()
431 	return (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index);  in hv_pkt_iter_first_raw()
438  * If ring buffer is empty, returns NULL and no other action needed.
442 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_first()
450 	bytes_avail = min(rbi->pkt_buffer_size, hv_pkt_iter_avail(rbi));  in hv_pkt_iter_first()
453 	 * Ensure the compiler does not use references to incoming Hyper-V values (which  in hv_pkt_iter_first()
456 	pkt_len = READ_ONCE(desc->len8) << 3;  in hv_pkt_iter_first()
457 	pkt_offset = READ_ONCE(desc->offset8) << 3;  in hv_pkt_iter_first()
461 	 * rbi->pkt_buffer_size  in hv_pkt_iter_first()
473 	/* Copy the Hyper-V packet out of the ring buffer */  in hv_pkt_iter_first()
474 	desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer;  in hv_pkt_iter_first()
478 	 * Hyper-V could still change len8 and offset8 after the earlier read.  in hv_pkt_iter_first()
482 	desc_copy->len8 = pkt_len >> 3;  in hv_pkt_iter_first()
483 	desc_copy->offset8 = pkt_offset >> 3;  in hv_pkt_iter_first()
500 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in __hv_pkt_iter_next()
501 	u32 packetlen = desc->len8 << 3;  in __hv_pkt_iter_next()
502 	u32 dsize = rbi->ring_datasize;  in __hv_pkt_iter_next()
506 	rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;  in __hv_pkt_iter_next()
507 	if (rbi->priv_read_index >= dsize)  in __hv_pkt_iter_next()
508 		rbi->priv_read_index -= dsize;  in __hv_pkt_iter_next()
519 	if (rbi->priv_read_index >= start_read_index)  in hv_pkt_iter_bytes_read()
520 		return rbi->priv_read_index - start_read_index;  in hv_pkt_iter_bytes_read()
522 		return rbi->ring_datasize - start_read_index +  in hv_pkt_iter_bytes_read()
523 			rbi->priv_read_index;  in hv_pkt_iter_bytes_read()
531  * because Hyper-V protects itself by throttling guests that signal
542  * interrupt_mask is used only on the guest->host ring buffer when
543  * sending requests to the host. The host does not use it on the host->
548 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_close()
557 	start_read_index = rbi->ring_buffer->read_index;  in hv_pkt_iter_close()
558 	rbi->ring_buffer->read_index = rbi->priv_read_index;  in hv_pkt_iter_close()
561 	 * Older versions of Hyper-V (before WS2102 and Win8) do not  in hv_pkt_iter_close()
562 	 * implement pending_send_sz and simply poll if the host->guest  in hv_pkt_iter_close()
565 	if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)  in hv_pkt_iter_close()
584 	pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);  in hv_pkt_iter_close()
608 	 * empty so the empty and full conditions are distinguishable.  in hv_pkt_iter_close()
616 	if (curr_write_sz - bytes_read > pending_sz)  in hv_pkt_iter_close()
626 	++channel->intr_in_full;  in hv_pkt_iter_close()