1 // SPDX-License-Identifier: GPL-2.0-only
26  * When we write to the ring buffer, check if the host needs to
29  *	1. The host guarantees that while it is draining the
34  *	2. The host guarantees that it will completely drain
37  *	   interrupt_mask and re-check to see if new data has
43  * the interrupt. The host expects interrupts only when the ring
44  * transitions from empty to non-empty (or full to non full on the guest
45  * to host ring).
47  * host logic is fixed.
52 	struct hv_ring_buffer_info *rbi = &channel->outbound;  in hv_signal_on_write()
55 	if (READ_ONCE(rbi->ring_buffer->interrupt_mask))  in hv_signal_on_write()
61 	 * This is the only case we need to signal when the  in hv_signal_on_write()
62 	 * ring transitions from being empty to non-empty.  in hv_signal_on_write()
64 	if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) {  in hv_signal_on_write()
65 		++channel->intr_out_empty;  in hv_signal_on_write()
74 	u32 next = ring_info->ring_buffer->write_index;  in hv_get_next_write_location()
84 	ring_info->ring_buffer->write_index = next_write_location;  in hv_set_next_write_location()
92 	ring_info->ring_buffer->read_index = next_read_location;  in hv_set_next_read_location()
93 	ring_info->priv_read_index = next_read_location;  in hv_set_next_read_location()
100 	return ring_info->ring_datasize;  in hv_get_ring_buffersize()
107 	return (u64)ring_info->ring_buffer->write_index << 32;  in hv_get_ring_bufferindices()
112  * Assume there is enough room. Handles wrap-around in dest case only!!
127 		start_write_offset -= ring_buffer_size;  in hv_copyto_ringbuffer()
146 	read_loc = READ_ONCE(rbi->ring_buffer->read_index);  in hv_get_ringbuffer_availbytes()
147 	write_loc = READ_ONCE(rbi->ring_buffer->write_index);  in hv_get_ringbuffer_availbytes()
148 	dsize = rbi->ring_datasize;  in hv_get_ringbuffer_availbytes()
150 	*write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :  in hv_get_ringbuffer_availbytes()
151 		read_loc - write_loc;  in hv_get_ringbuffer_availbytes()
152 	*read = dsize - *write;  in hv_get_ringbuffer_availbytes()
162 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
164 	if (!ring_info->ring_buffer) {  in hv_ringbuffer_get_debuginfo()
165 		mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
166 		return -EINVAL;  in hv_ringbuffer_get_debuginfo()
172 	debug_info->bytes_avail_toread = bytes_avail_toread;  in hv_ringbuffer_get_debuginfo()
173 	debug_info->bytes_avail_towrite = bytes_avail_towrite;  in hv_ringbuffer_get_debuginfo()
174 	debug_info->current_read_index = ring_info->ring_buffer->read_index;  in hv_ringbuffer_get_debuginfo()
175 	debug_info->current_write_index = ring_info->ring_buffer->write_index;  in hv_ringbuffer_get_debuginfo()
176 	debug_info->current_interrupt_mask  in hv_ringbuffer_get_debuginfo()
177 		= ring_info->ring_buffer->interrupt_mask;  in hv_ringbuffer_get_debuginfo()
178 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
187 	mutex_init(&channel->inbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
188 	mutex_init(&channel->outbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
204 	pages_wraparound = kcalloc(page_cnt * 2 - 1, sizeof(struct page *),  in hv_ringbuffer_init()
207 		return -ENOMEM;  in hv_ringbuffer_init()
210 	for (i = 0; i < 2 * (page_cnt - 1); i++)  in hv_ringbuffer_init()
211 		pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];  in hv_ringbuffer_init()
213 	ring_info->ring_buffer = (struct hv_ring_buffer *)  in hv_ringbuffer_init()
214 		vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);  in hv_ringbuffer_init()
219 	if (!ring_info->ring_buffer)  in hv_ringbuffer_init()
220 		return -ENOMEM;  in hv_ringbuffer_init()
222 	ring_info->ring_buffer->read_index =  in hv_ringbuffer_init()
223 		ring_info->ring_buffer->write_index = 0;  in hv_ringbuffer_init()
226 	ring_info->ring_buffer->feature_bits.value = 1;  in hv_ringbuffer_init()
228 	ring_info->ring_size = page_cnt << PAGE_SHIFT;  in hv_ringbuffer_init()
229 	ring_info->ring_size_div10_reciprocal =  in hv_ringbuffer_init()
230 		reciprocal_value(ring_info->ring_size / 10);  in hv_ringbuffer_init()
231 	ring_info->ring_datasize = ring_info->ring_size -  in hv_ringbuffer_init()
233 	ring_info->priv_read_index = 0;  in hv_ringbuffer_init()
235 	spin_lock_init(&ring_info->ring_lock);  in hv_ringbuffer_init()
243 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
244 	vunmap(ring_info->ring_buffer);  in hv_ringbuffer_cleanup()
245 	ring_info->ring_buffer = NULL;  in hv_ringbuffer_cleanup()
246 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
260 	struct hv_ring_buffer_info *outring_info = &channel->outbound;  in hv_ringbuffer_write()
262 	if (channel->rescind)  in hv_ringbuffer_write()
263 		return -ENODEV;  in hv_ringbuffer_write()
268 	spin_lock_irqsave(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
273 	 * If there is only room for the packet, assume it is full.  in hv_ringbuffer_write()
278 		++channel->out_full_total;  in hv_ringbuffer_write()
280 		if (!channel->out_full_flag) {  in hv_ringbuffer_write()
281 			++channel->out_full_first;  in hv_ringbuffer_write()
282 			channel->out_full_flag = true;  in hv_ringbuffer_write()
285 		spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
286 		return -EAGAIN;  in hv_ringbuffer_write()
289 	channel->out_full_flag = false;  in hv_ringbuffer_write()
318 	spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
322 	if (channel->rescind)  in hv_ringbuffer_write()
323 		return -ENODEV;  in hv_ringbuffer_write()
336 		return -EINVAL;  in hv_ringbuffer_read()
351 	offset = raw ? 0 : (desc->offset8 << 3);  in hv_ringbuffer_read()
352 	packetlen = (desc->len8 << 3) - offset;  in hv_ringbuffer_read()
354 	*requestid = desc->trans_id;  in hv_ringbuffer_read()
357 		return -ENOBUFS;  in hv_ringbuffer_read()
359 	/* since ring is double mapped, only one copy is necessary */  in hv_ringbuffer_read()
365 	/* Notify host of update */  in hv_ringbuffer_read()
380 	u32 priv_read_loc = rbi->priv_read_index;  in hv_pkt_iter_avail()
381 	u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index);  in hv_pkt_iter_avail()
384 		return write_loc - priv_read_loc;  in hv_pkt_iter_avail()
386 		return (rbi->ring_datasize - priv_read_loc) + write_loc;  in hv_pkt_iter_avail()
396 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_first()
403 	desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index;  in hv_pkt_iter_first()
405 		prefetch((char *)desc + (desc->len8 << 3));  in hv_pkt_iter_first()
421 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in __hv_pkt_iter_next()
422 	u32 packetlen = desc->len8 << 3;  in __hv_pkt_iter_next()
423 	u32 dsize = rbi->ring_datasize;  in __hv_pkt_iter_next()
427 	rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;  in __hv_pkt_iter_next()
428 	if (rbi->priv_read_index >= dsize)  in __hv_pkt_iter_next()
429 		rbi->priv_read_index -= dsize;  in __hv_pkt_iter_next()
440 	if (rbi->priv_read_index >= start_read_index)  in hv_pkt_iter_bytes_read()
441 		return rbi->priv_read_index - start_read_index;  in hv_pkt_iter_bytes_read()
443 		return rbi->ring_datasize - start_read_index +  in hv_pkt_iter_bytes_read()
444 			rbi->priv_read_index;  in hv_pkt_iter_bytes_read()
448  * Update host ring buffer after iterating over packets. If the host has
450  * sufficient space is being freed up, signal the host. But be careful to
451  * only signal the host when necessary, both for performance reasons and
452  * because Hyper-V protects itself by throttling guests that signal
463  * interrupt_mask is used only on the guest->host ring buffer when
464  * sending requests to the host. The host does not use it on the host->
469 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_close()
478 	start_read_index = rbi->ring_buffer->read_index;  in hv_pkt_iter_close()
479 	rbi->ring_buffer->read_index = rbi->priv_read_index;  in hv_pkt_iter_close()
482 	 * Older versions of Hyper-V (before WS2102 and Win8) do not  in hv_pkt_iter_close()
483 	 * implement pending_send_sz and simply poll if the host->guest  in hv_pkt_iter_close()
486 	if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)  in hv_pkt_iter_close()
493 	 * host were to set the pending_send_sz after we have sampled  in hv_pkt_iter_close()
505 	pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);  in hv_pkt_iter_close()
518 	 * We want to signal the host only if we're transitioning  in hv_pkt_iter_close()
521 	 * could run and free up enough space to signal the host, and then  in hv_pkt_iter_close()
522 	 * run again and free up additional space before the host has a  in hv_pkt_iter_close()
534 	 * before we began the iteration. If so, the host was not  in hv_pkt_iter_close()
537 	if (curr_write_sz - bytes_read > pending_sz)  in hv_pkt_iter_close()
547 	++channel->intr_in_full;  in hv_pkt_iter_close()