35 	struct rchan_buf *buf = vma->vm_private_data;  in relay_file_mmap_close()  local
36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
45 	struct rchan_buf *buf = vmf->vma->vm_private_data;  in relay_buf_fault()  local
48 	if (!buf)  in relay_buf_fault()
51 	page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));  in relay_buf_fault()
96 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)  in relay_mmap_buf()  argument
101 	if (!buf)  in relay_mmap_buf()
104 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
109 	vma->vm_private_data = buf;  in relay_mmap_buf()
110 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
123 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)  in relay_alloc_buf()  argument
131 	buf->page_array = relay_alloc_page_array(n_pages);  in relay_alloc_buf()
132 	if (!buf->page_array)  in relay_alloc_buf()
136 		buf->page_array[i] = alloc_page(GFP_KERNEL);  in relay_alloc_buf()
137 		if (unlikely(!buf->page_array[i]))  in relay_alloc_buf()
139 		set_page_private(buf->page_array[i], (unsigned long)buf);  in relay_alloc_buf()
141 	mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);  in relay_alloc_buf()
146 	buf->page_count = n_pages;  in relay_alloc_buf()
151 		__free_page(buf->page_array[j]);  in relay_alloc_buf()
152 	relay_free_page_array(buf->page_array);  in relay_alloc_buf()
164 	struct rchan_buf *buf;  in relay_create_buf()  local
169 	buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);  in relay_create_buf()
170 	if (!buf)  in relay_create_buf()
172 	buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t *),  in relay_create_buf()
174 	if (!buf->padding)  in relay_create_buf()
177 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
178 	if (!buf->start)  in relay_create_buf()
181 	buf->chan = chan;  in relay_create_buf()
182 	kref_get(&buf->chan->kref);  in relay_create_buf()
183 	return buf;  in relay_create_buf()
186 	kfree(buf->padding);  in relay_create_buf()
187 	kfree(buf);  in relay_create_buf()
200 	free_percpu(chan->buf);  in relay_destroy_channel()
208 static void relay_destroy_buf(struct rchan_buf *buf)  in relay_destroy_buf()  argument
210 	struct rchan *chan = buf->chan;  in relay_destroy_buf()
213 	if (likely(buf->start)) {  in relay_destroy_buf()
214 		vunmap(buf->start);  in relay_destroy_buf()
215 		for (i = 0; i < buf->page_count; i++)  in relay_destroy_buf()
216 			__free_page(buf->page_array[i]);  in relay_destroy_buf()
217 		relay_free_page_array(buf->page_array);  in relay_destroy_buf()
219 	*per_cpu_ptr(chan->buf, buf->cpu) = NULL;  in relay_destroy_buf()
220 	kfree(buf->padding);  in relay_destroy_buf()
221 	kfree(buf);  in relay_destroy_buf()
235 	struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);  in relay_remove_buf()  local
236 	relay_destroy_buf(buf);  in relay_remove_buf()
245 static int relay_buf_empty(struct rchan_buf *buf)  in relay_buf_empty()  argument
247 	return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;  in relay_buf_empty()
256 int relay_buf_full(struct rchan_buf *buf)  in relay_buf_full()  argument
258 	size_t ready = buf->subbufs_produced - buf->subbufs_consumed;  in relay_buf_full()
259 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
275 static int subbuf_start_default_callback (struct rchan_buf *buf,  in subbuf_start_default_callback()  argument
280 	if (relay_buf_full(buf))  in subbuf_start_default_callback()
289 static void buf_mapped_default_callback(struct rchan_buf *buf,  in buf_mapped_default_callback()  argument
297 static void buf_unmapped_default_callback(struct rchan_buf *buf,  in buf_unmapped_default_callback()  argument
308 						       struct rchan_buf *buf,  in create_buf_file_default_callback()  argument
339 	struct rchan_buf *buf;  in wakeup_readers()  local
341 	buf = container_of(work, struct rchan_buf, wakeup_work);  in wakeup_readers()
342 	wake_up_interruptible(&buf->read_wait);  in wakeup_readers()
352 static void __relay_reset(struct rchan_buf *buf, unsigned int init)  in __relay_reset()  argument
357 		init_waitqueue_head(&buf->read_wait);  in __relay_reset()
358 		kref_init(&buf->kref);  in __relay_reset()
359 		init_irq_work(&buf->wakeup_work, wakeup_readers);  in __relay_reset()
361 		irq_work_sync(&buf->wakeup_work);  in __relay_reset()
364 	buf->subbufs_produced = 0;  in __relay_reset()
365 	buf->subbufs_consumed = 0;  in __relay_reset()
366 	buf->bytes_consumed = 0;  in __relay_reset()
367 	buf->finalized = 0;  in __relay_reset()
368 	buf->data = buf->start;  in __relay_reset()
369 	buf->offset = 0;  in __relay_reset()
371 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
372 		buf->padding[i] = 0;  in __relay_reset()
374 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
390 	struct rchan_buf *buf;  in relay_reset()  local
396 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {  in relay_reset()
397 		__relay_reset(buf, 0);  in relay_reset()
403 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_reset()
404 			__relay_reset(buf, 0);  in relay_reset()
409 static inline void relay_set_buf_dentry(struct rchan_buf *buf,  in relay_set_buf_dentry()  argument
412 	buf->dentry = dentry;  in relay_set_buf_dentry()
413 	d_inode(buf->dentry)->i_size = buf->early_bytes;  in relay_set_buf_dentry()
417 					    struct rchan_buf *buf,  in relay_create_buf_file()  argument
430 					   S_IRUSR, buf,  in relay_create_buf_file()
447  	struct rchan_buf *buf = NULL;  in relay_open_buf()  local
451 		return *per_cpu_ptr(chan->buf, 0);  in relay_open_buf()
453 	buf = relay_create_buf(chan);  in relay_open_buf()
454 	if (!buf)  in relay_open_buf()
458 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
461 		relay_set_buf_dentry(buf, dentry);  in relay_open_buf()
465 						   S_IRUSR, buf,  in relay_open_buf()
471  	buf->cpu = cpu;  in relay_open_buf()
472  	__relay_reset(buf, 1);  in relay_open_buf()
475 		*per_cpu_ptr(chan->buf, 0) = buf;  in relay_open_buf()
476  		buf->cpu = 0;  in relay_open_buf()
479 	return buf;  in relay_open_buf()
482  	relay_destroy_buf(buf);  in relay_open_buf()
494 static void relay_close_buf(struct rchan_buf *buf)  in relay_close_buf()  argument
496 	buf->finalized = 1;  in relay_close_buf()
497 	irq_work_sync(&buf->wakeup_work);  in relay_close_buf()
498 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
499 	kref_put(&buf->kref, relay_remove_buf);  in relay_close_buf()
526 	struct rchan_buf *buf;  in relay_prepare_cpu()  local
530 		if ((buf = *per_cpu_ptr(chan->buf, cpu)))  in relay_prepare_cpu()
532 		buf = relay_open_buf(chan, cpu);  in relay_prepare_cpu()
533 		if (!buf) {  in relay_prepare_cpu()
538 		*per_cpu_ptr(chan->buf, cpu) = buf;  in relay_prepare_cpu()
573 	struct rchan_buf *buf;  in relay_open()  local
584 	chan->buf = alloc_percpu(struct rchan_buf *);  in relay_open()
585 	if (!chan->buf) {  in relay_open()
605 		buf = relay_open_buf(chan, i);  in relay_open()
606 		if (!buf)  in relay_open()
608 		*per_cpu_ptr(chan->buf, i) = buf;  in relay_open()
617 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_open()
618 			relay_close_buf(buf);  in relay_open()
628 	struct rchan_buf *buf;  member
637 	relay_set_buf_dentry(p->buf, p->dentry);  in __relay_set_buf_dentry()
663 	struct rchan_buf *buf;  in relay_late_setup_files()  local
682 		buf = *per_cpu_ptr(chan->buf, 0);  in relay_late_setup_files()
683 		if (!WARN_ON_ONCE(!buf)) {  in relay_late_setup_files()
684 			dentry = relay_create_buf_file(chan, buf, 0);  in relay_late_setup_files()
686 				relay_set_buf_dentry(buf, dentry);  in relay_late_setup_files()
701 		buf = *per_cpu_ptr(chan->buf, i);  in relay_late_setup_files()
702 		if (unlikely(!buf)) {  in relay_late_setup_files()
708 		dentry = relay_create_buf_file(chan, buf, i);  in relay_late_setup_files()
716 			relay_set_buf_dentry(buf, dentry);  in relay_late_setup_files()
719 			disp.buf = buf;  in relay_late_setup_files()
747 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)  in relay_switch_subbuf()  argument
752 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
755 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
756 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
757 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
758 		buf->padding[old_subbuf] = buf->prev_padding;  in relay_switch_subbuf()
759 		buf->subbufs_produced++;  in relay_switch_subbuf()
760 		if (buf->dentry)  in relay_switch_subbuf()
761 			d_inode(buf->dentry)->i_size +=  in relay_switch_subbuf()
762 				buf->chan->subbuf_size -  in relay_switch_subbuf()
763 				buf->padding[old_subbuf];  in relay_switch_subbuf()
765 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
766 					    buf->padding[old_subbuf];  in relay_switch_subbuf()
768 		if (waitqueue_active(&buf->read_wait)) {  in relay_switch_subbuf()
775 			irq_work_queue(&buf->wakeup_work);  in relay_switch_subbuf()
779 	old = buf->data;  in relay_switch_subbuf()
780 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
781 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
782 	buf->offset = 0;  in relay_switch_subbuf()
783 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
784 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
787 	buf->data = new;  in relay_switch_subbuf()
788 	buf->padding[new_subbuf] = 0;  in relay_switch_subbuf()
790 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
796 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
818 	struct rchan_buf *buf;  in relay_subbufs_consumed()  local
823 	buf = *per_cpu_ptr(chan->buf, cpu);  in relay_subbufs_consumed()
824 	if (!buf || subbufs_consumed > chan->n_subbufs)  in relay_subbufs_consumed()
827 	if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)  in relay_subbufs_consumed()
828 		buf->subbufs_consumed = buf->subbufs_produced;  in relay_subbufs_consumed()
830 		buf->subbufs_consumed += subbufs_consumed;  in relay_subbufs_consumed()
842 	struct rchan_buf *buf;  in relay_close()  local
849 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0)))  in relay_close()
850 		relay_close_buf(buf);  in relay_close()
853 			if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_close()
854 				relay_close_buf(buf);  in relay_close()
875 	struct rchan_buf *buf;  in relay_flush()  local
881 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {  in relay_flush()
882 		relay_switch_subbuf(buf, 0);  in relay_flush()
888 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_flush()
889 			relay_switch_subbuf(buf, 0);  in relay_flush()
903 	struct rchan_buf *buf = inode->i_private;  in relay_file_open()  local
904 	kref_get(&buf->kref);  in relay_file_open()
905 	filp->private_data = buf;  in relay_file_open()
919 	struct rchan_buf *buf = filp->private_data;  in relay_file_mmap()  local
920 	return relay_mmap_buf(buf, vma);  in relay_file_mmap()
933 	struct rchan_buf *buf = filp->private_data;  in relay_file_poll()  local
935 	if (buf->finalized)  in relay_file_poll()
939 		poll_wait(filp, &buf->read_wait, wait);  in relay_file_poll()
940 		if (!relay_buf_empty(buf))  in relay_file_poll()
957 	struct rchan_buf *buf = filp->private_data;  in relay_file_release()  local
958 	kref_put(&buf->kref, relay_remove_buf);  in relay_file_release()
966 static void relay_file_read_consume(struct rchan_buf *buf,  in relay_file_read_consume()  argument
970 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
971 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
974 	if (buf->subbufs_produced == buf->subbufs_consumed &&  in relay_file_read_consume()
975 	    buf->offset == buf->bytes_consumed)  in relay_file_read_consume()
978 	if (buf->bytes_consumed + bytes_consumed > subbuf_size) {  in relay_file_read_consume()
979 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
980 		buf->bytes_consumed = 0;  in relay_file_read_consume()
983 	buf->bytes_consumed += bytes_consumed;  in relay_file_read_consume()
985 		read_subbuf = buf->subbufs_consumed % n_subbufs;  in relay_file_read_consume()
987 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
988 	if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {  in relay_file_read_consume()
989 		if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&  in relay_file_read_consume()
990 		    (buf->offset == subbuf_size))  in relay_file_read_consume()
992 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
993 		buf->bytes_consumed = 0;  in relay_file_read_consume()
1000 static int relay_file_read_avail(struct rchan_buf *buf)  in relay_file_read_avail()  argument
1002 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
1003 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
1004 	size_t produced = buf->subbufs_produced;  in relay_file_read_avail()
1007 	relay_file_read_consume(buf, 0, 0);  in relay_file_read_avail()
1009 	consumed = buf->subbufs_consumed;  in relay_file_read_avail()
1011 	if (unlikely(buf->offset > subbuf_size)) {  in relay_file_read_avail()
1019 		buf->subbufs_consumed = consumed;  in relay_file_read_avail()
1020 		buf->bytes_consumed = 0;  in relay_file_read_avail()
1023 	produced = (produced % n_subbufs) * subbuf_size + buf->offset;  in relay_file_read_avail()
1024 	consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;  in relay_file_read_avail()
1030 		if (buf->offset == subbuf_size &&  in relay_file_read_avail()
1031 		    buf->subbufs_produced > buf->subbufs_consumed)  in relay_file_read_avail()
1045 					   struct rchan_buf *buf)  in relay_file_read_subbuf_avail()  argument
1049 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1051 	write_subbuf = (buf->data - buf->start) / subbuf_size;  in relay_file_read_subbuf_avail()
1052 	write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;  in relay_file_read_subbuf_avail()
1055 	padding = buf->padding[read_subbuf];  in relay_file_read_subbuf_avail()
1074 static size_t relay_file_read_start_pos(struct rchan_buf *buf)  in relay_file_read_start_pos()  argument
1077 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1078 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1079 	size_t consumed = buf->subbufs_consumed % n_subbufs;  in relay_file_read_start_pos()
1080 	size_t read_pos = consumed * subbuf_size + buf->bytes_consumed;  in relay_file_read_start_pos()
1083 	padding = buf->padding[read_subbuf];  in relay_file_read_start_pos()
1100 static size_t relay_file_read_end_pos(struct rchan_buf *buf,  in relay_file_read_end_pos()  argument
1105 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1106 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1109 	padding = buf->padding[read_subbuf];  in relay_file_read_end_pos()
1125 	struct rchan_buf *buf = filp->private_data;  in relay_file_read()  local
1137 		if (!relay_file_read_avail(buf))  in relay_file_read()
1140 		read_start = relay_file_read_start_pos(buf);  in relay_file_read()
1141 		avail = relay_file_read_subbuf_avail(read_start, buf);  in relay_file_read()
1146 		from = buf->start + read_start;  in relay_file_read()
1155 		relay_file_read_consume(buf, read_start, ret);  in relay_file_read()
1156 		*ppos = relay_file_read_end_pos(buf, read_start, ret);  in relay_file_read()
1174 				   struct pipe_buffer *buf)  in relay_pipe_buf_release()  argument
1178 	rbuf = (struct rchan_buf *)page_private(buf->page);  in relay_pipe_buf_release()
1179 	relay_consume_bytes(rbuf, buf->private);  in relay_pipe_buf_release()