35 	struct rchan_buf *buf = vma->vm_private_data;  in relay_file_mmap_close()  local
36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
45 	struct rchan_buf *buf = vmf->vma->vm_private_data;  in relay_buf_fault()  local
48 	if (!buf)  in relay_buf_fault()
51 	page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));  in relay_buf_fault()
96 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)  in relay_mmap_buf()  argument
101 	if (!buf)  in relay_mmap_buf()
104 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
109 	vma->vm_private_data = buf;  in relay_mmap_buf()
110 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
123 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)  in relay_alloc_buf()  argument
131 	buf->page_array = relay_alloc_page_array(n_pages);  in relay_alloc_buf()
132 	if (!buf->page_array)  in relay_alloc_buf()
136 		buf->page_array[i] = alloc_page(GFP_KERNEL);  in relay_alloc_buf()
137 		if (unlikely(!buf->page_array[i]))  in relay_alloc_buf()
139 		set_page_private(buf->page_array[i], (unsigned long)buf);  in relay_alloc_buf()
141 	mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);  in relay_alloc_buf()
146 	buf->page_count = n_pages;  in relay_alloc_buf()
151 		__free_page(buf->page_array[j]);  in relay_alloc_buf()
152 	relay_free_page_array(buf->page_array);  in relay_alloc_buf()
164 	struct rchan_buf *buf;  in relay_create_buf()  local
169 	buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);  in relay_create_buf()
170 	if (!buf)  in relay_create_buf()
172 	buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t *),  in relay_create_buf()
174 	if (!buf->padding)  in relay_create_buf()
177 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
178 	if (!buf->start)  in relay_create_buf()
181 	buf->chan = chan;  in relay_create_buf()
182 	kref_get(&buf->chan->kref);  in relay_create_buf()
183 	return buf;  in relay_create_buf()
186 	kfree(buf->padding);  in relay_create_buf()
187 	kfree(buf);  in relay_create_buf()
207 static void relay_destroy_buf(struct rchan_buf *buf)  in relay_destroy_buf()  argument
209 	struct rchan *chan = buf->chan;  in relay_destroy_buf()
212 	if (likely(buf->start)) {  in relay_destroy_buf()
213 		vunmap(buf->start);  in relay_destroy_buf()
214 		for (i = 0; i < buf->page_count; i++)  in relay_destroy_buf()
215 			__free_page(buf->page_array[i]);  in relay_destroy_buf()
216 		relay_free_page_array(buf->page_array);  in relay_destroy_buf()
218 	*per_cpu_ptr(chan->buf, buf->cpu) = NULL;  in relay_destroy_buf()
219 	kfree(buf->padding);  in relay_destroy_buf()
220 	kfree(buf);  in relay_destroy_buf()
234 	struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);  in relay_remove_buf()  local
235 	relay_destroy_buf(buf);  in relay_remove_buf()
244 static int relay_buf_empty(struct rchan_buf *buf)  in relay_buf_empty()  argument
246 	return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;  in relay_buf_empty()
255 int relay_buf_full(struct rchan_buf *buf)  in relay_buf_full()  argument
257 	size_t ready = buf->subbufs_produced - buf->subbufs_consumed;  in relay_buf_full()
258 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
274 static int subbuf_start_default_callback (struct rchan_buf *buf,  in subbuf_start_default_callback()  argument
279 	if (relay_buf_full(buf))  in subbuf_start_default_callback()
288 static void buf_mapped_default_callback(struct rchan_buf *buf,  in buf_mapped_default_callback()  argument
296 static void buf_unmapped_default_callback(struct rchan_buf *buf,  in buf_unmapped_default_callback()  argument
307 						       struct rchan_buf *buf,  in create_buf_file_default_callback()  argument
338 	struct rchan_buf *buf;  in wakeup_readers()  local
340 	buf = container_of(work, struct rchan_buf, wakeup_work);  in wakeup_readers()
341 	wake_up_interruptible(&buf->read_wait);  in wakeup_readers()
351 static void __relay_reset(struct rchan_buf *buf, unsigned int init)  in __relay_reset()  argument
356 		init_waitqueue_head(&buf->read_wait);  in __relay_reset()
357 		kref_init(&buf->kref);  in __relay_reset()
358 		init_irq_work(&buf->wakeup_work, wakeup_readers);  in __relay_reset()
360 		irq_work_sync(&buf->wakeup_work);  in __relay_reset()
363 	buf->subbufs_produced = 0;  in __relay_reset()
364 	buf->subbufs_consumed = 0;  in __relay_reset()
365 	buf->bytes_consumed = 0;  in __relay_reset()
366 	buf->finalized = 0;  in __relay_reset()
367 	buf->data = buf->start;  in __relay_reset()
368 	buf->offset = 0;  in __relay_reset()
370 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
371 		buf->padding[i] = 0;  in __relay_reset()
373 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
389 	struct rchan_buf *buf;  in relay_reset()  local
395 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {  in relay_reset()
396 		__relay_reset(buf, 0);  in relay_reset()
402 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_reset()
403 			__relay_reset(buf, 0);  in relay_reset()
408 static inline void relay_set_buf_dentry(struct rchan_buf *buf,  in relay_set_buf_dentry()  argument
411 	buf->dentry = dentry;  in relay_set_buf_dentry()
412 	d_inode(buf->dentry)->i_size = buf->early_bytes;  in relay_set_buf_dentry()
416 					    struct rchan_buf *buf,  in relay_create_buf_file()  argument
429 					   S_IRUSR, buf,  in relay_create_buf_file()
444  	struct rchan_buf *buf = NULL;  in relay_open_buf()  local
448 		return *per_cpu_ptr(chan->buf, 0);  in relay_open_buf()
450 	buf = relay_create_buf(chan);  in relay_open_buf()
451 	if (!buf)  in relay_open_buf()
455 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
458 		relay_set_buf_dentry(buf, dentry);  in relay_open_buf()
462 						   S_IRUSR, buf,  in relay_open_buf()
468  	buf->cpu = cpu;  in relay_open_buf()
469  	__relay_reset(buf, 1);  in relay_open_buf()
472 		*per_cpu_ptr(chan->buf, 0) = buf;  in relay_open_buf()
473  		buf->cpu = 0;  in relay_open_buf()
476 	return buf;  in relay_open_buf()
479  	relay_destroy_buf(buf);  in relay_open_buf()
491 static void relay_close_buf(struct rchan_buf *buf)  in relay_close_buf()  argument
493 	buf->finalized = 1;  in relay_close_buf()
494 	irq_work_sync(&buf->wakeup_work);  in relay_close_buf()
495 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
496 	kref_put(&buf->kref, relay_remove_buf);  in relay_close_buf()
523 	struct rchan_buf *buf;  in relay_prepare_cpu()  local
527 		if ((buf = *per_cpu_ptr(chan->buf, cpu)))  in relay_prepare_cpu()
529 		buf = relay_open_buf(chan, cpu);  in relay_prepare_cpu()
530 		if (!buf) {  in relay_prepare_cpu()
535 		*per_cpu_ptr(chan->buf, cpu) = buf;  in relay_prepare_cpu()
570 	struct rchan_buf *buf;  in relay_open()  local
581 	chan->buf = alloc_percpu(struct rchan_buf *);  in relay_open()
597 		buf = relay_open_buf(chan, i);  in relay_open()
598 		if (!buf)  in relay_open()
600 		*per_cpu_ptr(chan->buf, i) = buf;  in relay_open()
609 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_open()
610 			relay_close_buf(buf);  in relay_open()
620 	struct rchan_buf *buf;  member
629 	relay_set_buf_dentry(p->buf, p->dentry);  in __relay_set_buf_dentry()
655 	struct rchan_buf *buf;  in relay_late_setup_files()  local
674 		buf = *per_cpu_ptr(chan->buf, 0);  in relay_late_setup_files()
675 		if (!WARN_ON_ONCE(!buf)) {  in relay_late_setup_files()
676 			dentry = relay_create_buf_file(chan, buf, 0);  in relay_late_setup_files()
678 				relay_set_buf_dentry(buf, dentry);  in relay_late_setup_files()
693 		buf = *per_cpu_ptr(chan->buf, i);  in relay_late_setup_files()
694 		if (unlikely(!buf)) {  in relay_late_setup_files()
700 		dentry = relay_create_buf_file(chan, buf, i);  in relay_late_setup_files()
708 			relay_set_buf_dentry(buf, dentry);  in relay_late_setup_files()
711 			disp.buf = buf;  in relay_late_setup_files()
739 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)  in relay_switch_subbuf()  argument
744 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
747 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
748 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
749 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
750 		buf->padding[old_subbuf] = buf->prev_padding;  in relay_switch_subbuf()
751 		buf->subbufs_produced++;  in relay_switch_subbuf()
752 		if (buf->dentry)  in relay_switch_subbuf()
753 			d_inode(buf->dentry)->i_size +=  in relay_switch_subbuf()
754 				buf->chan->subbuf_size -  in relay_switch_subbuf()
755 				buf->padding[old_subbuf];  in relay_switch_subbuf()
757 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
758 					    buf->padding[old_subbuf];  in relay_switch_subbuf()
760 		if (waitqueue_active(&buf->read_wait)) {  in relay_switch_subbuf()
767 			irq_work_queue(&buf->wakeup_work);  in relay_switch_subbuf()
771 	old = buf->data;  in relay_switch_subbuf()
772 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
773 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
774 	buf->offset = 0;  in relay_switch_subbuf()
775 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
776 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
779 	buf->data = new;  in relay_switch_subbuf()
780 	buf->padding[new_subbuf] = 0;  in relay_switch_subbuf()
782 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
788 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
810 	struct rchan_buf *buf;  in relay_subbufs_consumed()  local
815 	buf = *per_cpu_ptr(chan->buf, cpu);  in relay_subbufs_consumed()
816 	if (!buf || subbufs_consumed > chan->n_subbufs)  in relay_subbufs_consumed()
819 	if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)  in relay_subbufs_consumed()
820 		buf->subbufs_consumed = buf->subbufs_produced;  in relay_subbufs_consumed()
822 		buf->subbufs_consumed += subbufs_consumed;  in relay_subbufs_consumed()
834 	struct rchan_buf *buf;  in relay_close()  local
841 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0)))  in relay_close()
842 		relay_close_buf(buf);  in relay_close()
845 			if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_close()
846 				relay_close_buf(buf);  in relay_close()
867 	struct rchan_buf *buf;  in relay_flush()  local
873 	if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {  in relay_flush()
874 		relay_switch_subbuf(buf, 0);  in relay_flush()
880 		if ((buf = *per_cpu_ptr(chan->buf, i)))  in relay_flush()
881 			relay_switch_subbuf(buf, 0);  in relay_flush()
895 	struct rchan_buf *buf = inode->i_private;  in relay_file_open()  local
896 	kref_get(&buf->kref);  in relay_file_open()
897 	filp->private_data = buf;  in relay_file_open()
911 	struct rchan_buf *buf = filp->private_data;  in relay_file_mmap()  local
912 	return relay_mmap_buf(buf, vma);  in relay_file_mmap()
925 	struct rchan_buf *buf = filp->private_data;  in relay_file_poll()  local
927 	if (buf->finalized)  in relay_file_poll()
931 		poll_wait(filp, &buf->read_wait, wait);  in relay_file_poll()
932 		if (!relay_buf_empty(buf))  in relay_file_poll()
949 	struct rchan_buf *buf = filp->private_data;  in relay_file_release()  local
950 	kref_put(&buf->kref, relay_remove_buf);  in relay_file_release()
958 static void relay_file_read_consume(struct rchan_buf *buf,  in relay_file_read_consume()  argument
962 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
963 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
966 	if (buf->subbufs_produced == buf->subbufs_consumed &&  in relay_file_read_consume()
967 	    buf->offset == buf->bytes_consumed)  in relay_file_read_consume()
970 	if (buf->bytes_consumed + bytes_consumed > subbuf_size) {  in relay_file_read_consume()
971 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
972 		buf->bytes_consumed = 0;  in relay_file_read_consume()
975 	buf->bytes_consumed += bytes_consumed;  in relay_file_read_consume()
977 		read_subbuf = buf->subbufs_consumed % n_subbufs;  in relay_file_read_consume()
979 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
980 	if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {  in relay_file_read_consume()
981 		if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&  in relay_file_read_consume()
982 		    (buf->offset == subbuf_size))  in relay_file_read_consume()
984 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
985 		buf->bytes_consumed = 0;  in relay_file_read_consume()
992 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos)  in relay_file_read_avail()  argument
994 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
995 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
996 	size_t produced = buf->subbufs_produced;  in relay_file_read_avail()
997 	size_t consumed = buf->subbufs_consumed;  in relay_file_read_avail()
999 	relay_file_read_consume(buf, read_pos, 0);  in relay_file_read_avail()
1001 	consumed = buf->subbufs_consumed;  in relay_file_read_avail()
1003 	if (unlikely(buf->offset > subbuf_size)) {  in relay_file_read_avail()
1011 		buf->subbufs_consumed = consumed;  in relay_file_read_avail()
1012 		buf->bytes_consumed = 0;  in relay_file_read_avail()
1015 	produced = (produced % n_subbufs) * subbuf_size + buf->offset;  in relay_file_read_avail()
1016 	consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;  in relay_file_read_avail()
1022 		if (buf->offset == subbuf_size &&  in relay_file_read_avail()
1023 		    buf->subbufs_produced > buf->subbufs_consumed)  in relay_file_read_avail()
1037 					   struct rchan_buf *buf)  in relay_file_read_subbuf_avail()  argument
1041 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1043 	write_subbuf = (buf->data - buf->start) / subbuf_size;  in relay_file_read_subbuf_avail()
1044 	write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;  in relay_file_read_subbuf_avail()
1047 	padding = buf->padding[read_subbuf];  in relay_file_read_subbuf_avail()
1068 					struct rchan_buf *buf)  in relay_file_read_start_pos()  argument
1071 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1072 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1073 	size_t consumed = buf->subbufs_consumed % n_subbufs;  in relay_file_read_start_pos()
1076 		read_pos = consumed * subbuf_size + buf->bytes_consumed;  in relay_file_read_start_pos()
1078 	padding = buf->padding[read_subbuf];  in relay_file_read_start_pos()
1095 static size_t relay_file_read_end_pos(struct rchan_buf *buf,  in relay_file_read_end_pos()  argument
1100 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1101 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1104 	padding = buf->padding[read_subbuf];  in relay_file_read_end_pos()
1120 	struct rchan_buf *buf = filp->private_data;  in relay_file_read()  local
1132 		if (!relay_file_read_avail(buf, *ppos))  in relay_file_read()
1135 		read_start = relay_file_read_start_pos(*ppos, buf);  in relay_file_read()
1136 		avail = relay_file_read_subbuf_avail(read_start, buf);  in relay_file_read()
1141 		from = buf->start + read_start;  in relay_file_read()
1150 		relay_file_read_consume(buf, read_start, ret);  in relay_file_read()
1151 		*ppos = relay_file_read_end_pos(buf, read_start, ret);  in relay_file_read()
1169 				   struct pipe_buffer *buf)  in relay_pipe_buf_release()  argument
1173 	rbuf = (struct rchan_buf *)page_private(buf->page);  in relay_pipe_buf_release()
1174 	relay_consume_bytes(rbuf, buf->private);  in relay_pipe_buf_release()