110 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_read()
177 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_poll()
193 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_read_wrapper()
206 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_poll_wrapper()
225 	struct iio_buffer *buffer;  in iio_buffer_wakeup_poll()  local
229 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_buffer_wakeup_poll()
230 		wake_up(&buffer->pollq);  in iio_buffer_wakeup_poll()
234 void iio_buffer_init(struct iio_buffer *buffer)  in iio_buffer_init()  argument
236 	INIT_LIST_HEAD(&buffer->demux_list);  in iio_buffer_init()
237 	INIT_LIST_HEAD(&buffer->buffer_list);  in iio_buffer_init()
238 	init_waitqueue_head(&buffer->pollq);  in iio_buffer_init()
239 	kref_init(&buffer->ref);  in iio_buffer_init()
240 	if (!buffer->watermark)  in iio_buffer_init()
241 		buffer->watermark = 1;  in iio_buffer_init()
248 	struct iio_buffer *buffer;  in iio_device_detach_buffers()  local
252 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_device_detach_buffers()
253 		iio_buffer_put(buffer);  in iio_device_detach_buffers()
302 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_show()  local
306 		       buffer->scan_mask);  in iio_scan_el_show()
352 		      struct iio_buffer *buffer, int bit)  in iio_scan_mask_set()  argument
365 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
378 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
389 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)  in iio_scan_mask_clear()  argument
391 	clear_bit(bit, buffer->scan_mask);  in iio_scan_mask_clear()
396 			       struct iio_buffer *buffer, int bit)  in iio_scan_mask_query()  argument
401 	if (!buffer->scan_mask)  in iio_scan_mask_query()
405 	return !!test_bit(bit, buffer->scan_mask);  in iio_scan_mask_query()
417 	struct iio_buffer *buffer = this_attr->buffer;  in iio_scan_el_store()  local
423 	if (iio_buffer_is_active(buffer)) {  in iio_scan_el_store()
427 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
431 		ret = iio_scan_mask_clear(buffer, this_attr->address);  in iio_scan_el_store()
435 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
451 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_ts_show()  local
453 	return sysfs_emit(buf, "%d\n", buffer->scan_timestamp);  in iio_scan_el_ts_show()
463 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_ts_store()  local
471 	if (iio_buffer_is_active(buffer)) {  in iio_scan_el_ts_store()
475 	buffer->scan_timestamp = state;  in iio_scan_el_ts_store()
483 					struct iio_buffer *buffer,  in iio_buffer_add_channel_sysfs()  argument
495 				     buffer,  in iio_buffer_add_channel_sysfs()
496 				     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
507 				     buffer,  in iio_buffer_add_channel_sysfs()
508 				     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
520 					     buffer,  in iio_buffer_add_channel_sysfs()
521 					     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
530 					     buffer,  in iio_buffer_add_channel_sysfs()
531 					     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
543 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_read_length()  local
545 	return sysfs_emit(buf, "%d\n", buffer->length);  in iio_buffer_read_length()
553 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_write_length()  local
561 	if (val == buffer->length)  in iio_buffer_write_length()
565 	if (iio_buffer_is_active(buffer)) {  in iio_buffer_write_length()
568 		buffer->access->set_length(buffer, val);  in iio_buffer_write_length()
573 	if (buffer->length && buffer->length < buffer->watermark)  in iio_buffer_write_length()
574 		buffer->watermark = buffer->length;  in iio_buffer_write_length()
585 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_show_enable()  local
587 	return sysfs_emit(buf, "%d\n", iio_buffer_is_active(buffer));  in iio_buffer_show_enable()
638 	struct iio_buffer *buffer)  in iio_buffer_activate()  argument
642 	iio_buffer_get(buffer);  in iio_buffer_activate()
643 	list_add(&buffer->buffer_list, &iio_dev_opaque->buffer_list);  in iio_buffer_activate()
646 static void iio_buffer_deactivate(struct iio_buffer *buffer)  in iio_buffer_deactivate()  argument
648 	list_del_init(&buffer->buffer_list);  in iio_buffer_deactivate()
649 	wake_up_interruptible(&buffer->pollq);  in iio_buffer_deactivate()
650 	iio_buffer_put(buffer);  in iio_buffer_deactivate()
656 	struct iio_buffer *buffer, *_buffer;  in iio_buffer_deactivate_all()  local
658 	list_for_each_entry_safe(buffer, _buffer,  in iio_buffer_deactivate_all()
660 		iio_buffer_deactivate(buffer);  in iio_buffer_deactivate_all()
663 static int iio_buffer_enable(struct iio_buffer *buffer,  in iio_buffer_enable()  argument
666 	if (!buffer->access->enable)  in iio_buffer_enable()
668 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
671 static int iio_buffer_disable(struct iio_buffer *buffer,  in iio_buffer_disable()  argument
674 	if (!buffer->access->disable)  in iio_buffer_disable()
676 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
680 	struct iio_buffer *buffer)  in iio_buffer_update_bytes_per_datum()  argument
684 	if (!buffer->access->set_bytes_per_datum)  in iio_buffer_update_bytes_per_datum()
687 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
688 		buffer->scan_timestamp);  in iio_buffer_update_bytes_per_datum()
690 	buffer->access->set_bytes_per_datum(buffer, bytes);  in iio_buffer_update_bytes_per_datum()
694 	struct iio_buffer *buffer)  in iio_buffer_request_update()  argument
698 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
699 	if (buffer->access->request_update) {  in iio_buffer_request_update()
700 		ret = buffer->access->request_update(buffer);  in iio_buffer_request_update()
736 	struct iio_buffer *buffer;  in iio_verify_update()  local
760 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_verify_update()
761 		if (buffer == remove_buffer)  in iio_verify_update()
763 		modes &= buffer->access->modes;  in iio_verify_update()
764 		config->watermark = min(config->watermark, buffer->watermark);  in iio_verify_update()
801 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_verify_update()
802 		if (buffer == remove_buffer)  in iio_verify_update()
804 		bitmap_or(compound_mask, compound_mask, buffer->scan_mask,  in iio_verify_update()
806 		scan_timestamp |= buffer->scan_timestamp;  in iio_verify_update()
849 static void iio_buffer_demux_free(struct iio_buffer *buffer)  in iio_buffer_demux_free()  argument
852 	list_for_each_entry_safe(p, q, &buffer->demux_list, l) {  in iio_buffer_demux_free()
858 static int iio_buffer_add_demux(struct iio_buffer *buffer,  in iio_buffer_add_demux()  argument
873 		list_add_tail(&(*p)->l, &buffer->demux_list);  in iio_buffer_add_demux()
880 				   struct iio_buffer *buffer)  in iio_buffer_update_demux()  argument
887 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
888 	kfree(buffer->demux_bounce);  in iio_buffer_update_demux()
889 	buffer->demux_bounce = NULL;  in iio_buffer_update_demux()
893 			 buffer->scan_mask,  in iio_buffer_update_demux()
899 			 buffer->scan_mask,  in iio_buffer_update_demux()
915 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
922 	if (buffer->scan_timestamp) {  in iio_buffer_update_demux()
926 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
931 	buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);  in iio_buffer_update_demux()
932 	if (buffer->demux_bounce == NULL) {  in iio_buffer_update_demux()
939 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
947 	struct iio_buffer *buffer;  in iio_update_demux()  local
950 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_update_demux()
951 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
958 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list)  in iio_update_demux()
959 		iio_buffer_demux_free(buffer);  in iio_update_demux()
968 	struct iio_buffer *buffer;  in iio_enable_buffers()  local
1004 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_enable_buffers()
1005 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
1034 	list_for_each_entry_continue_reverse(buffer, &iio_dev_opaque->buffer_list,  in iio_enable_buffers()
1036 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
1050 	struct iio_buffer *buffer;  in iio_disable_buffers()  local
1076 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_disable_buffers()
1077 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1202 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_store_enable()  local
1212 	inlist = iio_buffer_is_active(buffer);  in iio_buffer_store_enable()
1218 		ret = __iio_update_buffers(indio_dev, buffer, NULL);  in iio_buffer_store_enable()
1220 		ret = __iio_update_buffers(indio_dev, NULL, buffer);  in iio_buffer_store_enable()
1231 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_show_watermark()  local
1233 	return sysfs_emit(buf, "%u\n", buffer->watermark);  in iio_buffer_show_watermark()
1242 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_buffer_store_watermark()  local
1254 	if (val > buffer->length) {  in iio_buffer_store_watermark()
1259 	if (iio_buffer_is_active(buffer)) {  in iio_buffer_store_watermark()
1264 	buffer->watermark = val;  in iio_buffer_store_watermark()
1275 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_dma_show_data_available()  local
1277 	return sysfs_emit(buf, "%zu\n", iio_buffer_data_available(buffer));  in iio_dma_show_data_available()
1302 static struct attribute *iio_buffer_wrap_attr(struct iio_buffer *buffer,  in iio_buffer_wrap_attr()  argument
1312 	iio_attr->buffer = buffer;  in iio_buffer_wrap_attr()
1317 	list_add(&iio_attr->l, &buffer->buffer_attr_list);  in iio_buffer_wrap_attr()
1385 	struct iio_buffer *buffer = ib->buffer;  in iio_buffer_chrdev_release()  local
1387 	wake_up(&buffer->pollq);  in iio_buffer_chrdev_release()
1390 	clear_bit(IIO_BUSY_BIT_POS, &buffer->flags);  in iio_buffer_chrdev_release()
1409 	struct iio_buffer *buffer;  in iio_device_buffer_getfd()  local
1420 	buffer = iio_dev_opaque->attached_buffers[idx];  in iio_device_buffer_getfd()
1422 	if (test_and_set_bit(IIO_BUSY_BIT_POS, &buffer->flags)) {  in iio_device_buffer_getfd()
1434 	ib->buffer = buffer;  in iio_device_buffer_getfd()
1454 	clear_bit(IIO_BUSY_BIT_POS, &buffer->flags);  in iio_device_buffer_getfd()
1471 static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,  in __iio_buffer_alloc_sysfs_and_mask()  argument
1482 	if (buffer->attrs) {  in __iio_buffer_alloc_sysfs_and_mask()
1483 		while (buffer->attrs[buffer_attrcount] != NULL)  in __iio_buffer_alloc_sysfs_and_mask()
1488 	INIT_LIST_HEAD(&buffer->buffer_attr_list);  in __iio_buffer_alloc_sysfs_and_mask()
1496 			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,  in __iio_buffer_alloc_sysfs_and_mask()
1505 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in __iio_buffer_alloc_sysfs_and_mask()
1506 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,  in __iio_buffer_alloc_sysfs_and_mask()
1508 			if (buffer->scan_mask == NULL) {  in __iio_buffer_alloc_sysfs_and_mask()
1523 	if (!buffer->access->set_length)  in __iio_buffer_alloc_sysfs_and_mask()
1526 	if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)  in __iio_buffer_alloc_sysfs_and_mask()
1529 	if (buffer->attrs)  in __iio_buffer_alloc_sysfs_and_mask()
1530 		memcpy(&attr[ARRAY_SIZE(iio_buffer_attrs)], buffer->attrs,  in __iio_buffer_alloc_sysfs_and_mask()
1538 		wrapped = iio_buffer_wrap_attr(buffer, attr[i]);  in __iio_buffer_alloc_sysfs_and_mask()
1547 	list_for_each_entry(p, &buffer->buffer_attr_list, l)  in __iio_buffer_alloc_sysfs_and_mask()
1550 	buffer->buffer_group.name = kasprintf(GFP_KERNEL, "buffer%d", index);  in __iio_buffer_alloc_sysfs_and_mask()
1551 	if (!buffer->buffer_group.name) {  in __iio_buffer_alloc_sysfs_and_mask()
1556 	buffer->buffer_group.attrs = attr;  in __iio_buffer_alloc_sysfs_and_mask()
1558 	ret = iio_device_register_sysfs_group(indio_dev, &buffer->buffer_group);  in __iio_buffer_alloc_sysfs_and_mask()
1575 	kfree(buffer->buffer_group.name);  in __iio_buffer_alloc_sysfs_and_mask()
1577 	kfree(buffer->buffer_group.attrs);  in __iio_buffer_alloc_sysfs_and_mask()
1579 	bitmap_free(buffer->scan_mask);  in __iio_buffer_alloc_sysfs_and_mask()
1581 	iio_free_chan_devattr_list(&buffer->buffer_attr_list);  in __iio_buffer_alloc_sysfs_and_mask()
1586 static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer)  in __iio_buffer_free_sysfs_and_mask()  argument
1588 	bitmap_free(buffer->scan_mask);  in __iio_buffer_free_sysfs_and_mask()
1589 	kfree(buffer->buffer_group.name);  in __iio_buffer_free_sysfs_and_mask()
1590 	kfree(buffer->buffer_group.attrs);  in __iio_buffer_free_sysfs_and_mask()
1591 	iio_free_chan_devattr_list(&buffer->buffer_attr_list);  in __iio_buffer_free_sysfs_and_mask()
1598 	struct iio_buffer *buffer;  in iio_buffers_alloc_sysfs_and_mask()  local
1616 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_buffers_alloc_sysfs_and_mask()
1617 		ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, i);  in iio_buffers_alloc_sysfs_and_mask()
1640 		buffer = iio_dev_opaque->attached_buffers[unwind_idx];  in iio_buffers_alloc_sysfs_and_mask()
1641 		__iio_buffer_free_sysfs_and_mask(buffer);  in iio_buffers_alloc_sysfs_and_mask()
1649 	struct iio_buffer *buffer;  in iio_buffers_free_sysfs_and_mask()  local
1661 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_buffers_free_sysfs_and_mask()
1662 		__iio_buffer_free_sysfs_and_mask(buffer);  in iio_buffers_free_sysfs_and_mask()
1682 static const void *iio_demux(struct iio_buffer *buffer,  in iio_demux()  argument
1687 	if (list_empty(&buffer->demux_list))  in iio_demux()
1689 	list_for_each_entry(t, &buffer->demux_list, l)  in iio_demux()
1690 		memcpy(buffer->demux_bounce + t->to,  in iio_demux()
1693 	return buffer->demux_bounce;  in iio_demux()
1696 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)  in iio_push_to_buffer()  argument
1698 	const void *dataout = iio_demux(buffer, data);  in iio_push_to_buffer()
1701 	ret = buffer->access->store_to(buffer, dataout);  in iio_push_to_buffer()
1709 	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);  in iio_push_to_buffer()
1745 	struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);  in iio_buffer_release()  local
1747 	buffer->access->release(buffer);  in iio_buffer_release()
1756 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)  in iio_buffer_get()  argument
1758 	if (buffer)  in iio_buffer_get()
1759 		kref_get(&buffer->ref);  in iio_buffer_get()
1761 	return buffer;  in iio_buffer_get()
1769 void iio_buffer_put(struct iio_buffer *buffer)  in iio_buffer_put()  argument
1771 	if (buffer)  in iio_buffer_put()
1772 		kref_put(&buffer->ref, iio_buffer_release);  in iio_buffer_put()
1790 			     struct iio_buffer *buffer)  in iio_device_attach_buffer()  argument
1803 	buffer = iio_buffer_get(buffer);  in iio_device_attach_buffer()
1806 	if (!indio_dev->buffer)  in iio_device_attach_buffer()
1807 		indio_dev->buffer = buffer;  in iio_device_attach_buffer()
1809 	iio_dev_opaque->attached_buffers[cnt - 1] = buffer;  in iio_device_attach_buffer()