110 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_read_first_n_outer()
176 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_poll()
196 	if (!indio_dev->buffer)  in iio_buffer_wakeup_poll()
199 	wake_up(&indio_dev->buffer->pollq);  in iio_buffer_wakeup_poll()
202 void iio_buffer_init(struct iio_buffer *buffer)  in iio_buffer_init()  argument
204 	INIT_LIST_HEAD(&buffer->demux_list);  in iio_buffer_init()
205 	INIT_LIST_HEAD(&buffer->buffer_list);  in iio_buffer_init()
206 	init_waitqueue_head(&buffer->pollq);  in iio_buffer_init()
207 	kref_init(&buffer->ref);  in iio_buffer_init()
208 	if (!buffer->watermark)  in iio_buffer_init()
209 		buffer->watermark = 1;  in iio_buffer_init()
218 void iio_buffer_set_attrs(struct iio_buffer *buffer,  in iio_buffer_set_attrs()  argument
221 	buffer->attrs = attrs;  in iio_buffer_set_attrs()
272 		       indio_dev->buffer->scan_mask);  in iio_scan_el_show()
318 		      struct iio_buffer *buffer, int bit)  in iio_scan_mask_set()  argument
332 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
345 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
356 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)  in iio_scan_mask_clear()  argument
358 	clear_bit(bit, buffer->scan_mask);  in iio_scan_mask_clear()
363 			       struct iio_buffer *buffer, int bit)  in iio_scan_mask_query()  argument
368 	if (!buffer->scan_mask)  in iio_scan_mask_query()
372 	return !!test_bit(bit, buffer->scan_mask);  in iio_scan_mask_query()
383 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_store()  local
390 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_store()
394 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
398 		ret = iio_scan_mask_clear(buffer, this_attr->address);  in iio_scan_el_store()
402 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
419 	return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);  in iio_scan_el_ts_show()
436 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_ts_store()
440 	indio_dev->buffer->scan_timestamp = state;  in iio_scan_el_ts_store()
451 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_add_channel_sysfs()  local
460 				     &buffer->scan_el_dev_attr_list);  in iio_buffer_add_channel_sysfs()
471 				     &buffer->scan_el_dev_attr_list);  in iio_buffer_add_channel_sysfs()
483 					     &buffer->scan_el_dev_attr_list);  in iio_buffer_add_channel_sysfs()
492 					     &buffer->scan_el_dev_attr_list);  in iio_buffer_add_channel_sysfs()
505 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_read_length()  local
507 	return sprintf(buf, "%d\n", buffer->length);  in iio_buffer_read_length()
515 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_write_length()  local
523 	if (val == buffer->length)  in iio_buffer_write_length()
527 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_write_length()
530 		buffer->access->set_length(buffer, val);  in iio_buffer_write_length()
535 	if (buffer->length && buffer->length < buffer->watermark)  in iio_buffer_write_length()
536 		buffer->watermark = buffer->length;  in iio_buffer_write_length()
548 	return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));  in iio_buffer_show_enable()
593 	struct iio_buffer *buffer)  in iio_buffer_activate()  argument
595 	iio_buffer_get(buffer);  in iio_buffer_activate()
596 	list_add(&buffer->buffer_list, &indio_dev->buffer_list);  in iio_buffer_activate()
599 static void iio_buffer_deactivate(struct iio_buffer *buffer)  in iio_buffer_deactivate()  argument
601 	list_del_init(&buffer->buffer_list);  in iio_buffer_deactivate()
602 	wake_up_interruptible(&buffer->pollq);  in iio_buffer_deactivate()
603 	iio_buffer_put(buffer);  in iio_buffer_deactivate()
608 	struct iio_buffer *buffer, *_buffer;  in iio_buffer_deactivate_all()  local
610 	list_for_each_entry_safe(buffer, _buffer,  in iio_buffer_deactivate_all()
612 		iio_buffer_deactivate(buffer);  in iio_buffer_deactivate_all()
615 static int iio_buffer_enable(struct iio_buffer *buffer,  in iio_buffer_enable()  argument
618 	if (!buffer->access->enable)  in iio_buffer_enable()
620 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
623 static int iio_buffer_disable(struct iio_buffer *buffer,  in iio_buffer_disable()  argument
626 	if (!buffer->access->disable)  in iio_buffer_disable()
628 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
632 	struct iio_buffer *buffer)  in iio_buffer_update_bytes_per_datum()  argument
636 	if (!buffer->access->set_bytes_per_datum)  in iio_buffer_update_bytes_per_datum()
639 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
640 		buffer->scan_timestamp);  in iio_buffer_update_bytes_per_datum()
642 	buffer->access->set_bytes_per_datum(buffer, bytes);  in iio_buffer_update_bytes_per_datum()
646 	struct iio_buffer *buffer)  in iio_buffer_request_update()  argument
650 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
651 	if (buffer->access->request_update) {  in iio_buffer_request_update()
652 		ret = buffer->access->request_update(buffer);  in iio_buffer_request_update()
687 	struct iio_buffer *buffer;  in iio_verify_update()  local
704 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
705 		if (buffer == remove_buffer)  in iio_verify_update()
707 		modes &= buffer->access->modes;  in iio_verify_update()
708 		config->watermark = min(config->watermark, buffer->watermark);  in iio_verify_update()
746 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
747 		if (buffer == remove_buffer)  in iio_verify_update()
749 		bitmap_or(compound_mask, compound_mask, buffer->scan_mask,  in iio_verify_update()
751 		scan_timestamp |= buffer->scan_timestamp;  in iio_verify_update()
794 static void iio_buffer_demux_free(struct iio_buffer *buffer)  in iio_buffer_demux_free()  argument
797 	list_for_each_entry_safe(p, q, &buffer->demux_list, l) {  in iio_buffer_demux_free()
803 static int iio_buffer_add_demux(struct iio_buffer *buffer,  in iio_buffer_add_demux()  argument
818 		list_add_tail(&(*p)->l, &buffer->demux_list);  in iio_buffer_add_demux()
825 				   struct iio_buffer *buffer)  in iio_buffer_update_demux()  argument
832 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
833 	kfree(buffer->demux_bounce);  in iio_buffer_update_demux()
834 	buffer->demux_bounce = NULL;  in iio_buffer_update_demux()
838 			 buffer->scan_mask,  in iio_buffer_update_demux()
844 			 buffer->scan_mask,  in iio_buffer_update_demux()
860 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
867 	if (buffer->scan_timestamp) {  in iio_buffer_update_demux()
871 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
877 	buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);  in iio_buffer_update_demux()
878 	if (buffer->demux_bounce == NULL) {  in iio_buffer_update_demux()
885 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
892 	struct iio_buffer *buffer;  in iio_update_demux()  local
895 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_update_demux()
896 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
903 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list)  in iio_update_demux()
904 		iio_buffer_demux_free(buffer);  in iio_update_demux()
912 	struct iio_buffer *buffer;  in iio_enable_buffers()  local
947 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_enable_buffers()
948 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
967 	list_for_each_entry_continue_reverse(buffer, &indio_dev->buffer_list,  in iio_enable_buffers()
969 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
982 	struct iio_buffer *buffer;  in iio_disable_buffers()  local
1003 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_disable_buffers()
1004 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1137 	inlist = iio_buffer_is_active(indio_dev->buffer);  in iio_buffer_store_enable()
1144 					 indio_dev->buffer, NULL);  in iio_buffer_store_enable()
1147 					 NULL, indio_dev->buffer);  in iio_buffer_store_enable()
1161 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_show_watermark()  local
1163 	return sprintf(buf, "%u\n", buffer->watermark);  in iio_buffer_show_watermark()
1172 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_store_watermark()  local
1184 	if (val > buffer->length) {  in iio_buffer_store_watermark()
1189 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_store_watermark()
1194 	buffer->watermark = val;  in iio_buffer_store_watermark()
1208 	bytes = iio_buffer_data_available(indio_dev->buffer);  in iio_dma_show_data_available()
1237 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_alloc_sysfs_and_mask()  local
1250 	if (!buffer)  in iio_buffer_alloc_sysfs_and_mask()
1254 	if (buffer->attrs) {  in iio_buffer_alloc_sysfs_and_mask()
1255 		while (buffer->attrs[attrcount] != NULL)  in iio_buffer_alloc_sysfs_and_mask()
1265 	if (!buffer->access->set_length)  in iio_buffer_alloc_sysfs_and_mask()
1268 	if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)  in iio_buffer_alloc_sysfs_and_mask()
1271 	if (buffer->attrs)  in iio_buffer_alloc_sysfs_and_mask()
1272 		memcpy(&attr[ARRAY_SIZE(iio_buffer_attrs)], buffer->attrs,  in iio_buffer_alloc_sysfs_and_mask()
1277 	buffer->buffer_group.name = "buffer";  in iio_buffer_alloc_sysfs_and_mask()
1278 	buffer->buffer_group.attrs = attr;  in iio_buffer_alloc_sysfs_and_mask()
1280 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;  in iio_buffer_alloc_sysfs_and_mask()
1282 	if (buffer->scan_el_attrs != NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1283 		attr = buffer->scan_el_attrs->attrs;  in iio_buffer_alloc_sysfs_and_mask()
1288 	INIT_LIST_HEAD(&buffer->scan_el_dev_attr_list);  in iio_buffer_alloc_sysfs_and_mask()
1305 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1306 			buffer->scan_mask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),  in iio_buffer_alloc_sysfs_and_mask()
1307 						    sizeof(*buffer->scan_mask),  in iio_buffer_alloc_sysfs_and_mask()
1309 			if (buffer->scan_mask == NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1316 	buffer->scan_el_group.name = iio_scan_elements_group_name;  in iio_buffer_alloc_sysfs_and_mask()
1318 	buffer->scan_el_group.attrs = kcalloc(attrcount + 1,  in iio_buffer_alloc_sysfs_and_mask()
1319 					      sizeof(buffer->scan_el_group.attrs[0]),  in iio_buffer_alloc_sysfs_and_mask()
1321 	if (buffer->scan_el_group.attrs == NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1325 	if (buffer->scan_el_attrs)  in iio_buffer_alloc_sysfs_and_mask()
1326 		memcpy(buffer->scan_el_group.attrs, buffer->scan_el_attrs,  in iio_buffer_alloc_sysfs_and_mask()
1327 		       sizeof(buffer->scan_el_group.attrs[0])*attrcount_orig);  in iio_buffer_alloc_sysfs_and_mask()
1330 	list_for_each_entry(p, &buffer->scan_el_dev_attr_list, l)  in iio_buffer_alloc_sysfs_and_mask()
1331 		buffer->scan_el_group.attrs[attrn++] = &p->dev_attr.attr;  in iio_buffer_alloc_sysfs_and_mask()
1332 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;  in iio_buffer_alloc_sysfs_and_mask()
1337 	kfree(buffer->scan_mask);  in iio_buffer_alloc_sysfs_and_mask()
1339 	iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);  in iio_buffer_alloc_sysfs_and_mask()
1340 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_alloc_sysfs_and_mask()
1347 	if (!indio_dev->buffer)  in iio_buffer_free_sysfs_and_mask()
1350 	kfree(indio_dev->buffer->scan_mask);  in iio_buffer_free_sysfs_and_mask()
1351 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1352 	kfree(indio_dev->buffer->scan_el_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1353 	iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);  in iio_buffer_free_sysfs_and_mask()
1372 static const void *iio_demux(struct iio_buffer *buffer,  in iio_demux()  argument
1377 	if (list_empty(&buffer->demux_list))  in iio_demux()
1379 	list_for_each_entry(t, &buffer->demux_list, l)  in iio_demux()
1380 		memcpy(buffer->demux_bounce + t->to,  in iio_demux()
1383 	return buffer->demux_bounce;  in iio_demux()
1386 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)  in iio_push_to_buffer()  argument
1388 	const void *dataout = iio_demux(buffer, data);  in iio_push_to_buffer()
1391 	ret = buffer->access->store_to(buffer, dataout);  in iio_push_to_buffer()
1399 	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);  in iio_push_to_buffer()
1434 	struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);  in iio_buffer_release()  local
1436 	buffer->access->release(buffer);  in iio_buffer_release()
1445 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)  in iio_buffer_get()  argument
1447 	if (buffer)  in iio_buffer_get()
1448 		kref_get(&buffer->ref);  in iio_buffer_get()
1450 	return buffer;  in iio_buffer_get()
1458 void iio_buffer_put(struct iio_buffer *buffer)  in iio_buffer_put()  argument
1460 	if (buffer)  in iio_buffer_put()
1461 		kref_put(&buffer->ref, iio_buffer_release);  in iio_buffer_put()
1475 			      struct iio_buffer *buffer)  in iio_device_attach_buffer()  argument
1477 	indio_dev->buffer = iio_buffer_get(buffer);  in iio_device_attach_buffer()