46 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,  in iio_buffer_flush_hwfifo()  argument
49 	if (!indio_dev->info->hwfifo_flush_to_buffer)  in iio_buffer_flush_hwfifo()
52 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);  in iio_buffer_flush_hwfifo()
55 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,  in iio_buffer_ready()  argument
62 	if (!indio_dev->info)  in iio_buffer_ready()
76 			iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
82 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
109 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_read_first_n_outer()  local
110 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_read_first_n_outer()
116 	if (!indio_dev->info)  in iio_buffer_read_first_n_outer()
138 		if (!indio_dev->info) {  in iio_buffer_read_first_n_outer()
143 		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {  in iio_buffer_read_first_n_outer()
175 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_poll()  local
176 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_poll()
178 	if (!indio_dev->info || rb == NULL)  in iio_buffer_poll()
182 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))  in iio_buffer_poll()
194 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)  in iio_buffer_wakeup_poll()  argument
196 	if (!indio_dev->buffer)  in iio_buffer_wakeup_poll()
199 	wake_up(&indio_dev->buffer->pollq);  in iio_buffer_wakeup_poll()
268 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_show()  local
272 		       indio_dev->buffer->scan_mask);  in iio_scan_el_show()
298 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,  in iio_validate_scan_mask()  argument
301 	if (!indio_dev->setup_ops->validate_scan_mask)  in iio_validate_scan_mask()
304 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);  in iio_validate_scan_mask()
317 static int iio_scan_mask_set(struct iio_dev *indio_dev,  in iio_scan_mask_set()  argument
323 	trialmask = kmalloc_array(BITS_TO_LONGS(indio_dev->masklength),  in iio_scan_mask_set()
328 	if (!indio_dev->masklength) {  in iio_scan_mask_set()
332 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
335 	if (!iio_validate_scan_mask(indio_dev, trialmask))  in iio_scan_mask_set()
338 	if (indio_dev->available_scan_masks) {  in iio_scan_mask_set()
339 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_scan_mask_set()
340 					   indio_dev->masklength,  in iio_scan_mask_set()
345 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
362 static int iio_scan_mask_query(struct iio_dev *indio_dev,  in iio_scan_mask_query()  argument
365 	if (bit > indio_dev->masklength)  in iio_scan_mask_query()
382 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_store()  local
383 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_store()
389 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_store()
390 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_store()
394 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
402 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
408 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_store()
418 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_show()  local
419 	return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);  in iio_scan_el_ts_show()
428 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_store()  local
435 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_ts_store()
436 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_ts_store()
440 	indio_dev->buffer->scan_timestamp = state;  in iio_scan_el_ts_store()
442 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_ts_store()
447 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,  in iio_buffer_add_channel_sysfs()  argument
451 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_add_channel_sysfs()
459 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
470 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
482 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
491 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
504 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_read_length()  local
505 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_read_length()
514 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_write_length()  local
515 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_write_length()
526 	mutex_lock(&indio_dev->mlock);  in iio_buffer_write_length()
527 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_write_length()
538 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_write_length()
547 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_enable()  local
548 	return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));  in iio_buffer_show_enable()
551 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,  in iio_storage_bytes_for_si()  argument
557 	ch = iio_find_channel_from_si(indio_dev, scan_index);  in iio_storage_bytes_for_si()
564 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)  in iio_storage_bytes_for_timestamp()  argument
566 	return iio_storage_bytes_for_si(indio_dev,  in iio_storage_bytes_for_timestamp()
567 					indio_dev->scan_index_timestamp);  in iio_storage_bytes_for_timestamp()
570 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,  in iio_compute_scan_bytes()  argument
578 			 indio_dev->masklength) {  in iio_compute_scan_bytes()
579 		length = iio_storage_bytes_for_si(indio_dev, i);  in iio_compute_scan_bytes()
585 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_compute_scan_bytes()
592 static void iio_buffer_activate(struct iio_dev *indio_dev,  in iio_buffer_activate()  argument
596 	list_add(&buffer->buffer_list, &indio_dev->buffer_list);  in iio_buffer_activate()
606 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)  in iio_buffer_deactivate_all()  argument
611 			&indio_dev->buffer_list, buffer_list)  in iio_buffer_deactivate_all()
616 	struct iio_dev *indio_dev)  in iio_buffer_enable()  argument
620 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
624 	struct iio_dev *indio_dev)  in iio_buffer_disable()  argument
628 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
631 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,  in iio_buffer_update_bytes_per_datum()  argument
639 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
645 static int iio_buffer_request_update(struct iio_dev *indio_dev,  in iio_buffer_request_update()  argument
650 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
654 			dev_dbg(&indio_dev->dev,  in iio_buffer_request_update()
664 static void iio_free_scan_mask(struct iio_dev *indio_dev,  in iio_free_scan_mask()  argument
668 	if (!indio_dev->available_scan_masks)  in iio_free_scan_mask()
680 static int iio_verify_update(struct iio_dev *indio_dev,  in iio_verify_update()  argument
699 		list_is_singular(&indio_dev->buffer_list))  in iio_verify_update()
702 	modes = indio_dev->modes;  in iio_verify_update()
704 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
718 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {  in iio_verify_update()
725 		if (insert_buffer && !list_empty(&indio_dev->buffer_list))  in iio_verify_update()
733 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)  in iio_verify_update()
734 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");  in iio_verify_update()
739 	compound_mask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),  in iio_verify_update()
746 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
750 			  indio_dev->masklength);  in iio_verify_update()
756 			  insert_buffer->scan_mask, indio_dev->masklength);  in iio_verify_update()
760 	if (indio_dev->available_scan_masks) {  in iio_verify_update()
761 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_verify_update()
762 				    indio_dev->masklength,  in iio_verify_update()
772 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,  in iio_verify_update()
824 static int iio_buffer_update_demux(struct iio_dev *indio_dev,  in iio_buffer_update_demux()  argument
837 	if (bitmap_equal(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
839 			 indio_dev->masklength))  in iio_buffer_update_demux()
845 			 indio_dev->masklength) {  in iio_buffer_update_demux()
846 		in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
847 				       indio_dev->masklength,  in iio_buffer_update_demux()
850 			in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
851 					       indio_dev->masklength,  in iio_buffer_update_demux()
853 			length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
857 		length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
868 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_buffer_update_demux()
890 static int iio_update_demux(struct iio_dev *indio_dev)  in iio_update_demux()  argument
895 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_update_demux()
896 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
903 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list)  in iio_update_demux()
909 static int iio_enable_buffers(struct iio_dev *indio_dev,  in iio_enable_buffers()  argument
915 	indio_dev->active_scan_mask = config->scan_mask;  in iio_enable_buffers()
916 	indio_dev->scan_timestamp = config->scan_timestamp;  in iio_enable_buffers()
917 	indio_dev->scan_bytes = config->scan_bytes;  in iio_enable_buffers()
919 	iio_update_demux(indio_dev);  in iio_enable_buffers()
922 	if (indio_dev->setup_ops->preenable) {  in iio_enable_buffers()
923 		ret = indio_dev->setup_ops->preenable(indio_dev);  in iio_enable_buffers()
925 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
931 	if (indio_dev->info->update_scan_mode) {  in iio_enable_buffers()
932 		ret = indio_dev->info  in iio_enable_buffers()
933 			->update_scan_mode(indio_dev,  in iio_enable_buffers()
934 					   indio_dev->active_scan_mask);  in iio_enable_buffers()
936 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
943 	if (indio_dev->info->hwfifo_set_watermark)  in iio_enable_buffers()
944 		indio_dev->info->hwfifo_set_watermark(indio_dev,  in iio_enable_buffers()
947 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_enable_buffers()
948 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
953 	indio_dev->currentmode = config->mode;  in iio_enable_buffers()
955 	if (indio_dev->setup_ops->postenable) {  in iio_enable_buffers()
956 		ret = indio_dev->setup_ops->postenable(indio_dev);  in iio_enable_buffers()
958 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
967 	list_for_each_entry_continue_reverse(buffer, &indio_dev->buffer_list,  in iio_enable_buffers()
969 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
971 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_enable_buffers()
972 	if (indio_dev->setup_ops->postdisable)  in iio_enable_buffers()
973 		indio_dev->setup_ops->postdisable(indio_dev);  in iio_enable_buffers()
975 	indio_dev->active_scan_mask = NULL;  in iio_enable_buffers()
980 static int iio_disable_buffers(struct iio_dev *indio_dev)  in iio_disable_buffers()  argument
987 	if (list_empty(&indio_dev->buffer_list))  in iio_disable_buffers()
997 	if (indio_dev->setup_ops->predisable) {  in iio_disable_buffers()
998 		ret2 = indio_dev->setup_ops->predisable(indio_dev);  in iio_disable_buffers()
1003 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_disable_buffers()
1004 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1009 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_disable_buffers()
1011 	if (indio_dev->setup_ops->postdisable) {  in iio_disable_buffers()
1012 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);  in iio_disable_buffers()
1017 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);  in iio_disable_buffers()
1018 	indio_dev->active_scan_mask = NULL;  in iio_disable_buffers()
1023 static int __iio_update_buffers(struct iio_dev *indio_dev,  in __iio_update_buffers()  argument
1030 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,  in __iio_update_buffers()
1036 		ret = iio_buffer_request_update(indio_dev, insert_buffer);  in __iio_update_buffers()
1041 	ret = iio_disable_buffers(indio_dev);  in __iio_update_buffers()
1048 		iio_buffer_activate(indio_dev, insert_buffer);  in __iio_update_buffers()
1051 	if (list_empty(&indio_dev->buffer_list))  in __iio_update_buffers()
1054 	ret = iio_enable_buffers(indio_dev, &new_config);  in __iio_update_buffers()
1069 	iio_buffer_deactivate_all(indio_dev);  in __iio_update_buffers()
1072 	iio_free_scan_mask(indio_dev, new_config.scan_mask);  in __iio_update_buffers()
1076 int iio_update_buffers(struct iio_dev *indio_dev,  in iio_update_buffers()  argument
1085 	mutex_lock(&indio_dev->info_exist_lock);  in iio_update_buffers()
1086 	mutex_lock(&indio_dev->mlock);  in iio_update_buffers()
1099 	if (indio_dev->info == NULL) {  in iio_update_buffers()
1104 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);  in iio_update_buffers()
1107 	mutex_unlock(&indio_dev->mlock);  in iio_update_buffers()
1108 	mutex_unlock(&indio_dev->info_exist_lock);  in iio_update_buffers()
1114 void iio_disable_all_buffers(struct iio_dev *indio_dev)  in iio_disable_all_buffers()  argument
1116 	iio_disable_buffers(indio_dev);  in iio_disable_all_buffers()
1117 	iio_buffer_deactivate_all(indio_dev);  in iio_disable_all_buffers()
1127 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_enable()  local
1134 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_enable()
1137 	inlist = iio_buffer_is_active(indio_dev->buffer);  in iio_buffer_store_enable()
1143 		ret = __iio_update_buffers(indio_dev,  in iio_buffer_store_enable()
1144 					 indio_dev->buffer, NULL);  in iio_buffer_store_enable()
1146 		ret = __iio_update_buffers(indio_dev,  in iio_buffer_store_enable()
1147 					 NULL, indio_dev->buffer);  in iio_buffer_store_enable()
1150 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_enable()
1160 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_watermark()  local
1161 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_show_watermark()
1171 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_watermark()  local
1172 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_store_watermark()
1182 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1189 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_store_watermark()
1196 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1205 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_dma_show_data_available()  local
1208 	bytes = iio_buffer_data_available(indio_dev->buffer);  in iio_dma_show_data_available()
1233 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_alloc_sysfs_and_mask()  argument
1237 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_alloc_sysfs_and_mask()
1241 	channels = indio_dev->channels;  in iio_buffer_alloc_sysfs_and_mask()
1243 		int ml = indio_dev->masklength;  in iio_buffer_alloc_sysfs_and_mask()
1245 		for (i = 0; i < indio_dev->num_channels; i++)  in iio_buffer_alloc_sysfs_and_mask()
1247 		indio_dev->masklength = ml;  in iio_buffer_alloc_sysfs_and_mask()
1280 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;  in iio_buffer_alloc_sysfs_and_mask()
1289 	channels = indio_dev->channels;  in iio_buffer_alloc_sysfs_and_mask()
1292 		for (i = 0; i < indio_dev->num_channels; i++) {  in iio_buffer_alloc_sysfs_and_mask()
1296 			ret = iio_buffer_add_channel_sysfs(indio_dev,  in iio_buffer_alloc_sysfs_and_mask()
1302 				indio_dev->scan_index_timestamp =  in iio_buffer_alloc_sysfs_and_mask()
1305 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1306 			buffer->scan_mask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),  in iio_buffer_alloc_sysfs_and_mask()
1332 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;  in iio_buffer_alloc_sysfs_and_mask()
1340 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_alloc_sysfs_and_mask()
1345 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_free_sysfs_and_mask()  argument
1347 	if (!indio_dev->buffer)  in iio_buffer_free_sysfs_and_mask()
1350 	kfree(indio_dev->buffer->scan_mask);  in iio_buffer_free_sysfs_and_mask()
1351 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1352 	kfree(indio_dev->buffer->scan_el_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1353 	iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);  in iio_buffer_free_sysfs_and_mask()
1365 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,  in iio_validate_scan_mask_onehot()  argument
1368 	return bitmap_weight(mask, indio_dev->masklength) == 1;  in iio_validate_scan_mask_onehot()
1408 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)  in iio_push_to_buffers()  argument
1413 	list_for_each_entry(buf, &indio_dev->buffer_list, buffer_list) {  in iio_push_to_buffers()
1474 void iio_device_attach_buffer(struct iio_dev *indio_dev,  in iio_device_attach_buffer()  argument
1477 	indio_dev->buffer = iio_buffer_get(buffer);  in iio_device_attach_buffer()