44 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,  in iio_buffer_flush_hwfifo()  argument
47 	if (!indio_dev->info->hwfifo_flush_to_buffer)  in iio_buffer_flush_hwfifo()
50 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);  in iio_buffer_flush_hwfifo()
53 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,  in iio_buffer_ready()  argument
60 	if (!indio_dev->info)  in iio_buffer_ready()
74 			iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
80 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
107 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_read_outer()  local
108 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_read_outer()
114 	if (!indio_dev->info)  in iio_buffer_read_outer()
136 		if (!indio_dev->info) {  in iio_buffer_read_outer()
141 		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {  in iio_buffer_read_outer()
173 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_poll()  local
174 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_poll()
176 	if (!indio_dev->info || rb == NULL)  in iio_buffer_poll()
180 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))  in iio_buffer_poll()
192 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)  in iio_buffer_wakeup_poll()  argument
194 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_wakeup_poll()
268 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_show()  local
269 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_show()
299 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,  in iio_validate_scan_mask()  argument
302 	if (!indio_dev->setup_ops->validate_scan_mask)  in iio_validate_scan_mask()
305 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);  in iio_validate_scan_mask()
318 static int iio_scan_mask_set(struct iio_dev *indio_dev,  in iio_scan_mask_set()  argument
324 	trialmask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);  in iio_scan_mask_set()
327 	if (!indio_dev->masklength) {  in iio_scan_mask_set()
331 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
334 	if (!iio_validate_scan_mask(indio_dev, trialmask))  in iio_scan_mask_set()
337 	if (indio_dev->available_scan_masks) {  in iio_scan_mask_set()
338 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_scan_mask_set()
339 					   indio_dev->masklength,  in iio_scan_mask_set()
344 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
361 static int iio_scan_mask_query(struct iio_dev *indio_dev,  in iio_scan_mask_query()  argument
364 	if (bit > indio_dev->masklength)  in iio_scan_mask_query()
381 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_store()  local
382 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_store()
388 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_store()
393 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
401 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
407 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_store()
417 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_show()  local
418 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_ts_show()
429 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_store()  local
430 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_ts_store()
437 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_ts_store()
444 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_ts_store()
449 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,  in iio_buffer_add_channel_sysfs()  argument
461 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
472 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
484 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
493 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
506 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_read_length()  local
507 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_read_length()
516 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_write_length()  local
517 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_write_length()
528 	mutex_lock(&indio_dev->mlock);  in iio_buffer_write_length()
540 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_write_length()
549 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_enable()  local
550 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_show_enable()
555 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,  in iio_storage_bytes_for_si()  argument
561 	ch = iio_find_channel_from_si(indio_dev, scan_index);  in iio_storage_bytes_for_si()
568 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)  in iio_storage_bytes_for_timestamp()  argument
570 	return iio_storage_bytes_for_si(indio_dev,  in iio_storage_bytes_for_timestamp()
571 					indio_dev->scan_index_timestamp);  in iio_storage_bytes_for_timestamp()
574 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,  in iio_compute_scan_bytes()  argument
582 			 indio_dev->masklength) {  in iio_compute_scan_bytes()
583 		length = iio_storage_bytes_for_si(indio_dev, i);  in iio_compute_scan_bytes()
590 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_compute_scan_bytes()
600 static void iio_buffer_activate(struct iio_dev *indio_dev,  in iio_buffer_activate()  argument
603 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_activate()
616 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)  in iio_buffer_deactivate_all()  argument
618 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_deactivate_all()
627 	struct iio_dev *indio_dev)  in iio_buffer_enable()  argument
631 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
635 	struct iio_dev *indio_dev)  in iio_buffer_disable()  argument
639 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
642 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,  in iio_buffer_update_bytes_per_datum()  argument
650 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
656 static int iio_buffer_request_update(struct iio_dev *indio_dev,  in iio_buffer_request_update()  argument
661 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
665 			dev_dbg(&indio_dev->dev,  in iio_buffer_request_update()
675 static void iio_free_scan_mask(struct iio_dev *indio_dev,  in iio_free_scan_mask()  argument
679 	if (!indio_dev->available_scan_masks)  in iio_free_scan_mask()
691 static int iio_verify_update(struct iio_dev *indio_dev,  in iio_verify_update()  argument
695 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_verify_update()
704 	    bitmap_empty(insert_buffer->scan_mask, indio_dev->masklength)) {  in iio_verify_update()
705 		dev_dbg(&indio_dev->dev,  in iio_verify_update()
721 	modes = indio_dev->modes;  in iio_verify_update()
737 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {  in iio_verify_update()
752 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)  in iio_verify_update()
753 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");  in iio_verify_update()
758 	compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);  in iio_verify_update()
768 			  indio_dev->masklength);  in iio_verify_update()
774 			  insert_buffer->scan_mask, indio_dev->masklength);  in iio_verify_update()
778 	if (indio_dev->available_scan_masks) {  in iio_verify_update()
779 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_verify_update()
780 				    indio_dev->masklength,  in iio_verify_update()
790 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,  in iio_verify_update()
842 static int iio_buffer_update_demux(struct iio_dev *indio_dev,  in iio_buffer_update_demux()  argument
855 	if (bitmap_equal(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
857 			 indio_dev->masklength))  in iio_buffer_update_demux()
863 			 indio_dev->masklength) {  in iio_buffer_update_demux()
864 		in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
865 				       indio_dev->masklength,  in iio_buffer_update_demux()
868 			in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
869 					       indio_dev->masklength,  in iio_buffer_update_demux()
871 			length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
875 		length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
886 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_buffer_update_demux()
908 static int iio_update_demux(struct iio_dev *indio_dev)  in iio_update_demux()  argument
910 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_update_demux()
915 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
928 static int iio_enable_buffers(struct iio_dev *indio_dev,  in iio_enable_buffers()  argument
931 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_enable_buffers()
935 	indio_dev->active_scan_mask = config->scan_mask;  in iio_enable_buffers()
936 	indio_dev->scan_timestamp = config->scan_timestamp;  in iio_enable_buffers()
937 	indio_dev->scan_bytes = config->scan_bytes;  in iio_enable_buffers()
938 	indio_dev->currentmode = config->mode;  in iio_enable_buffers()
940 	iio_update_demux(indio_dev);  in iio_enable_buffers()
943 	if (indio_dev->setup_ops->preenable) {  in iio_enable_buffers()
944 		ret = indio_dev->setup_ops->preenable(indio_dev);  in iio_enable_buffers()
946 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
952 	if (indio_dev->info->update_scan_mode) {  in iio_enable_buffers()
953 		ret = indio_dev->info  in iio_enable_buffers()
954 			->update_scan_mode(indio_dev,  in iio_enable_buffers()
955 					   indio_dev->active_scan_mask);  in iio_enable_buffers()
957 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
964 	if (indio_dev->info->hwfifo_set_watermark)  in iio_enable_buffers()
965 		indio_dev->info->hwfifo_set_watermark(indio_dev,  in iio_enable_buffers()
969 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
974 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_enable_buffers()
975 		ret = iio_trigger_attach_poll_func(indio_dev->trig,  in iio_enable_buffers()
976 						   indio_dev->pollfunc);  in iio_enable_buffers()
981 	if (indio_dev->setup_ops->postenable) {  in iio_enable_buffers()
982 		ret = indio_dev->setup_ops->postenable(indio_dev);  in iio_enable_buffers()
984 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
993 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_enable_buffers()
994 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_enable_buffers()
995 					     indio_dev->pollfunc);  in iio_enable_buffers()
1000 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
1002 	if (indio_dev->setup_ops->postdisable)  in iio_enable_buffers()
1003 		indio_dev->setup_ops->postdisable(indio_dev);  in iio_enable_buffers()
1005 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_enable_buffers()
1006 	indio_dev->active_scan_mask = NULL;  in iio_enable_buffers()
1011 static int iio_disable_buffers(struct iio_dev *indio_dev)  in iio_disable_buffers()  argument
1013 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_disable_buffers()
1029 	if (indio_dev->setup_ops->predisable) {  in iio_disable_buffers()
1030 		ret2 = indio_dev->setup_ops->predisable(indio_dev);  in iio_disable_buffers()
1035 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_disable_buffers()
1036 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_disable_buffers()
1037 					     indio_dev->pollfunc);  in iio_disable_buffers()
1041 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1046 	if (indio_dev->setup_ops->postdisable) {  in iio_disable_buffers()
1047 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);  in iio_disable_buffers()
1052 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);  in iio_disable_buffers()
1053 	indio_dev->active_scan_mask = NULL;  in iio_disable_buffers()
1054 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_disable_buffers()
1059 static int __iio_update_buffers(struct iio_dev *indio_dev,  in __iio_update_buffers()  argument
1063 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in __iio_update_buffers()
1067 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,  in __iio_update_buffers()
1073 		ret = iio_buffer_request_update(indio_dev, insert_buffer);  in __iio_update_buffers()
1078 	ret = iio_disable_buffers(indio_dev);  in __iio_update_buffers()
1085 		iio_buffer_activate(indio_dev, insert_buffer);  in __iio_update_buffers()
1091 	ret = iio_enable_buffers(indio_dev, &new_config);  in __iio_update_buffers()
1106 	iio_buffer_deactivate_all(indio_dev);  in __iio_update_buffers()
1109 	iio_free_scan_mask(indio_dev, new_config.scan_mask);  in __iio_update_buffers()
1113 int iio_update_buffers(struct iio_dev *indio_dev,  in iio_update_buffers()  argument
1122 	mutex_lock(&indio_dev->info_exist_lock);  in iio_update_buffers()
1123 	mutex_lock(&indio_dev->mlock);  in iio_update_buffers()
1136 	if (indio_dev->info == NULL) {  in iio_update_buffers()
1141 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);  in iio_update_buffers()
1144 	mutex_unlock(&indio_dev->mlock);  in iio_update_buffers()
1145 	mutex_unlock(&indio_dev->info_exist_lock);  in iio_update_buffers()
1151 void iio_disable_all_buffers(struct iio_dev *indio_dev)  in iio_disable_all_buffers()  argument
1153 	iio_disable_buffers(indio_dev);  in iio_disable_all_buffers()
1154 	iio_buffer_deactivate_all(indio_dev);  in iio_disable_all_buffers()
1164 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_enable()  local
1165 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_store_enable()
1172 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_enable()
1181 		ret = __iio_update_buffers(indio_dev, buffer, NULL);  in iio_buffer_store_enable()
1183 		ret = __iio_update_buffers(indio_dev, NULL, buffer);  in iio_buffer_store_enable()
1186 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_enable()
1196 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_watermark()  local
1197 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_show_watermark()
1207 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_watermark()  local
1208 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_store_watermark()
1218 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1232 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1241 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_dma_show_data_available()  local
1242 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_dma_show_data_available()
1268 					     struct iio_dev *indio_dev)  in __iio_buffer_alloc_sysfs_and_mask()  argument
1302 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;  in __iio_buffer_alloc_sysfs_and_mask()
1306 	channels = indio_dev->channels;  in __iio_buffer_alloc_sysfs_and_mask()
1309 		for (i = 0; i < indio_dev->num_channels; i++) {  in __iio_buffer_alloc_sysfs_and_mask()
1313 			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,  in __iio_buffer_alloc_sysfs_and_mask()
1319 				indio_dev->scan_index_timestamp =  in __iio_buffer_alloc_sysfs_and_mask()
1322 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in __iio_buffer_alloc_sysfs_and_mask()
1323 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,  in __iio_buffer_alloc_sysfs_and_mask()
1345 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;  in __iio_buffer_alloc_sysfs_and_mask()
1358 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_alloc_sysfs_and_mask()  argument
1360 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_alloc_sysfs_and_mask()
1364 	channels = indio_dev->channels;  in iio_buffer_alloc_sysfs_and_mask()
1366 		int ml = indio_dev->masklength;  in iio_buffer_alloc_sysfs_and_mask()
1368 		for (i = 0; i < indio_dev->num_channels; i++)  in iio_buffer_alloc_sysfs_and_mask()
1370 		indio_dev->masklength = ml;  in iio_buffer_alloc_sysfs_and_mask()
1376 	return __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev);  in iio_buffer_alloc_sysfs_and_mask()
1387 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_free_sysfs_and_mask()  argument
1389 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_free_sysfs_and_mask()
1406 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,  in iio_validate_scan_mask_onehot()  argument
1409 	return bitmap_weight(mask, indio_dev->masklength) == 1;  in iio_validate_scan_mask_onehot()
1449 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)  in iio_push_to_buffers()  argument
1451 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_push_to_buffers()
1516 void iio_device_attach_buffer(struct iio_dev *indio_dev,  in iio_device_attach_buffer()  argument
1519 	indio_dev->buffer = iio_buffer_get(buffer);  in iio_device_attach_buffer()