46 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,  in iio_buffer_flush_hwfifo()  argument
49 	if (!indio_dev->info->hwfifo_flush_to_buffer)  in iio_buffer_flush_hwfifo()
52 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);  in iio_buffer_flush_hwfifo()
55 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,  in iio_buffer_ready()  argument
62 	if (!indio_dev->info)  in iio_buffer_ready()
76 			iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
82 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
111 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_read()  local
117 	if (!indio_dev->info)  in iio_buffer_read()
142 		if (!indio_dev->info) {  in iio_buffer_read()
147 		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {  in iio_buffer_read()
180 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_write()  local
185 	if (!indio_dev->info)  in iio_buffer_write()
197 		if (!indio_dev->info)  in iio_buffer_write()
243 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_poll()  local
245 	if (!indio_dev->info || !rb)  in iio_buffer_poll()
252 		if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))  in iio_buffer_poll()
310 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)  in iio_buffer_wakeup_poll()  argument
312 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_wakeup_poll()
342 void iio_device_detach_buffers(struct iio_dev *indio_dev)  in iio_device_detach_buffers()  argument
344 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_detach_buffers()
429 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,  in iio_validate_scan_mask()  argument
432 	if (!indio_dev->setup_ops->validate_scan_mask)  in iio_validate_scan_mask()
435 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);  in iio_validate_scan_mask()
448 static int iio_scan_mask_set(struct iio_dev *indio_dev,  in iio_scan_mask_set()  argument
454 	if (!indio_dev->masklength) {  in iio_scan_mask_set()
459 	trialmask = bitmap_alloc(indio_dev->masklength, GFP_KERNEL);  in iio_scan_mask_set()
462 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
465 	if (!iio_validate_scan_mask(indio_dev, trialmask))  in iio_scan_mask_set()
468 	if (indio_dev->available_scan_masks) {  in iio_scan_mask_set()
469 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_scan_mask_set()
470 					   indio_dev->masklength,  in iio_scan_mask_set()
475 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
492 static int iio_scan_mask_query(struct iio_dev *indio_dev,  in iio_scan_mask_query()  argument
495 	if (bit > indio_dev->masklength)  in iio_scan_mask_query()
512 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_store()  local
513 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_scan_el_store()
525 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
533 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
559 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_store()  local
560 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_scan_el_ts_store()
580 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,  in iio_buffer_add_channel_sysfs()  argument
592 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
604 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
617 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
627 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
648 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in length_store()  local
649 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in length_store()
686 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,  in iio_storage_bytes_for_si()  argument
692 	ch = iio_find_channel_from_si(indio_dev, scan_index);  in iio_storage_bytes_for_si()
699 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)  in iio_storage_bytes_for_timestamp()  argument
701 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_storage_bytes_for_timestamp()
703 	return iio_storage_bytes_for_si(indio_dev,  in iio_storage_bytes_for_timestamp()
707 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,  in iio_compute_scan_bytes()  argument
715 			 indio_dev->masklength) {  in iio_compute_scan_bytes()
716 		length = iio_storage_bytes_for_si(indio_dev, i);  in iio_compute_scan_bytes()
723 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_compute_scan_bytes()
733 static void iio_buffer_activate(struct iio_dev *indio_dev,  in iio_buffer_activate()  argument
736 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_activate()
749 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)  in iio_buffer_deactivate_all()  argument
751 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_deactivate_all()
760 			     struct iio_dev *indio_dev)  in iio_buffer_enable()  argument
764 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
768 			      struct iio_dev *indio_dev)  in iio_buffer_disable()  argument
772 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
775 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,  in iio_buffer_update_bytes_per_datum()  argument
783 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
789 static int iio_buffer_request_update(struct iio_dev *indio_dev,  in iio_buffer_request_update()  argument
794 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
798 			dev_dbg(&indio_dev->dev,  in iio_buffer_request_update()
808 static void iio_free_scan_mask(struct iio_dev *indio_dev,  in iio_free_scan_mask()  argument
812 	if (!indio_dev->available_scan_masks)  in iio_free_scan_mask()
824 static int iio_verify_update(struct iio_dev *indio_dev,  in iio_verify_update()  argument
829 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_verify_update()
838 	    bitmap_empty(insert_buffer->scan_mask, indio_dev->masklength)) {  in iio_verify_update()
839 		dev_dbg(&indio_dev->dev,  in iio_verify_update()
855 	modes = indio_dev->modes;  in iio_verify_update()
871 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {  in iio_verify_update()
886 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)  in iio_verify_update()
887 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");  in iio_verify_update()
892 	compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);  in iio_verify_update()
902 			  indio_dev->masklength);  in iio_verify_update()
908 			  insert_buffer->scan_mask, indio_dev->masklength);  in iio_verify_update()
912 	if (indio_dev->available_scan_masks) {  in iio_verify_update()
913 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_verify_update()
914 						indio_dev->masklength,  in iio_verify_update()
924 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,  in iio_verify_update()
977 static int iio_buffer_update_demux(struct iio_dev *indio_dev,  in iio_buffer_update_demux()  argument
990 	if (bitmap_equal(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
992 			 indio_dev->masklength))  in iio_buffer_update_demux()
998 			 indio_dev->masklength) {  in iio_buffer_update_demux()
999 		in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
1000 				       indio_dev->masklength,  in iio_buffer_update_demux()
1003 			length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
1006 			in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
1007 					       indio_dev->masklength,  in iio_buffer_update_demux()
1010 		length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
1021 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_buffer_update_demux()
1042 static int iio_update_demux(struct iio_dev *indio_dev)  in iio_update_demux()  argument
1044 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_update_demux()
1049 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
1062 static int iio_enable_buffers(struct iio_dev *indio_dev,  in iio_enable_buffers()  argument
1065 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_enable_buffers()
1069 	indio_dev->active_scan_mask = config->scan_mask;  in iio_enable_buffers()
1070 	indio_dev->scan_timestamp = config->scan_timestamp;  in iio_enable_buffers()
1071 	indio_dev->scan_bytes = config->scan_bytes;  in iio_enable_buffers()
1074 	iio_update_demux(indio_dev);  in iio_enable_buffers()
1077 	if (indio_dev->setup_ops->preenable) {  in iio_enable_buffers()
1078 		ret = indio_dev->setup_ops->preenable(indio_dev);  in iio_enable_buffers()
1080 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
1086 	if (indio_dev->info->update_scan_mode) {  in iio_enable_buffers()
1087 		ret = indio_dev->info  in iio_enable_buffers()
1088 			->update_scan_mode(indio_dev,  in iio_enable_buffers()
1089 					   indio_dev->active_scan_mask);  in iio_enable_buffers()
1091 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
1098 	if (indio_dev->info->hwfifo_set_watermark)  in iio_enable_buffers()
1099 		indio_dev->info->hwfifo_set_watermark(indio_dev,  in iio_enable_buffers()
1103 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
1111 		ret = iio_trigger_attach_poll_func(indio_dev->trig,  in iio_enable_buffers()
1112 						   indio_dev->pollfunc);  in iio_enable_buffers()
1117 	if (indio_dev->setup_ops->postenable) {  in iio_enable_buffers()
1118 		ret = indio_dev->setup_ops->postenable(indio_dev);  in iio_enable_buffers()
1120 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
1130 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_enable_buffers()
1131 					     indio_dev->pollfunc);  in iio_enable_buffers()
1137 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
1139 	if (indio_dev->setup_ops->postdisable)  in iio_enable_buffers()
1140 		indio_dev->setup_ops->postdisable(indio_dev);  in iio_enable_buffers()
1143 	indio_dev->active_scan_mask = NULL;  in iio_enable_buffers()
1148 static int iio_disable_buffers(struct iio_dev *indio_dev)  in iio_disable_buffers()  argument
1150 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_disable_buffers()
1166 	if (indio_dev->setup_ops->predisable) {  in iio_disable_buffers()
1167 		ret2 = indio_dev->setup_ops->predisable(indio_dev);  in iio_disable_buffers()
1173 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_disable_buffers()
1174 					     indio_dev->pollfunc);  in iio_disable_buffers()
1178 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1183 	if (indio_dev->setup_ops->postdisable) {  in iio_disable_buffers()
1184 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);  in iio_disable_buffers()
1189 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);  in iio_disable_buffers()
1190 	indio_dev->active_scan_mask = NULL;  in iio_disable_buffers()
1196 static int __iio_update_buffers(struct iio_dev *indio_dev,  in __iio_update_buffers()  argument
1200 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in __iio_update_buffers()
1204 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,  in __iio_update_buffers()
1210 		ret = iio_buffer_request_update(indio_dev, insert_buffer);  in __iio_update_buffers()
1215 	ret = iio_disable_buffers(indio_dev);  in __iio_update_buffers()
1222 		iio_buffer_activate(indio_dev, insert_buffer);  in __iio_update_buffers()
1228 	ret = iio_enable_buffers(indio_dev, &new_config);  in __iio_update_buffers()
1243 	iio_buffer_deactivate_all(indio_dev);  in __iio_update_buffers()
1246 	iio_free_scan_mask(indio_dev, new_config.scan_mask);  in __iio_update_buffers()
1250 int iio_update_buffers(struct iio_dev *indio_dev,  in iio_update_buffers()  argument
1254 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_update_buffers()
1278 	if (!indio_dev->info) {  in iio_update_buffers()
1283 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);  in iio_update_buffers()
1293 void iio_disable_all_buffers(struct iio_dev *indio_dev)  in iio_disable_all_buffers()  argument
1295 	iio_disable_buffers(indio_dev);  in iio_disable_all_buffers()
1296 	iio_buffer_deactivate_all(indio_dev);  in iio_disable_all_buffers()
1304 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in enable_store()  local
1305 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in enable_store()
1322 		ret = __iio_update_buffers(indio_dev, buffer, NULL);  in enable_store()
1324 		ret = __iio_update_buffers(indio_dev, NULL, buffer);  in enable_store()
1343 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in watermark_store()  local
1344 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in watermark_store()
1447 static int iio_buffer_register_legacy_sysfs_groups(struct iio_dev *indio_dev,  in iio_buffer_register_legacy_sysfs_groups()  argument
1452 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_register_legacy_sysfs_groups()
1467 	ret = iio_device_register_sysfs_group(indio_dev, group);  in iio_buffer_register_legacy_sysfs_groups()
1484 	ret = iio_device_register_sysfs_group(indio_dev, group);  in iio_buffer_register_legacy_sysfs_groups()
1498 static void iio_buffer_unregister_legacy_sysfs_groups(struct iio_dev *indio_dev)  in iio_buffer_unregister_legacy_sysfs_groups()  argument
1500 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_unregister_legacy_sysfs_groups()
1509 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_chrdev_release()  local
1516 	iio_device_put(indio_dev);  in iio_buffer_chrdev_release()
1530 static long iio_device_buffer_getfd(struct iio_dev *indio_dev, unsigned long arg)  in iio_device_buffer_getfd()  argument
1532 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_buffer_getfd()
1544 	iio_device_get(indio_dev);  in iio_device_buffer_getfd()
1559 	ib->indio_dev = indio_dev;  in iio_device_buffer_getfd()
1590 	iio_device_put(indio_dev);  in iio_device_buffer_getfd()
1594 static long iio_device_buffer_ioctl(struct iio_dev *indio_dev, struct file *filp,  in iio_device_buffer_ioctl()  argument
1599 		return iio_device_buffer_getfd(indio_dev, arg);  in iio_device_buffer_ioctl()
1606 					     struct iio_dev *indio_dev,  in __iio_buffer_alloc_sysfs_and_mask()  argument
1609 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in __iio_buffer_alloc_sysfs_and_mask()
1625 	channels = indio_dev->channels;  in __iio_buffer_alloc_sysfs_and_mask()
1628 		for (i = 0; i < indio_dev->num_channels; i++) {  in __iio_buffer_alloc_sysfs_and_mask()
1636 				dev_err(&indio_dev->dev,  in __iio_buffer_alloc_sysfs_and_mask()
1645 			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,  in __iio_buffer_alloc_sysfs_and_mask()
1654 		if (indio_dev->masklength && !buffer->scan_mask) {  in __iio_buffer_alloc_sysfs_and_mask()
1655 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,  in __iio_buffer_alloc_sysfs_and_mask()
1707 	ret = iio_device_register_sysfs_group(indio_dev, &buffer->buffer_group);  in __iio_buffer_alloc_sysfs_and_mask()
1715 	ret = iio_buffer_register_legacy_sysfs_groups(indio_dev, attr,  in __iio_buffer_alloc_sysfs_and_mask()
1736 					     struct iio_dev *indio_dev,  in __iio_buffer_free_sysfs_and_mask()  argument
1740 		iio_buffer_unregister_legacy_sysfs_groups(indio_dev);  in __iio_buffer_free_sysfs_and_mask()
1747 int iio_buffers_alloc_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffers_alloc_sysfs_and_mask()  argument
1749 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffers_alloc_sysfs_and_mask()
1755 	channels = indio_dev->channels;  in iio_buffers_alloc_sysfs_and_mask()
1757 		int ml = indio_dev->masklength;  in iio_buffers_alloc_sysfs_and_mask()
1759 		for (i = 0; i < indio_dev->num_channels; i++)  in iio_buffers_alloc_sysfs_and_mask()
1761 		indio_dev->masklength = ml;  in iio_buffers_alloc_sysfs_and_mask()
1769 		ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, idx);  in iio_buffers_alloc_sysfs_and_mask()
1782 	iio_device_ioctl_handler_register(indio_dev,  in iio_buffers_alloc_sysfs_and_mask()
1790 		__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, idx);  in iio_buffers_alloc_sysfs_and_mask()
1795 void iio_buffers_free_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffers_free_sysfs_and_mask()  argument
1797 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffers_free_sysfs_and_mask()
1809 		__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, i);  in iio_buffers_free_sysfs_and_mask()
1822 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,  in iio_validate_scan_mask_onehot()  argument
1825 	return bitmap_weight(mask, indio_dev->masklength) == 1;  in iio_validate_scan_mask_onehot()
1865 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)  in iio_push_to_buffers()  argument
1867 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_push_to_buffers()
1894 int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev,  in iio_push_to_buffers_with_ts_unaligned()  argument
1899 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_push_to_buffers_with_ts_unaligned()
1908 	data_sz = min_t(size_t, indio_dev->scan_bytes, data_sz);  in iio_push_to_buffers_with_ts_unaligned()
1909 	if (iio_dev_opaque->bounce_buffer_size !=  indio_dev->scan_bytes) {  in iio_push_to_buffers_with_ts_unaligned()
1912 		bb = devm_krealloc(&indio_dev->dev,  in iio_push_to_buffers_with_ts_unaligned()
1914 				   indio_dev->scan_bytes, GFP_KERNEL);  in iio_push_to_buffers_with_ts_unaligned()
1918 		iio_dev_opaque->bounce_buffer_size = indio_dev->scan_bytes;  in iio_push_to_buffers_with_ts_unaligned()
1921 	return iio_push_to_buffers_with_timestamp(indio_dev,  in iio_push_to_buffers_with_ts_unaligned()
1982 int iio_device_attach_buffer(struct iio_dev *indio_dev,  in iio_device_attach_buffer()  argument
1985 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_attach_buffer()
1999 	if (!indio_dev->buffer)  in iio_device_attach_buffer()
2000 		indio_dev->buffer = buffer;  in iio_device_attach_buffer()