46 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,  in iio_buffer_flush_hwfifo()  argument
49 	if (!indio_dev->info->hwfifo_flush_to_buffer)  in iio_buffer_flush_hwfifo()
52 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);  in iio_buffer_flush_hwfifo()
55 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,  in iio_buffer_ready()  argument
62 	if (!indio_dev->info)  in iio_buffer_ready()
76 			iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
82 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
111 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_read()  local
117 	if (!indio_dev->info)  in iio_buffer_read()
139 		if (!indio_dev->info) {  in iio_buffer_read()
144 		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {  in iio_buffer_read()
178 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_poll()  local
180 	if (!indio_dev->info || rb == NULL)  in iio_buffer_poll()
184 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))  in iio_buffer_poll()
222 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)  in iio_buffer_wakeup_poll()  argument
224 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_wakeup_poll()
245 void iio_device_detach_buffers(struct iio_dev *indio_dev)  in iio_device_detach_buffers()  argument
247 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_detach_buffers()
332 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,  in iio_validate_scan_mask()  argument
335 	if (!indio_dev->setup_ops->validate_scan_mask)  in iio_validate_scan_mask()
338 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);  in iio_validate_scan_mask()
351 static int iio_scan_mask_set(struct iio_dev *indio_dev,  in iio_scan_mask_set()  argument
357 	if (!indio_dev->masklength) {  in iio_scan_mask_set()
362 	trialmask = bitmap_alloc(indio_dev->masklength, GFP_KERNEL);  in iio_scan_mask_set()
365 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
368 	if (!iio_validate_scan_mask(indio_dev, trialmask))  in iio_scan_mask_set()
371 	if (indio_dev->available_scan_masks) {  in iio_scan_mask_set()
372 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_scan_mask_set()
373 					   indio_dev->masklength,  in iio_scan_mask_set()
378 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
395 static int iio_scan_mask_query(struct iio_dev *indio_dev,  in iio_scan_mask_query()  argument
398 	if (bit > indio_dev->masklength)  in iio_scan_mask_query()
415 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_store()  local
422 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_store()
427 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
435 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
441 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_store()
462 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_store()  local
470 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_ts_store()
477 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_ts_store()
482 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,  in iio_buffer_add_channel_sysfs()  argument
494 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
506 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
519 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
529 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
552 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_write_length()  local
564 	mutex_lock(&indio_dev->mlock);  in iio_buffer_write_length()
576 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_write_length()
590 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,  in iio_storage_bytes_for_si()  argument
596 	ch = iio_find_channel_from_si(indio_dev, scan_index);  in iio_storage_bytes_for_si()
603 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)  in iio_storage_bytes_for_timestamp()  argument
605 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_storage_bytes_for_timestamp()
607 	return iio_storage_bytes_for_si(indio_dev,  in iio_storage_bytes_for_timestamp()
611 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,  in iio_compute_scan_bytes()  argument
619 			 indio_dev->masklength) {  in iio_compute_scan_bytes()
620 		length = iio_storage_bytes_for_si(indio_dev, i);  in iio_compute_scan_bytes()
627 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_compute_scan_bytes()
637 static void iio_buffer_activate(struct iio_dev *indio_dev,  in iio_buffer_activate()  argument
640 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_activate()
653 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)  in iio_buffer_deactivate_all()  argument
655 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_deactivate_all()
664 	struct iio_dev *indio_dev)  in iio_buffer_enable()  argument
668 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
672 	struct iio_dev *indio_dev)  in iio_buffer_disable()  argument
676 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
679 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,  in iio_buffer_update_bytes_per_datum()  argument
687 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
693 static int iio_buffer_request_update(struct iio_dev *indio_dev,  in iio_buffer_request_update()  argument
698 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
702 			dev_dbg(&indio_dev->dev,  in iio_buffer_request_update()
712 static void iio_free_scan_mask(struct iio_dev *indio_dev,  in iio_free_scan_mask()  argument
716 	if (!indio_dev->available_scan_masks)  in iio_free_scan_mask()
728 static int iio_verify_update(struct iio_dev *indio_dev,  in iio_verify_update()  argument
732 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_verify_update()
741 	    bitmap_empty(insert_buffer->scan_mask, indio_dev->masklength)) {  in iio_verify_update()
742 		dev_dbg(&indio_dev->dev,  in iio_verify_update()
758 	modes = indio_dev->modes;  in iio_verify_update()
774 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {  in iio_verify_update()
789 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)  in iio_verify_update()
790 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");  in iio_verify_update()
795 	compound_mask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);  in iio_verify_update()
805 			  indio_dev->masklength);  in iio_verify_update()
811 			  insert_buffer->scan_mask, indio_dev->masklength);  in iio_verify_update()
815 	if (indio_dev->available_scan_masks) {  in iio_verify_update()
816 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_verify_update()
817 				    indio_dev->masklength,  in iio_verify_update()
827 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,  in iio_verify_update()
879 static int iio_buffer_update_demux(struct iio_dev *indio_dev,  in iio_buffer_update_demux()  argument
892 	if (bitmap_equal(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
894 			 indio_dev->masklength))  in iio_buffer_update_demux()
900 			 indio_dev->masklength) {  in iio_buffer_update_demux()
901 		in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
902 				       indio_dev->masklength,  in iio_buffer_update_demux()
905 			length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
908 			in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
909 					       indio_dev->masklength,  in iio_buffer_update_demux()
912 		length = iio_storage_bytes_for_si(indio_dev, in_ind);  in iio_buffer_update_demux()
923 		length = iio_storage_bytes_for_timestamp(indio_dev);  in iio_buffer_update_demux()
944 static int iio_update_demux(struct iio_dev *indio_dev)  in iio_update_demux()  argument
946 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_update_demux()
951 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
964 static int iio_enable_buffers(struct iio_dev *indio_dev,  in iio_enable_buffers()  argument
967 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_enable_buffers()
971 	indio_dev->active_scan_mask = config->scan_mask;  in iio_enable_buffers()
972 	indio_dev->scan_timestamp = config->scan_timestamp;  in iio_enable_buffers()
973 	indio_dev->scan_bytes = config->scan_bytes;  in iio_enable_buffers()
974 	indio_dev->currentmode = config->mode;  in iio_enable_buffers()
976 	iio_update_demux(indio_dev);  in iio_enable_buffers()
979 	if (indio_dev->setup_ops->preenable) {  in iio_enable_buffers()
980 		ret = indio_dev->setup_ops->preenable(indio_dev);  in iio_enable_buffers()
982 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
988 	if (indio_dev->info->update_scan_mode) {  in iio_enable_buffers()
989 		ret = indio_dev->info  in iio_enable_buffers()
990 			->update_scan_mode(indio_dev,  in iio_enable_buffers()
991 					   indio_dev->active_scan_mask);  in iio_enable_buffers()
993 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
1000 	if (indio_dev->info->hwfifo_set_watermark)  in iio_enable_buffers()
1001 		indio_dev->info->hwfifo_set_watermark(indio_dev,  in iio_enable_buffers()
1005 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
1010 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_enable_buffers()
1011 		ret = iio_trigger_attach_poll_func(indio_dev->trig,  in iio_enable_buffers()
1012 						   indio_dev->pollfunc);  in iio_enable_buffers()
1017 	if (indio_dev->setup_ops->postenable) {  in iio_enable_buffers()
1018 		ret = indio_dev->setup_ops->postenable(indio_dev);  in iio_enable_buffers()
1020 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
1029 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_enable_buffers()
1030 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_enable_buffers()
1031 					     indio_dev->pollfunc);  in iio_enable_buffers()
1036 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
1038 	if (indio_dev->setup_ops->postdisable)  in iio_enable_buffers()
1039 		indio_dev->setup_ops->postdisable(indio_dev);  in iio_enable_buffers()
1041 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_enable_buffers()
1042 	indio_dev->active_scan_mask = NULL;  in iio_enable_buffers()
1047 static int iio_disable_buffers(struct iio_dev *indio_dev)  in iio_disable_buffers()  argument
1049 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_disable_buffers()
1065 	if (indio_dev->setup_ops->predisable) {  in iio_disable_buffers()
1066 		ret2 = indio_dev->setup_ops->predisable(indio_dev);  in iio_disable_buffers()
1071 	if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {  in iio_disable_buffers()
1072 		iio_trigger_detach_poll_func(indio_dev->trig,  in iio_disable_buffers()
1073 					     indio_dev->pollfunc);  in iio_disable_buffers()
1077 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1082 	if (indio_dev->setup_ops->postdisable) {  in iio_disable_buffers()
1083 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);  in iio_disable_buffers()
1088 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);  in iio_disable_buffers()
1089 	indio_dev->active_scan_mask = NULL;  in iio_disable_buffers()
1090 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_disable_buffers()
1095 static int __iio_update_buffers(struct iio_dev *indio_dev,  in __iio_update_buffers()  argument
1099 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in __iio_update_buffers()
1103 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,  in __iio_update_buffers()
1109 		ret = iio_buffer_request_update(indio_dev, insert_buffer);  in __iio_update_buffers()
1114 	ret = iio_disable_buffers(indio_dev);  in __iio_update_buffers()
1121 		iio_buffer_activate(indio_dev, insert_buffer);  in __iio_update_buffers()
1127 	ret = iio_enable_buffers(indio_dev, &new_config);  in __iio_update_buffers()
1142 	iio_buffer_deactivate_all(indio_dev);  in __iio_update_buffers()
1145 	iio_free_scan_mask(indio_dev, new_config.scan_mask);  in __iio_update_buffers()
1149 int iio_update_buffers(struct iio_dev *indio_dev,  in iio_update_buffers()  argument
1153 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_update_buffers()
1160 	mutex_lock(&indio_dev->mlock);  in iio_update_buffers()
1173 	if (indio_dev->info == NULL) {  in iio_update_buffers()
1178 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);  in iio_update_buffers()
1181 	mutex_unlock(&indio_dev->mlock);  in iio_update_buffers()
1188 void iio_disable_all_buffers(struct iio_dev *indio_dev)  in iio_disable_all_buffers()  argument
1190 	iio_disable_buffers(indio_dev);  in iio_disable_all_buffers()
1191 	iio_buffer_deactivate_all(indio_dev);  in iio_disable_all_buffers()
1201 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_enable()  local
1209 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_enable()
1218 		ret = __iio_update_buffers(indio_dev, buffer, NULL);  in iio_buffer_store_enable()
1220 		ret = __iio_update_buffers(indio_dev, NULL, buffer);  in iio_buffer_store_enable()
1223 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_enable()
1241 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_watermark()  local
1252 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1266 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1322 static int iio_buffer_register_legacy_sysfs_groups(struct iio_dev *indio_dev,  in iio_buffer_register_legacy_sysfs_groups()  argument
1327 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_register_legacy_sysfs_groups()
1342 	ret = iio_device_register_sysfs_group(indio_dev, group);  in iio_buffer_register_legacy_sysfs_groups()
1359 	ret = iio_device_register_sysfs_group(indio_dev, group);  in iio_buffer_register_legacy_sysfs_groups()
1373 static void iio_buffer_unregister_legacy_sysfs_groups(struct iio_dev *indio_dev)  in iio_buffer_unregister_legacy_sysfs_groups()  argument
1375 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffer_unregister_legacy_sysfs_groups()
1384 	struct iio_dev *indio_dev = ib->indio_dev;  in iio_buffer_chrdev_release()  local
1391 	iio_device_put(indio_dev);  in iio_buffer_chrdev_release()
1404 static long iio_device_buffer_getfd(struct iio_dev *indio_dev, unsigned long arg)  in iio_device_buffer_getfd()  argument
1406 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_buffer_getfd()
1418 	iio_device_get(indio_dev);  in iio_device_buffer_getfd()
1433 	ib->indio_dev = indio_dev;  in iio_device_buffer_getfd()
1456 	iio_device_put(indio_dev);  in iio_device_buffer_getfd()
1460 static long iio_device_buffer_ioctl(struct iio_dev *indio_dev, struct file *filp,  in iio_device_buffer_ioctl()  argument
1465 		return iio_device_buffer_getfd(indio_dev, arg);  in iio_device_buffer_ioctl()
1472 					     struct iio_dev *indio_dev,  in __iio_buffer_alloc_sysfs_and_mask()  argument
1475 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in __iio_buffer_alloc_sysfs_and_mask()
1489 	channels = indio_dev->channels;  in __iio_buffer_alloc_sysfs_and_mask()
1492 		for (i = 0; i < indio_dev->num_channels; i++) {  in __iio_buffer_alloc_sysfs_and_mask()
1496 			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,  in __iio_buffer_alloc_sysfs_and_mask()
1505 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in __iio_buffer_alloc_sysfs_and_mask()
1506 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,  in __iio_buffer_alloc_sysfs_and_mask()
1558 	ret = iio_device_register_sysfs_group(indio_dev, &buffer->buffer_group);  in __iio_buffer_alloc_sysfs_and_mask()
1566 	ret = iio_buffer_register_legacy_sysfs_groups(indio_dev, attr,  in __iio_buffer_alloc_sysfs_and_mask()
1594 int iio_buffers_alloc_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffers_alloc_sysfs_and_mask()  argument
1596 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffers_alloc_sysfs_and_mask()
1603 	channels = indio_dev->channels;  in iio_buffers_alloc_sysfs_and_mask()
1605 		int ml = indio_dev->masklength;  in iio_buffers_alloc_sysfs_and_mask()
1607 		for (i = 0; i < indio_dev->num_channels; i++)  in iio_buffers_alloc_sysfs_and_mask()
1609 		indio_dev->masklength = ml;  in iio_buffers_alloc_sysfs_and_mask()
1617 		ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, i);  in iio_buffers_alloc_sysfs_and_mask()
1633 	iio_device_ioctl_handler_register(indio_dev,  in iio_buffers_alloc_sysfs_and_mask()
1646 void iio_buffers_free_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffers_free_sysfs_and_mask()  argument
1648 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_buffers_free_sysfs_and_mask()
1658 	iio_buffer_unregister_legacy_sysfs_groups(indio_dev);  in iio_buffers_free_sysfs_and_mask()
1675 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,  in iio_validate_scan_mask_onehot()  argument
1678 	return bitmap_weight(mask, indio_dev->masklength) == 1;  in iio_validate_scan_mask_onehot()
1718 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)  in iio_push_to_buffers()  argument
1720 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_push_to_buffers()
1789 int iio_device_attach_buffer(struct iio_dev *indio_dev,  in iio_device_attach_buffer()  argument
1792 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);  in iio_device_attach_buffer()
1806 	if (!indio_dev->buffer)  in iio_device_attach_buffer()
1807 		indio_dev->buffer = buffer;  in iio_device_attach_buffer()