110 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_read()
179 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_write()
242 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_poll()
268 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_read_wrapper()
281 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_write_wrapper()
294 	struct iio_buffer *rb = ib->buffer;  in iio_buffer_poll_wrapper()
313 	struct iio_buffer *buffer;  in iio_buffer_wakeup_poll()  local
317 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_buffer_wakeup_poll()
318 		wake_up(&buffer->pollq);  in iio_buffer_wakeup_poll()
322 int iio_pop_from_buffer(struct iio_buffer *buffer, void *data)  in iio_pop_from_buffer()  argument
324 	if (!buffer || !buffer->access || !buffer->access->remove_from)  in iio_pop_from_buffer()
327 	return buffer->access->remove_from(buffer, data);  in iio_pop_from_buffer()
331 void iio_buffer_init(struct iio_buffer *buffer)  in iio_buffer_init()  argument
333 	INIT_LIST_HEAD(&buffer->demux_list);  in iio_buffer_init()
334 	INIT_LIST_HEAD(&buffer->buffer_list);  in iio_buffer_init()
335 	init_waitqueue_head(&buffer->pollq);  in iio_buffer_init()
336 	kref_init(&buffer->ref);  in iio_buffer_init()
337 	if (!buffer->watermark)  in iio_buffer_init()
338 		buffer->watermark = 1;  in iio_buffer_init()
345 	struct iio_buffer *buffer;  in iio_device_detach_buffers()  local
349 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_device_detach_buffers()
350 		iio_buffer_put(buffer);  in iio_device_detach_buffers()
399 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_show()  local
403 		       buffer->scan_mask);  in iio_scan_el_show()
449 			     struct iio_buffer *buffer, int bit)  in iio_scan_mask_set()  argument
462 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
475 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
486 static int iio_scan_mask_clear(struct iio_buffer *buffer, int bit)  in iio_scan_mask_clear()  argument
488 	clear_bit(bit, buffer->scan_mask);  in iio_scan_mask_clear()
493 			       struct iio_buffer *buffer, int bit)  in iio_scan_mask_query()  argument
498 	if (!buffer->scan_mask)  in iio_scan_mask_query()
502 	return !!test_bit(bit, buffer->scan_mask);  in iio_scan_mask_query()
515 	struct iio_buffer *buffer = this_attr->buffer;  in iio_scan_el_store()  local
521 	if (iio_buffer_is_active(buffer)) {  in iio_scan_el_store()
525 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
529 		ret = iio_scan_mask_clear(buffer, this_attr->address);  in iio_scan_el_store()
533 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
548 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_ts_show()  local
550 	return sysfs_emit(buf, "%d\n", buffer->scan_timestamp);  in iio_scan_el_ts_show()
561 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in iio_scan_el_ts_store()  local
569 	if (iio_buffer_is_active(buffer)) {  in iio_scan_el_ts_store()
573 	buffer->scan_timestamp = state;  in iio_scan_el_ts_store()
581 					struct iio_buffer *buffer,  in iio_buffer_add_channel_sysfs()  argument
593 				     buffer,  in iio_buffer_add_channel_sysfs()
594 				     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
605 				     buffer,  in iio_buffer_add_channel_sysfs()
606 				     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
618 					     buffer,  in iio_buffer_add_channel_sysfs()
619 					     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
628 					     buffer,  in iio_buffer_add_channel_sysfs()
629 					     &buffer->buffer_attr_list);  in iio_buffer_add_channel_sysfs()
640 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in length_show()  local
642 	return sysfs_emit(buf, "%d\n", buffer->length);  in length_show()
650 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in length_store()  local
658 	if (val == buffer->length)  in length_store()
662 	if (iio_buffer_is_active(buffer)) {  in length_store()
665 		buffer->access->set_length(buffer, val);  in length_store()
670 	if (buffer->length && buffer->length < buffer->watermark)  in length_store()
671 		buffer->watermark = buffer->length;  in length_store()
681 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in enable_show()  local
683 	return sysfs_emit(buf, "%d\n", iio_buffer_is_active(buffer));  in enable_show()
734 				struct iio_buffer *buffer)  in iio_buffer_activate()  argument
738 	iio_buffer_get(buffer);  in iio_buffer_activate()
739 	list_add(&buffer->buffer_list, &iio_dev_opaque->buffer_list);  in iio_buffer_activate()
742 static void iio_buffer_deactivate(struct iio_buffer *buffer)  in iio_buffer_deactivate()  argument
744 	list_del_init(&buffer->buffer_list);  in iio_buffer_deactivate()
745 	wake_up_interruptible(&buffer->pollq);  in iio_buffer_deactivate()
746 	iio_buffer_put(buffer);  in iio_buffer_deactivate()
752 	struct iio_buffer *buffer, *_buffer;  in iio_buffer_deactivate_all()  local
754 	list_for_each_entry_safe(buffer, _buffer,  in iio_buffer_deactivate_all()
756 		iio_buffer_deactivate(buffer);  in iio_buffer_deactivate_all()
759 static int iio_buffer_enable(struct iio_buffer *buffer,  in iio_buffer_enable()  argument
762 	if (!buffer->access->enable)  in iio_buffer_enable()
764 	return buffer->access->enable(buffer, indio_dev);  in iio_buffer_enable()
767 static int iio_buffer_disable(struct iio_buffer *buffer,  in iio_buffer_disable()  argument
770 	if (!buffer->access->disable)  in iio_buffer_disable()
772 	return buffer->access->disable(buffer, indio_dev);  in iio_buffer_disable()
776 					      struct iio_buffer *buffer)  in iio_buffer_update_bytes_per_datum()  argument
780 	if (!buffer->access->set_bytes_per_datum)  in iio_buffer_update_bytes_per_datum()
783 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
784 				       buffer->scan_timestamp);  in iio_buffer_update_bytes_per_datum()
786 	buffer->access->set_bytes_per_datum(buffer, bytes);  in iio_buffer_update_bytes_per_datum()
790 				     struct iio_buffer *buffer)  in iio_buffer_request_update()  argument
794 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
795 	if (buffer->access->request_update) {  in iio_buffer_request_update()
796 		ret = buffer->access->request_update(buffer);  in iio_buffer_request_update()
833 	struct iio_buffer *buffer;  in iio_verify_update()  local
857 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_verify_update()
858 		if (buffer == remove_buffer)  in iio_verify_update()
860 		modes &= buffer->access->modes;  in iio_verify_update()
861 		config->watermark = min(config->watermark, buffer->watermark);  in iio_verify_update()
898 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_verify_update()
899 		if (buffer == remove_buffer)  in iio_verify_update()
901 		bitmap_or(compound_mask, compound_mask, buffer->scan_mask,  in iio_verify_update()
903 		scan_timestamp |= buffer->scan_timestamp;  in iio_verify_update()
946 static void iio_buffer_demux_free(struct iio_buffer *buffer)  in iio_buffer_demux_free()  argument
950 	list_for_each_entry_safe(p, q, &buffer->demux_list, l) {  in iio_buffer_demux_free()
956 static int iio_buffer_add_demux(struct iio_buffer *buffer,  in iio_buffer_add_demux()  argument
971 		list_add_tail(&(*p)->l, &buffer->demux_list);  in iio_buffer_add_demux()
978 				   struct iio_buffer *buffer)  in iio_buffer_update_demux()  argument
985 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
986 	kfree(buffer->demux_bounce);  in iio_buffer_update_demux()
987 	buffer->demux_bounce = NULL;  in iio_buffer_update_demux()
991 			 buffer->scan_mask,  in iio_buffer_update_demux()
997 			 buffer->scan_mask,  in iio_buffer_update_demux()
1013 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
1020 	if (buffer->scan_timestamp) {  in iio_buffer_update_demux()
1024 		ret = iio_buffer_add_demux(buffer, &p, in_loc, out_loc, length);  in iio_buffer_update_demux()
1029 	buffer->demux_bounce = kzalloc(out_loc, GFP_KERNEL);  in iio_buffer_update_demux()
1030 	if (!buffer->demux_bounce) {  in iio_buffer_update_demux()
1037 	iio_buffer_demux_free(buffer);  in iio_buffer_update_demux()
1045 	struct iio_buffer *buffer;  in iio_update_demux()  local
1048 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_update_demux()
1049 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
1056 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list)  in iio_update_demux()
1057 		iio_buffer_demux_free(buffer);  in iio_update_demux()
1066 	struct iio_buffer *buffer, *tmp = NULL;  in iio_enable_buffers()  local
1102 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_enable_buffers()
1103 		ret = iio_buffer_enable(buffer, indio_dev);  in iio_enable_buffers()
1105 			tmp = buffer;  in iio_enable_buffers()
1134 	buffer = list_prepare_entry(tmp, &iio_dev_opaque->buffer_list, buffer_list);  in iio_enable_buffers()
1135 	list_for_each_entry_continue_reverse(buffer, &iio_dev_opaque->buffer_list,  in iio_enable_buffers()
1137 		iio_buffer_disable(buffer, indio_dev);  in iio_enable_buffers()
1151 	struct iio_buffer *buffer;  in iio_disable_buffers()  local
1177 	list_for_each_entry(buffer, &iio_dev_opaque->buffer_list, buffer_list) {  in iio_disable_buffers()
1178 		ret2 = iio_buffer_disable(buffer, indio_dev);  in iio_disable_buffers()
1306 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in enable_store()  local
1316 	inlist = iio_buffer_is_active(buffer);  in enable_store()
1322 		ret = __iio_update_buffers(indio_dev, buffer, NULL);  in enable_store()
1324 		ret = __iio_update_buffers(indio_dev, NULL, buffer);  in enable_store()
1334 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in watermark_show()  local
1336 	return sysfs_emit(buf, "%u\n", buffer->watermark);  in watermark_show()
1345 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in watermark_store()  local
1357 	if (val > buffer->length) {  in watermark_store()
1362 	if (iio_buffer_is_active(buffer)) {  in watermark_store()
1367 	buffer->watermark = val;  in watermark_store()
1377 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in data_available_show()  local
1379 	return sysfs_emit(buf, "%zu\n", iio_buffer_data_available(buffer));  in data_available_show()
1386 	struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;  in direction_show()  local
1388 	switch (buffer->direction) {  in direction_show()
1422 static struct attribute *iio_buffer_wrap_attr(struct iio_buffer *buffer,  in iio_buffer_wrap_attr()  argument
1432 	iio_attr->buffer = buffer;  in iio_buffer_wrap_attr()
1442 	list_add(&iio_attr->l, &buffer->buffer_attr_list);  in iio_buffer_wrap_attr()
1510 	struct iio_buffer *buffer = ib->buffer;  in iio_buffer_chrdev_release()  local
1512 	wake_up(&buffer->pollq);  in iio_buffer_chrdev_release()
1515 	clear_bit(IIO_BUSY_BIT_POS, &buffer->flags);  in iio_buffer_chrdev_release()
1535 	struct iio_buffer *buffer;  in iio_device_buffer_getfd()  local
1546 	buffer = iio_dev_opaque->attached_buffers[idx];  in iio_device_buffer_getfd()
1548 	if (test_and_set_bit(IIO_BUSY_BIT_POS, &buffer->flags)) {  in iio_device_buffer_getfd()
1560 	ib->buffer = buffer;  in iio_device_buffer_getfd()
1588 	clear_bit(IIO_BUSY_BIT_POS, &buffer->flags);  in iio_device_buffer_getfd()
1605 static int __iio_buffer_alloc_sysfs_and_mask(struct iio_buffer *buffer,  in __iio_buffer_alloc_sysfs_and_mask()  argument
1617 	if (buffer->attrs) {  in __iio_buffer_alloc_sysfs_and_mask()
1618 		while (buffer->attrs[buffer_attrcount])  in __iio_buffer_alloc_sysfs_and_mask()
1624 	INIT_LIST_HEAD(&buffer->buffer_attr_list);  in __iio_buffer_alloc_sysfs_and_mask()
1645 			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,  in __iio_buffer_alloc_sysfs_and_mask()
1654 		if (indio_dev->masklength && !buffer->scan_mask) {  in __iio_buffer_alloc_sysfs_and_mask()
1655 			buffer->scan_mask = bitmap_zalloc(indio_dev->masklength,  in __iio_buffer_alloc_sysfs_and_mask()
1657 			if (!buffer->scan_mask) {  in __iio_buffer_alloc_sysfs_and_mask()
1672 	if (!buffer->access->set_length)  in __iio_buffer_alloc_sysfs_and_mask()
1675 	if (buffer->access->flags & INDIO_BUFFER_FLAG_FIXED_WATERMARK)  in __iio_buffer_alloc_sysfs_and_mask()
1678 	if (buffer->attrs)  in __iio_buffer_alloc_sysfs_and_mask()
1679 		for (i = 0, id_attr = buffer->attrs[i];  in __iio_buffer_alloc_sysfs_and_mask()
1680 		     (id_attr = buffer->attrs[i]); i++)  in __iio_buffer_alloc_sysfs_and_mask()
1684 	buffer->buffer_group.attrs = attr;  in __iio_buffer_alloc_sysfs_and_mask()
1689 		wrapped = iio_buffer_wrap_attr(buffer, attr[i]);  in __iio_buffer_alloc_sysfs_and_mask()
1698 	list_for_each_entry(p, &buffer->buffer_attr_list, l)  in __iio_buffer_alloc_sysfs_and_mask()
1701 	buffer->buffer_group.name = kasprintf(GFP_KERNEL, "buffer%d", index);  in __iio_buffer_alloc_sysfs_and_mask()
1702 	if (!buffer->buffer_group.name) {  in __iio_buffer_alloc_sysfs_and_mask()
1707 	ret = iio_device_register_sysfs_group(indio_dev, &buffer->buffer_group);  in __iio_buffer_alloc_sysfs_and_mask()
1724 	kfree(buffer->buffer_group.name);  in __iio_buffer_alloc_sysfs_and_mask()
1726 	kfree(buffer->buffer_group.attrs);  in __iio_buffer_alloc_sysfs_and_mask()
1728 	bitmap_free(buffer->scan_mask);  in __iio_buffer_alloc_sysfs_and_mask()
1730 	iio_free_chan_devattr_list(&buffer->buffer_attr_list);  in __iio_buffer_alloc_sysfs_and_mask()
1735 static void __iio_buffer_free_sysfs_and_mask(struct iio_buffer *buffer,  in __iio_buffer_free_sysfs_and_mask()  argument
1741 	bitmap_free(buffer->scan_mask);  in __iio_buffer_free_sysfs_and_mask()
1742 	kfree(buffer->buffer_group.name);  in __iio_buffer_free_sysfs_and_mask()
1743 	kfree(buffer->buffer_group.attrs);  in __iio_buffer_free_sysfs_and_mask()
1744 	iio_free_chan_devattr_list(&buffer->buffer_attr_list);  in __iio_buffer_free_sysfs_and_mask()
1751 	struct iio_buffer *buffer;  in iio_buffers_alloc_sysfs_and_mask()  local
1768 		buffer = iio_dev_opaque->attached_buffers[idx];  in iio_buffers_alloc_sysfs_and_mask()
1769 		ret = __iio_buffer_alloc_sysfs_and_mask(buffer, indio_dev, idx);  in iio_buffers_alloc_sysfs_and_mask()
1789 		buffer = iio_dev_opaque->attached_buffers[idx];  in iio_buffers_alloc_sysfs_and_mask()
1790 		__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, idx);  in iio_buffers_alloc_sysfs_and_mask()
1798 	struct iio_buffer *buffer;  in iio_buffers_free_sysfs_and_mask()  local
1808 		buffer = iio_dev_opaque->attached_buffers[i];  in iio_buffers_free_sysfs_and_mask()
1809 		__iio_buffer_free_sysfs_and_mask(buffer, indio_dev, i);  in iio_buffers_free_sysfs_and_mask()
1829 static const void *iio_demux(struct iio_buffer *buffer,  in iio_demux()  argument
1834 	if (list_empty(&buffer->demux_list))  in iio_demux()
1836 	list_for_each_entry(t, &buffer->demux_list, l)  in iio_demux()
1837 		memcpy(buffer->demux_bounce + t->to,  in iio_demux()
1840 	return buffer->demux_bounce;  in iio_demux()
1843 static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)  in iio_push_to_buffer()  argument
1845 	const void *dataout = iio_demux(buffer, data);  in iio_push_to_buffer()
1848 	ret = buffer->access->store_to(buffer, dataout);  in iio_push_to_buffer()
1856 	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);  in iio_push_to_buffer()
1938 	struct iio_buffer *buffer = container_of(ref, struct iio_buffer, ref);  in iio_buffer_release()  local
1940 	buffer->access->release(buffer);  in iio_buffer_release()
1949 struct iio_buffer *iio_buffer_get(struct iio_buffer *buffer)  in iio_buffer_get()  argument
1951 	if (buffer)  in iio_buffer_get()
1952 		kref_get(&buffer->ref);  in iio_buffer_get()
1954 	return buffer;  in iio_buffer_get()
1962 void iio_buffer_put(struct iio_buffer *buffer)  in iio_buffer_put()  argument
1964 	if (buffer)  in iio_buffer_put()
1965 		kref_put(&buffer->ref, iio_buffer_release);  in iio_buffer_put()
1983 			     struct iio_buffer *buffer)  in iio_device_attach_buffer()  argument
1996 	buffer = iio_buffer_get(buffer);  in iio_device_attach_buffer()
1999 	if (!indio_dev->buffer)  in iio_device_attach_buffer()
2000 		indio_dev->buffer = buffer;  in iio_device_attach_buffer()
2002 	iio_dev_opaque->attached_buffers[cnt - 1] = buffer;  in iio_device_attach_buffer()