240 	struct scmi_raw_mode_info *raw;  member
249 scmi_raw_queue_select(struct scmi_raw_mode_info *raw, unsigned int idx,  in scmi_raw_queue_select()  argument
253 		return raw->q[idx];  in scmi_raw_queue_select()
255 	return xa_load(&raw->chans_q, chan_id);  in scmi_raw_queue_select()
336 scmi_xfer_raw_waiter_get(struct scmi_raw_mode_info *raw, struct scmi_xfer *xfer,  in scmi_xfer_raw_waiter_get()  argument
341 	mutex_lock(&raw->free_mtx);  in scmi_xfer_raw_waiter_get()
342 	if (!list_empty(&raw->free_waiters)) {  in scmi_xfer_raw_waiter_get()
343 		rw = list_first_entry(&raw->free_waiters,  in scmi_xfer_raw_waiter_get()
355 	mutex_unlock(&raw->free_mtx);  in scmi_xfer_raw_waiter_get()
360 static void scmi_xfer_raw_waiter_put(struct scmi_raw_mode_info *raw,  in scmi_xfer_raw_waiter_put()  argument
368 	mutex_lock(&raw->free_mtx);  in scmi_xfer_raw_waiter_put()
369 	list_add_tail(&rw->node, &raw->free_waiters);  in scmi_xfer_raw_waiter_put()
370 	mutex_unlock(&raw->free_mtx);  in scmi_xfer_raw_waiter_put()
373 static void scmi_xfer_raw_waiter_enqueue(struct scmi_raw_mode_info *raw,  in scmi_xfer_raw_waiter_enqueue()  argument
382 				      raw->desc->max_rx_timeout_ms,  in scmi_xfer_raw_waiter_enqueue()
385 	mutex_lock(&raw->active_mtx);  in scmi_xfer_raw_waiter_enqueue()
386 	list_add_tail(&rw->node, &raw->active_waiters);  in scmi_xfer_raw_waiter_enqueue()
387 	mutex_unlock(&raw->active_mtx);  in scmi_xfer_raw_waiter_enqueue()
390 	queue_work(raw->wait_wq, &raw->waiters_work);  in scmi_xfer_raw_waiter_enqueue()
394 scmi_xfer_raw_waiter_dequeue(struct scmi_raw_mode_info *raw)  in scmi_xfer_raw_waiter_dequeue()  argument
398 	mutex_lock(&raw->active_mtx);  in scmi_xfer_raw_waiter_dequeue()
399 	if (!list_empty(&raw->active_waiters)) {  in scmi_xfer_raw_waiter_dequeue()
400 		rw = list_first_entry(&raw->active_waiters,  in scmi_xfer_raw_waiter_dequeue()
404 	mutex_unlock(&raw->active_mtx);  in scmi_xfer_raw_waiter_dequeue()
434 	struct scmi_raw_mode_info *raw;  in scmi_xfer_raw_worker()  local
438 	raw = container_of(work, struct scmi_raw_mode_info, waiters_work);  in scmi_xfer_raw_worker()
439 	dev = raw->handle->dev;  in scmi_xfer_raw_worker()
440 	max_tmo = msecs_to_jiffies(raw->desc->max_rx_timeout_ms);  in scmi_xfer_raw_worker()
450 		rw = scmi_xfer_raw_waiter_dequeue(raw);  in scmi_xfer_raw_worker()
474 		if (raw->desc->ops->mark_txdone)  in scmi_xfer_raw_worker()
475 			raw->desc->ops->mark_txdone(rw->cinfo, ret, xfer);  in scmi_xfer_raw_worker()
491 		scmi_xfer_raw_put(raw->handle, xfer);  in scmi_xfer_raw_worker()
492 		scmi_xfer_raw_waiter_put(raw, rw);  in scmi_xfer_raw_worker()
496 static void scmi_xfer_raw_reset(struct scmi_raw_mode_info *raw)  in scmi_xfer_raw_reset()  argument
500 	dev_info(raw->handle->dev, "Resetting SCMI Raw stack.\n");  in scmi_xfer_raw_reset()
503 		scmi_raw_buffer_queue_flush(raw->q[i]);  in scmi_xfer_raw_reset()
527 static int scmi_xfer_raw_get_init(struct scmi_raw_mode_info *raw, void *buf,  in scmi_xfer_raw_get_init()  argument
534 	struct device *dev = raw->handle->dev;  in scmi_xfer_raw_get_init()
541 	if (tx_size > raw->desc->max_msg_size)  in scmi_xfer_raw_get_init()
544 	xfer = scmi_xfer_raw_get(raw->handle);  in scmi_xfer_raw_get_init()
558 	xfer->rx.len = raw->desc->max_msg_size;  in scmi_xfer_raw_get_init()
560 	memset(xfer->tx.buf, 0x00, raw->desc->max_msg_size);  in scmi_xfer_raw_get_init()
572 		ret = scmi_xfer_raw_inflight_register(raw->handle, xfer);  in scmi_xfer_raw_get_init()
577 			msleep(raw->desc->max_rx_timeout_ms /  in scmi_xfer_raw_get_init()
586 		scmi_xfer_raw_put(raw->handle, xfer);  in scmi_xfer_raw_get_init()
611 static int scmi_do_xfer_raw_start(struct scmi_raw_mode_info *raw,  in scmi_do_xfer_raw_start()  argument
618 	struct device *dev = raw->handle->dev;  in scmi_do_xfer_raw_start()
625 	cinfo = scmi_xfer_raw_channel_get(raw->handle, chan_id);  in scmi_do_xfer_raw_start()
629 	rw = scmi_xfer_raw_waiter_get(raw, xfer, cinfo, async);  in scmi_do_xfer_raw_start()
636 	if (is_polling_enabled(cinfo, raw->desc))  in scmi_do_xfer_raw_start()
647 	ret = raw->desc->ops->send_message(rw->cinfo, xfer);  in scmi_do_xfer_raw_start()
650 		scmi_xfer_raw_waiter_put(raw, rw);  in scmi_do_xfer_raw_start()
654 	trace_scmi_msg_dump(raw->id, cinfo->id, xfer->hdr.protocol_id,  in scmi_do_xfer_raw_start()
659 	scmi_xfer_raw_waiter_enqueue(raw, rw);  in scmi_do_xfer_raw_start()
677 static int scmi_raw_message_send(struct scmi_raw_mode_info *raw,  in scmi_raw_message_send()  argument
683 	ret = scmi_xfer_raw_get_init(raw, buf, len, &xfer);  in scmi_raw_message_send()
687 	ret = scmi_do_xfer_raw_start(raw, xfer, chan_id, async);  in scmi_raw_message_send()
689 		scmi_xfer_raw_put(raw->handle, xfer);  in scmi_raw_message_send()
735 static int scmi_raw_message_receive(struct scmi_raw_mode_info *raw,  in scmi_raw_message_receive()  argument
744 	q = scmi_raw_queue_select(raw, idx, chan_id);  in scmi_raw_message_receive()
750 		dev_dbg(raw->handle->dev, "RAW - No message available!\n");  in scmi_raw_message_receive()
779 		ret = scmi_raw_message_receive(rd->raw, rd->rx.buf, rd->rx.len,  in scmi_dbg_raw_mode_common_read()
833 	ret = scmi_raw_message_send(rd->raw, rd->tx.buf, rd->tx_size,  in scmi_dbg_raw_mode_common_write()
852 	q = scmi_raw_queue_select(rd->raw, idx, rd->chan_id);  in scmi_test_dbg_raw_common_poll()
890 	struct scmi_raw_mode_info *raw;  in scmi_dbg_raw_mode_open()  local
897 	raw = inode->i_private;  in scmi_dbg_raw_mode_open()
902 	rd->rx.len = raw->desc->max_msg_size + sizeof(u32);  in scmi_dbg_raw_mode_open()
909 	rd->tx.len = raw->desc->max_msg_size + sizeof(u32);  in scmi_dbg_raw_mode_open()
921 	rd->raw = raw;  in scmi_dbg_raw_mode_open()
944 	scmi_xfer_raw_reset(rd->raw);  in scmi_dbg_raw_mode_reset_write()
1028 scmi_raw_queue_init(struct scmi_raw_mode_info *raw)  in scmi_raw_queue_init()  argument
1032 	struct device *dev = raw->handle->dev;  in scmi_raw_queue_init()
1039 	rb = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rb), GFP_KERNEL);  in scmi_raw_queue_init()
1045 	for (i = 0; i < raw->tx_max_msg; i++, rb++) {  in scmi_raw_queue_init()
1046 		rb->max_len = raw->desc->max_msg_size + sizeof(u32);  in scmi_raw_queue_init()
1060 static int scmi_xfer_raw_worker_init(struct scmi_raw_mode_info *raw)  in scmi_xfer_raw_worker_init()  argument
1064 	struct device *dev = raw->handle->dev;  in scmi_xfer_raw_worker_init()
1066 	rw = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rw), GFP_KERNEL);  in scmi_xfer_raw_worker_init()
1070 	raw->wait_wq = alloc_workqueue("scmi-raw-wait-wq-%d",  in scmi_xfer_raw_worker_init()
1072 				       WQ_HIGHPRI | WQ_SYSFS, 0, raw->id);  in scmi_xfer_raw_worker_init()
1073 	if (!raw->wait_wq)  in scmi_xfer_raw_worker_init()
1076 	mutex_init(&raw->free_mtx);  in scmi_xfer_raw_worker_init()
1077 	INIT_LIST_HEAD(&raw->free_waiters);  in scmi_xfer_raw_worker_init()
1078 	mutex_init(&raw->active_mtx);  in scmi_xfer_raw_worker_init()
1079 	INIT_LIST_HEAD(&raw->active_waiters);  in scmi_xfer_raw_worker_init()
1081 	for (i = 0; i < raw->tx_max_msg; i++, rw++) {  in scmi_xfer_raw_worker_init()
1083 		scmi_xfer_raw_waiter_put(raw, rw);  in scmi_xfer_raw_worker_init()
1085 	INIT_WORK(&raw->waiters_work, scmi_xfer_raw_worker);  in scmi_xfer_raw_worker_init()
1090 static int scmi_raw_mode_setup(struct scmi_raw_mode_info *raw,  in scmi_raw_mode_setup()  argument
1095 	struct device *dev = raw->handle->dev;  in scmi_raw_mode_setup()
1102 		raw->q[idx] = scmi_raw_queue_init(raw);  in scmi_raw_mode_setup()
1103 		if (IS_ERR(raw->q[idx])) {  in scmi_raw_mode_setup()
1104 			ret = PTR_ERR(raw->q[idx]);  in scmi_raw_mode_setup()
1109 	xa_init(&raw->chans_q);  in scmi_raw_mode_setup()
1117 			q = scmi_raw_queue_init(raw);  in scmi_raw_mode_setup()
1123 			xret = xa_store(&raw->chans_q, channels[i], q,  in scmi_raw_mode_setup()
1135 	ret = scmi_xfer_raw_worker_init(raw);  in scmi_raw_mode_setup()
1140 	raw->gid = gid;  in scmi_raw_mode_setup()
1145 	xa_destroy(&raw->chans_q);  in scmi_raw_mode_setup()
1173 	struct scmi_raw_mode_info *raw;  in scmi_raw_mode_init()  local
1180 	raw = devm_kzalloc(dev, sizeof(*raw), GFP_KERNEL);  in scmi_raw_mode_init()
1181 	if (!raw)  in scmi_raw_mode_init()
1184 	raw->handle = handle;  in scmi_raw_mode_init()
1185 	raw->desc = desc;  in scmi_raw_mode_init()
1186 	raw->tx_max_msg = tx_max_msg;  in scmi_raw_mode_init()
1187 	raw->id = instance_id;  in scmi_raw_mode_init()
1189 	ret = scmi_raw_mode_setup(raw, channels, num_chans);  in scmi_raw_mode_init()
1191 		devm_kfree(dev, raw);  in scmi_raw_mode_init()
1195 	raw->dentry = debugfs_create_dir("raw", top_dentry);  in scmi_raw_mode_init()
1197 	debugfs_create_file("reset", 0200, raw->dentry, raw,  in scmi_raw_mode_init()
1200 	debugfs_create_file("message", 0600, raw->dentry, raw,  in scmi_raw_mode_init()
1203 	debugfs_create_file("message_async", 0600, raw->dentry, raw,  in scmi_raw_mode_init()
1206 	debugfs_create_file("notification", 0400, raw->dentry, raw,  in scmi_raw_mode_init()
1209 	debugfs_create_file("errors", 0400, raw->dentry, raw,  in scmi_raw_mode_init()
1221 		top_chans = debugfs_create_dir("channels", raw->dentry);  in scmi_raw_mode_init()
1230 			debugfs_create_file("message", 0600, chd, raw,  in scmi_raw_mode_init()
1233 			debugfs_create_file("message_async", 0600, chd, raw,  in scmi_raw_mode_init()
1238 	dev_info(dev, "SCMI RAW Mode initialized for instance %d\n", raw->id);  in scmi_raw_mode_init()
1240 	return raw;  in scmi_raw_mode_init()
1250 	struct scmi_raw_mode_info *raw = r;  in scmi_raw_mode_cleanup()  local
1252 	if (!raw)  in scmi_raw_mode_cleanup()
1255 	debugfs_remove_recursive(raw->dentry);  in scmi_raw_mode_cleanup()
1257 	cancel_work_sync(&raw->waiters_work);  in scmi_raw_mode_cleanup()
1258 	destroy_workqueue(raw->wait_wq);  in scmi_raw_mode_cleanup()
1259 	xa_destroy(&raw->chans_q);  in scmi_raw_mode_cleanup()
1317 	struct scmi_raw_mode_info *raw = r;  in scmi_raw_message_report()  local
1319 	if (!raw || (idx == SCMI_RAW_REPLY_QUEUE && !SCMI_XFER_IS_RAW(xfer)))  in scmi_raw_message_report()
1322 	dev = raw->handle->dev;  in scmi_raw_message_report()
1323 	q = scmi_raw_queue_select(raw, idx,  in scmi_raw_message_report()
1388 static void scmi_xfer_raw_fill(struct scmi_raw_mode_info *raw,  in scmi_xfer_raw_fill()  argument
1398 	raw->desc->ops->fetch_response(cinfo, xfer);  in scmi_xfer_raw_fill()
1423 	struct scmi_raw_mode_info *raw = r;  in scmi_raw_error_report()  local
1425 	if (!raw)  in scmi_raw_error_report()
1428 	xfer.rx.len = raw->desc->max_msg_size;  in scmi_raw_error_report()
1431 		dev_info(raw->handle->dev,  in scmi_raw_error_report()
1442 	scmi_xfer_raw_fill(raw, cinfo, &xfer, msg_hdr);  in scmi_raw_error_report()
1443 	scmi_raw_message_report(raw, &xfer, SCMI_RAW_ERRS_QUEUE, 0);  in scmi_raw_error_report()