34 channel_to_ops(struct tegra_bpmp_channel *channel)  in channel_to_ops()  argument
36 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()
78 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_get_thread_index()  argument
80 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()
86 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
101 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_ready()  argument
103 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_ready()
105 	return ops->is_response_ready(channel);  in tegra_bpmp_is_response_ready()
108 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_ready()  argument
110 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_ready()
112 	return ops->is_request_ready(channel);  in tegra_bpmp_is_request_ready()
115 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_response()  argument
117 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
123 		if (tegra_bpmp_is_response_ready(channel))  in tegra_bpmp_wait_response()
130 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_response()  argument
132 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_response()
134 	return ops->ack_response(channel);  in tegra_bpmp_ack_response()
137 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_request()  argument
139 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_request()
141 	return ops->ack_request(channel);  in tegra_bpmp_ack_request()
145 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_channel_free()  argument
147 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_channel_free()
149 	return ops->is_request_channel_free(channel);  in tegra_bpmp_is_request_channel_free()
153 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_channel_free()  argument
155 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_channel_free()
157 	return ops->is_response_channel_free(channel);  in tegra_bpmp_is_response_channel_free()
161 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_request_channel_free()  argument
163 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
169 		if (tegra_bpmp_is_request_channel_free(channel))  in tegra_bpmp_wait_request_channel_free()
178 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_request()  argument
180 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_request()
182 	return ops->post_request(channel);  in tegra_bpmp_post_request()
185 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_response()  argument
187 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_response()
189 	return ops->post_response(channel);  in tegra_bpmp_post_response()
197 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_read()  argument
203 		memcpy(data, channel->ib->data, size);  in __tegra_bpmp_channel_read()
205 	err = tegra_bpmp_ack_response(channel);  in __tegra_bpmp_channel_read()
209 	*ret = channel->ib->code;  in __tegra_bpmp_channel_read()
214 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_read()  argument
217 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()
222 	index = tegra_bpmp_channel_get_thread_index(channel);  in tegra_bpmp_channel_read()
229 	err = __tegra_bpmp_channel_read(channel, data, size, ret);  in tegra_bpmp_channel_read()
239 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_write()  argument
243 	channel->ob->code = mrq;  in __tegra_bpmp_channel_write()
244 	channel->ob->flags = flags;  in __tegra_bpmp_channel_write()
247 		memcpy(channel->ob->data, data, size);  in __tegra_bpmp_channel_write()
249 	return tegra_bpmp_post_request(channel);  in __tegra_bpmp_channel_write()
258 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_write_threaded()  local
275 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
277 	if (!tegra_bpmp_is_request_channel_free(channel)) {  in tegra_bpmp_write_threaded()
284 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,  in tegra_bpmp_write_threaded()
292 	return channel;  in tegra_bpmp_write_threaded()
303 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_write()  argument
309 	err = tegra_bpmp_wait_request_channel_free(channel);  in tegra_bpmp_channel_write()
313 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);  in tegra_bpmp_channel_write()
319 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer_atomic()  local
328 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
332 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,  in tegra_bpmp_transfer_atomic()
345 	err = tegra_bpmp_wait_response(channel);  in tegra_bpmp_transfer_atomic()
349 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer_atomic()
357 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer()  local
367 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
369 	if (IS_ERR(channel))  in tegra_bpmp_transfer()
370 		return PTR_ERR(channel);  in tegra_bpmp_transfer()
378 	err = wait_for_completion_timeout(&channel->completion, timeout);  in tegra_bpmp_transfer()
382 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer()
399 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,  in tegra_bpmp_mrq_return()  argument
402 	unsigned long flags = channel->ib->flags;  in tegra_bpmp_mrq_return()
403 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()
409 	err = tegra_bpmp_ack_request(channel);  in tegra_bpmp_mrq_return()
416 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))  in tegra_bpmp_mrq_return()
419 	channel->ob->code = code;  in tegra_bpmp_mrq_return()
422 		memcpy(channel->ob->data, data, size);  in tegra_bpmp_mrq_return()
424 	err = tegra_bpmp_post_response(channel);  in tegra_bpmp_mrq_return()
438 				  struct tegra_bpmp_channel *channel)  in tegra_bpmp_handle_mrq()  argument
448 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));  in tegra_bpmp_handle_mrq()
452 	entry->handler(mrq, channel, entry->data);  in tegra_bpmp_handle_mrq()
528 				       struct tegra_bpmp_channel *channel,  in tegra_bpmp_mrq_handle_ping()  argument
534 	request = (struct mrq_ping_request *)channel->ib->data;  in tegra_bpmp_mrq_handle_ping()
539 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));  in tegra_bpmp_mrq_handle_ping()
648 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_signal()  argument
650 	unsigned long flags = channel->ob->flags;  in tegra_bpmp_channel_signal()
655 	complete(&channel->completion);  in tegra_bpmp_channel_signal()
660 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
664 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
668 	if (tegra_bpmp_is_request_ready(channel))  in tegra_bpmp_handle_rx()
669 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);  in tegra_bpmp_handle_rx()
674 		struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
676 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
678 		if (tegra_bpmp_is_response_ready(channel)) {  in tegra_bpmp_handle_rx()
679 			tegra_bpmp_channel_signal(channel);  in tegra_bpmp_handle_rx()