21 	struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);  in mmc_hsq_retry_handler()  local
22 	struct mmc_host *mmc = hsq->mmc;  in mmc_hsq_retry_handler()
24 	mmc->ops->request(mmc, hsq->mrq);  in mmc_hsq_retry_handler()
27 static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)  in mmc_hsq_pump_requests()  argument
29 	struct mmc_host *mmc = hsq->mmc;  in mmc_hsq_pump_requests()
34 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_pump_requests()
37 	if (hsq->mrq || hsq->recovery_halt) {  in mmc_hsq_pump_requests()
38 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
43 	if (!hsq->qcnt || !hsq->enabled) {  in mmc_hsq_pump_requests()
44 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
48 	slot = &hsq->slot[hsq->next_tag];  in mmc_hsq_pump_requests()
49 	hsq->mrq = slot->mrq;  in mmc_hsq_pump_requests()
50 	hsq->qcnt--;  in mmc_hsq_pump_requests()
52 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_pump_requests()
55 		ret = mmc->ops->request_atomic(mmc, hsq->mrq);  in mmc_hsq_pump_requests()
57 		mmc->ops->request(mmc, hsq->mrq);  in mmc_hsq_pump_requests()
69 		schedule_work(&hsq->retry_work);  in mmc_hsq_pump_requests()
74 static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)  in mmc_hsq_update_next_tag()  argument
84 		hsq->next_tag = HSQ_INVALID_TAG;  in mmc_hsq_update_next_tag()
92 	if (++hsq->next_tag != HSQ_INVALID_TAG) {  in mmc_hsq_update_next_tag()
93 		slot = &hsq->slot[hsq->next_tag];  in mmc_hsq_update_next_tag()
100 		slot = &hsq->slot[tag];  in mmc_hsq_update_next_tag()
108 	hsq->next_tag = tag;  in mmc_hsq_update_next_tag()
111 static void mmc_hsq_post_request(struct mmc_hsq *hsq)  in mmc_hsq_post_request()  argument
116 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_post_request()
118 	remains = hsq->qcnt;  in mmc_hsq_post_request()
119 	hsq->mrq = NULL;  in mmc_hsq_post_request()
122 	mmc_hsq_update_next_tag(hsq, remains);  in mmc_hsq_post_request()
124 	if (hsq->waiting_for_idle && !remains) {  in mmc_hsq_post_request()
125 		hsq->waiting_for_idle = false;  in mmc_hsq_post_request()
126 		wake_up(&hsq->wait_queue);  in mmc_hsq_post_request()
130 	if (hsq->recovery_halt) {  in mmc_hsq_post_request()
131 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_post_request()
135 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_post_request()
142 		mmc_hsq_pump_requests(hsq);  in mmc_hsq_post_request()
155 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_finalize_request()  local
158 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_finalize_request()
160 	if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {  in mmc_hsq_finalize_request()
161 		spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_finalize_request()
168 	hsq->slot[hsq->next_tag].mrq = NULL;  in mmc_hsq_finalize_request()
170 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_finalize_request()
172 	mmc_cqe_request_done(mmc, hsq->mrq);  in mmc_hsq_finalize_request()
174 	mmc_hsq_post_request(hsq);  in mmc_hsq_finalize_request()
182 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_recovery_start()  local
185 	spin_lock_irqsave(&hsq->lock, flags);  in mmc_hsq_recovery_start()
187 	hsq->recovery_halt = true;  in mmc_hsq_recovery_start()
189 	spin_unlock_irqrestore(&hsq->lock, flags);  in mmc_hsq_recovery_start()
194 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_recovery_finish()  local
197 	spin_lock_irq(&hsq->lock);  in mmc_hsq_recovery_finish()
199 	hsq->recovery_halt = false;  in mmc_hsq_recovery_finish()
200 	remains = hsq->qcnt;  in mmc_hsq_recovery_finish()
202 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_recovery_finish()
209 		mmc_hsq_pump_requests(hsq);  in mmc_hsq_recovery_finish()
214 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_request()  local
217 	spin_lock_irq(&hsq->lock);  in mmc_hsq_request()
219 	if (!hsq->enabled) {  in mmc_hsq_request()
220 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
225 	if (hsq->recovery_halt) {  in mmc_hsq_request()
226 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
230 	hsq->slot[tag].mrq = mrq;  in mmc_hsq_request()
236 	if (hsq->next_tag == HSQ_INVALID_TAG)  in mmc_hsq_request()
237 		hsq->next_tag = tag;  in mmc_hsq_request()
239 	hsq->qcnt++;  in mmc_hsq_request()
241 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_request()
243 	mmc_hsq_pump_requests(hsq);  in mmc_hsq_request()
254 static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)  in mmc_hsq_queue_is_idle()  argument
258 	spin_lock_irq(&hsq->lock);  in mmc_hsq_queue_is_idle()
260 	is_idle = (!hsq->mrq && !hsq->qcnt) ||  in mmc_hsq_queue_is_idle()
261 		hsq->recovery_halt;  in mmc_hsq_queue_is_idle()
263 	*ret = hsq->recovery_halt ? -EBUSY : 0;  in mmc_hsq_queue_is_idle()
264 	hsq->waiting_for_idle = !is_idle;  in mmc_hsq_queue_is_idle()
266 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_queue_is_idle()
273 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_wait_for_idle()  local
276 	wait_event(hsq->wait_queue,  in mmc_hsq_wait_for_idle()
277 		   mmc_hsq_queue_is_idle(hsq, &ret));  in mmc_hsq_wait_for_idle()
284 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_disable()  local
288 	spin_lock_irq(&hsq->lock);  in mmc_hsq_disable()
290 	if (!hsq->enabled) {  in mmc_hsq_disable()
291 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
295 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
297 	ret = wait_event_timeout(hsq->wait_queue,  in mmc_hsq_disable()
298 				 mmc_hsq_queue_is_idle(hsq, &ret),  in mmc_hsq_disable()
305 	spin_lock_irq(&hsq->lock);  in mmc_hsq_disable()
307 	hsq->enabled = false;  in mmc_hsq_disable()
309 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_disable()
314 	struct mmc_hsq *hsq = mmc->cqe_private;  in mmc_hsq_enable()  local
316 	spin_lock_irq(&hsq->lock);  in mmc_hsq_enable()
318 	if (hsq->enabled) {  in mmc_hsq_enable()
319 		spin_unlock_irq(&hsq->lock);  in mmc_hsq_enable()
323 	hsq->enabled = true;  in mmc_hsq_enable()
325 	spin_unlock_irq(&hsq->lock);  in mmc_hsq_enable()
340 int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)  in mmc_hsq_init()  argument
342 	hsq->num_slots = HSQ_NUM_SLOTS;  in mmc_hsq_init()
343 	hsq->next_tag = HSQ_INVALID_TAG;  in mmc_hsq_init()
345 	hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,  in mmc_hsq_init()
347 	if (!hsq->slot)  in mmc_hsq_init()
350 	hsq->mmc = mmc;  in mmc_hsq_init()
351 	hsq->mmc->cqe_private = hsq;  in mmc_hsq_init()
354 	INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);  in mmc_hsq_init()
355 	spin_lock_init(&hsq->lock);  in mmc_hsq_init()
356 	init_waitqueue_head(&hsq->wait_queue);  in mmc_hsq_init()