40 		"	lgr	1,%[schid]\n"  in do_siga_sync()
49 		: "cc", "0", "1", "2", "3");  in do_siga_sync()
60 		"	lgr	1,%[schid]\n"  in do_siga_input()
67 		: "cc", "0", "1", "2");  in do_siga_input()
90 		"	lgr	1,%[schid]\n"  in do_siga_output()
99 		: "cc", "0", "1", "2", "3");  in do_siga_output()
106  * @q: queue to manipulate
115 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,  in qdio_do_eqbs()  argument
118 	int tmp_count = count, tmp_start = start, nr = q->nr;  in qdio_do_eqbs()
121 	qperf_inc(q, eqbs);  in qdio_do_eqbs()
123 	if (!q->is_input_q)  in qdio_do_eqbs()
124 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_eqbs()
126 	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,  in qdio_do_eqbs()
136 		qperf_inc(q, eqbs_partial);  in qdio_do_eqbs()
137 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",  in qdio_do_eqbs()
142 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);  in qdio_do_eqbs()
145 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);  in qdio_do_eqbs()
146 		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));  in qdio_do_eqbs()
148 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,  in qdio_do_eqbs()
149 			   q->first_to_check, count, q->irq_ptr->int_parm);  in qdio_do_eqbs()
156  * @q: queue to manipulate
165 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,  in qdio_do_sqbs()  argument
170 	int nr = q->nr;  in qdio_do_sqbs()
174 	qperf_inc(q, sqbs);  in qdio_do_sqbs()
176 	if (!q->is_input_q)  in qdio_do_sqbs()
177 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_sqbs()
179 	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);  in qdio_do_sqbs()
189 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);  in qdio_do_sqbs()
190 		qperf_inc(q, sqbs_partial);  in qdio_do_sqbs()
193 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);  in qdio_do_sqbs()
194 		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));  in qdio_do_sqbs()
196 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,  in qdio_do_sqbs()
197 			   q->first_to_check, count, q->irq_ptr->int_parm);  in qdio_do_sqbs()
206 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,  in get_buf_states()  argument
211 	int i = 1;  in get_buf_states()
213 	if (is_qebsm(q))  in get_buf_states()
214 		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);  in get_buf_states()
217 	__state = q->slsb.val[bufnr];  in get_buf_states()
227 		if (q->slsb.val[bufnr] != __state)  in get_buf_states()
236 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,  in get_buf_state()  argument
239 	return get_buf_states(q, bufnr, state, 1, auto_ack);  in get_buf_state()
243 static inline int set_buf_states(struct qdio_q *q, int bufnr,  in set_buf_states()  argument
248 	if (is_qebsm(q))  in set_buf_states()
249 		return qdio_do_sqbs(q, state, bufnr, count);  in set_buf_states()
255 		WRITE_ONCE(q->slsb.val[bufnr], state);  in set_buf_states()
265 static inline int set_buf_state(struct qdio_q *q, int bufnr,  in set_buf_state()  argument
268 	return set_buf_states(q, bufnr, state, 1);  in set_buf_state()
274 	struct qdio_q *q;  in qdio_init_buf_states()  local
277 	for_each_input_queue(irq_ptr, q, i)  in qdio_init_buf_states()
278 		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,  in qdio_init_buf_states()
280 	for_each_output_queue(irq_ptr, q, i)  in qdio_init_buf_states()
281 		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,  in qdio_init_buf_states()
285 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,  in qdio_siga_sync()  argument
288 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_sync()
292 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);  in qdio_siga_sync()
293 	qperf_inc(q, siga_sync);  in qdio_siga_sync()
295 	if (is_qebsm(q)) {  in qdio_siga_sync()
296 		schid = q->irq_ptr->sch_token;  in qdio_siga_sync()
302 		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);  in qdio_siga_sync()
306 static inline int qdio_sync_input_queue(struct qdio_q *q)  in qdio_sync_input_queue()  argument
308 	return qdio_siga_sync(q, 0, q->mask);  in qdio_sync_input_queue()
311 static inline int qdio_sync_output_queue(struct qdio_q *q)  in qdio_sync_output_queue()  argument
313 	return qdio_siga_sync(q, q->mask, 0);  in qdio_sync_output_queue()
316 static inline int qdio_siga_sync_q(struct qdio_q *q)  in qdio_siga_sync_q()  argument
318 	if (q->is_input_q)  in qdio_siga_sync_q()
319 		return qdio_sync_input_queue(q);  in qdio_siga_sync_q()
321 		return qdio_sync_output_queue(q);  in qdio_siga_sync_q()
324 static int qdio_siga_output(struct qdio_q *q, unsigned int count,  in qdio_siga_output()  argument
327 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_output()
332 	if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {  in qdio_siga_output()
333 		if (count > 1)  in qdio_siga_output()
339 	if (is_qebsm(q)) {  in qdio_siga_output()
340 		schid = q->irq_ptr->sch_token;  in qdio_siga_output()
344 	cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);  in qdio_siga_output()
358 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,  in qdio_siga_output()
359 			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);  in qdio_siga_output()
360 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);  in qdio_siga_output()
365 static inline int qdio_siga_input(struct qdio_q *q)  in qdio_siga_input()  argument
367 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_input()
371 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);  in qdio_siga_input()
372 	qperf_inc(q, siga_read);  in qdio_siga_input()
374 	if (is_qebsm(q)) {  in qdio_siga_input()
375 		schid = q->irq_ptr->sch_token;  in qdio_siga_input()
379 	cc = do_siga_input(schid, q->mask, fc);  in qdio_siga_input()
381 		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);  in qdio_siga_input()
385 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,  in debug_get_buf_state()  argument
388 	if (qdio_need_siga_sync(q->irq_ptr))  in debug_get_buf_state()
389 		qdio_siga_sync_q(q);  in debug_get_buf_state()
390 	return get_buf_state(q, bufnr, state, 0);  in debug_get_buf_state()
393 static inline void qdio_stop_polling(struct qdio_q *q)  in qdio_stop_polling()  argument
395 	if (!q->u.in.batch_count)  in qdio_stop_polling()
398 	qperf_inc(q, stop_polling);  in qdio_stop_polling()
401 	set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,  in qdio_stop_polling()
402 		       q->u.in.batch_count);  in qdio_stop_polling()
403 	q->u.in.batch_count = 0;  in qdio_stop_polling()
406 static inline void account_sbals(struct qdio_q *q, unsigned int count)  in account_sbals()  argument
408 	q->q_stats.nr_sbal_total += count;  in account_sbals()
409 	q->q_stats.nr_sbals[ilog2(count)]++;  in account_sbals()
412 static void process_buffer_error(struct qdio_q *q, unsigned int start,  in process_buffer_error()  argument
416 	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&  in process_buffer_error()
417 	    q->sbal[start]->element[15].sflags == 0x10) {  in process_buffer_error()
418 		qperf_inc(q, target_full);  in process_buffer_error()
419 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);  in process_buffer_error()
423 	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));  in process_buffer_error()
424 	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);  in process_buffer_error()
427 		  q->sbal[start]->element[14].sflags,  in process_buffer_error()
428 		  q->sbal[start]->element[15].sflags);  in process_buffer_error()
431 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,  in inbound_handle_work()  argument
436 		set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);  in inbound_handle_work()
438 	if (!q->u.in.batch_count)  in inbound_handle_work()
439 		q->u.in.batch_start = start;  in inbound_handle_work()
440 	q->u.in.batch_count += count;  in inbound_handle_work()
443 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start,  in get_inbound_buffer_frontier()  argument
449 	q->timestamp = get_tod_clock_fast();  in get_inbound_buffer_frontier()
451 	count = atomic_read(&q->nr_buf_used);  in get_inbound_buffer_frontier()
455 	if (qdio_need_siga_sync(q->irq_ptr))  in get_inbound_buffer_frontier()
456 		qdio_sync_input_queue(q);  in get_inbound_buffer_frontier()
458 	count = get_buf_states(q, start, &state, count, 1);  in get_inbound_buffer_frontier()
464 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,  in get_inbound_buffer_frontier()
467 		inbound_handle_work(q, start, count, is_qebsm(q));  in get_inbound_buffer_frontier()
468 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)  in get_inbound_buffer_frontier()
469 			qperf_inc(q, inbound_queue_full);  in get_inbound_buffer_frontier()
470 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
471 			account_sbals(q, count);  in get_inbound_buffer_frontier()
474 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,  in get_inbound_buffer_frontier()
478 		process_buffer_error(q, start, count);  in get_inbound_buffer_frontier()
479 		inbound_handle_work(q, start, count, false);  in get_inbound_buffer_frontier()
480 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)  in get_inbound_buffer_frontier()
481 			qperf_inc(q, inbound_queue_full);  in get_inbound_buffer_frontier()
482 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
483 			account_sbals_error(q, count);  in get_inbound_buffer_frontier()
486 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
487 			q->q_stats.nr_sbal_nop++;  in get_inbound_buffer_frontier()
488 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",  in get_inbound_buffer_frontier()
489 			      q->nr, start);  in get_inbound_buffer_frontier()
495 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,  in get_inbound_buffer_frontier()
497 			      state, start, q->nr);  in get_inbound_buffer_frontier()
502 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)  in qdio_inbound_q_done()  argument
506 	if (!atomic_read(&q->nr_buf_used))  in qdio_inbound_q_done()
507 		return 1;  in qdio_inbound_q_done()
509 	if (qdio_need_siga_sync(q->irq_ptr))  in qdio_inbound_q_done()
510 		qdio_sync_input_queue(q);  in qdio_inbound_q_done()
511 	get_buf_state(q, start, &state, 0);  in qdio_inbound_q_done()
517 	return 1;  in qdio_inbound_q_done()
520 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start,  in get_outbound_buffer_frontier()  argument
526 	q->timestamp = get_tod_clock_fast();  in get_outbound_buffer_frontier()
528 	count = atomic_read(&q->nr_buf_used);  in get_outbound_buffer_frontier()
532 	if (qdio_need_siga_sync(q->irq_ptr))  in get_outbound_buffer_frontier()
533 		qdio_sync_output_queue(q);  in get_outbound_buffer_frontier()
535 	count = get_buf_states(q, start, &state, count, 0);  in get_outbound_buffer_frontier()
545 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,  in get_outbound_buffer_frontier()
546 			"out empty:%1d %02x", q->nr, count);  in get_outbound_buffer_frontier()
548 		atomic_sub(count, &q->nr_buf_used);  in get_outbound_buffer_frontier()
549 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
550 			account_sbals(q, count);  in get_outbound_buffer_frontier()
553 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",  in get_outbound_buffer_frontier()
554 			      q->nr, count);  in get_outbound_buffer_frontier()
557 		process_buffer_error(q, start, count);  in get_outbound_buffer_frontier()
558 		atomic_sub(count, &q->nr_buf_used);  in get_outbound_buffer_frontier()
559 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
560 			account_sbals_error(q, count);  in get_outbound_buffer_frontier()
564 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
565 			q->q_stats.nr_sbal_nop++;  in get_outbound_buffer_frontier()
566 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",  in get_outbound_buffer_frontier()
567 			      q->nr);  in get_outbound_buffer_frontier()
574 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,  in get_outbound_buffer_frontier()
576 			      state, start, q->nr);  in get_outbound_buffer_frontier()
581 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,  in qdio_kick_outbound_q()  argument
587 	if (!qdio_need_siga_out(q->irq_ptr))  in qdio_kick_outbound_q()
590 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);  in qdio_kick_outbound_q()
592 	qperf_inc(q, siga_write);  in qdio_kick_outbound_q()
594 	cc = qdio_siga_output(q, count, &busy_bit, aob);  in qdio_kick_outbound_q()
604 			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);  in qdio_kick_outbound_q()
607 			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);  in qdio_kick_outbound_q()
611 	case 1:  in qdio_kick_outbound_q()
613 		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);  in qdio_kick_outbound_q()
618 		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);  in qdio_kick_outbound_q()
627 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);  in qdio_set_state()
656 	struct qdio_q *q;  in qdio_handle_activate_check()  local
663 		q = irq_ptr->input_qs[0];  in qdio_handle_activate_check()
665 		q = irq_ptr->output_qs[0];  in qdio_handle_activate_check()
671 	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE,  in qdio_handle_activate_check()
672 		   q->nr, q->first_to_check, 0, irq_ptr->int_parm);  in qdio_handle_activate_check()
749 		WARN_ON_ONCE(1);  in qdio_int_handler()
912 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,  in qdio_allocate()
952 	DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);  in qdio_trace_init_data()
955 	DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,  in qdio_trace_init_data()
1007 	/* establish q */  in qdio_establish()
1119  * @q: queue containing the buffers
1123 static int handle_inbound(struct qdio_q *q, int bufnr, int count)  in handle_inbound()  argument
1127 	qperf_inc(q, inbound_call);  in handle_inbound()
1130 	overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),  in handle_inbound()
1131 			     q->u.in.batch_count);  in handle_inbound()
1133 		q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);  in handle_inbound()
1134 		q->u.in.batch_count -= overlap;  in handle_inbound()
1137 	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);  in handle_inbound()
1138 	atomic_add(count, &q->nr_buf_used);  in handle_inbound()
1140 	if (qdio_need_siga_in(q->irq_ptr))  in handle_inbound()
1141 		return qdio_siga_input(q);  in handle_inbound()
1148  * @q: queue containing the buffers
1153 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count,  in handle_outbound()  argument
1159 	qperf_inc(q, outbound_call);  in handle_outbound()
1161 	count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);  in handle_outbound()
1162 	used = atomic_add_return(count, &q->nr_buf_used);  in handle_outbound()
1165 		qperf_inc(q, outbound_queue_full);  in handle_outbound()
1167 	if (queue_type(q) == QDIO_IQDIO_QFMT) {  in handle_outbound()
1171 		rc = qdio_kick_outbound_q(q, count, phys_aob);  in handle_outbound()
1172 	} else if (qdio_need_siga_sync(q->irq_ptr)) {  in handle_outbound()
1173 		rc = qdio_sync_output_queue(q);  in handle_outbound()
1175 		   get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 &&  in handle_outbound()
1178 		qperf_inc(q, fast_requeue);  in handle_outbound()
1180 		rc = qdio_kick_outbound_q(q, count, 0);  in handle_outbound()
1227  *   1 - irqs not started since new data is available
1231 	struct qdio_q *q;  in qdio_start_irq()  local
1238 	for_each_input_queue(irq_ptr, q, i)  in qdio_start_irq()
1239 		qdio_stop_polling(q);  in qdio_start_irq()
1250 	for_each_input_queue(irq_ptr, q, i) {  in qdio_start_irq()
1251 		if (!qdio_inbound_q_done(q, q->first_to_check))  in qdio_start_irq()
1261 		return 1;  in qdio_start_irq()
1266 static int __qdio_inspect_queue(struct qdio_q *q, unsigned int *bufnr,  in __qdio_inspect_queue()  argument
1269 	unsigned int start = q->first_to_check;  in __qdio_inspect_queue()
1273 	count = q->is_input_q ? get_inbound_buffer_frontier(q, start, error) :  in __qdio_inspect_queue()
1274 				get_outbound_buffer_frontier(q, start, error);  in __qdio_inspect_queue()
1281 	q->first_to_check = add_buf(start, count);  in __qdio_inspect_queue()
1290 	struct qdio_q *q;  in qdio_inspect_queue()  local
1294 	q = is_input ? irq_ptr->input_qs[nr] : irq_ptr->output_qs[nr];  in qdio_inspect_queue()
1296 	return __qdio_inspect_queue(q, bufnr, error);  in qdio_inspect_queue()
1306  *   1 - interrupts successfully disabled
1318 		return 1;  in qdio_stop_irq()