163 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)  in print_ddcb_info()  argument
168 	struct pci_dev *pci_dev = cd->pci_dev;  in print_ddcb_info()
170 	spin_lock_irqsave(&cd->print_lock, flags);  in print_ddcb_info()
174 		 cd->card_idx, queue->ddcb_act, queue->ddcb_next);  in print_ddcb_info()
190 	spin_unlock_irqrestore(&cd->print_lock, flags);  in print_ddcb_info()
241 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)  in ddcb_requ_finished()  argument
244 		(cd->card_state != GENWQE_CARD_USED);  in ddcb_requ_finished()
265 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,  in enqueue_ddcb()  argument
312 	__genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */  in enqueue_ddcb()
362 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,  in genwqe_check_ddcb_queue()  argument
367 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_check_ddcb_queue()
407 			errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);  in genwqe_check_ddcb_queue()
408 			status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);  in genwqe_check_ddcb_queue()
474 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)  in __genwqe_wait_ddcb()  argument
479 	struct pci_dev *pci_dev = cd->pci_dev;  in __genwqe_wait_ddcb()
493 				ddcb_requ_finished(cd, req),  in __genwqe_wait_ddcb()
511 		genwqe_check_ddcb_queue(cd, req->queue);  in __genwqe_wait_ddcb()
512 		if (ddcb_requ_finished(cd, req))  in __genwqe_wait_ddcb()
521 			__genwqe_readq(cd, queue->IO_QUEUE_STATUS));  in __genwqe_wait_ddcb()
526 		print_ddcb_info(cd, req->queue);  in __genwqe_wait_ddcb()
544 	if (cd->card_state != GENWQE_CARD_USED) {  in __genwqe_wait_ddcb()
562 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,  in get_next_ddcb()  argument
615 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)  in __genwqe_purge_ddcb()  argument
621 	struct pci_dev *pci_dev = cd->pci_dev;  in __genwqe_purge_ddcb()
705 	queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);  in __genwqe_purge_ddcb()
715 	print_ddcb_info(cd, req->queue);  in __genwqe_purge_ddcb()
720 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)  in genwqe_init_debug_data()  argument
723 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_init_debug_data()
734 	d->slu_unitcfg = cd->slu_unitcfg;  in genwqe_init_debug_data()
735 	d->app_unitcfg = cd->app_unitcfg;  in genwqe_init_debug_data()
749 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,  in __genwqe_enqueue_ddcb()  argument
755 	struct pci_dev *pci_dev = cd->pci_dev;  in __genwqe_enqueue_ddcb()
759 	if (cd->card_state != GENWQE_CARD_USED) {  in __genwqe_enqueue_ddcb()
767 	queue = req->queue = &cd->queue;  in __genwqe_enqueue_ddcb()
773 		genwqe_check_ddcb_queue(cd, queue);  in __genwqe_enqueue_ddcb()
782 	pddcb = get_next_ddcb(cd, queue, &req->num);	/* get ptr and num */  in __genwqe_enqueue_ddcb()
827 	if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)  in __genwqe_enqueue_ddcb()
850 	if (genwqe_get_slu_id(cd) <= 0x2) {  in __genwqe_enqueue_ddcb()
882 		genwqe_init_debug_data(cd, &req->debug_data);  in __genwqe_enqueue_ddcb()
887 	enqueue_ddcb(cd, queue, pddcb, req->num);  in __genwqe_enqueue_ddcb()
895 	wake_up_interruptible(&cd->queue_waitq);  in __genwqe_enqueue_ddcb()
906 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,  in __genwqe_execute_raw_ddcb()  argument
911 	struct pci_dev *pci_dev = cd->pci_dev;  in __genwqe_execute_raw_ddcb()
924 	rc = __genwqe_enqueue_ddcb(cd, req, f_flags);  in __genwqe_execute_raw_ddcb()
928 	rc = __genwqe_wait_ddcb(cd, req);  in __genwqe_execute_raw_ddcb()
954 	__genwqe_purge_ddcb(cd, req);  in __genwqe_execute_raw_ddcb()
971 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)  in genwqe_next_ddcb_ready()  argument
975 	struct ddcb_queue *queue = &cd->queue;  in genwqe_next_ddcb_ready()
1001 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)  in genwqe_ddcbs_in_flight()  argument
1005 	struct ddcb_queue *queue = &cd->queue;  in genwqe_ddcbs_in_flight()
1014 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)  in setup_ddcb_queue()  argument
1020 	struct pci_dev *pci_dev = cd->pci_dev;  in setup_ddcb_queue()
1035 	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,  in setup_ddcb_queue()
1073 	__genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */  in setup_ddcb_queue()
1074 	__genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);  in setup_ddcb_queue()
1075 	__genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);  in setup_ddcb_queue()
1076 	__genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);  in setup_ddcb_queue()
1083 	__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,  in setup_ddcb_queue()
1096 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)  in free_ddcb_queue()  argument
1106 		__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,  in free_ddcb_queue()
1116 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;  in genwqe_pf_isr()  local
1117 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_pf_isr()
1123 	cd->irqs_processed++;  in genwqe_pf_isr()
1124 	wake_up_interruptible(&cd->queue_waitq);  in genwqe_pf_isr()
1130 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);  in genwqe_pf_isr()
1134 		if (cd->use_platform_recovery) {  in genwqe_pf_isr()
1140 			readq(cd->mmio + IO_SLC_CFGREG_GFIR);  in genwqe_pf_isr()
1147 		wake_up_interruptible(&cd->health_waitq);  in genwqe_pf_isr()
1164 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;  in genwqe_vf_isr()  local
1166 	cd->irqs_processed++;  in genwqe_vf_isr()
1167 	wake_up_interruptible(&cd->queue_waitq);  in genwqe_vf_isr()
1183 	struct genwqe_dev *cd = (struct genwqe_dev *)data;  in genwqe_card_thread()  local
1187 		genwqe_check_ddcb_queue(cd, &cd->queue);  in genwqe_card_thread()
1191 				cd->queue_waitq,  in genwqe_card_thread()
1192 				genwqe_ddcbs_in_flight(cd) ||  in genwqe_card_thread()
1196 				cd->queue_waitq,  in genwqe_card_thread()
1197 				genwqe_next_ddcb_ready(cd) ||  in genwqe_card_thread()
1220 int genwqe_setup_service_layer(struct genwqe_dev *cd)  in genwqe_setup_service_layer()  argument
1224 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_setup_service_layer()
1226 	if (genwqe_is_privileged(cd)) {  in genwqe_setup_service_layer()
1227 		rc = genwqe_card_reset(cd);  in genwqe_setup_service_layer()
1233 		genwqe_read_softreset(cd);  in genwqe_setup_service_layer()
1236 	queue = &cd->queue;  in genwqe_setup_service_layer()
1247 	rc = setup_ddcb_queue(cd, queue);  in genwqe_setup_service_layer()
1253 	init_waitqueue_head(&cd->queue_waitq);  in genwqe_setup_service_layer()
1254 	cd->card_thread = kthread_run(genwqe_card_thread, cd,  in genwqe_setup_service_layer()
1256 				      cd->card_idx);  in genwqe_setup_service_layer()
1257 	if (IS_ERR(cd->card_thread)) {  in genwqe_setup_service_layer()
1258 		rc = PTR_ERR(cd->card_thread);  in genwqe_setup_service_layer()
1259 		cd->card_thread = NULL;  in genwqe_setup_service_layer()
1263 	rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);  in genwqe_setup_service_layer()
1272 	init_waitqueue_head(&cd->health_waitq);  in genwqe_setup_service_layer()
1274 	if (genwqe_is_privileged(cd)) {  in genwqe_setup_service_layer()
1276 				 GENWQE_DEVNAME, cd);  in genwqe_setup_service_layer()
1279 				 GENWQE_DEVNAME, cd);  in genwqe_setup_service_layer()
1286 	cd->card_state = GENWQE_CARD_USED;  in genwqe_setup_service_layer()
1290 	genwqe_reset_interrupt_capability(cd);  in genwqe_setup_service_layer()
1292 	kthread_stop(cd->card_thread);  in genwqe_setup_service_layer()
1293 	cd->card_thread = NULL;  in genwqe_setup_service_layer()
1295 	free_ddcb_queue(cd, queue);  in genwqe_setup_service_layer()
1307 static int queue_wake_up_all(struct genwqe_dev *cd)  in queue_wake_up_all()  argument
1311 	struct ddcb_queue *queue = &cd->queue;  in queue_wake_up_all()
1332 int genwqe_finish_queue(struct genwqe_dev *cd)  in genwqe_finish_queue()  argument
1336 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_finish_queue()
1337 	struct ddcb_queue *queue = &cd->queue;  in genwqe_finish_queue()
1343 	if (cd->card_state == GENWQE_CARD_USED)  in genwqe_finish_queue()
1344 		cd->card_state = GENWQE_CARD_UNUSED;  in genwqe_finish_queue()
1348 	queue_wake_up_all(cd);  in genwqe_finish_queue()
1352 		in_flight = genwqe_ddcbs_in_flight(cd);  in genwqe_finish_queue()
1384 int genwqe_release_service_layer(struct genwqe_dev *cd)  in genwqe_release_service_layer()  argument
1386 	struct pci_dev *pci_dev = cd->pci_dev;  in genwqe_release_service_layer()
1388 	if (!ddcb_queue_initialized(&cd->queue))  in genwqe_release_service_layer()
1391 	free_irq(pci_dev->irq, cd);  in genwqe_release_service_layer()
1392 	genwqe_reset_interrupt_capability(cd);  in genwqe_release_service_layer()
1394 	if (cd->card_thread != NULL) {  in genwqe_release_service_layer()
1395 		kthread_stop(cd->card_thread);  in genwqe_release_service_layer()
1396 		cd->card_thread = NULL;  in genwqe_release_service_layer()
1399 	free_ddcb_queue(cd, &cd->queue);  in genwqe_release_service_layer()