62 	struct pqi_queue_group *queue_group, enum pqi_io_path path,
69 	unsigned int cdb_length, struct pqi_queue_group *queue_group,
72 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
76 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
2722 	struct pqi_queue_group *queue_group)  in pqi_raid_bypass_submit_scsi_cmd()  argument
2803 			return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
2807 			return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
2813 		rmd.cdb, rmd.cdb_length, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3140 static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)  in pqi_process_io_intr()  argument
3150 	oq_ci = queue_group->oq_ci_copy;  in pqi_process_io_intr()
3153 		oq_pi = readl(queue_group->oq_pi);  in pqi_process_io_intr()
3165 		response = queue_group->oq_element_array +  in pqi_process_io_intr()
3232 		queue_group->oq_ci_copy = oq_ci;  in pqi_process_io_intr()
3233 		writel(oq_ci, queue_group->oq_ci);  in pqi_process_io_intr()
3259 	struct pqi_queue_group *queue_group;  in pqi_send_event_ack()  local
3261 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
3262 	put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);  in pqi_send_event_ack()
3265 		spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3267 		iq_pi = queue_group->iq_pi_copy[RAID_PATH];  in pqi_send_event_ack()
3268 		iq_ci = readl(queue_group->iq_ci[RAID_PATH]);  in pqi_send_event_ack()
3275 			&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3281 	next_element = queue_group->iq_element_array[RAID_PATH] +  in pqi_send_event_ack()
3287 	queue_group->iq_pi_copy[RAID_PATH] = iq_pi;  in pqi_send_event_ack()
3293 	writel(iq_pi, queue_group->iq_pi[RAID_PATH]);  in pqi_send_event_ack()
3295 	spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3721 	struct pqi_queue_group *queue_group;  in pqi_irq_handler()  local
3725 	queue_group = data;  in pqi_irq_handler()
3726 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3731 	num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
3746 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
3747 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
3826 	struct pqi_queue_group *queue_group;  in pqi_alloc_operational_queues()  local
3882 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3883 		queue_group->iq_element_array[RAID_PATH] = element_array;  in pqi_alloc_operational_queues()
3884 		queue_group->iq_element_array_bus_addr[RAID_PATH] =  in pqi_alloc_operational_queues()
3890 		queue_group->iq_element_array[AIO_PATH] = element_array;  in pqi_alloc_operational_queues()
3891 		queue_group->iq_element_array_bus_addr[AIO_PATH] =  in pqi_alloc_operational_queues()
3900 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3901 		queue_group->oq_element_array = element_array;  in pqi_alloc_operational_queues()
3902 		queue_group->oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
3921 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3922 		queue_group->iq_ci[RAID_PATH] = next_queue_index;  in pqi_alloc_operational_queues()
3923 		queue_group->iq_ci_bus_addr[RAID_PATH] =  in pqi_alloc_operational_queues()
3930 		queue_group->iq_ci[AIO_PATH] = next_queue_index;  in pqi_alloc_operational_queues()
3931 		queue_group->iq_ci_bus_addr[AIO_PATH] =  in pqi_alloc_operational_queues()
3938 		queue_group->oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
3939 		queue_group->oq_pi_bus_addr =  in pqi_alloc_operational_queues()
4170 	struct pqi_queue_group *queue_group, enum pqi_io_path path,  in pqi_start_io()  argument
4184 	spin_lock_irqsave(&queue_group->submit_lock[path], flags);  in pqi_start_io()
4187 		io_request->queue_group = queue_group;  in pqi_start_io()
4189 			&queue_group->request_list[path]);  in pqi_start_io()
4192 	iq_pi = queue_group->iq_pi_copy[path];  in pqi_start_io()
4195 		&queue_group->request_list[path], request_list_entry) {  in pqi_start_io()
4205 		iq_ci = readl(queue_group->iq_ci[path]);  in pqi_start_io()
4211 		put_unaligned_le16(queue_group->oq_id,  in pqi_start_io()
4214 		next_element = queue_group->iq_element_array[path] +  in pqi_start_io()
4226 			memcpy(queue_group->iq_element_array[path],  in pqi_start_io()
4237 	if (iq_pi != queue_group->iq_pi_copy[path]) {  in pqi_start_io()
4238 		queue_group->iq_pi_copy[path] = iq_pi;  in pqi_start_io()
4243 		writel(iq_pi, queue_group->iq_pi[path]);  in pqi_start_io()
4246 	spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);  in pqi_start_io()
4581 	struct pqi_queue_group *queue_group;  in pqi_create_queue_group()  local
4585 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
4596 	put_unaligned_le16(queue_group->iq_id[RAID_PATH],  in pqi_create_queue_group()
4599 		(u64)queue_group->iq_element_array_bus_addr[RAID_PATH],  in pqi_create_queue_group()
4601 	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],  in pqi_create_queue_group()
4617 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4631 	put_unaligned_le16(queue_group->iq_id[AIO_PATH],  in pqi_create_queue_group()
4633 	put_unaligned_le64((u64)queue_group->  in pqi_create_queue_group()
4636 	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],  in pqi_create_queue_group()
4652 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4667 	put_unaligned_le16(queue_group->iq_id[AIO_PATH],  in pqi_create_queue_group()
4688 	put_unaligned_le16(queue_group->oq_id,  in pqi_create_queue_group()
4690 	put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,  in pqi_create_queue_group()
4692 	put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,  in pqi_create_queue_group()
4699 	put_unaligned_le16(queue_group->int_msg_num,  in pqi_create_queue_group()
4710 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
5237 	struct pqi_queue_group *queue_group)  in pqi_raid_submit_scsi_cmd_with_io_request()  argument
5307 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
5314 	struct pqi_queue_group *queue_group)  in pqi_raid_submit_scsi_cmd()  argument
5321 		device, scmd, queue_group);  in pqi_raid_submit_scsi_cmd()
5368 	struct pqi_queue_group *queue_group)  in pqi_aio_submit_scsi_cmd()  argument
5371 		scmd->cmnd, scmd->cmd_len, queue_group, NULL, false);  in pqi_aio_submit_scsi_cmd()
5376 	unsigned int cdb_length, struct pqi_queue_group *queue_group,  in pqi_aio_submit_io()  argument
5438 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
5444 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,  in pqi_aio_submit_r1_write_io()  argument
5496 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r1_write_io()
5502 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,  in pqi_aio_submit_r56_write_io()  argument
5561 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r56_write_io()
5690 	struct pqi_queue_group *queue_group;  in pqi_scsi_queue_command()  local
5723 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
5730 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5737 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5740 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5742 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
5753 	struct pqi_queue_group *queue_group)  in pqi_wait_until_queued_io_drained()  argument
5762 				&queue_group->submit_lock[path], flags);  in pqi_wait_until_queued_io_drained()
5764 				list_empty(&queue_group->request_list[path]);  in pqi_wait_until_queued_io_drained()
5766 				&queue_group->submit_lock[path], flags);  in pqi_wait_until_queued_io_drained()
5784 	struct pqi_queue_group *queue_group;  in pqi_wait_until_inbound_queues_empty()  local
5789 		queue_group = &ctrl_info->queue_groups[i];  in pqi_wait_until_inbound_queues_empty()
5791 		rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);  in pqi_wait_until_inbound_queues_empty()
5796 			iq_pi = queue_group->iq_pi_copy[path];  in pqi_wait_until_inbound_queues_empty()
5799 				iq_ci = readl(queue_group->iq_ci[path]);  in pqi_wait_until_inbound_queues_empty()
5818 	struct pqi_queue_group *queue_group;  in pqi_fail_io_queued_for_device()  local
5826 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
5830 				&queue_group->submit_lock[path], flags);  in pqi_fail_io_queued_for_device()
5833 				&queue_group->request_list[path],  in pqi_fail_io_queued_for_device()
5852 				&queue_group->submit_lock[path], flags);  in pqi_fail_io_queued_for_device()