75 	struct pqi_queue_group *queue_group, enum pqi_io_path path,
82 	unsigned int cdb_length, struct pqi_queue_group *queue_group,
85 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
89 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,
2952 	struct pqi_queue_group *queue_group)  in pqi_raid_bypass_submit_scsi_cmd()  argument
3033 			return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3037 			return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3043 		rmd.cdb, rmd.cdb_length, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3382 static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)  in pqi_process_io_intr()  argument
3392 	oq_ci = queue_group->oq_ci_copy;  in pqi_process_io_intr()
3395 		oq_pi = readl(queue_group->oq_pi);  in pqi_process_io_intr()
3407 		response = queue_group->oq_element_array +  in pqi_process_io_intr()
3474 		queue_group->oq_ci_copy = oq_ci;  in pqi_process_io_intr()
3475 		writel(oq_ci, queue_group->oq_ci);  in pqi_process_io_intr()
3501 	struct pqi_queue_group *queue_group;  in pqi_send_event_ack()  local
3503 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
3504 	put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);  in pqi_send_event_ack()
3507 		spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3509 		iq_pi = queue_group->iq_pi_copy[RAID_PATH];  in pqi_send_event_ack()
3510 		iq_ci = readl(queue_group->iq_ci[RAID_PATH]);  in pqi_send_event_ack()
3517 			&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3523 	next_element = queue_group->iq_element_array[RAID_PATH] +  in pqi_send_event_ack()
3529 	queue_group->iq_pi_copy[RAID_PATH] = iq_pi;  in pqi_send_event_ack()
3535 	writel(iq_pi, queue_group->iq_pi[RAID_PATH]);  in pqi_send_event_ack()
3537 	spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);  in pqi_send_event_ack()
3984 	struct pqi_queue_group *queue_group;  in pqi_irq_handler()  local
3988 	queue_group = data;  in pqi_irq_handler()
3989 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3994 	num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
4009 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
4010 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
4093 	struct pqi_queue_group *queue_group;  in pqi_alloc_operational_queues()  local
4149 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4150 		queue_group->iq_element_array[RAID_PATH] = element_array;  in pqi_alloc_operational_queues()
4151 		queue_group->iq_element_array_bus_addr[RAID_PATH] =  in pqi_alloc_operational_queues()
4157 		queue_group->iq_element_array[AIO_PATH] = element_array;  in pqi_alloc_operational_queues()
4158 		queue_group->iq_element_array_bus_addr[AIO_PATH] =  in pqi_alloc_operational_queues()
4167 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4168 		queue_group->oq_element_array = element_array;  in pqi_alloc_operational_queues()
4169 		queue_group->oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
4188 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4189 		queue_group->iq_ci[RAID_PATH] = next_queue_index;  in pqi_alloc_operational_queues()
4190 		queue_group->iq_ci_bus_addr[RAID_PATH] =  in pqi_alloc_operational_queues()
4197 		queue_group->iq_ci[AIO_PATH] = next_queue_index;  in pqi_alloc_operational_queues()
4198 		queue_group->iq_ci_bus_addr[AIO_PATH] =  in pqi_alloc_operational_queues()
4205 		queue_group->oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
4206 		queue_group->oq_pi_bus_addr =  in pqi_alloc_operational_queues()
4437 	struct pqi_queue_group *queue_group, enum pqi_io_path path,  in pqi_start_io()  argument
4451 	spin_lock_irqsave(&queue_group->submit_lock[path], flags);  in pqi_start_io()
4454 		io_request->queue_group = queue_group;  in pqi_start_io()
4456 			&queue_group->request_list[path]);  in pqi_start_io()
4459 	iq_pi = queue_group->iq_pi_copy[path];  in pqi_start_io()
4462 		&queue_group->request_list[path], request_list_entry) {  in pqi_start_io()
4472 		iq_ci = readl(queue_group->iq_ci[path]);  in pqi_start_io()
4478 		put_unaligned_le16(queue_group->oq_id,  in pqi_start_io()
4481 		next_element = queue_group->iq_element_array[path] +  in pqi_start_io()
4493 			memcpy(queue_group->iq_element_array[path],  in pqi_start_io()
4504 	if (iq_pi != queue_group->iq_pi_copy[path]) {  in pqi_start_io()
4505 		queue_group->iq_pi_copy[path] = iq_pi;  in pqi_start_io()
4510 		writel(iq_pi, queue_group->iq_pi[path]);  in pqi_start_io()
4513 	spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);  in pqi_start_io()
4848 	struct pqi_queue_group *queue_group;  in pqi_create_queue_group()  local
4852 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
4863 	put_unaligned_le16(queue_group->iq_id[RAID_PATH],  in pqi_create_queue_group()
4866 		(u64)queue_group->iq_element_array_bus_addr[RAID_PATH],  in pqi_create_queue_group()
4868 	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],  in pqi_create_queue_group()
4884 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4898 	put_unaligned_le16(queue_group->iq_id[AIO_PATH],  in pqi_create_queue_group()
4900 	put_unaligned_le64((u64)queue_group->  in pqi_create_queue_group()
4903 	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],  in pqi_create_queue_group()
4919 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4934 	put_unaligned_le16(queue_group->iq_id[AIO_PATH],  in pqi_create_queue_group()
4955 	put_unaligned_le16(queue_group->oq_id,  in pqi_create_queue_group()
4957 	put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,  in pqi_create_queue_group()
4959 	put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,  in pqi_create_queue_group()
4966 	put_unaligned_le16(queue_group->int_msg_num,  in pqi_create_queue_group()
4977 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
5502 	struct pqi_queue_group *queue_group, bool io_high_prio)  in pqi_raid_submit_io()  argument
5579 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_io()
5586 	struct pqi_queue_group *queue_group)  in pqi_raid_submit_scsi_cmd()  argument
5592 	return pqi_raid_submit_io(ctrl_info, device, scmd, queue_group, io_high_prio);  in pqi_raid_submit_scsi_cmd()
5639 	struct pqi_queue_group *queue_group)  in pqi_aio_submit_scsi_cmd()  argument
5646 		scmd->cmnd, scmd->cmd_len, queue_group, NULL,  in pqi_aio_submit_scsi_cmd()
5652 	unsigned int cdb_length, struct pqi_queue_group *queue_group,  in pqi_aio_submit_io()  argument
5721 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
5727 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,  in pqi_aio_submit_r1_write_io()  argument
5782 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r1_write_io()
5788 	struct scsi_cmnd *scmd, struct pqi_queue_group *queue_group,  in pqi_aio_submit_r56_write_io()  argument
5849 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r56_write_io()
5980 	struct pqi_queue_group *queue_group;  in pqi_scsi_queue_command()  local
6018 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
6025 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6032 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6035 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6037 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6055 	struct pqi_queue_group *queue_group;  in pqi_queued_io_count()  local
6061 		queue_group = &ctrl_info->queue_groups[i];  in pqi_queued_io_count()
6063 			spin_lock_irqsave(&queue_group->submit_lock[path], flags);  in pqi_queued_io_count()
6064 			list_for_each_entry(io_request, &queue_group->request_list[path], request_list_entry)  in pqi_queued_io_count()
6066 			spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);  in pqi_queued_io_count()
6078 	struct pqi_queue_group *queue_group;  in pqi_nonempty_inbound_queue_count()  local
6085 		queue_group = &ctrl_info->queue_groups[i];  in pqi_nonempty_inbound_queue_count()
6087 			iq_pi = queue_group->iq_pi_copy[path];  in pqi_nonempty_inbound_queue_count()
6088 			iq_ci = readl(queue_group->iq_ci[path]);  in pqi_nonempty_inbound_queue_count()
6142 	struct pqi_queue_group *queue_group;  in pqi_fail_io_queued_for_device()  local
6150 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
6154 				&queue_group->submit_lock[path], flags);  in pqi_fail_io_queued_for_device()
6157 				&queue_group->request_list[path],  in pqi_fail_io_queued_for_device()
6179 				&queue_group->submit_lock[path], flags);  in pqi_fail_io_queued_for_device()