Lines Matching +full:fis +full:- +full:index +full:- +full:block
1 // SPDX-License-Identifier: GPL-2.0-or-later
24 #include <linux/blk-mq.h>
26 #include <linux/dma-mapping.h>
38 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
103 * Global variable used to hold the major block device number
144 if (dd->sr) in mtip_check_surprise_removal()
150 dd->sr = true; in mtip_check_surprise_removal()
151 if (dd->queue) in mtip_check_surprise_removal()
152 blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue); in mtip_check_surprise_removal()
154 dev_warn(&dd->pdev->dev, in mtip_check_surprise_removal()
155 "%s: dd->queue is NULL\n", __func__); in mtip_check_surprise_removal()
165 struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0]; in mtip_cmd_from_tag()
167 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag)); in mtip_cmd_from_tag()
177 * -1 The HBA Reset bit did not clear.
184 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
187 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
196 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
197 return -1; in mtip_hba_reset()
199 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
202 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
203 return -1; in mtip_hba_reset()
225 spin_lock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
227 port->s_active[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
229 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
230 spin_unlock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
234 * Enable/disable the reception of FIS
246 /* enable FIS reception */ in mtip_enable_fis()
247 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_fis()
249 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
251 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
254 readl(port->mmio + PORT_CMD); in mtip_enable_fis()
272 /* enable FIS reception */ in mtip_enable_engine()
273 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_engine()
275 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
277 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
279 readl(port->mmio + PORT_CMD); in mtip_enable_engine()
284 * Enables the port DMA engine and FIS reception.
291 /* Enable FIS reception */ in mtip_start_port()
300 * and FIS reception.
310 writel(0, port->mmio + PORT_IRQ_MASK); in mtip_deinit_port()
315 /* Disable FIS reception */ in mtip_deinit_port()
323 * then initializes it by setting the command header and RX FIS addresses,
325 * re-enabling the default set of port interrupts.
337 /* Program the command list base and FIS base addresses */ in mtip_init_port()
338 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
339 writel((port->command_list_dma >> 16) >> 16, in mtip_init_port()
340 port->mmio + PORT_LST_ADDR_HI); in mtip_init_port()
341 writel((port->rxfis_dma >> 16) >> 16, in mtip_init_port()
342 port->mmio + PORT_FIS_ADDR_HI); in mtip_init_port()
343 set_bit(MTIP_PF_HOST_CAP_64, &port->flags); in mtip_init_port()
346 writel(port->command_list_dma & 0xFFFFFFFF, in mtip_init_port()
347 port->mmio + PORT_LST_ADDR); in mtip_init_port()
348 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); in mtip_init_port()
351 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); in mtip_init_port()
354 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
355 writel(0xFFFFFFFF, port->completed[i]); in mtip_init_port()
358 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); in mtip_init_port()
361 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
362 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
365 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); in mtip_init_port()
385 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) in mtip_restart_port()
389 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
396 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { in mtip_restart_port()
397 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
400 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
401 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
408 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
411 writel(readl(port->mmio + PORT_SCR_CTL) | in mtip_restart_port()
412 1, port->mmio + PORT_SCR_CTL); in mtip_restart_port()
413 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
420 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
424 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, in mtip_restart_port()
425 port->mmio + PORT_SCR_CTL); in mtip_restart_port()
426 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
430 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
434 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
437 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
438 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
450 if (mtip_check_surprise_removal(dd->pdev)) in mtip_device_reset()
454 rv = -EFAULT; in mtip_device_reset()
457 mtip_init_port(dd->port); in mtip_device_reset()
458 mtip_start_port(dd->port); in mtip_device_reset()
461 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
462 dd->mmio + HOST_CTL); in mtip_device_reset()
478 for (group = SLOTBITS_IN_LONGS; group > 0; group--) in print_tags()
480 tagbits[group-1]); in print_tags()
481 dev_warn(&dd->pdev->dev, in print_tags()
494 cmd->status = status; in mtip_complete_command()
495 if (likely(!blk_should_fake_timeout(req->q))) in mtip_complete_command()
513 struct host_to_dev_fis *fis; in mtip_handle_tfe() local
520 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
522 port = dd->port; in mtip_handle_tfe()
524 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { in mtip_handle_tfe()
535 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
536 completed = readl(port->completed[group]); in mtip_handle_tfe()
538 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); in mtip_handle_tfe()
541 writel(completed, port->completed[group]); in mtip_handle_tfe()
567 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
568 dd->port->log_buf, in mtip_handle_tfe()
569 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
571 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
573 /* non-critical error, don't fail the load */ in mtip_handle_tfe()
575 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
577 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
579 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
584 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
586 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
591 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_handle_tfe()
592 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
603 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
609 fis = (struct host_to_dev_fis *)cmd->command; in mtip_handle_tfe()
611 /* Should re-issue? */ in mtip_handle_tfe()
613 fis->command == ATA_CMD_SET_FEATURES) in mtip_handle_tfe()
618 fis->command == ATA_CMD_FPDMA_WRITE)) { in mtip_handle_tfe()
619 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
621 fis->command == ATA_CMD_FPDMA_WRITE ? in mtip_handle_tfe()
635 if (reissue && (cmd->retries-- > 0)) { in mtip_handle_tfe()
639 /* Re-issue the command. */ in mtip_handle_tfe()
646 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
661 struct driver_data *dd = port->dd; in mtip_workq_sdbfx()
670 writel(completed, port->completed[group]); in mtip_workq_sdbfx()
687 /* If last, re-enable interrupts */ in mtip_workq_sdbfx()
688 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
689 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
697 struct mtip_port *port = dd->port; in mtip_process_legacy()
700 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) { in mtip_process_legacy()
702 int status = readl(port->cmd_issue[group]); in mtip_process_legacy()
715 dev_warn(&dd->pdev->dev, in mtip_process_errors()
717 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
721 dev_warn(&dd->pdev->dev, in mtip_process_errors()
723 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
727 dev_warn(&dd->pdev->dev, in mtip_process_errors()
730 if (mtip_check_surprise_removal(dd->pdev)) in mtip_process_errors()
734 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
735 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
742 struct mtip_port *port = dd->port; in mtip_handle_irq()
748 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
753 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
755 mtip_check_surprise_removal(dd->pdev); in mtip_handle_irq()
758 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
763 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
768 twork = &dd->work[i]; in mtip_handle_irq()
769 twork->completed = readl(port->completed[i]); in mtip_handle_irq()
770 if (twork->completed) in mtip_handle_irq()
774 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
777 twork = &dd->work[i]; in mtip_handle_irq()
778 if (twork->completed) in mtip_handle_irq()
780 twork->cpu_binding, in mtip_handle_irq()
781 dd->isr_workq, in mtip_handle_irq()
782 &twork->work); in mtip_handle_irq()
785 if (likely(dd->work[0].completed)) in mtip_handle_irq()
787 dd->work[0].completed); in mtip_handle_irq()
799 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_handle_irq()
804 &dd->dd_flag)) in mtip_handle_irq()
816 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
840 writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_non_ncq_command()
844 struct host_to_dev_fis *fis) in mtip_pause_ncq() argument
848 task_file_data = readl(port->mmio+PORT_TFDATA); in mtip_pause_ncq()
852 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { in mtip_pause_ncq()
853 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
855 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && in mtip_pause_ncq()
856 (fis->features == 0x03)) { in mtip_pause_ncq()
857 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
858 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
860 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || in mtip_pause_ncq()
861 ((fis->command == 0xFC) && in mtip_pause_ncq()
862 (fis->features == 0x27 || fis->features == 0x72 || in mtip_pause_ncq()
863 fis->features == 0x62 || fis->features == 0x26))) { in mtip_pause_ncq()
864 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
865 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
868 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
884 active = readl(port->s_active[0]) & 0xFFFFFFFE; in mtip_commands_active()
885 for (n = 1; n < port->dd->slot_groups; n++) in mtip_commands_active()
886 active |= readl(port->s_active[n]); in mtip_commands_active()
899 * -EBUSY Commands still active
906 blk_mq_quiesce_queue(port->dd->queue); in mtip_quiesce_io()
910 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && in mtip_quiesce_io()
911 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_quiesce_io()
918 if (mtip_check_surprise_removal(port->dd->pdev)) in mtip_quiesce_io()
926 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
927 return active ? -EBUSY : 0; in mtip_quiesce_io()
929 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
930 return -EFAULT; in mtip_quiesce_io()
944 * @fis Pointer to the FIS that describes the command.
945 * @fis_len Length in WORDS of the FIS.
948 * @opts Command header options, excluding the FIS length
954 * -EFAULT The buffer address is not correctly aligned.
955 * -EBUSY Internal command or other IO in progress.
956 * -EAGAIN Time out waiting for command to complete.
959 struct host_to_dev_fis *fis, in mtip_exec_internal_command() argument
967 struct driver_data *dd = port->dd; in mtip_exec_internal_command()
979 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
980 return -EFAULT; in mtip_exec_internal_command()
983 if (mtip_check_surprise_removal(dd->pdev)) in mtip_exec_internal_command()
984 return -EFAULT; in mtip_exec_internal_command()
986 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); in mtip_exec_internal_command()
989 return -EFAULT; in mtip_exec_internal_command()
992 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
994 if (fis->command == ATA_CMD_SEC_ERASE_PREP) in mtip_exec_internal_command()
995 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
997 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
999 if (fis->command != ATA_CMD_STANDBYNOW1) { in mtip_exec_internal_command()
1002 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); in mtip_exec_internal_command()
1004 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1005 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1006 return -EBUSY; in mtip_exec_internal_command()
1012 int_cmd->icmd = &icmd; in mtip_exec_internal_command()
1013 memcpy(int_cmd->command, fis, fis_len*4); in mtip_exec_internal_command()
1015 rq->timeout = timeout; in mtip_exec_internal_command()
1020 if (int_cmd->status) { in mtip_exec_internal_command()
1021 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", in mtip_exec_internal_command()
1022 fis->command, int_cmd->status); in mtip_exec_internal_command()
1023 rv = -EIO; in mtip_exec_internal_command()
1025 if (mtip_check_surprise_removal(dd->pdev) || in mtip_exec_internal_command()
1027 &dd->dd_flag)) { in mtip_exec_internal_command()
1028 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1030 fis->command); in mtip_exec_internal_command()
1031 rv = -ENXIO; in mtip_exec_internal_command()
1035 rv = -EAGAIN; in mtip_exec_internal_command()
1039 if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)]) in mtip_exec_internal_command()
1041 rv = -ENXIO; in mtip_exec_internal_command()
1042 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1044 rv = -EAGAIN; in mtip_exec_internal_command()
1050 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1051 if (rv >= 0 && mtip_pause_ncq(port, fis)) { in mtip_exec_internal_command()
1055 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1061 * Byte-swap ATA ID strings.
1063 * ATA identify data contains strings in byte-swapped 16-bit words.
1065 * This function swaps bytes in-place.
1081 struct host_to_dev_fis *fis, in mtip_set_timeout() argument
1084 switch (fis->command) { in mtip_set_timeout()
1091 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1093 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1116 * into the identify data buffer (@e port->identify) in the
1119 * port->identify_valid is non-zero.
1127 * -EFAULT An error occurred while coping data to the user buffer.
1128 * -1 Command failed.
1133 struct host_to_dev_fis fis; in mtip_get_identify() local
1135 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1136 return -EFAULT; in mtip_get_identify()
1138 /* Build the FIS. */ in mtip_get_identify()
1139 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_get_identify()
1140 fis.type = 0x27; in mtip_get_identify()
1141 fis.opts = 1 << 7; in mtip_get_identify()
1142 fis.command = ATA_CMD_ID_ATA; in mtip_get_identify()
1145 port->identify_valid = 0; in mtip_get_identify()
1148 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); in mtip_get_identify()
1152 &fis, in mtip_get_identify()
1154 port->identify_dma, in mtip_get_identify()
1159 rv = -1; in mtip_get_identify()
1164 * Perform any necessary byte-swapping. Yes, the kernel does in fact in mtip_get_identify()
1165 * perform field-sensitive swapping on the string fields. in mtip_get_identify()
1169 ata_swap_string(port->identify + 27, 40); /* model string*/ in mtip_get_identify()
1170 ata_swap_string(port->identify + 23, 8); /* firmware string*/ in mtip_get_identify()
1171 ata_swap_string(port->identify + 10, 20); /* serial# string*/ in mtip_get_identify()
1176 port->identify[i] = le16_to_cpu(port->identify[i]); in mtip_get_identify()
1181 if (port->identify[128] & 0x4) in mtip_get_identify()
1182 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1184 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1187 port->identify_valid = 1; in mtip_get_identify()
1192 port->identify, in mtip_get_identify()
1194 rv = -EFAULT; in mtip_get_identify()
1210 * -1 An error occurred while executing the command.
1215 struct host_to_dev_fis fis; in mtip_standby_immediate() local
1219 /* Build the FIS. */ in mtip_standby_immediate()
1220 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_standby_immediate()
1221 fis.type = 0x27; in mtip_standby_immediate()
1222 fis.opts = 1 << 7; in mtip_standby_immediate()
1223 fis.command = ATA_CMD_STANDBYNOW1; in mtip_standby_immediate()
1225 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1229 &fis, in mtip_standby_immediate()
1236 jiffies_to_msecs(jiffies - start)); in mtip_standby_immediate()
1238 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1259 struct host_to_dev_fis fis; in mtip_read_log_page() local
1261 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_read_log_page()
1262 fis.type = 0x27; in mtip_read_log_page()
1263 fis.opts = 1 << 7; in mtip_read_log_page()
1264 fis.command = ATA_CMD_READ_LOG_EXT; in mtip_read_log_page()
1265 fis.sect_count = sectors & 0xFF; in mtip_read_log_page()
1266 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; in mtip_read_log_page()
1267 fis.lba_low = page; in mtip_read_log_page()
1268 fis.lba_mid = 0; in mtip_read_log_page()
1269 fis.device = ATA_DEVICE_OBS; in mtip_read_log_page()
1274 &fis, in mtip_read_log_page()
1295 struct host_to_dev_fis fis; in mtip_get_smart_data() local
1297 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_get_smart_data()
1298 fis.type = 0x27; in mtip_get_smart_data()
1299 fis.opts = 1 << 7; in mtip_get_smart_data()
1300 fis.command = ATA_CMD_SMART; in mtip_get_smart_data()
1301 fis.features = 0xD0; in mtip_get_smart_data()
1302 fis.sect_count = 1; in mtip_get_smart_data()
1303 fis.lba_mid = 0x4F; in mtip_get_smart_data()
1304 fis.lba_hi = 0xC2; in mtip_get_smart_data()
1305 fis.device = ATA_DEVICE_OBS; in mtip_get_smart_data()
1308 &fis, in mtip_get_smart_data()
1324 * -EINVAL NULL buffer passed or unsupported attribute @id.
1325 * -EPERM Identify data not valid, SMART not supported or not enabled
1334 return -EINVAL; in mtip_get_smart_attr()
1336 if (!port->identify_valid) { in mtip_get_smart_attr()
1337 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1338 return -EPERM; in mtip_get_smart_attr()
1340 if (!(port->identify[82] & 0x1)) { in mtip_get_smart_attr()
1341 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1342 return -EPERM; in mtip_get_smart_attr()
1344 if (!(port->identify[85] & 0x1)) { in mtip_get_smart_attr()
1345 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1346 return -EPERM; in mtip_get_smart_attr()
1349 memset(port->smart_buf, 0, ATA_SECT_SIZE); in mtip_get_smart_attr()
1350 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); in mtip_get_smart_attr()
1352 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1356 pattr = (struct smart_attr *)(port->smart_buf + 2); in mtip_get_smart_attr()
1358 if (pattr->attr_id == id) { in mtip_get_smart_attr()
1364 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1366 rv = -EINVAL; in mtip_get_smart_attr()
1384 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1386 raw0 = port->identify[100]; in mtip_hw_get_capacity()
1387 raw1 = port->identify[101]; in mtip_hw_get_capacity()
1388 raw2 = port->identify[102]; in mtip_hw_get_capacity()
1389 raw3 = port->identify[103]; in mtip_hw_get_capacity()
1392 return (bool) !!port->identify_valid; in mtip_hw_get_capacity()
1409 if (!port->identify_valid) in mtip_dump_identify()
1412 strlcpy(cbuf, (char *)(port->identify+10), 21); in mtip_dump_identify()
1413 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1416 strlcpy(cbuf, (char *)(port->identify+23), 9); in mtip_dump_identify()
1417 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1420 strlcpy(cbuf, (char *)(port->identify+27), 41); in mtip_dump_identify()
1421 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1423 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1424 port->identify[128], in mtip_dump_identify()
1425 port->identify[128] & 0x4 ? "(LOCKED)" : ""); in mtip_dump_identify()
1427 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1428 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1433 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1445 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1467 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; in fill_command_sg()
1469 for_each_sg(command->sg, sg, nents, n) { in fill_command_sg()
1472 dev_err(&dd->pdev->dev, in fill_command_sg()
1474 command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF); in fill_command_sg()
1475 command_sg->dba = cpu_to_le32(sg_dma_address(sg)); in fill_command_sg()
1476 command_sg->dba_upper = in fill_command_sg()
1486 * return value -1 An error occurred while executing the command.
1490 struct host_to_dev_fis fis; in exec_drive_task() local
1491 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_task()
1494 /* Build the FIS. */ in exec_drive_task()
1495 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_task()
1496 fis.type = 0x27; in exec_drive_task()
1497 fis.opts = 1 << 7; in exec_drive_task()
1498 fis.command = command[0]; in exec_drive_task()
1499 fis.features = command[1]; in exec_drive_task()
1500 fis.sect_count = command[2]; in exec_drive_task()
1501 fis.sector = command[3]; in exec_drive_task()
1502 fis.cyl_low = command[4]; in exec_drive_task()
1503 fis.cyl_hi = command[5]; in exec_drive_task()
1504 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ in exec_drive_task()
1506 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1520 &fis, in exec_drive_task()
1526 return -1; in exec_drive_task()
1529 command[0] = reply->command; /* Status*/ in exec_drive_task()
1530 command[1] = reply->features; /* Error*/ in exec_drive_task()
1531 command[4] = reply->cyl_low; in exec_drive_task()
1532 command[5] = reply->cyl_hi; in exec_drive_task()
1553 * return value -EFAULT An error occurred while copying the completion
1555 * return value -1 An error occurred while executing the command.
1560 struct host_to_dev_fis fis; in exec_drive_command() local
1569 return -EFAULT; in exec_drive_command()
1571 buf = dma_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1576 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1579 return -ENOMEM; in exec_drive_command()
1583 /* Build the FIS. */ in exec_drive_command()
1584 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_command()
1585 fis.type = 0x27; in exec_drive_command()
1586 fis.opts = 1 << 7; in exec_drive_command()
1587 fis.command = command[0]; in exec_drive_command()
1588 fis.features = command[2]; in exec_drive_command()
1589 fis.sect_count = command[3]; in exec_drive_command()
1590 if (fis.command == ATA_CMD_SMART) { in exec_drive_command()
1591 fis.sector = command[1]; in exec_drive_command()
1592 fis.cyl_low = 0x4F; in exec_drive_command()
1593 fis.cyl_hi = 0xC2; in exec_drive_command()
1596 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1599 reply = (port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_command()
1601 reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_command()
1614 &fis, in exec_drive_command()
1621 rv = -EFAULT; in exec_drive_command()
1626 command[0] = reply->command; /* Status*/ in exec_drive_command()
1627 command[1] = reply->features; /* Error*/ in exec_drive_command()
1628 command[2] = reply->sect_count; in exec_drive_command()
1642 rv = -EFAULT; in exec_drive_command()
1648 dma_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
1709 struct host_to_dev_fis fis; in exec_drive_taskfile() local
1724 int intotal = outtotal + req_task->out_size; in exec_drive_taskfile()
1727 taskout = req_task->out_size; in exec_drive_taskfile()
1728 taskin = req_task->in_size; in exec_drive_taskfile()
1731 return -EINVAL; in exec_drive_taskfile()
1738 outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf, in exec_drive_taskfile()
1740 if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) { in exec_drive_taskfile()
1741 err = -ENOMEM; in exec_drive_taskfile()
1754 inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf, in exec_drive_taskfile()
1756 if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) { in exec_drive_taskfile()
1757 err = -ENOMEM; in exec_drive_taskfile()
1763 /* only supports PIO and non-data commands from this ioctl. */ in exec_drive_taskfile()
1764 switch (req_task->data_phase) { in exec_drive_taskfile()
1767 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1770 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1773 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
1776 err = -EINVAL; in exec_drive_taskfile()
1780 /* Build the FIS. */ in exec_drive_taskfile()
1781 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_taskfile()
1783 fis.type = 0x27; in exec_drive_taskfile()
1784 fis.opts = 1 << 7; in exec_drive_taskfile()
1785 fis.command = req_task->io_ports[7]; in exec_drive_taskfile()
1786 fis.features = req_task->io_ports[1]; in exec_drive_taskfile()
1787 fis.sect_count = req_task->io_ports[2]; in exec_drive_taskfile()
1788 fis.lba_low = req_task->io_ports[3]; in exec_drive_taskfile()
1789 fis.lba_mid = req_task->io_ports[4]; in exec_drive_taskfile()
1790 fis.lba_hi = req_task->io_ports[5]; in exec_drive_taskfile()
1792 fis.device = req_task->io_ports[6] & ~0x10; in exec_drive_taskfile()
1794 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { in exec_drive_taskfile()
1795 req_task->in_flags.all = in exec_drive_taskfile()
1798 fis.lba_low_ex = req_task->hob_ports[3]; in exec_drive_taskfile()
1799 fis.lba_mid_ex = req_task->hob_ports[4]; in exec_drive_taskfile()
1800 fis.lba_hi_ex = req_task->hob_ports[5]; in exec_drive_taskfile()
1801 fis.features_ex = req_task->hob_ports[1]; in exec_drive_taskfile()
1802 fis.sect_cnt_ex = req_task->hob_ports[2]; in exec_drive_taskfile()
1805 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; in exec_drive_taskfile()
1808 force_single_sector = implicit_sector(fis.command, fis.features); in exec_drive_taskfile()
1810 if ((taskin || taskout) && (!fis.sect_count)) { in exec_drive_taskfile()
1812 fis.sect_count = nsect; in exec_drive_taskfile()
1815 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
1818 err = -EINVAL; in exec_drive_taskfile()
1829 fis.command, in exec_drive_taskfile()
1830 fis.features, in exec_drive_taskfile()
1831 fis.sect_count, in exec_drive_taskfile()
1832 fis.lba_low, in exec_drive_taskfile()
1833 fis.lba_mid, in exec_drive_taskfile()
1834 fis.lba_hi, in exec_drive_taskfile()
1835 fis.device); in exec_drive_taskfile()
1838 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && in exec_drive_taskfile()
1843 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
1849 transfer_size = ATA_SECT_SIZE * fis.sect_count; in exec_drive_taskfile()
1852 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
1853 &fis, in exec_drive_taskfile()
1859 err = -EIO; in exec_drive_taskfile()
1863 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
1865 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { in exec_drive_taskfile()
1866 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
1867 req_task->io_ports[7] = reply->control; in exec_drive_taskfile()
1869 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
1870 req_task->io_ports[7] = reply->command; in exec_drive_taskfile()
1875 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1878 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1884 req_task->io_ports[1] = reply->features; in exec_drive_taskfile()
1885 req_task->io_ports[2] = reply->sect_count; in exec_drive_taskfile()
1886 req_task->io_ports[3] = reply->lba_low; in exec_drive_taskfile()
1887 req_task->io_ports[4] = reply->lba_mid; in exec_drive_taskfile()
1888 req_task->io_ports[5] = reply->lba_hi; in exec_drive_taskfile()
1889 req_task->io_ports[6] = reply->device; in exec_drive_taskfile()
1891 if (req_task->out_flags.all & 1) { in exec_drive_taskfile()
1893 req_task->hob_ports[3] = reply->lba_low_ex; in exec_drive_taskfile()
1894 req_task->hob_ports[4] = reply->lba_mid_ex; in exec_drive_taskfile()
1895 req_task->hob_ports[5] = reply->lba_hi_ex; in exec_drive_taskfile()
1896 req_task->hob_ports[1] = reply->features_ex; in exec_drive_taskfile()
1897 req_task->hob_ports[2] = reply->sect_cnt_ex; in exec_drive_taskfile()
1904 req_task->io_ports[7], in exec_drive_taskfile()
1905 req_task->io_ports[1], in exec_drive_taskfile()
1906 req_task->io_ports[2], in exec_drive_taskfile()
1907 req_task->io_ports[3], in exec_drive_taskfile()
1908 req_task->io_ports[4], in exec_drive_taskfile()
1909 req_task->io_ports[5], in exec_drive_taskfile()
1910 req_task->io_ports[6]); in exec_drive_taskfile()
1914 err = -EFAULT; in exec_drive_taskfile()
1920 err = -EFAULT; in exec_drive_taskfile()
1926 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1929 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1938 * Handle IOCTL calls from the Block Layer.
1940 * This function is called by the Block Layer when it receives an IOCTL
1942 * this function returns -ENOTTY.
1945 * @cmd IOCTL command passed from the Block Layer.
1946 * @arg IOCTL argument passed from the Block Layer.
1950 * -ENOTTY The specified command is not supported.
1951 * -EFAULT An error occurred copying data to a user space buffer.
1952 * -EIO An error occurred while executing the command.
1960 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
1962 return -EFAULT; in mtip_hw_ioctl()
1973 return -EFAULT; in mtip_hw_ioctl()
1976 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
1979 return -EIO; in mtip_hw_ioctl()
1985 return -EFAULT; in mtip_hw_ioctl()
1997 return -EFAULT; in mtip_hw_ioctl()
2000 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2001 return -EIO; in mtip_hw_ioctl()
2007 return -EFAULT; in mtip_hw_ioctl()
2017 return -EFAULT; in mtip_hw_ioctl()
2026 return -EFAULT; in mtip_hw_ioctl()
2032 return -EINVAL; in mtip_hw_ioctl()
2040 * This function is called by the block layer to issue an io
2062 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_hw_submit_io()
2063 struct host_to_dev_fis *fis; in mtip_hw_submit_io() local
2064 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2071 nents = blk_rq_map_sg(hctx->queue, rq, command->sg); in mtip_hw_submit_io()
2072 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2074 prefetch(&port->flags); in mtip_hw_submit_io()
2076 command->scatter_ents = nents; in mtip_hw_submit_io()
2082 command->retries = MTIP_MAX_RETRIES; in mtip_hw_submit_io()
2084 /* Fill out fis */ in mtip_hw_submit_io()
2085 fis = command->command; in mtip_hw_submit_io()
2086 fis->type = 0x27; in mtip_hw_submit_io()
2087 fis->opts = 1 << 7; in mtip_hw_submit_io()
2089 fis->command = ATA_CMD_FPDMA_READ; in mtip_hw_submit_io()
2091 fis->command = ATA_CMD_FPDMA_WRITE; in mtip_hw_submit_io()
2092 fis->lba_low = start & 0xFF; in mtip_hw_submit_io()
2093 fis->lba_mid = (start >> 8) & 0xFF; in mtip_hw_submit_io()
2094 fis->lba_hi = (start >> 16) & 0xFF; in mtip_hw_submit_io()
2095 fis->lba_low_ex = (start >> 24) & 0xFF; in mtip_hw_submit_io()
2096 fis->lba_mid_ex = (start >> 32) & 0xFF; in mtip_hw_submit_io()
2097 fis->lba_hi_ex = (start >> 40) & 0xFF; in mtip_hw_submit_io()
2098 fis->device = 1 << 6; in mtip_hw_submit_io()
2099 fis->features = nsect & 0xFF; in mtip_hw_submit_io()
2100 fis->features_ex = (nsect >> 8) & 0xFF; in mtip_hw_submit_io()
2101 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); in mtip_hw_submit_io()
2102 fis->sect_cnt_ex = 0; in mtip_hw_submit_io()
2103 fis->control = 0; in mtip_hw_submit_io()
2104 fis->res2 = 0; in mtip_hw_submit_io()
2105 fis->res3 = 0; in mtip_hw_submit_io()
2108 if (unlikely(command->unaligned)) in mtip_hw_submit_io()
2109 fis->device |= 1 << 7; in mtip_hw_submit_io()
2112 hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF); in mtip_hw_submit_io()
2113 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_hw_submit_io()
2114 hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16); in mtip_hw_submit_io()
2115 hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH); in mtip_hw_submit_io()
2116 hdr->byte_count = 0; in mtip_hw_submit_io()
2118 command->direction = dma_dir; in mtip_hw_submit_io()
2124 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { in mtip_hw_submit_io()
2125 set_bit(rq->tag, port->cmds_to_issue); in mtip_hw_submit_io()
2126 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_hw_submit_io()
2131 mtip_issue_ncq_command(port, rq->tag); in mtip_hw_submit_io()
2148 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status()
2151 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2153 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2190 if (dd->pdev) { in show_device_status()
2191 if (dd->port && in show_device_status()
2192 dd->port->identify && in show_device_status()
2193 dd->port->identify_valid) { in show_device_status()
2195 (char *) (dd->port->identify + 10), 21); in show_device_status()
2196 status = *(dd->port->identify + 141); in show_device_status()
2202 if (dd->port && in show_device_status()
2203 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2206 dev_name(&dd->pdev->dev), in show_device_status()
2212 dev_name(&dd->pdev->dev), in show_device_status()
2220 if (dd->pdev) { in show_device_status()
2221 if (dd->port && in show_device_status()
2222 dd->port->identify && in show_device_status()
2223 dd->port->identify_valid) { in show_device_status()
2225 (char *) (dd->port->identify+10), 21); in show_device_status()
2226 status = *(dd->port->identify + 141); in show_device_status()
2232 if (dd->port && in show_device_status()
2233 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { in show_device_status()
2236 dev_name(&dd->pdev->dev), in show_device_status()
2242 dev_name(&dd->pdev->dev), in show_device_status()
2264 return -ENOMEM; in mtip_hw_read_device_status()
2271 rv = -EFAULT; in mtip_hw_read_device_status()
2280 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers()
2291 return -ENOMEM; in mtip_hw_read_registers()
2295 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2297 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2302 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2304 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2309 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2311 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2315 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2317 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2322 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2325 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2327 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2335 rv = -EFAULT; in mtip_hw_read_registers()
2344 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags()
2354 return -ENOMEM; in mtip_hw_read_flags()
2356 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", in mtip_hw_read_flags()
2357 dd->port->flags); in mtip_hw_read_flags()
2358 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", in mtip_hw_read_flags()
2359 dd->dd_flag); in mtip_hw_read_flags()
2364 rv = -EFAULT; in mtip_hw_read_flags()
2394 return -1; in mtip_hw_debugfs_init()
2396 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2397 if (IS_ERR_OR_NULL(dd->dfs_node)) { in mtip_hw_debugfs_init()
2398 dev_warn(&dd->pdev->dev, in mtip_hw_debugfs_init()
2400 dd->disk->disk_name); in mtip_hw_debugfs_init()
2401 dd->dfs_node = NULL; in mtip_hw_debugfs_init()
2402 return -1; in mtip_hw_debugfs_init()
2405 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); in mtip_hw_debugfs_init()
2406 debugfs_create_file("registers", 0444, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2414 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
2428 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
2434 dd->mmio + HOST_HSORG); in hba_setup()
2439 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
2458 * HBA base + 0xFC [15:0] - vendor-specific hardware interface in mtip_detect_product()
2461 * [ 3] asic-style interface in mtip_detect_product()
2462 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). in mtip_detect_product()
2464 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
2466 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
2467 dd->slot_groups = 1; in mtip_detect_product()
2470 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
2473 dev_info(&dd->pdev->dev, in mtip_detect_product()
2474 "ASIC-FPGA design, HS rev 0x%x, " in mtip_detect_product()
2481 dev_warn(&dd->pdev->dev, in mtip_detect_product()
2486 dd->slot_groups = slotgroups; in mtip_detect_product()
2490 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
2500 * -EFAULT FTL rebuild error/timeout/interruption
2506 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2514 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
2515 return -EFAULT; in mtip_ftl_rebuild_poll()
2516 if (mtip_check_surprise_removal(dd->pdev)) in mtip_ftl_rebuild_poll()
2517 return -EFAULT; in mtip_ftl_rebuild_poll()
2519 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2520 return -EFAULT; in mtip_ftl_rebuild_poll()
2522 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2527 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2529 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2533 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2535 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2542 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2544 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2545 return -EFAULT; in mtip_ftl_rebuild_poll()
2551 struct driver_data *dd = rq->q->queuedata; in mtip_softirq_done_fn()
2554 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, in mtip_softirq_done_fn()
2555 cmd->direction); in mtip_softirq_done_fn()
2557 if (unlikely(cmd->unaligned)) in mtip_softirq_done_fn()
2558 atomic_inc(&dd->port->cmd_slot_unal); in mtip_softirq_done_fn()
2560 blk_mq_end_request(rq, cmd->status); in mtip_softirq_done_fn()
2568 dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag); in mtip_abort_cmd()
2570 clear_bit(req->tag, dd->port->cmds_to_issue); in mtip_abort_cmd()
2571 cmd->status = BLK_STS_IOERR; in mtip_abort_cmd()
2580 set_bit(req->tag, dd->port->cmds_to_issue); in mtip_queue_cmd()
2598 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
2599 struct mtip_port *port = dd->port; in mtip_service_thread()
2603 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2605 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2611 wait_event_interruptible(port->svc_wait, (port->flags) && in mtip_service_thread()
2612 (port->flags & MTIP_PF_SVC_THD_WORK)); in mtip_service_thread()
2615 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2619 &dd->dd_flag))) in mtip_service_thread()
2622 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2626 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2628 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2631 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) in mtip_service_thread()
2634 if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2639 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_service_thread()
2642 if (atomic_read(&dd->irq_workers_active) != 0) in mtip_service_thread()
2643 dev_warn(&dd->pdev->dev, in mtip_service_thread()
2646 blk_mq_quiesce_queue(dd->queue); in mtip_service_thread()
2648 blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd); in mtip_service_thread()
2650 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); in mtip_service_thread()
2653 blk_mq_tagset_busy_iter(&dd->tags, in mtip_service_thread()
2656 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); in mtip_service_thread()
2658 blk_mq_unquiesce_queue(dd->queue); in mtip_service_thread()
2661 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_service_thread()
2667 slot = find_next_bit(port->cmds_to_issue, in mtip_service_thread()
2686 clear_bit(slot, port->cmds_to_issue); in mtip_service_thread()
2689 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_service_thread()
2692 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { in mtip_service_thread()
2694 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); in mtip_service_thread()
2712 struct mtip_port *port = dd->port; in mtip_dma_free()
2714 if (port->block1) in mtip_dma_free()
2715 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_free()
2716 port->block1, port->block1_dma); in mtip_dma_free()
2718 if (port->command_list) { in mtip_dma_free()
2719 dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_free()
2720 port->command_list, port->command_list_dma); in mtip_dma_free()
2730 * -ENOMEM Not enough free DMA region space to initialize driver
2734 struct mtip_port *port = dd->port; in mtip_dma_alloc()
2736 /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */ in mtip_dma_alloc()
2737 port->block1 = in mtip_dma_alloc()
2738 dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2739 &port->block1_dma, GFP_KERNEL); in mtip_dma_alloc()
2740 if (!port->block1) in mtip_dma_alloc()
2741 return -ENOMEM; in mtip_dma_alloc()
2744 port->command_list = in mtip_dma_alloc()
2745 dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_alloc()
2746 &port->command_list_dma, GFP_KERNEL); in mtip_dma_alloc()
2747 if (!port->command_list) { in mtip_dma_alloc()
2748 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2749 port->block1, port->block1_dma); in mtip_dma_alloc()
2750 port->block1 = NULL; in mtip_dma_alloc()
2751 port->block1_dma = 0; in mtip_dma_alloc()
2752 return -ENOMEM; in mtip_dma_alloc()
2756 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
2757 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
2758 port->identify = port->block1 + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
2759 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
2760 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
2761 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
2762 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
2763 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
2774 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
2775 return -EFAULT; in mtip_hw_get_identify()
2777 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
2779 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
2782 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
2785 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
2786 dd->port->log_buf, in mtip_hw_get_identify()
2787 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
2789 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2791 /* non-critical error, don't fail the load */ in mtip_hw_get_identify()
2793 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
2795 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2797 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2800 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2802 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2805 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2807 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2813 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
2814 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2817 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2838 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; in mtip_hw_init()
2841 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
2842 rv = -EIO; in mtip_hw_init()
2848 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
2849 dd->numa_node); in mtip_hw_init()
2850 if (!dd->port) in mtip_hw_init()
2851 return -ENOMEM; in mtip_hw_init()
2855 dd->work[i].port = dd->port; in mtip_hw_init()
2859 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
2861 dd->unal_qdepth = 0; in mtip_hw_init()
2863 atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
2867 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
2870 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
2871 dd->port->dd = dd; in mtip_hw_init()
2879 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
2880 dd->port->s_active[i] = in mtip_hw_init()
2881 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
2882 dd->port->cmd_issue[i] = in mtip_hw_init()
2883 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
2884 dd->port->completed[i] = in mtip_hw_init()
2885 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
2890 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
2894 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { in mtip_hw_init()
2895 timetaken = jiffies - timetaken; in mtip_hw_init()
2896 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2899 rv = -ENODEV; in mtip_hw_init()
2902 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
2903 timetaken = jiffies - timetaken; in mtip_hw_init()
2904 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2907 rv = -EFAULT; in mtip_hw_init()
2912 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
2914 dev_err(&dd->pdev->dev, in mtip_hw_init()
2916 rv = -EIO; in mtip_hw_init()
2921 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
2922 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
2925 mtip_init_port(dd->port); in mtip_hw_init()
2926 mtip_start_port(dd->port); in mtip_hw_init()
2929 rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED, in mtip_hw_init()
2930 dev_driver_string(&dd->pdev->dev), dd); in mtip_hw_init()
2932 dev_err(&dd->pdev->dev, in mtip_hw_init()
2933 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
2936 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
2939 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
2940 dd->mmio + HOST_CTL); in mtip_hw_init()
2942 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
2944 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
2945 rv = -EFAULT; in mtip_hw_init()
2953 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
2954 dd->mmio + HOST_CTL); in mtip_hw_init()
2957 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
2958 free_irq(dd->pdev->irq, dd); in mtip_hw_init()
2961 mtip_deinit_port(dd->port); in mtip_hw_init()
2966 kfree(dd->port); in mtip_hw_init()
2975 if (dd->sr || !dd->port) in mtip_standby_drive()
2976 return -ENODEV; in mtip_standby_drive()
2981 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
2982 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && in mtip_standby_drive()
2983 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { in mtip_standby_drive()
2984 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
2986 dev_warn(&dd->pdev->dev, in mtip_standby_drive()
3002 if (!dd->sr) { in mtip_hw_exit()
3003 /* de-initialize the port. */ in mtip_hw_exit()
3004 mtip_deinit_port(dd->port); in mtip_hw_exit()
3007 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
3008 dd->mmio + HOST_CTL); in mtip_hw_exit()
3012 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
3013 free_irq(dd->pdev->irq, dd); in mtip_hw_exit()
3020 kfree(dd->port); in mtip_hw_exit()
3021 dd->port = NULL; in mtip_hw_exit()
3029 * This function is called by the Block Layer just before the
3051 * This function is called by the Block Layer just before the
3058 * -EFAULT Suspend was not successful
3067 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
3068 "Failed standby-immediate command\n"); in mtip_hw_suspend()
3069 return -EFAULT; in mtip_hw_suspend()
3073 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
3074 dd->mmio + HOST_CTL); in mtip_hw_suspend()
3075 mtip_deinit_port(dd->port); in mtip_hw_suspend()
3083 * This function is called by the Block Layer as the
3090 * -EFAULT Resume was not successful
3099 dev_err(&dd->pdev->dev, in mtip_hw_resume()
3101 return -EFAULT; in mtip_hw_resume()
3105 * Enable the port, DMA engine, and FIS reception specific in mtip_hw_resume()
3108 mtip_init_port(dd->port); in mtip_hw_resume()
3109 mtip_start_port(dd->port); in mtip_hw_resume()
3112 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
3113 dd->mmio + HOST_CTL); in mtip_hw_resume()
3123 int index, in rssd_disk_name_format() argument
3127 const int base = 'z' - 'a' + 1; in rssd_disk_name_format()
3133 p = end - 1; in rssd_disk_name_format()
3138 return -EINVAL; in rssd_disk_name_format()
3139 *--p = 'a' + (index % unit); in rssd_disk_name_format()
3140 index = (index / unit) - 1; in rssd_disk_name_format()
3141 } while (index >= 0); in rssd_disk_name_format()
3143 memmove(begin, p, end - p); in rssd_disk_name_format()
3150 * Block layer IOCTL handler.
3159 * -ENOTTY IOCTL not supported or invalid driver data
3167 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl()
3170 return -EACCES; in mtip_block_ioctl()
3173 return -ENOTTY; in mtip_block_ioctl()
3175 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3176 return -ENOTTY; in mtip_block_ioctl()
3180 return -ENOTTY; in mtip_block_ioctl()
3188 * Block layer compat IOCTL handler.
3197 * -ENOTTY IOCTL not supported or invalid driver data
3205 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl()
3208 return -EACCES; in mtip_block_compat_ioctl()
3211 return -ENOTTY; in mtip_block_compat_ioctl()
3213 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3214 return -ENOTTY; in mtip_block_compat_ioctl()
3218 return -ENOTTY; in mtip_block_compat_ioctl()
3231 compat_tasksize - (2 * sizeof(compat_long_t)))) in mtip_block_compat_ioctl()
3232 return -EFAULT; in mtip_block_compat_ioctl()
3234 if (get_user(req_task.out_size, &compat_req_task->out_size)) in mtip_block_compat_ioctl()
3235 return -EFAULT; in mtip_block_compat_ioctl()
3237 if (get_user(req_task.in_size, &compat_req_task->in_size)) in mtip_block_compat_ioctl()
3238 return -EFAULT; in mtip_block_compat_ioctl()
3246 compat_tasksize - in mtip_block_compat_ioctl()
3248 return -EFAULT; in mtip_block_compat_ioctl()
3250 if (put_user(req_task.out_size, &compat_req_task->out_size)) in mtip_block_compat_ioctl()
3251 return -EFAULT; in mtip_block_compat_ioctl()
3253 if (put_user(req_task.in_size, &compat_req_task->in_size)) in mtip_block_compat_ioctl()
3254 return -EFAULT; in mtip_block_compat_ioctl()
3272 * that each partition is also 4KB aligned. Non-aligned partitions adversely
3280 * -ENOTTY An error occurred while reading the drive capacity.
3285 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo()
3289 return -ENOTTY; in mtip_block_getgeo()
3292 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3294 return -ENOTTY; in mtip_block_getgeo()
3297 geo->heads = 224; in mtip_block_getgeo()
3298 geo->sectors = 56; in mtip_block_getgeo()
3299 sector_div(capacity, (geo->heads * geo->sectors)); in mtip_block_getgeo()
3300 geo->cylinders = capacity; in mtip_block_getgeo()
3308 if (dev && dev->bd_disk) { in mtip_block_open()
3309 dd = (struct driver_data *) dev->bd_disk->private_data; in mtip_block_open()
3313 &dd->dd_flag)) { in mtip_block_open()
3314 return -ENODEV; in mtip_block_open()
3319 return -ENODEV; in mtip_block_open()
3327 * Block device operation function.
3329 * This structure contains pointers to the functions required by the block
3345 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { in is_se_active()
3346 if (dd->port->ic_pause_timer) { in is_se_active()
3347 unsigned long to = dd->port->ic_pause_timer + in is_se_active()
3351 &dd->port->flags); in is_se_active()
3352 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); in is_se_active()
3353 dd->port->ic_pause_timer = 0; in is_se_active()
3354 wake_up_interruptible(&dd->port->svc_wait); in is_se_active()
3365 if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO))) in is_stopped()
3368 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in is_stopped()
3370 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in is_stopped()
3372 if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) && in is_stopped()
3375 if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) in is_stopped()
3377 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) in is_stopped()
3386 struct driver_data *dd = hctx->queue->queuedata; in mtip_check_unal_depth()
3389 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) in mtip_check_unal_depth()
3398 cmd->unaligned = 1; in mtip_check_unal_depth()
3401 if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0) in mtip_check_unal_depth()
3410 struct driver_data *dd = hctx->queue->queuedata; in mtip_issue_reserved_cmd()
3412 struct mtip_int_cmd *icmd = cmd->icmd; in mtip_issue_reserved_cmd()
3414 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_issue_reserved_cmd()
3417 if (mtip_commands_active(dd->port)) in mtip_issue_reserved_cmd()
3420 hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); in mtip_issue_reserved_cmd()
3421 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_issue_reserved_cmd()
3422 hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16); in mtip_issue_reserved_cmd()
3424 hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len); in mtip_issue_reserved_cmd()
3425 if (icmd->buf_len) { in mtip_issue_reserved_cmd()
3426 command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ; in mtip_issue_reserved_cmd()
3428 command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF); in mtip_issue_reserved_cmd()
3429 command_sg->dba = cpu_to_le32(icmd->buffer & 0xFFFFFFFF); in mtip_issue_reserved_cmd()
3430 command_sg->dba_upper = in mtip_issue_reserved_cmd()
3431 cpu_to_le32((icmd->buffer >> 16) >> 16); in mtip_issue_reserved_cmd()
3433 hdr->opts |= cpu_to_le32((1 << 16)); in mtip_issue_reserved_cmd()
3437 hdr->byte_count = 0; in mtip_issue_reserved_cmd()
3440 mtip_issue_non_ncq_command(dd->port, rq->tag); in mtip_issue_reserved_cmd()
3447 struct driver_data *dd = hctx->queue->queuedata; in mtip_queue_rq()
3448 struct request *rq = bd->rq; in mtip_queue_rq()
3469 struct driver_data *dd = set->driver_data; in mtip_free_cmd()
3472 if (!cmd->command) in mtip_free_cmd()
3475 dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command, in mtip_free_cmd()
3476 cmd->command_dma); in mtip_free_cmd()
3482 struct driver_data *dd = set->driver_data; in mtip_init_cmd()
3485 cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_init_cmd()
3486 &cmd->command_dma, GFP_KERNEL); in mtip_init_cmd()
3487 if (!cmd->command) in mtip_init_cmd()
3488 return -ENOMEM; in mtip_init_cmd()
3490 sg_init_table(cmd->sg, MTIP_MAX_SG); in mtip_init_cmd()
3497 struct driver_data *dd = req->q->queuedata; in mtip_cmd_timeout()
3502 cmd->status = BLK_STS_TIMEOUT; in mtip_cmd_timeout()
3507 if (test_bit(req->tag, dd->port->cmds_to_issue)) in mtip_cmd_timeout()
3510 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) in mtip_cmd_timeout()
3513 wake_up_interruptible(&dd->port->svc_wait); in mtip_cmd_timeout()
3527 * Block layer initialization function.
3541 unsigned int index = 0; in mtip_block_initialize() local
3543 if (dd->disk) in mtip_block_initialize()
3547 rv = -EINVAL; in mtip_block_initialize()
3551 memset(&dd->tags, 0, sizeof(dd->tags)); in mtip_block_initialize()
3552 dd->tags.ops = &mtip_mq_ops; in mtip_block_initialize()
3553 dd->tags.nr_hw_queues = 1; in mtip_block_initialize()
3554 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; in mtip_block_initialize()
3555 dd->tags.reserved_tags = 1; in mtip_block_initialize()
3556 dd->tags.cmd_size = sizeof(struct mtip_cmd); in mtip_block_initialize()
3557 dd->tags.numa_node = dd->numa_node; in mtip_block_initialize()
3558 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; in mtip_block_initialize()
3559 dd->tags.driver_data = dd; in mtip_block_initialize()
3560 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; in mtip_block_initialize()
3562 rv = blk_mq_alloc_tag_set(&dd->tags); in mtip_block_initialize()
3564 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3569 dd->disk = blk_mq_alloc_disk(&dd->tags, dd); in mtip_block_initialize()
3570 if (IS_ERR(dd->disk)) { in mtip_block_initialize()
3571 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3573 rv = -ENOMEM; in mtip_block_initialize()
3576 dd->queue = dd->disk->queue; in mtip_block_initialize()
3581 index = rv; in mtip_block_initialize()
3584 index, in mtip_block_initialize()
3585 dd->disk->disk_name, in mtip_block_initialize()
3590 dd->disk->major = dd->major; in mtip_block_initialize()
3591 dd->disk->first_minor = index * MTIP_MAX_MINORS; in mtip_block_initialize()
3592 dd->disk->minors = MTIP_MAX_MINORS; in mtip_block_initialize()
3593 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
3594 dd->disk->private_data = dd; in mtip_block_initialize()
3595 dd->index = index; in mtip_block_initialize()
3603 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3605 rv = -EINVAL; in mtip_block_initialize()
3610 * if rebuild pending, start the service thread, and delay the block in mtip_block_initialize()
3617 blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue); in mtip_block_initialize()
3618 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue); in mtip_block_initialize()
3619 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); in mtip_block_initialize()
3620 blk_queue_physical_block_size(dd->queue, 4096); in mtip_block_initialize()
3621 blk_queue_max_hw_sectors(dd->queue, 0xffff); in mtip_block_initialize()
3622 blk_queue_max_segment_size(dd->queue, 0x400000); in mtip_block_initialize()
3623 dma_set_max_seg_size(&dd->pdev->dev, 0x400000); in mtip_block_initialize()
3624 blk_queue_io_min(dd->queue, 4096); in mtip_block_initialize()
3628 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
3630 rv = -EIO; in mtip_block_initialize()
3633 set_capacity(dd->disk, capacity); in mtip_block_initialize()
3635 /* Enable the block device and add it to /dev */ in mtip_block_initialize()
3636 device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); in mtip_block_initialize()
3638 if (dd->mtip_svc_handler) { in mtip_block_initialize()
3639 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
3644 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
3645 dd, dd->numa_node, in mtip_block_initialize()
3646 "mtip_svc_thd_%02d", index); in mtip_block_initialize()
3648 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
3649 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
3650 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
3651 rv = -EFAULT; in mtip_block_initialize()
3654 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
3662 del_gendisk(dd->disk); in mtip_block_initialize()
3667 ida_free(&rssd_index_ida, index); in mtip_block_initialize()
3669 blk_cleanup_disk(dd->disk); in mtip_block_initialize()
3671 blk_mq_free_tag_set(&dd->tags); in mtip_block_initialize()
3673 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
3682 cmd->status = BLK_STS_IOERR; in mtip_no_dev_cleanup()
3688 * Block layer deinitialization function.
3701 if (dd->mtip_svc_handler) { in mtip_block_remove()
3702 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_block_remove()
3703 wake_up_interruptible(&dd->port->svc_wait); in mtip_block_remove()
3704 kthread_stop(dd->mtip_svc_handler); in mtip_block_remove()
3707 if (!dd->sr) { in mtip_block_remove()
3712 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) in mtip_block_remove()
3716 dev_info(&dd->pdev->dev, "device %s surprise removal\n", in mtip_block_remove()
3717 dd->disk->disk_name); in mtip_block_remove()
3719 blk_freeze_queue_start(dd->queue); in mtip_block_remove()
3720 blk_mq_quiesce_queue(dd->queue); in mtip_block_remove()
3721 blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd); in mtip_block_remove()
3722 blk_mq_unquiesce_queue(dd->queue); in mtip_block_remove()
3724 if (dd->disk) { in mtip_block_remove()
3725 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_block_remove()
3726 del_gendisk(dd->disk); in mtip_block_remove()
3727 if (dd->disk->queue) { in mtip_block_remove()
3728 blk_cleanup_queue(dd->queue); in mtip_block_remove()
3729 blk_mq_free_tag_set(&dd->tags); in mtip_block_remove()
3730 dd->queue = NULL; in mtip_block_remove()
3732 put_disk(dd->disk); in mtip_block_remove()
3734 dd->disk = NULL; in mtip_block_remove()
3736 ida_free(&rssd_index_ida, dd->index); in mtip_block_remove()
3738 /* De-initialize the protocol layer. */ in mtip_block_remove()
3761 if (dd->disk) { in mtip_block_shutdown()
3762 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
3763 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
3765 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_block_shutdown()
3766 del_gendisk(dd->disk); in mtip_block_shutdown()
3767 if (dd->disk->queue) { in mtip_block_shutdown()
3768 blk_cleanup_queue(dd->queue); in mtip_block_shutdown()
3769 blk_mq_free_tag_set(&dd->tags); in mtip_block_shutdown()
3771 put_disk(dd->disk); in mtip_block_shutdown()
3772 dd->disk = NULL; in mtip_block_shutdown()
3773 dd->queue = NULL; in mtip_block_shutdown()
3776 ida_free(&rssd_index_ida, dd->index); in mtip_block_shutdown()
3782 dev_info(&dd->pdev->dev, in mtip_block_suspend()
3783 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
3790 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
3791 dd->disk->disk_name); in mtip_block_resume()
3798 cpu_use[cpu]--; in drop_cpu()
3854 dev_info(&dd->pdev->dev, in mtip_disable_link_opts()
3855 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", in mtip_disable_link_opts()
3856 pdev->vendor, pdev->device); in mtip_disable_link_opts()
3871 if (pdev->bus && pdev->bus->self) { in mtip_fix_ero_nosnoop()
3872 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && in mtip_fix_ero_nosnoop()
3873 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { in mtip_fix_ero_nosnoop()
3874 mtip_disable_link_opts(dd, pdev->bus->self); in mtip_fix_ero_nosnoop()
3877 struct pci_dev *parent_dev = pdev->bus->self; in mtip_fix_ero_nosnoop()
3878 if (parent_dev->bus && in mtip_fix_ero_nosnoop()
3879 parent_dev->bus->parent && in mtip_fix_ero_nosnoop()
3880 parent_dev->bus->parent->self && in mtip_fix_ero_nosnoop()
3881 parent_dev->bus->parent->self->vendor == in mtip_fix_ero_nosnoop()
3883 (parent_dev->bus->parent->self->device & in mtip_fix_ero_nosnoop()
3886 parent_dev->bus->parent->self); in mtip_fix_ero_nosnoop()
3896 * PCI device and then calls the block layer initialization function.
3913 my_node = pcibus_to_node(pdev->bus); in mtip_pci_probe()
3918 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); in mtip_pci_probe()
3921 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", in mtip_pci_probe()
3922 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), in mtip_pci_probe()
3927 return -ENOMEM; in mtip_pci_probe()
3934 dev_err(&pdev->dev, "Unable to enable device\n"); in mtip_pci_probe()
3941 dev_err(&pdev->dev, "Unable to map regions\n"); in mtip_pci_probe()
3945 rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in mtip_pci_probe()
3947 dev_warn(&pdev->dev, "64-bit DMA enable failed\n"); in mtip_pci_probe()
3952 dd->major = mtip_major; in mtip_pci_probe()
3953 dd->instance = instance; in mtip_pci_probe()
3954 dd->pdev = pdev; in mtip_pci_probe()
3955 dd->numa_node = my_node; in mtip_pci_probe()
3957 INIT_LIST_HEAD(&dd->online_list); in mtip_pci_probe()
3958 INIT_LIST_HEAD(&dd->remove_list); in mtip_pci_probe()
3960 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
3961 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
3963 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
3964 if (!dd->isr_workq) { in mtip_pci_probe()
3965 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
3966 rv = -ENOMEM; in mtip_pci_probe()
3972 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
3976 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); in mtip_pci_probe()
3980 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", in mtip_pci_probe()
3981 dd->numa_node, in mtip_pci_probe()
3983 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
3986 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); in mtip_pci_probe()
3988 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3989 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", in mtip_pci_probe()
3990 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
3993 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
3994 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3995 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3996 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
3997 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3998 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3999 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
4000 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
4006 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
4007 snprintf(&cpu_list[j], 256 - j, "%d ", i); in mtip_pci_probe()
4012 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); in mtip_pci_probe()
4015 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
4016 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
4017 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
4018 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
4019 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
4020 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
4021 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
4022 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
4027 dev_warn(&pdev->dev, in mtip_pci_probe()
4034 /* Initialize the block layer. */ in mtip_pci_probe()
4037 dev_err(&pdev->dev, in mtip_pci_probe()
4038 "Unable to initialize block layer\n"); in mtip_pci_probe()
4048 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
4054 list_add(&dd->online_list, &online_list); in mtip_pci_probe()
4063 if (dd->isr_workq) { in mtip_pci_probe()
4064 flush_workqueue(dd->isr_workq); in mtip_pci_probe()
4065 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
4066 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
4067 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
4068 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
4093 set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag); in mtip_pci_remove()
4096 list_del_init(&dd->online_list); in mtip_pci_remove()
4097 list_add(&dd->remove_list, &removing_list); in mtip_pci_remove()
4101 synchronize_irq(dd->pdev->irq); in mtip_pci_remove()
4107 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_pci_remove()
4110 if (atomic_read(&dd->irq_workers_active) != 0) { in mtip_pci_remove()
4111 dev_warn(&dd->pdev->dev, in mtip_pci_remove()
4115 blk_set_queue_dying(dd->queue); in mtip_pci_remove()
4116 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
4118 /* Clean up the block layer. */ in mtip_pci_remove()
4121 if (dd->isr_workq) { in mtip_pci_remove()
4122 flush_workqueue(dd->isr_workq); in mtip_pci_remove()
4123 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
4124 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
4125 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
4126 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
4132 list_del_init(&dd->remove_list); in mtip_pci_remove()
4154 dev_err(&pdev->dev, in mtip_pci_suspend()
4156 return -EFAULT; in mtip_pci_suspend()
4159 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
4164 dev_err(&pdev->dev, in mtip_pci_suspend()
4196 dev_err(&pdev->dev, in mtip_pci_resume()
4198 return -EFAULT; in mtip_pci_resume()
4210 dev_err(&pdev->dev, in mtip_pci_resume()
4222 dev_err(&pdev->dev, "Unable to resume\n"); in mtip_pci_resume()
4225 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
4272 * block device number to the Cyclone devices and registers the PCI layer
4284 /* Allocate a major block device number to use with this driver. */ in mtip_init()
4287 pr_err("Unable to register block device (%d)\n", in mtip_init()
4289 return -EBUSY; in mtip_init()
4319 * Module de-initialization function.
4322 * the major block device number allocated by mtip_init() and
4330 /* Release the allocated major block device number. */ in mtip_exit()
4340 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");