129 ioat_init_channel(struct ioatdma_device *ioat_dma,
131 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
132 static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
133 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
309 static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)  in ioat_dma_self_test()  argument
314 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_dma_self_test()
315 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_dma_self_test()
360 	tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,  in ioat_dma_self_test()
412 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)  in ioat_dma_setup_interrupts()  argument
415 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat_dma_setup_interrupts()
433 	msixcnt = ioat_dma->dma_dev.chancnt;  in ioat_dma_setup_interrupts()
435 		ioat_dma->msix_entries[i].entry = i;  in ioat_dma_setup_interrupts()
437 	err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);  in ioat_dma_setup_interrupts()
442 		msix = &ioat_dma->msix_entries[i];  in ioat_dma_setup_interrupts()
443 		ioat_chan = ioat_chan_by_index(ioat_dma, i);  in ioat_dma_setup_interrupts()
449 				msix = &ioat_dma->msix_entries[j];  in ioat_dma_setup_interrupts()
450 				ioat_chan = ioat_chan_by_index(ioat_dma, j);  in ioat_dma_setup_interrupts()
457 	ioat_dma->irq_mode = IOAT_MSIX;  in ioat_dma_setup_interrupts()
466 			       "ioat-msi", ioat_dma);  in ioat_dma_setup_interrupts()
471 	ioat_dma->irq_mode = IOAT_MSI;  in ioat_dma_setup_interrupts()
476 			       IRQF_SHARED, "ioat-intx", ioat_dma);  in ioat_dma_setup_interrupts()
480 	ioat_dma->irq_mode = IOAT_INTX;  in ioat_dma_setup_interrupts()
483 		ioat_intr_quirk(ioat_dma);  in ioat_dma_setup_interrupts()
485 	writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
490 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
491 	ioat_dma->irq_mode = IOAT_NOIRQ;  in ioat_dma_setup_interrupts()
496 static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)  in ioat_disable_interrupts()  argument
499 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_disable_interrupts()
502 static int ioat_probe(struct ioatdma_device *ioat_dma)  in ioat_probe()  argument
505 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_probe()
506 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat_probe()
509 	ioat_dma->completion_pool = dma_pool_create("completion_pool", dev,  in ioat_probe()
514 	if (!ioat_dma->completion_pool) {  in ioat_probe()
519 	ioat_enumerate_channels(ioat_dma);  in ioat_probe()
529 	err = ioat_dma_setup_interrupts(ioat_dma);  in ioat_probe()
533 	err = ioat3_dma_self_test(ioat_dma);  in ioat_probe()
540 	ioat_disable_interrupts(ioat_dma);  in ioat_probe()
542 	dma_pool_destroy(ioat_dma->completion_pool);  in ioat_probe()
547 static int ioat_register(struct ioatdma_device *ioat_dma)  in ioat_register()  argument
549 	int err = dma_async_device_register(&ioat_dma->dma_dev);  in ioat_register()
552 		ioat_disable_interrupts(ioat_dma);  in ioat_register()
553 		dma_pool_destroy(ioat_dma->completion_pool);  in ioat_register()
559 static void ioat_dma_remove(struct ioatdma_device *ioat_dma)  in ioat_dma_remove()  argument
561 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_dma_remove()
563 	ioat_disable_interrupts(ioat_dma);  in ioat_dma_remove()
565 	ioat_kobject_del(ioat_dma);  in ioat_dma_remove()
569 	dma_pool_destroy(ioat_dma->completion_pool);  in ioat_dma_remove()
578 static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma)  in ioat_enumerate_channels()  argument
581 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_enumerate_channels()
582 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_enumerate_channels()
587 	dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);  in ioat_enumerate_channels()
589 	if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {  in ioat_enumerate_channels()
591 			 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));  in ioat_enumerate_channels()
592 		dma->chancnt = ARRAY_SIZE(ioat_dma->idx);  in ioat_enumerate_channels()
594 	xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);  in ioat_enumerate_channels()
605 		ioat_init_channel(ioat_dma, ioat_chan, i);  in ioat_enumerate_channels()
624 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;  in ioat_free_chan_resources()  local
670 	dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion,  in ioat_free_chan_resources()
702 		dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,  in ioat_alloc_chan_resources()
749 ioat_init_channel(struct ioatdma_device *ioat_dma,  in ioat_init_channel()  argument
752 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_init_channel()
756 	ioat_chan->ioat_dma = ioat_dma;  in ioat_init_channel()
757 	ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));  in ioat_init_channel()
762 	ioat_dma->idx[idx] = ioat_chan;  in ioat_init_channel()
768 static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)  in ioat_xor_val_self_test()  argument
785 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_xor_val_self_test()
786 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_xor_val_self_test()
1033 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)  in ioat3_dma_self_test()  argument
1037 	rc = ioat_dma_self_test(ioat_dma);  in ioat3_dma_self_test()
1041 	rc = ioat_xor_val_self_test(ioat_dma);  in ioat3_dma_self_test()
1046 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)  in ioat_intr_quirk()  argument
1053 	dma = &ioat_dma->dma_dev;  in ioat_intr_quirk()
1059 	if (ioat_dma->cap & IOAT_CAP_DWBES) {  in ioat_intr_quirk()
1072 static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)  in ioat3_dma_probe()  argument
1074 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat3_dma_probe()
1082 	dma = &ioat_dma->dma_dev;  in ioat3_dma_probe()
1091 	ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);  in ioat3_dma_probe()
1094 		ioat_dma->cap &=  in ioat3_dma_probe()
1098 	if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))  in ioat3_dma_probe()
1099 		ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);  in ioat3_dma_probe()
1101 	if (ioat_dma->cap & IOAT_CAP_XOR) {  in ioat3_dma_probe()
1111 	if (ioat_dma->cap & IOAT_CAP_PQ) {  in ioat3_dma_probe()
1118 		if (ioat_dma->cap & IOAT_CAP_RAID16SS)  in ioat3_dma_probe()
1123 		if (!(ioat_dma->cap & IOAT_CAP_XOR)) {  in ioat3_dma_probe()
1129 			if (ioat_dma->cap & IOAT_CAP_RAID16SS)  in ioat3_dma_probe()
1139 	if (ioat_dma->cap & IOAT_CAP_RAID16SS) {  in ioat3_dma_probe()
1147 			ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,  in ioat3_dma_probe()
1150 			if (!ioat_dma->sed_hw_pool[i])  in ioat3_dma_probe()
1156 	if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))  in ioat3_dma_probe()
1159 	err = ioat_probe(ioat_dma);  in ioat3_dma_probe()
1169 	err = ioat_register(ioat_dma);  in ioat3_dma_probe()
1173 	ioat_kobject_add(ioat_dma, &ioat_ktype);  in ioat3_dma_probe()
1176 		ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);  in ioat3_dma_probe()
1194 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);  in ioat_shutdown()  local
1198 	if (!ioat_dma)  in ioat_shutdown()
1202 		ioat_chan = ioat_dma->idx[i];  in ioat_shutdown()
1214 	ioat_disable_interrupts(ioat_dma);  in ioat_shutdown()
1217 static void ioat_resume(struct ioatdma_device *ioat_dma)  in ioat_resume()  argument
1224 		ioat_chan = ioat_dma->idx[i];  in ioat_resume()
1281 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);  in ioat_pcie_error_resume()  local
1286 	ioat_resume(ioat_dma);  in ioat_pcie_error_resume()