119 static void vmci_read_data(struct vmci_guest_device *vmci_dev,  in vmci_read_data()  argument
122 	if (vmci_dev->mmio_base == NULL)  in vmci_read_data()
123 		ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR,  in vmci_read_data()
132 		struct vmci_data_in_out_header *buffer_header = vmci_dev->data_buffer;  in vmci_read_data()
134 		size_t buffer_offset = dest - vmci_dev->data_buffer;  in vmci_read_data()
139 		sg_array[0].addr = vmci_dev->data_buffer_base + buffer_offset;  in vmci_read_data()
142 		vmci_write_reg(vmci_dev, lower_32_bits(vmci_dev->data_buffer_base),  in vmci_read_data()
145 		wait_event(vmci_dev->inout_wq, buffer_header->busy == 1);  in vmci_read_data()
309 	struct vmci_guest_device *vmci_dev = (struct vmci_guest_device *)data;  in vmci_dispatch_dgs()  local
310 	u8 *dg_in_buffer = vmci_dev->data_buffer;  in vmci_dispatch_dgs()
315 	bool is_io_port = vmci_dev->mmio_base == NULL;  in vmci_dispatch_dgs()
331 	vmci_read_data(vmci_dev, dg_in_buffer, current_dg_in_buffer_size);  in vmci_dispatch_dgs()
397 				vmci_read_data(vmci_dev,  in vmci_dispatch_dgs()
415 				dev_dbg(vmci_dev->dev,  in vmci_dispatch_dgs()
429 			dev_dbg(vmci_dev->dev,  in vmci_dispatch_dgs()
438 				vmci_read_data(vmci_dev, dg_in_buffer,  in vmci_dispatch_dgs()
456 			vmci_read_data(vmci_dev, dg_in_buffer,  in vmci_dispatch_dgs()
558 static void vmci_free_dg_buffers(struct vmci_guest_device *vmci_dev)  in vmci_free_dg_buffers()  argument
560 	if (vmci_dev->mmio_base != NULL) {  in vmci_free_dg_buffers()
561 		if (vmci_dev->tx_buffer != NULL)  in vmci_free_dg_buffers()
562 			dma_free_coherent(vmci_dev->dev,  in vmci_free_dg_buffers()
564 					  vmci_dev->tx_buffer,  in vmci_free_dg_buffers()
565 					  vmci_dev->tx_buffer_base);  in vmci_free_dg_buffers()
566 		if (vmci_dev->data_buffer != NULL)  in vmci_free_dg_buffers()
567 			dma_free_coherent(vmci_dev->dev,  in vmci_free_dg_buffers()
569 					  vmci_dev->data_buffer,  in vmci_free_dg_buffers()
570 					  vmci_dev->data_buffer_base);  in vmci_free_dg_buffers()
572 		vfree(vmci_dev->data_buffer);  in vmci_free_dg_buffers()
582 	struct vmci_guest_device *vmci_dev;  in vmci_guest_probe_device()  local
629 	vmci_dev = devm_kzalloc(&pdev->dev, sizeof(*vmci_dev), GFP_KERNEL);  in vmci_guest_probe_device()
630 	if (!vmci_dev) {  in vmci_guest_probe_device()
636 	vmci_dev->dev = &pdev->dev;  in vmci_guest_probe_device()
637 	vmci_dev->exclusive_vectors = false;  in vmci_guest_probe_device()
638 	vmci_dev->iobase = iobase;  in vmci_guest_probe_device()
639 	vmci_dev->mmio_base = mmio_base;  in vmci_guest_probe_device()
641 	tasklet_init(&vmci_dev->datagram_tasklet,  in vmci_guest_probe_device()
642 		     vmci_dispatch_dgs, (unsigned long)vmci_dev);  in vmci_guest_probe_device()
643 	tasklet_init(&vmci_dev->bm_tasklet,  in vmci_guest_probe_device()
644 		     vmci_process_bitmap, (unsigned long)vmci_dev);  in vmci_guest_probe_device()
645 	init_waitqueue_head(&vmci_dev->inout_wq);  in vmci_guest_probe_device()
648 		vmci_dev->tx_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE,  in vmci_guest_probe_device()
649 							 &vmci_dev->tx_buffer_base,  in vmci_guest_probe_device()
651 		if (!vmci_dev->tx_buffer) {  in vmci_guest_probe_device()
657 		vmci_dev->data_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE,  in vmci_guest_probe_device()
658 							   &vmci_dev->data_buffer_base,  in vmci_guest_probe_device()
661 		vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE);  in vmci_guest_probe_device()
663 	if (!vmci_dev->data_buffer) {  in vmci_guest_probe_device()
681 	capabilities = vmci_read_reg(vmci_dev, VMCI_CAPS_ADDR);  in vmci_guest_probe_device()
710 		vmci_dev->notification_bitmap = dma_alloc_coherent(  in vmci_guest_probe_device()
711 			&pdev->dev, PAGE_SIZE, &vmci_dev->notification_base,  in vmci_guest_probe_device()
713 		if (!vmci_dev->notification_bitmap)  in vmci_guest_probe_device()
734 	vmci_write_reg(vmci_dev, caps_in_use, VMCI_CAPS_ADDR);  in vmci_guest_probe_device()
738 		vmci_write_reg(vmci_dev, PAGE_SHIFT, VMCI_GUEST_PAGE_SHIFT);  in vmci_guest_probe_device()
741 		vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->data_buffer_base),  in vmci_guest_probe_device()
743 		vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->tx_buffer_base),  in vmci_guest_probe_device()
749 	vmci_dev_g = vmci_dev;  in vmci_guest_probe_device()
759 			vmci_dev->notification_base >> PAGE_SHIFT;  in vmci_guest_probe_device()
792 	if (vmci_dev->mmio_base != NULL)  in vmci_guest_probe_device()
804 		vmci_dev->exclusive_vectors = true;  in vmci_guest_probe_device()
812 			    IRQF_SHARED, KBUILD_MODNAME, vmci_dev);  in vmci_guest_probe_device()
825 	if (vmci_dev->exclusive_vectors) {  in vmci_guest_probe_device()
828 				    vmci_dev);  in vmci_guest_probe_device()
838 					    0, KBUILD_MODNAME, vmci_dev);  in vmci_guest_probe_device()
858 	vmci_write_reg(vmci_dev, cmd, VMCI_IMR_ADDR);  in vmci_guest_probe_device()
861 	vmci_write_reg(vmci_dev, VMCI_CONTROL_INT_ENABLE, VMCI_CONTROL_ADDR);  in vmci_guest_probe_device()
863 	pci_set_drvdata(pdev, vmci_dev);  in vmci_guest_probe_device()
869 	if (vmci_dev->exclusive_vectors)  in vmci_guest_probe_device()
870 		free_irq(pci_irq_vector(pdev, 1), vmci_dev);  in vmci_guest_probe_device()
873 	free_irq(pci_irq_vector(pdev, 0), vmci_dev);  in vmci_guest_probe_device()
874 	tasklet_kill(&vmci_dev->datagram_tasklet);  in vmci_guest_probe_device()
875 	tasklet_kill(&vmci_dev->bm_tasklet);  in vmci_guest_probe_device()
894 	if (vmci_dev->notification_bitmap) {  in vmci_guest_probe_device()
895 		vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR);  in vmci_guest_probe_device()
897 				  vmci_dev->notification_bitmap,  in vmci_guest_probe_device()
898 				  vmci_dev->notification_base);  in vmci_guest_probe_device()
902 	vmci_free_dg_buffers(vmci_dev);  in vmci_guest_probe_device()
910 	struct vmci_guest_device *vmci_dev = pci_get_drvdata(pdev);  in vmci_guest_remove_device()  local
931 	vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR);  in vmci_guest_remove_device()
938 	if (vmci_dev->exclusive_vectors) {  in vmci_guest_remove_device()
939 		free_irq(pci_irq_vector(pdev, 1), vmci_dev);  in vmci_guest_remove_device()
940 		if (vmci_dev->mmio_base != NULL)  in vmci_guest_remove_device()
941 			free_irq(pci_irq_vector(pdev, 2), vmci_dev);  in vmci_guest_remove_device()
943 	free_irq(pci_irq_vector(pdev, 0), vmci_dev);  in vmci_guest_remove_device()
946 	tasklet_kill(&vmci_dev->datagram_tasklet);  in vmci_guest_remove_device()
947 	tasklet_kill(&vmci_dev->bm_tasklet);  in vmci_guest_remove_device()
949 	if (vmci_dev->notification_bitmap) {  in vmci_guest_remove_device()
956 				  vmci_dev->notification_bitmap,  in vmci_guest_remove_device()
957 				  vmci_dev->notification_base);  in vmci_guest_remove_device()
960 	vmci_free_dg_buffers(vmci_dev);  in vmci_guest_remove_device()
962 	if (vmci_dev->mmio_base != NULL)  in vmci_guest_remove_device()
963 		pci_iounmap(pdev, vmci_dev->mmio_base);  in vmci_guest_remove_device()