72 	struct zpci_dev *tmp, *zdev = NULL;  in get_zdev_by_fid()  local
77 			zdev = tmp;  in get_zdev_by_fid()
78 			zpci_zdev_get(zdev);  in get_zdev_by_fid()
83 	return zdev;  in get_zdev_by_fid()
88 	struct zpci_dev *tmp, *zdev;  in zpci_remove_reserved_devices()  local
93 	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {  in zpci_remove_reserved_devices()
94 		if (zdev->state == ZPCI_FN_STATE_STANDBY &&  in zpci_remove_reserved_devices()
95 		    !clp_get_state(zdev->fid, &state) &&  in zpci_remove_reserved_devices()
97 			list_move_tail(&zdev->entry, &remove);  in zpci_remove_reserved_devices()
101 	list_for_each_entry_safe(zdev, tmp, &remove, entry)  in zpci_remove_reserved_devices()
102 		zpci_device_reserved(zdev);  in zpci_remove_reserved_devices()
118 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,  in zpci_register_ioat()  argument
121 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);  in zpci_register_ioat()
129 	fib.gd = zdev->gisa;  in zpci_register_ioat()
132 		zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status);  in zpci_register_ioat()
138 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)  in zpci_unregister_ioat()  argument
140 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);  in zpci_unregister_ioat()
144 	fib.gd = zdev->gisa;  in zpci_unregister_ioat()
148 		zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);  in zpci_unregister_ioat()
153 int zpci_fmb_enable_device(struct zpci_dev *zdev)  in zpci_fmb_enable_device()  argument
155 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);  in zpci_fmb_enable_device()
159 	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)  in zpci_fmb_enable_device()
162 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);  in zpci_fmb_enable_device()
163 	if (!zdev->fmb)  in zpci_fmb_enable_device()
165 	WARN_ON((u64) zdev->fmb & 0xf);  in zpci_fmb_enable_device()
168 	atomic64_set(&zdev->allocated_pages, 0);  in zpci_fmb_enable_device()
169 	atomic64_set(&zdev->mapped_pages, 0);  in zpci_fmb_enable_device()
170 	atomic64_set(&zdev->unmapped_pages, 0);  in zpci_fmb_enable_device()
172 	fib.fmb_addr = virt_to_phys(zdev->fmb);  in zpci_fmb_enable_device()
173 	fib.gd = zdev->gisa;  in zpci_fmb_enable_device()
176 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);  in zpci_fmb_enable_device()
177 		zdev->fmb = NULL;  in zpci_fmb_enable_device()
183 int zpci_fmb_disable_device(struct zpci_dev *zdev)  in zpci_fmb_disable_device()  argument
185 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);  in zpci_fmb_disable_device()
189 	if (!zdev->fmb)  in zpci_fmb_disable_device()
192 	fib.gd = zdev->gisa;  in zpci_fmb_disable_device()
200 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);  in zpci_fmb_disable_device()
201 		zdev->fmb = NULL;  in zpci_fmb_disable_device()
206 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)  in zpci_cfg_load()  argument
208 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);  in zpci_cfg_load()
222 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)  in zpci_cfg_store()  argument
224 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);  in zpci_cfg_store()
273 	struct zpci_dev *zdev =	to_zpci(pdev);  in pci_iomap_range_fh()  local
276 	idx = zdev->bars[bar].map_idx;  in pci_iomap_range_fh()
280 	zpci_iomap_start[idx].fh = zdev->fh;  in pci_iomap_range_fh()
292 	struct zpci_dev *zdev = to_zpci(pdev);  in pci_iomap_range_mio()  local
295 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);  in pci_iomap_range_mio()
322 	struct zpci_dev *zdev = to_zpci(pdev);  in pci_iomap_wc_range_mio()  local
325 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);  in pci_iomap_wc_range_mio()
379 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);  in pci_read()  local
381 	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;  in pci_read()
387 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);  in pci_write()  local
389 	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;  in pci_write()
399 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_map_resources()  local
408 		if (zpci_use_mio(zdev))  in zpci_map_resources()
410 				(resource_size_t __force) zdev->bars[i].mio_wt;  in zpci_map_resources()
422 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_unmap_resources()  local
426 	if (zpci_use_mio(zdev))  in zpci_unmap_resources()
438 static int zpci_alloc_iomap(struct zpci_dev *zdev)  in zpci_alloc_iomap()  argument
453 static void zpci_free_iomap(struct zpci_dev *zdev, int entry)  in zpci_free_iomap()  argument
461 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh)  in zpci_do_update_iomap_fh()  argument
467 		if (!zdev->bars[bar].size)  in zpci_do_update_iomap_fh()
469 		idx = zdev->bars[bar].map_idx;  in zpci_do_update_iomap_fh()
472 		WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh);  in zpci_do_update_iomap_fh()
477 void zpci_update_fh(struct zpci_dev *zdev, u32 fh)  in zpci_update_fh()  argument
479 	if (!fh || zdev->fh == fh)  in zpci_update_fh()
482 	zdev->fh = fh;  in zpci_update_fh()
483 	if (zpci_use_mio(zdev))  in zpci_update_fh()
485 	if (zdev->has_resources && zdev_enabled(zdev))  in zpci_update_fh()
486 		zpci_do_update_iomap_fh(zdev, fh);  in zpci_update_fh()
489 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,  in __alloc_res()  argument
501 	r->name = zdev->res_name;  in __alloc_res()
510 int zpci_setup_bus_resources(struct zpci_dev *zdev)  in zpci_setup_bus_resources()  argument
516 	snprintf(zdev->res_name, sizeof(zdev->res_name),  in zpci_setup_bus_resources()
517 		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);  in zpci_setup_bus_resources()
520 		if (!zdev->bars[i].size)  in zpci_setup_bus_resources()
522 		entry = zpci_alloc_iomap(zdev);  in zpci_setup_bus_resources()
525 		zdev->bars[i].map_idx = entry;  in zpci_setup_bus_resources()
529 		if (zdev->bars[i].val & 8)  in zpci_setup_bus_resources()
531 		if (zdev->bars[i].val & 4)  in zpci_setup_bus_resources()
534 		if (zpci_use_mio(zdev))  in zpci_setup_bus_resources()
535 			addr = (unsigned long) zdev->bars[i].mio_wt;  in zpci_setup_bus_resources()
538 		size = 1UL << zdev->bars[i].size;  in zpci_setup_bus_resources()
540 		res = __alloc_res(zdev, addr, size, flags);  in zpci_setup_bus_resources()
542 			zpci_free_iomap(zdev, entry);  in zpci_setup_bus_resources()
545 		zdev->bars[i].res = res;  in zpci_setup_bus_resources()
547 	zdev->has_resources = 1;  in zpci_setup_bus_resources()
552 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)  in zpci_cleanup_bus_resources()  argument
559 		res = zdev->bars[i].res;  in zpci_cleanup_bus_resources()
564 		pci_bus_remove_resource(zdev->zbus->bus, res);  in zpci_cleanup_bus_resources()
565 		zpci_free_iomap(zdev, zdev->bars[i].map_idx);  in zpci_cleanup_bus_resources()
566 		zdev->bars[i].res = NULL;  in zpci_cleanup_bus_resources()
569 	zdev->has_resources = 0;  in zpci_cleanup_bus_resources()
575 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_device_add()  local
580 	zpci_zdev_get(zdev);  in pcibios_device_add()
600 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_release_device()  local
603 	zpci_zdev_put(zdev);  in pcibios_release_device()
608 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_enable_device()  local
610 	zpci_debug_init_device(zdev, dev_name(&pdev->dev));  in pcibios_enable_device()
611 	zpci_fmb_enable_device(zdev);  in pcibios_enable_device()
618 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_disable_device()  local
620 	zpci_fmb_disable_device(zdev);  in pcibios_disable_device()
621 	zpci_debug_exit_device(zdev);  in pcibios_disable_device()
672 int zpci_enable_device(struct zpci_dev *zdev)  in zpci_enable_device()  argument
674 	u32 fh = zdev->fh;  in zpci_enable_device()
677 	if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES))  in zpci_enable_device()
680 		zpci_update_fh(zdev, fh);  in zpci_enable_device()
685 int zpci_disable_device(struct zpci_dev *zdev)  in zpci_disable_device()  argument
687 	u32 fh = zdev->fh;  in zpci_disable_device()
690 	cc = clp_disable_fh(zdev, &fh);  in zpci_disable_device()
692 		zpci_update_fh(zdev, fh);  in zpci_disable_device()
695 			zdev->fid);  in zpci_disable_device()
697 		rc = clp_refresh_fh(zdev->fid, &fh);  in zpci_disable_device()
699 			zpci_update_fh(zdev, fh);  in zpci_disable_device()
732 int zpci_hot_reset_device(struct zpci_dev *zdev)  in zpci_hot_reset_device()  argument
737 	zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);  in zpci_hot_reset_device()
738 	if (zdev_enabled(zdev)) {  in zpci_hot_reset_device()
740 		rc = zpci_disable_device(zdev);  in zpci_hot_reset_device()
752 	rc = zpci_enable_device(zdev);  in zpci_hot_reset_device()
756 	if (zdev->dma_table)  in zpci_hot_reset_device()
757 		rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,  in zpci_hot_reset_device()
758 					virt_to_phys(zdev->dma_table), &status);  in zpci_hot_reset_device()
760 		rc = zpci_dma_init_device(zdev);  in zpci_hot_reset_device()
762 		zpci_disable_device(zdev);  in zpci_hot_reset_device()
782 	struct zpci_dev *zdev;  in zpci_create_device()  local
786 	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);  in zpci_create_device()
787 	if (!zdev)  in zpci_create_device()
791 	zdev->fid = fid;  in zpci_create_device()
792 	zdev->fh = fh;  in zpci_create_device()
795 	rc = clp_query_pci_fn(zdev);  in zpci_create_device()
798 	zdev->state =  state;  in zpci_create_device()
800 	kref_init(&zdev->kref);  in zpci_create_device()
801 	mutex_init(&zdev->lock);  in zpci_create_device()
802 	mutex_init(&zdev->kzdev_lock);  in zpci_create_device()
804 	rc = zpci_init_iommu(zdev);  in zpci_create_device()
808 	rc = zpci_bus_device_register(zdev, &pci_root_ops);  in zpci_create_device()
813 	list_add_tail(&zdev->entry, &zpci_list);  in zpci_create_device()
816 	return zdev;  in zpci_create_device()
819 	zpci_destroy_iommu(zdev);  in zpci_create_device()
822 	kfree(zdev);  in zpci_create_device()
826 bool zpci_is_device_configured(struct zpci_dev *zdev)  in zpci_is_device_configured()  argument
828 	enum zpci_state state = zdev->state;  in zpci_is_device_configured()
845 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh)  in zpci_scan_configured_device()  argument
847 	zpci_update_fh(zdev, fh);  in zpci_scan_configured_device()
848 	return zpci_bus_scan_device(zdev);  in zpci_scan_configured_device()
861 int zpci_deconfigure_device(struct zpci_dev *zdev)  in zpci_deconfigure_device()  argument
865 	if (zdev->zbus->bus)  in zpci_deconfigure_device()
866 		zpci_bus_remove_device(zdev, false);  in zpci_deconfigure_device()
868 	if (zdev->dma_table) {  in zpci_deconfigure_device()
869 		rc = zpci_dma_exit_device(zdev);  in zpci_deconfigure_device()
873 	if (zdev_enabled(zdev)) {  in zpci_deconfigure_device()
874 		rc = zpci_disable_device(zdev);  in zpci_deconfigure_device()
879 	rc = sclp_pci_deconfigure(zdev->fid);  in zpci_deconfigure_device()
880 	zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc);  in zpci_deconfigure_device()
883 	zdev->state = ZPCI_FN_STATE_STANDBY;  in zpci_deconfigure_device()
897 void zpci_device_reserved(struct zpci_dev *zdev)  in zpci_device_reserved()  argument
899 	if (zdev->has_hp_slot)  in zpci_device_reserved()
900 		zpci_exit_slot(zdev);  in zpci_device_reserved()
906 	list_del(&zdev->entry);  in zpci_device_reserved()
908 	zdev->state = ZPCI_FN_STATE_RESERVED;  in zpci_device_reserved()
909 	zpci_dbg(3, "rsv fid:%x\n", zdev->fid);  in zpci_device_reserved()
910 	zpci_zdev_put(zdev);  in zpci_device_reserved()
915 	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);  in zpci_release_device()  local
918 	if (zdev->zbus->bus)  in zpci_release_device()
919 		zpci_bus_remove_device(zdev, false);  in zpci_release_device()
921 	if (zdev->dma_table)  in zpci_release_device()
922 		zpci_dma_exit_device(zdev);  in zpci_release_device()
923 	if (zdev_enabled(zdev))  in zpci_release_device()
924 		zpci_disable_device(zdev);  in zpci_release_device()
926 	switch (zdev->state) {  in zpci_release_device()
928 		ret = sclp_pci_deconfigure(zdev->fid);  in zpci_release_device()
929 		zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret);  in zpci_release_device()
932 		if (zdev->has_hp_slot)  in zpci_release_device()
933 			zpci_exit_slot(zdev);  in zpci_release_device()
935 		list_del(&zdev->entry);  in zpci_release_device()
937 		zpci_dbg(3, "rsv fid:%x\n", zdev->fid);  in zpci_release_device()
940 		if (zdev->has_resources)  in zpci_release_device()
941 			zpci_cleanup_bus_resources(zdev);  in zpci_release_device()
942 		zpci_bus_device_unregister(zdev);  in zpci_release_device()
943 		zpci_destroy_iommu(zdev);  in zpci_release_device()
948 	zpci_dbg(3, "rem fid:%x\n", zdev->fid);  in zpci_release_device()
949 	kfree_rcu(zdev, rcu);  in zpci_release_device()
955 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_report_error()  local
957 	return sclp_pci_report(report, zdev->fh, zdev->fid);  in zpci_report_error()
971 int zpci_clear_error_state(struct zpci_dev *zdev)  in zpci_clear_error_state()  argument
973 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR);  in zpci_clear_error_state()
980 		zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);  in zpci_clear_error_state()
998 int zpci_reset_load_store_blocked(struct zpci_dev *zdev)  in zpci_reset_load_store_blocked()  argument
1000 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK);  in zpci_reset_load_store_blocked()
1007 		zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);  in zpci_reset_load_store_blocked()