72 	struct zpci_dev *tmp, *zdev = NULL;  in get_zdev_by_fid()  local
77 			zdev = tmp;  in get_zdev_by_fid()
78 			zpci_zdev_get(zdev);  in get_zdev_by_fid()
83 	return zdev;  in get_zdev_by_fid()
88 	struct zpci_dev *tmp, *zdev;  in zpci_remove_reserved_devices()  local
93 	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {  in zpci_remove_reserved_devices()
94 		if (zdev->state == ZPCI_FN_STATE_STANDBY &&  in zpci_remove_reserved_devices()
95 		    !clp_get_state(zdev->fid, &state) &&  in zpci_remove_reserved_devices()
97 			list_move_tail(&zdev->entry, &remove);  in zpci_remove_reserved_devices()
101 	list_for_each_entry_safe(zdev, tmp, &remove, entry)  in zpci_remove_reserved_devices()
102 		zpci_device_reserved(zdev);  in zpci_remove_reserved_devices()
118 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,  in zpci_register_ioat()  argument
121 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);  in zpci_register_ioat()
129 	fib.gd = zdev->gisa;  in zpci_register_ioat()
132 		zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);  in zpci_register_ioat()
138 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)  in zpci_unregister_ioat()  argument
140 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);  in zpci_unregister_ioat()
144 	fib.gd = zdev->gisa;  in zpci_unregister_ioat()
148 		zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);  in zpci_unregister_ioat()
153 int zpci_fmb_enable_device(struct zpci_dev *zdev)  in zpci_fmb_enable_device()  argument
155 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);  in zpci_fmb_enable_device()
159 	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)  in zpci_fmb_enable_device()
162 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);  in zpci_fmb_enable_device()
163 	if (!zdev->fmb)  in zpci_fmb_enable_device()
165 	WARN_ON((u64) zdev->fmb & 0xf);  in zpci_fmb_enable_device()
168 	atomic64_set(&zdev->allocated_pages, 0);  in zpci_fmb_enable_device()
169 	atomic64_set(&zdev->mapped_pages, 0);  in zpci_fmb_enable_device()
170 	atomic64_set(&zdev->unmapped_pages, 0);  in zpci_fmb_enable_device()
172 	fib.fmb_addr = virt_to_phys(zdev->fmb);  in zpci_fmb_enable_device()
173 	fib.gd = zdev->gisa;  in zpci_fmb_enable_device()
176 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);  in zpci_fmb_enable_device()
177 		zdev->fmb = NULL;  in zpci_fmb_enable_device()
183 int zpci_fmb_disable_device(struct zpci_dev *zdev)  in zpci_fmb_disable_device()  argument
185 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);  in zpci_fmb_disable_device()
189 	if (!zdev->fmb)  in zpci_fmb_disable_device()
192 	fib.gd = zdev->gisa;  in zpci_fmb_disable_device()
200 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);  in zpci_fmb_disable_device()
201 		zdev->fmb = NULL;  in zpci_fmb_disable_device()
206 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)  in zpci_cfg_load()  argument
208 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);  in zpci_cfg_load()
222 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)  in zpci_cfg_store()  argument
224 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);  in zpci_cfg_store()
310 	struct zpci_dev *zdev =	to_zpci(pdev);  in pci_iomap_range_fh()  local
313 	idx = zdev->bars[bar].map_idx;  in pci_iomap_range_fh()
317 	zpci_iomap_start[idx].fh = zdev->fh;  in pci_iomap_range_fh()
329 	struct zpci_dev *zdev = to_zpci(pdev);  in pci_iomap_range_mio()  local
332 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);  in pci_iomap_range_mio()
359 	struct zpci_dev *zdev = to_zpci(pdev);  in pci_iomap_wc_range_mio()  local
362 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);  in pci_iomap_wc_range_mio()
416 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);  in pci_read()  local
418 	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;  in pci_read()
424 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);  in pci_write()  local
426 	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;  in pci_write()
436 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_map_resources()  local
445 		if (zpci_use_mio(zdev))  in zpci_map_resources()
447 				(resource_size_t __force) zdev->bars[i].mio_wt;  in zpci_map_resources()
459 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_unmap_resources()  local
463 	if (zpci_use_mio(zdev))  in zpci_unmap_resources()
475 static int zpci_alloc_iomap(struct zpci_dev *zdev)  in zpci_alloc_iomap()  argument
490 static void zpci_free_iomap(struct zpci_dev *zdev, int entry)  in zpci_free_iomap()  argument
498 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh)  in zpci_do_update_iomap_fh()  argument
504 		if (!zdev->bars[bar].size)  in zpci_do_update_iomap_fh()
506 		idx = zdev->bars[bar].map_idx;  in zpci_do_update_iomap_fh()
509 		WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh);  in zpci_do_update_iomap_fh()
514 void zpci_update_fh(struct zpci_dev *zdev, u32 fh)  in zpci_update_fh()  argument
516 	if (!fh || zdev->fh == fh)  in zpci_update_fh()
519 	zdev->fh = fh;  in zpci_update_fh()
520 	if (zpci_use_mio(zdev))  in zpci_update_fh()
522 	if (zdev->has_resources && zdev_enabled(zdev))  in zpci_update_fh()
523 		zpci_do_update_iomap_fh(zdev, fh);  in zpci_update_fh()
526 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,  in __alloc_res()  argument
538 	r->name = zdev->res_name;  in __alloc_res()
547 int zpci_setup_bus_resources(struct zpci_dev *zdev,  in zpci_setup_bus_resources()  argument
554 	snprintf(zdev->res_name, sizeof(zdev->res_name),  in zpci_setup_bus_resources()
555 		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);  in zpci_setup_bus_resources()
558 		if (!zdev->bars[i].size)  in zpci_setup_bus_resources()
560 		entry = zpci_alloc_iomap(zdev);  in zpci_setup_bus_resources()
563 		zdev->bars[i].map_idx = entry;  in zpci_setup_bus_resources()
567 		if (zdev->bars[i].val & 8)  in zpci_setup_bus_resources()
569 		if (zdev->bars[i].val & 4)  in zpci_setup_bus_resources()
572 		if (zpci_use_mio(zdev))  in zpci_setup_bus_resources()
573 			addr = (unsigned long) zdev->bars[i].mio_wt;  in zpci_setup_bus_resources()
576 		size = 1UL << zdev->bars[i].size;  in zpci_setup_bus_resources()
578 		res = __alloc_res(zdev, addr, size, flags);  in zpci_setup_bus_resources()
580 			zpci_free_iomap(zdev, entry);  in zpci_setup_bus_resources()
583 		zdev->bars[i].res = res;  in zpci_setup_bus_resources()
586 	zdev->has_resources = 1;  in zpci_setup_bus_resources()
591 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)  in zpci_cleanup_bus_resources()  argument
596 		if (!zdev->bars[i].size || !zdev->bars[i].res)  in zpci_cleanup_bus_resources()
599 		zpci_free_iomap(zdev, zdev->bars[i].map_idx);  in zpci_cleanup_bus_resources()
600 		release_resource(zdev->bars[i].res);  in zpci_cleanup_bus_resources()
601 		kfree(zdev->bars[i].res);  in zpci_cleanup_bus_resources()
603 	zdev->has_resources = 0;  in zpci_cleanup_bus_resources()
608 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_device_add()  local
613 	zpci_zdev_get(zdev);  in pcibios_device_add()
633 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_release_device()  local
636 	zpci_zdev_put(zdev);  in pcibios_release_device()
641 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_enable_device()  local
643 	zpci_debug_init_device(zdev, dev_name(&pdev->dev));  in pcibios_enable_device()
644 	zpci_fmb_enable_device(zdev);  in pcibios_enable_device()
651 	struct zpci_dev *zdev = to_zpci(pdev);  in pcibios_disable_device()  local
653 	zpci_fmb_disable_device(zdev);  in pcibios_disable_device()
654 	zpci_debug_exit_device(zdev);  in pcibios_disable_device()
705 int zpci_enable_device(struct zpci_dev *zdev)  in zpci_enable_device()  argument
707 	u32 fh = zdev->fh;  in zpci_enable_device()
710 	if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES))  in zpci_enable_device()
713 		zpci_update_fh(zdev, fh);  in zpci_enable_device()
718 int zpci_disable_device(struct zpci_dev *zdev)  in zpci_disable_device()  argument
720 	u32 fh = zdev->fh;  in zpci_disable_device()
723 	cc = clp_disable_fh(zdev, &fh);  in zpci_disable_device()
725 		zpci_update_fh(zdev, fh);  in zpci_disable_device()
728 			zdev->fid);  in zpci_disable_device()
730 		rc = clp_refresh_fh(zdev->fid, &fh);  in zpci_disable_device()
732 			zpci_update_fh(zdev, fh);  in zpci_disable_device()
765 int zpci_hot_reset_device(struct zpci_dev *zdev)  in zpci_hot_reset_device()  argument
769 	zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);  in zpci_hot_reset_device()
770 	if (zdev_enabled(zdev)) {  in zpci_hot_reset_device()
772 		rc = zpci_disable_device(zdev);  in zpci_hot_reset_device()
784 	rc = zpci_enable_device(zdev);  in zpci_hot_reset_device()
788 	if (zdev->dma_table)  in zpci_hot_reset_device()
789 		rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,  in zpci_hot_reset_device()
790 					virt_to_phys(zdev->dma_table));  in zpci_hot_reset_device()
792 		rc = zpci_dma_init_device(zdev);  in zpci_hot_reset_device()
794 		zpci_disable_device(zdev);  in zpci_hot_reset_device()
814 	struct zpci_dev *zdev;  in zpci_create_device()  local
818 	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);  in zpci_create_device()
819 	if (!zdev)  in zpci_create_device()
823 	zdev->fid = fid;  in zpci_create_device()
824 	zdev->fh = fh;  in zpci_create_device()
827 	rc = clp_query_pci_fn(zdev);  in zpci_create_device()
830 	zdev->state =  state;  in zpci_create_device()
832 	kref_init(&zdev->kref);  in zpci_create_device()
833 	mutex_init(&zdev->lock);  in zpci_create_device()
834 	mutex_init(&zdev->kzdev_lock);  in zpci_create_device()
836 	rc = zpci_init_iommu(zdev);  in zpci_create_device()
840 	rc = zpci_bus_device_register(zdev, &pci_root_ops);  in zpci_create_device()
845 	list_add_tail(&zdev->entry, &zpci_list);  in zpci_create_device()
848 	return zdev;  in zpci_create_device()
851 	zpci_destroy_iommu(zdev);  in zpci_create_device()
854 	kfree(zdev);  in zpci_create_device()
858 bool zpci_is_device_configured(struct zpci_dev *zdev)  in zpci_is_device_configured()  argument
860 	enum zpci_state state = zdev->state;  in zpci_is_device_configured()
879 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh)  in zpci_scan_configured_device()  argument
883 	zpci_update_fh(zdev, fh);  in zpci_scan_configured_device()
885 	if (!zdev->zbus->bus)  in zpci_scan_configured_device()
892 	if (zdev->devfn == 0 && zdev->zbus->multifunction)  in zpci_scan_configured_device()
893 		rc = zpci_bus_scan_bus(zdev->zbus);  in zpci_scan_configured_device()
895 		rc = zpci_bus_scan_device(zdev);  in zpci_scan_configured_device()
910 int zpci_deconfigure_device(struct zpci_dev *zdev)  in zpci_deconfigure_device()  argument
914 	if (zdev->zbus->bus)  in zpci_deconfigure_device()
915 		zpci_bus_remove_device(zdev, false);  in zpci_deconfigure_device()
917 	if (zdev->dma_table) {  in zpci_deconfigure_device()
918 		rc = zpci_dma_exit_device(zdev);  in zpci_deconfigure_device()
922 	if (zdev_enabled(zdev)) {  in zpci_deconfigure_device()
923 		rc = zpci_disable_device(zdev);  in zpci_deconfigure_device()
928 	rc = sclp_pci_deconfigure(zdev->fid);  in zpci_deconfigure_device()
929 	zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc);  in zpci_deconfigure_device()
932 	zdev->state = ZPCI_FN_STATE_STANDBY;  in zpci_deconfigure_device()
946 void zpci_device_reserved(struct zpci_dev *zdev)  in zpci_device_reserved()  argument
948 	if (zdev->has_hp_slot)  in zpci_device_reserved()
949 		zpci_exit_slot(zdev);  in zpci_device_reserved()
955 	list_del(&zdev->entry);  in zpci_device_reserved()
957 	zdev->state = ZPCI_FN_STATE_RESERVED;  in zpci_device_reserved()
958 	zpci_dbg(3, "rsv fid:%x\n", zdev->fid);  in zpci_device_reserved()
959 	zpci_zdev_put(zdev);  in zpci_device_reserved()
964 	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);  in zpci_release_device()  local
967 	if (zdev->zbus->bus)  in zpci_release_device()
968 		zpci_bus_remove_device(zdev, false);  in zpci_release_device()
970 	if (zdev->dma_table)  in zpci_release_device()
971 		zpci_dma_exit_device(zdev);  in zpci_release_device()
972 	if (zdev_enabled(zdev))  in zpci_release_device()
973 		zpci_disable_device(zdev);  in zpci_release_device()
975 	switch (zdev->state) {  in zpci_release_device()
977 		ret = sclp_pci_deconfigure(zdev->fid);  in zpci_release_device()
978 		zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret);  in zpci_release_device()
981 		if (zdev->has_hp_slot)  in zpci_release_device()
982 			zpci_exit_slot(zdev);  in zpci_release_device()
984 		list_del(&zdev->entry);  in zpci_release_device()
986 		zpci_dbg(3, "rsv fid:%x\n", zdev->fid);  in zpci_release_device()
989 		if (zdev->has_resources)  in zpci_release_device()
990 			zpci_cleanup_bus_resources(zdev);  in zpci_release_device()
991 		zpci_bus_device_unregister(zdev);  in zpci_release_device()
992 		zpci_destroy_iommu(zdev);  in zpci_release_device()
997 	zpci_dbg(3, "rem fid:%x\n", zdev->fid);  in zpci_release_device()
998 	kfree(zdev);  in zpci_release_device()
1004 	struct zpci_dev *zdev = to_zpci(pdev);  in zpci_report_error()  local
1006 	return sclp_pci_report(report, zdev->fh, zdev->fid);  in zpci_report_error()
1020 int zpci_clear_error_state(struct zpci_dev *zdev)  in zpci_clear_error_state()  argument
1022 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR);  in zpci_clear_error_state()
1029 		zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);  in zpci_clear_error_state()
1047 int zpci_reset_load_store_blocked(struct zpci_dev *zdev)  in zpci_reset_load_store_blocked()  argument
1049 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK);  in zpci_reset_load_store_blocked()
1056 		zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);  in zpci_reset_load_store_blocked()