1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IOMMU API for s390 PCI devices
4  *
5  * Copyright IBM Corp. 2015
6  * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
7  */
8 
9 #include <linux/pci.h>
10 #include <linux/iommu.h>
11 #include <linux/iommu-helper.h>
12 #include <linux/sizes.h>
13 #include <asm/pci_dma.h>
14 
15 /*
16  * Physically contiguous memory regions can be mapped with 4 KiB alignment,
17  * we allow all page sizes that are an order of 4KiB (no special large page
18  * support so far).
19  */
20 #define S390_IOMMU_PGSIZES	(~0xFFFUL)
21 
22 static const struct iommu_ops s390_iommu_ops;
23 
24 struct s390_domain {
25 	struct iommu_domain	domain;
26 	struct list_head	devices;
27 	unsigned long		*dma_table;
28 	spinlock_t		dma_table_lock;
29 	spinlock_t		list_lock;
30 };
31 
32 struct s390_domain_device {
33 	struct list_head	list;
34 	struct zpci_dev		*zdev;
35 };
36 
to_s390_domain(struct iommu_domain * dom)37 static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
38 {
39 	return container_of(dom, struct s390_domain, domain);
40 }
41 
s390_iommu_capable(enum iommu_cap cap)42 static bool s390_iommu_capable(enum iommu_cap cap)
43 {
44 	switch (cap) {
45 	case IOMMU_CAP_CACHE_COHERENCY:
46 		return true;
47 	case IOMMU_CAP_INTR_REMAP:
48 		return true;
49 	default:
50 		return false;
51 	}
52 }
53 
s390_domain_alloc(unsigned domain_type)54 static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
55 {
56 	struct s390_domain *s390_domain;
57 
58 	if (domain_type != IOMMU_DOMAIN_UNMANAGED)
59 		return NULL;
60 
61 	s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
62 	if (!s390_domain)
63 		return NULL;
64 
65 	s390_domain->dma_table = dma_alloc_cpu_table();
66 	if (!s390_domain->dma_table) {
67 		kfree(s390_domain);
68 		return NULL;
69 	}
70 
71 	spin_lock_init(&s390_domain->dma_table_lock);
72 	spin_lock_init(&s390_domain->list_lock);
73 	INIT_LIST_HEAD(&s390_domain->devices);
74 
75 	return &s390_domain->domain;
76 }
77 
s390_domain_free(struct iommu_domain * domain)78 static void s390_domain_free(struct iommu_domain *domain)
79 {
80 	struct s390_domain *s390_domain = to_s390_domain(domain);
81 
82 	dma_cleanup_tables(s390_domain->dma_table);
83 	kfree(s390_domain);
84 }
85 
s390_iommu_attach_device(struct iommu_domain * domain,struct device * dev)86 static int s390_iommu_attach_device(struct iommu_domain *domain,
87 				    struct device *dev)
88 {
89 	struct s390_domain *s390_domain = to_s390_domain(domain);
90 	struct zpci_dev *zdev = to_zpci_dev(dev);
91 	struct s390_domain_device *domain_device;
92 	unsigned long flags;
93 	int rc;
94 
95 	if (!zdev)
96 		return -ENODEV;
97 
98 	domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
99 	if (!domain_device)
100 		return -ENOMEM;
101 
102 	if (zdev->dma_table)
103 		zpci_dma_exit_device(zdev);
104 
105 	zdev->dma_table = s390_domain->dma_table;
106 	rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
107 				(u64) zdev->dma_table);
108 	if (rc)
109 		goto out_restore;
110 
111 	spin_lock_irqsave(&s390_domain->list_lock, flags);
112 	/* First device defines the DMA range limits */
113 	if (list_empty(&s390_domain->devices)) {
114 		domain->geometry.aperture_start = zdev->start_dma;
115 		domain->geometry.aperture_end = zdev->end_dma;
116 		domain->geometry.force_aperture = true;
117 	/* Allow only devices with identical DMA range limits */
118 	} else if (domain->geometry.aperture_start != zdev->start_dma ||
119 		   domain->geometry.aperture_end != zdev->end_dma) {
120 		rc = -EINVAL;
121 		spin_unlock_irqrestore(&s390_domain->list_lock, flags);
122 		goto out_restore;
123 	}
124 	domain_device->zdev = zdev;
125 	zdev->s390_domain = s390_domain;
126 	list_add(&domain_device->list, &s390_domain->devices);
127 	spin_unlock_irqrestore(&s390_domain->list_lock, flags);
128 
129 	return 0;
130 
131 out_restore:
132 	zpci_dma_init_device(zdev);
133 	kfree(domain_device);
134 
135 	return rc;
136 }
137 
s390_iommu_detach_device(struct iommu_domain * domain,struct device * dev)138 static void s390_iommu_detach_device(struct iommu_domain *domain,
139 				     struct device *dev)
140 {
141 	struct s390_domain *s390_domain = to_s390_domain(domain);
142 	struct zpci_dev *zdev = to_zpci_dev(dev);
143 	struct s390_domain_device *domain_device, *tmp;
144 	unsigned long flags;
145 	int found = 0;
146 
147 	if (!zdev)
148 		return;
149 
150 	spin_lock_irqsave(&s390_domain->list_lock, flags);
151 	list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
152 				 list) {
153 		if (domain_device->zdev == zdev) {
154 			list_del(&domain_device->list);
155 			kfree(domain_device);
156 			found = 1;
157 			break;
158 		}
159 	}
160 	spin_unlock_irqrestore(&s390_domain->list_lock, flags);
161 
162 	if (found) {
163 		zdev->s390_domain = NULL;
164 		zpci_unregister_ioat(zdev, 0);
165 		zpci_dma_init_device(zdev);
166 	}
167 }
168 
s390_iommu_probe_device(struct device * dev)169 static struct iommu_device *s390_iommu_probe_device(struct device *dev)
170 {
171 	struct zpci_dev *zdev = to_zpci_dev(dev);
172 
173 	return &zdev->iommu_dev;
174 }
175 
s390_iommu_release_device(struct device * dev)176 static void s390_iommu_release_device(struct device *dev)
177 {
178 	struct zpci_dev *zdev = to_zpci_dev(dev);
179 	struct iommu_domain *domain;
180 
181 	/*
182 	 * This is a workaround for a scenario where the IOMMU API common code
183 	 * "forgets" to call the detach_dev callback: After binding a device
184 	 * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
185 	 * the attach_dev), removing the device via
186 	 * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
187 	 * only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
188 	 * notifier.
189 	 *
190 	 * So let's call detach_dev from here if it hasn't been called before.
191 	 */
192 	if (zdev && zdev->s390_domain) {
193 		domain = iommu_get_domain_for_dev(dev);
194 		if (domain)
195 			s390_iommu_detach_device(domain, dev);
196 	}
197 }
198 
s390_iommu_update_trans(struct s390_domain * s390_domain,unsigned long pa,dma_addr_t dma_addr,size_t size,int flags)199 static int s390_iommu_update_trans(struct s390_domain *s390_domain,
200 				   unsigned long pa, dma_addr_t dma_addr,
201 				   size_t size, int flags)
202 {
203 	struct s390_domain_device *domain_device;
204 	u8 *page_addr = (u8 *) (pa & PAGE_MASK);
205 	dma_addr_t start_dma_addr = dma_addr;
206 	unsigned long irq_flags, nr_pages, i;
207 	unsigned long *entry;
208 	int rc = 0;
209 
210 	if (dma_addr < s390_domain->domain.geometry.aperture_start ||
211 	    dma_addr + size > s390_domain->domain.geometry.aperture_end)
212 		return -EINVAL;
213 
214 	nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
215 	if (!nr_pages)
216 		return 0;
217 
218 	spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
219 	for (i = 0; i < nr_pages; i++) {
220 		entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
221 		if (!entry) {
222 			rc = -ENOMEM;
223 			goto undo_cpu_trans;
224 		}
225 		dma_update_cpu_trans(entry, page_addr, flags);
226 		page_addr += PAGE_SIZE;
227 		dma_addr += PAGE_SIZE;
228 	}
229 
230 	spin_lock(&s390_domain->list_lock);
231 	list_for_each_entry(domain_device, &s390_domain->devices, list) {
232 		rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
233 					start_dma_addr, nr_pages * PAGE_SIZE);
234 		if (rc)
235 			break;
236 	}
237 	spin_unlock(&s390_domain->list_lock);
238 
239 undo_cpu_trans:
240 	if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
241 		flags = ZPCI_PTE_INVALID;
242 		while (i-- > 0) {
243 			page_addr -= PAGE_SIZE;
244 			dma_addr -= PAGE_SIZE;
245 			entry = dma_walk_cpu_trans(s390_domain->dma_table,
246 						   dma_addr);
247 			if (!entry)
248 				break;
249 			dma_update_cpu_trans(entry, page_addr, flags);
250 		}
251 	}
252 	spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
253 
254 	return rc;
255 }
256 
s390_iommu_map(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t size,int prot,gfp_t gfp)257 static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
258 			  phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
259 {
260 	struct s390_domain *s390_domain = to_s390_domain(domain);
261 	int flags = ZPCI_PTE_VALID, rc = 0;
262 
263 	if (!(prot & IOMMU_READ))
264 		return -EINVAL;
265 
266 	if (!(prot & IOMMU_WRITE))
267 		flags |= ZPCI_TABLE_PROTECTED;
268 
269 	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
270 				     size, flags);
271 
272 	return rc;
273 }
274 
s390_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)275 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
276 					   dma_addr_t iova)
277 {
278 	struct s390_domain *s390_domain = to_s390_domain(domain);
279 	unsigned long *sto, *pto, *rto, flags;
280 	unsigned int rtx, sx, px;
281 	phys_addr_t phys = 0;
282 
283 	if (iova < domain->geometry.aperture_start ||
284 	    iova > domain->geometry.aperture_end)
285 		return 0;
286 
287 	rtx = calc_rtx(iova);
288 	sx = calc_sx(iova);
289 	px = calc_px(iova);
290 	rto = s390_domain->dma_table;
291 
292 	spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
293 	if (rto && reg_entry_isvalid(rto[rtx])) {
294 		sto = get_rt_sto(rto[rtx]);
295 		if (sto && reg_entry_isvalid(sto[sx])) {
296 			pto = get_st_pto(sto[sx]);
297 			if (pto && pt_entry_isvalid(pto[px]))
298 				phys = pto[px] & ZPCI_PTE_ADDR_MASK;
299 		}
300 	}
301 	spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
302 
303 	return phys;
304 }
305 
s390_iommu_unmap(struct iommu_domain * domain,unsigned long iova,size_t size,struct iommu_iotlb_gather * gather)306 static size_t s390_iommu_unmap(struct iommu_domain *domain,
307 			       unsigned long iova, size_t size,
308 			       struct iommu_iotlb_gather *gather)
309 {
310 	struct s390_domain *s390_domain = to_s390_domain(domain);
311 	int flags = ZPCI_PTE_INVALID;
312 	phys_addr_t paddr;
313 	int rc;
314 
315 	paddr = s390_iommu_iova_to_phys(domain, iova);
316 	if (!paddr)
317 		return 0;
318 
319 	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
320 				     size, flags);
321 	if (rc)
322 		return 0;
323 
324 	return size;
325 }
326 
zpci_init_iommu(struct zpci_dev * zdev)327 int zpci_init_iommu(struct zpci_dev *zdev)
328 {
329 	int rc = 0;
330 
331 	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
332 				    "s390-iommu.%08x", zdev->fid);
333 	if (rc)
334 		goto out_err;
335 
336 	iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
337 
338 	rc = iommu_device_register(&zdev->iommu_dev);
339 	if (rc)
340 		goto out_sysfs;
341 
342 	return 0;
343 
344 out_sysfs:
345 	iommu_device_sysfs_remove(&zdev->iommu_dev);
346 
347 out_err:
348 	return rc;
349 }
350 
zpci_destroy_iommu(struct zpci_dev * zdev)351 void zpci_destroy_iommu(struct zpci_dev *zdev)
352 {
353 	iommu_device_unregister(&zdev->iommu_dev);
354 	iommu_device_sysfs_remove(&zdev->iommu_dev);
355 }
356 
357 static const struct iommu_ops s390_iommu_ops = {
358 	.capable = s390_iommu_capable,
359 	.domain_alloc = s390_domain_alloc,
360 	.domain_free = s390_domain_free,
361 	.attach_dev = s390_iommu_attach_device,
362 	.detach_dev = s390_iommu_detach_device,
363 	.map = s390_iommu_map,
364 	.unmap = s390_iommu_unmap,
365 	.iova_to_phys = s390_iommu_iova_to_phys,
366 	.probe_device = s390_iommu_probe_device,
367 	.release_device = s390_iommu_release_device,
368 	.device_group = generic_device_group,
369 	.pgsize_bitmap = S390_IOMMU_PGSIZES,
370 };
371 
s390_iommu_init(void)372 static int __init s390_iommu_init(void)
373 {
374 	return bus_set_iommu(&pci_bus_type, &s390_iommu_ops);
375 }
376 subsys_initcall(s390_iommu_init);
377