1 /*
2  * SPDX-License-Identifier: Apache-2.0
3  *
4  * Copyright 2020 Broadcom
5  *
6  */
7 
8 #include <zephyr/drivers/pcie/endpoint/pcie_ep.h>
9 #include <string.h>
10 #include <zephyr/sys/util.h>
11 
12 /*
13  * During DEVICE_TO_HOST data transfer, in order to make sure that all
14  * PCIe writes (posted) have reached Host, i.e. to flush PCIe writes,
15  * we need to add a dummy PCIe read (non posted transaction) after each
16  * DEVICE_TO_HOST data transfer.
17  *
18  * There are a few possible scenarios, where we need to *place*
19  * a dummy PCIe read.
20  * All possible scenarios are captured in the table below.
21  *
22  * As can be seen in the table, for 64-bit systems, we could just do sys_read8
23  * on mapped Host address to generate a dummy PCIe read, before unmapping the
24  * address - irrespective of low/high outbound memory usage as core is capable
25  * of accessing highmem.
26  * Basically, we issue single byte PCIe read with sys_read8.
27  *
28  * For 32-bit systems, if using low outbound memory for memcpy/DMA,
29  * we could do sys_read8 on the mapped address.
30  * But, for 32-bit systems using high outbound memory for DMA operation,
31  * sys_read8 is not possible, as the core cannot access highmem.
32  * In this case, we need to *explicitly* perform PCIe read.
33  *
34  * +-------------+----------------------+-------------------------------------+
35  * |     Core    |  Data transfer with  | OB memory type | Dummy PCIe read    |
36  * +-------------+----------------------+----------------+--------------------+
37  * | 64-bit      |                      |    highmem     |     sys_read8      |
38  * |             |       memcpy         |----------------+--------------------+
39  * |             |                      |    lowmem      |     sys_read8      |
40  * | (e.g.       +----------------------+----------------+--------------------+
41  * | Cortex-A72) |                      |    highmem     |     sys_read8      |
42  * |             |       DMA            |----------------+--------------------+
43  * |             |                      |    lowmem      |     sys_read8      |
44  * +-------------+----------------------+----------------+--------------------+
45  * | 32-bit      |                      |    highmem     |        NA          |
46  * |             |       memcpy         |----------------+--------------------+
47  * |             |                      |    lowmem      |     sys_read8      |
48  * | (e.g.       +----------------------+----------------+--------------------+
49  * | Cortex-M7)  |                      |    highmem     | Explicit PCIe read |
50  * |             |       DMA            |----------------+--------------------+
51  * |             |                      |    lowmem      |     sys_read8      |
52  * +-------------+----------------------+----------------+--------------------+
53  *
54  * Based on this explanation, the 2 common APIs below, namely
55  * pcie_ep_xfer_data_memcpy and pcie_ep_xfer_data_dma
56  * are implemented with dummy PCIe read, phew!
57  */
58 
pcie_ep_mapped_copy(uint64_t mapped_addr,uintptr_t local_addr,const uint32_t size,const enum xfer_direction dir)59 static int pcie_ep_mapped_copy(uint64_t mapped_addr, uintptr_t local_addr,
60 			       const uint32_t size,
61 			       const enum xfer_direction dir)
62 {
63 	/*
64 	 * Make sure that address can be generated by core, this condition
65 	 * would not hit if proper pcie_ob_mem_type is passed by core
66 	 */
67 	if ((!IS_ENABLED(CONFIG_64BIT)) && (mapped_addr >> 32)) {
68 		return -EINVAL;
69 	}
70 
71 	if (dir == DEVICE_TO_HOST) {
72 		memcpy(UINT_TO_POINTER(mapped_addr),
73 		       UINT_TO_POINTER(local_addr), size);
74 	} else {
75 		memcpy(UINT_TO_POINTER(local_addr),
76 		       UINT_TO_POINTER(mapped_addr), size);
77 	}
78 
79 	return 0;
80 }
81 
82 /*
83  * Helper API to achieve data transfer with memcpy operation
84  * through PCIe outbound memory
85  */
pcie_ep_xfer_data_memcpy(const struct device * dev,uint64_t pcie_addr,uintptr_t * local_addr,uint32_t size,enum pcie_ob_mem_type ob_mem_type,enum xfer_direction dir)86 int pcie_ep_xfer_data_memcpy(const struct device *dev, uint64_t pcie_addr,
87 			     uintptr_t *local_addr, uint32_t size,
88 			     enum pcie_ob_mem_type ob_mem_type,
89 			     enum xfer_direction dir)
90 {
91 	uint64_t mapped_addr;
92 	int mapped_size, ret;
93 	uint32_t xfer_size, unmapped_size;
94 
95 	/* Map pcie_addr to outbound memory */
96 	mapped_size = pcie_ep_map_addr(dev, pcie_addr, &mapped_addr,
97 				       size, ob_mem_type);
98 	/* Check if outbound memory mapping succeeded */
99 	if (mapped_size < 0) {
100 		return mapped_size;
101 	}
102 
103 	ret = pcie_ep_mapped_copy(mapped_addr, (uintptr_t)local_addr,
104 				  mapped_size, dir);
105 
106 	/* Check if mapped_copy succeeded */
107 	if (ret < 0) {
108 		goto out_unmap;
109 	}
110 
111 	/* Flush the PCIe writes upon successful memcpy */
112 	if (dir == DEVICE_TO_HOST) {
113 		sys_read8(mapped_addr);
114 	}
115 
116 	/* Check if we achieved data transfer for given size */
117 	if (mapped_size == size) {
118 		ret = 0;
119 		goto out_unmap;
120 	}
121 
122 	/*
123 	 * In normal case, we are done with data transfer by now,
124 	 * but some PCIe address translation hardware requires us to
125 	 * align Host address to be mapped to the translation window size.
126 	 * So, even though translation window size is good enough for
127 	 * size of Host buffer, we may not be able to map entire Host buffer
128 	 * to given outbound window in one time, and we may need to map
129 	 * remaining size and complete remaining data transfer
130 	 */
131 
132 	pcie_ep_unmap_addr(dev, mapped_addr); /* unmap previous Host buffer */
133 	xfer_size = mapped_size; /* save already transferred data size */
134 
135 	unmapped_size = size - mapped_size;
136 	mapped_size = pcie_ep_map_addr(dev, pcie_addr + xfer_size,
137 				       &mapped_addr, unmapped_size,
138 				       ob_mem_type);
139 	/* Check if outbound memory mapping succeeded */
140 	if (mapped_size < 0) {
141 		return mapped_size;
142 	}
143 
144 	/*
145 	 * In second attempt, we must have mapped entire size,
146 	 * if not just quit here before attempting memcpy
147 	 */
148 	if (mapped_size != unmapped_size) {
149 		ret = -EIO;
150 		goto out_unmap;
151 	}
152 
153 	ret = pcie_ep_mapped_copy(mapped_addr,
154 				  ((uintptr_t)local_addr) + xfer_size,
155 				  mapped_size, dir);
156 	/* Flush the PCIe writes upon successful memcpy */
157 	if (!ret && (dir == DEVICE_TO_HOST)) {
158 		sys_read8(mapped_addr);
159 	}
160 out_unmap:
161 	pcie_ep_unmap_addr(dev, mapped_addr);
162 	return ret;
163 }
164 
165 /*
166  * Helper API to achieve data transfer with DMA operation through
167  * PCIe outbound memory, this API is based off pcie_ep_xfer_data_memcpy,
168  * here we use "system dma" instead of memcpy
169  */
pcie_ep_xfer_data_dma(const struct device * dev,uint64_t pcie_addr,uintptr_t * local_addr,uint32_t size,enum pcie_ob_mem_type ob_mem_type,enum xfer_direction dir)170 int pcie_ep_xfer_data_dma(const struct device *dev, uint64_t pcie_addr,
171 			  uintptr_t *local_addr, uint32_t size,
172 			  enum pcie_ob_mem_type ob_mem_type,
173 			  enum xfer_direction dir)
174 {
175 	uint64_t mapped_addr;
176 	int mapped_size, ret;
177 	uint32_t xfer_size, unmapped_size;
178 	uint32_t dummy_data;	/* For explicit dummy PCIe read */
179 
180 	/* Map pcie_addr to outbound memory */
181 	mapped_size = pcie_ep_map_addr(dev, pcie_addr, &mapped_addr,
182 				       size, ob_mem_type);
183 	/* Check if outbound memory mapping succeeded */
184 	if (mapped_size < 0) {
185 		return mapped_size;
186 	}
187 
188 	ret = pcie_ep_dma_xfer(dev, mapped_addr, (uintptr_t)local_addr,
189 			       mapped_size, dir);
190 	/* Check if dma succeeded */
191 	if (ret < 0) {
192 		goto out_unmap;
193 	}
194 
195 	/* Flush the PCIe writes upon successful DMA */
196 	if (dir == DEVICE_TO_HOST) {
197 		if (IS_ENABLED(CONFIG_64BIT) || !(mapped_addr >> 32)) {
198 			sys_read8(mapped_addr);
199 		}
200 	}
201 
202 	pcie_ep_unmap_addr(dev, mapped_addr);
203 
204 	/*
205 	 * Explicit PCIe read to flush PCIe writes for 32-bit system
206 	 * using high outbound memory for DMA operation
207 	 */
208 	if (dir == DEVICE_TO_HOST && (!IS_ENABLED(CONFIG_64BIT)) &&
209 	    (mapped_addr >> 32)) {
210 		ret = pcie_ep_xfer_data_memcpy(dev, pcie_addr,
211 					       (uintptr_t *)&dummy_data,
212 					       sizeof(dummy_data),
213 					       PCIE_OB_LOWMEM, HOST_TO_DEVICE);
214 		if (ret < 0) {
215 			return ret;
216 		}
217 	}
218 
219 	/* Check if we achieved data transfer for given size */
220 	if (mapped_size == size) {
221 		return 0;
222 	}
223 
224 	/* map remaining size and complete remaining data transfer */
225 
226 	xfer_size = mapped_size; /* save already transferred data size */
227 
228 	unmapped_size = size - mapped_size;
229 	mapped_size = pcie_ep_map_addr(dev, pcie_addr + xfer_size,
230 				       &mapped_addr, unmapped_size,
231 				       ob_mem_type);
232 	/* Check if outbound memory mapping succeeded */
233 	if (mapped_size < 0) {
234 		return mapped_size;
235 	}
236 
237 	/*
238 	 * In second attempt, we must have mapped entire size,
239 	 * if not just quit here before attempting dma
240 	 */
241 	if (mapped_size != unmapped_size) {
242 		ret = -EIO;
243 		goto out_unmap;
244 	}
245 
246 	ret = pcie_ep_dma_xfer(dev, mapped_addr,
247 			       ((uintptr_t)local_addr) + xfer_size,
248 			       mapped_size, dir);
249 	/* Check if dma copy succeeded */
250 	if (ret < 0) {
251 		goto out_unmap;
252 	}
253 
254 	/* Flush the PCIe writes upon successful DMA */
255 	if (dir == DEVICE_TO_HOST) {
256 		if (IS_ENABLED(CONFIG_64BIT) || !(mapped_addr >> 32)) {
257 			sys_read8(mapped_addr);
258 		}
259 	}
260 
261 	pcie_ep_unmap_addr(dev, mapped_addr);
262 
263 	/*
264 	 * Explicit PCIe read to flush PCIe writes for 32-bit system
265 	 * using high outbound memory for DMA operation
266 	 */
267 	if (dir == DEVICE_TO_HOST && (!IS_ENABLED(CONFIG_64BIT)) &&
268 	    (mapped_addr >> 32)) {
269 		ret = pcie_ep_xfer_data_memcpy(dev, pcie_addr,
270 					       (uintptr_t *)&dummy_data,
271 					       sizeof(dummy_data),
272 					       PCIE_OB_LOWMEM, HOST_TO_DEVICE);
273 	}
274 
275 	return ret;
276 out_unmap:
277 	pcie_ep_unmap_addr(dev, mapped_addr);
278 	return ret;
279 }
280