1 // SPDX-License-Identifier: GPL-2.0
2 /* XDP user-space packet buffer
3  * Copyright(c) 2018 Intel Corporation.
4  */
5 
6 #include <linux/init.h>
7 #include <linux/sched/mm.h>
8 #include <linux/sched/signal.h>
9 #include <linux/sched/task.h>
10 #include <linux/uaccess.h>
11 #include <linux/slab.h>
12 #include <linux/bpf.h>
13 #include <linux/mm.h>
14 #include <linux/netdevice.h>
15 #include <linux/rtnetlink.h>
16 
17 #include "xdp_umem.h"
18 #include "xsk_queue.h"
19 
20 #define XDP_UMEM_MIN_CHUNK_SIZE 2048
21 
xdp_add_sk_umem(struct xdp_umem * umem,struct xdp_sock * xs)22 void xdp_add_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
23 {
24 	unsigned long flags;
25 
26 	spin_lock_irqsave(&umem->xsk_list_lock, flags);
27 	list_add_rcu(&xs->list, &umem->xsk_list);
28 	spin_unlock_irqrestore(&umem->xsk_list_lock, flags);
29 }
30 
xdp_del_sk_umem(struct xdp_umem * umem,struct xdp_sock * xs)31 void xdp_del_sk_umem(struct xdp_umem *umem, struct xdp_sock *xs)
32 {
33 	unsigned long flags;
34 
35 	if (xs->dev) {
36 		spin_lock_irqsave(&umem->xsk_list_lock, flags);
37 		list_del_rcu(&xs->list);
38 		spin_unlock_irqrestore(&umem->xsk_list_lock, flags);
39 
40 		if (umem->zc)
41 			synchronize_net();
42 	}
43 }
44 
xdp_umem_query(struct net_device * dev,u16 queue_id)45 int xdp_umem_query(struct net_device *dev, u16 queue_id)
46 {
47 	struct netdev_bpf bpf;
48 
49 	ASSERT_RTNL();
50 
51 	memset(&bpf, 0, sizeof(bpf));
52 	bpf.command = XDP_QUERY_XSK_UMEM;
53 	bpf.xsk.queue_id = queue_id;
54 
55 	if (!dev->netdev_ops->ndo_bpf)
56 		return 0;
57 	return dev->netdev_ops->ndo_bpf(dev, &bpf) ?: !!bpf.xsk.umem;
58 }
59 
xdp_umem_assign_dev(struct xdp_umem * umem,struct net_device * dev,u32 queue_id,u16 flags)60 int xdp_umem_assign_dev(struct xdp_umem *umem, struct net_device *dev,
61 			u32 queue_id, u16 flags)
62 {
63 	bool force_zc, force_copy;
64 	struct netdev_bpf bpf;
65 	int err;
66 
67 	force_zc = flags & XDP_ZEROCOPY;
68 	force_copy = flags & XDP_COPY;
69 
70 	if (force_zc && force_copy)
71 		return -EINVAL;
72 
73 	if (force_copy)
74 		return 0;
75 
76 	if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xsk_async_xmit)
77 		return force_zc ? -EOPNOTSUPP : 0; /* fail or fallback */
78 
79 	bpf.command = XDP_QUERY_XSK_UMEM;
80 
81 	rtnl_lock();
82 	err = xdp_umem_query(dev, queue_id);
83 	if (err) {
84 		err = err < 0 ? -EOPNOTSUPP : -EBUSY;
85 		goto err_rtnl_unlock;
86 	}
87 
88 	bpf.command = XDP_SETUP_XSK_UMEM;
89 	bpf.xsk.umem = umem;
90 	bpf.xsk.queue_id = queue_id;
91 
92 	err = dev->netdev_ops->ndo_bpf(dev, &bpf);
93 	if (err)
94 		goto err_rtnl_unlock;
95 	rtnl_unlock();
96 
97 	dev_hold(dev);
98 	umem->dev = dev;
99 	umem->queue_id = queue_id;
100 	umem->zc = true;
101 	return 0;
102 
103 err_rtnl_unlock:
104 	rtnl_unlock();
105 	return force_zc ? err : 0; /* fail or fallback */
106 }
107 
xdp_umem_clear_dev(struct xdp_umem * umem)108 static void xdp_umem_clear_dev(struct xdp_umem *umem)
109 {
110 	struct netdev_bpf bpf;
111 	int err;
112 
113 	if (umem->dev) {
114 		bpf.command = XDP_SETUP_XSK_UMEM;
115 		bpf.xsk.umem = NULL;
116 		bpf.xsk.queue_id = umem->queue_id;
117 
118 		rtnl_lock();
119 		err = umem->dev->netdev_ops->ndo_bpf(umem->dev, &bpf);
120 		rtnl_unlock();
121 
122 		if (err)
123 			WARN(1, "failed to disable umem!\n");
124 
125 		dev_put(umem->dev);
126 		umem->dev = NULL;
127 	}
128 }
129 
xdp_umem_unpin_pages(struct xdp_umem * umem)130 static void xdp_umem_unpin_pages(struct xdp_umem *umem)
131 {
132 	unsigned int i;
133 
134 	for (i = 0; i < umem->npgs; i++) {
135 		struct page *page = umem->pgs[i];
136 
137 		set_page_dirty_lock(page);
138 		put_page(page);
139 	}
140 
141 	kfree(umem->pgs);
142 	umem->pgs = NULL;
143 }
144 
xdp_umem_unaccount_pages(struct xdp_umem * umem)145 static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
146 {
147 	if (umem->user) {
148 		atomic_long_sub(umem->npgs, &umem->user->locked_vm);
149 		free_uid(umem->user);
150 	}
151 }
152 
xdp_umem_release(struct xdp_umem * umem)153 static void xdp_umem_release(struct xdp_umem *umem)
154 {
155 	struct task_struct *task;
156 	struct mm_struct *mm;
157 
158 	xdp_umem_clear_dev(umem);
159 
160 	if (umem->fq) {
161 		xskq_destroy(umem->fq);
162 		umem->fq = NULL;
163 	}
164 
165 	if (umem->cq) {
166 		xskq_destroy(umem->cq);
167 		umem->cq = NULL;
168 	}
169 
170 	xdp_umem_unpin_pages(umem);
171 
172 	task = get_pid_task(umem->pid, PIDTYPE_PID);
173 	put_pid(umem->pid);
174 	if (!task)
175 		goto out;
176 	mm = get_task_mm(task);
177 	put_task_struct(task);
178 	if (!mm)
179 		goto out;
180 
181 	mmput(mm);
182 	kfree(umem->pages);
183 	umem->pages = NULL;
184 
185 	xdp_umem_unaccount_pages(umem);
186 out:
187 	kfree(umem);
188 }
189 
xdp_umem_release_deferred(struct work_struct * work)190 static void xdp_umem_release_deferred(struct work_struct *work)
191 {
192 	struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
193 
194 	xdp_umem_release(umem);
195 }
196 
xdp_get_umem(struct xdp_umem * umem)197 void xdp_get_umem(struct xdp_umem *umem)
198 {
199 	refcount_inc(&umem->users);
200 }
201 
xdp_put_umem(struct xdp_umem * umem)202 void xdp_put_umem(struct xdp_umem *umem)
203 {
204 	if (!umem)
205 		return;
206 
207 	if (refcount_dec_and_test(&umem->users)) {
208 		INIT_WORK(&umem->work, xdp_umem_release_deferred);
209 		schedule_work(&umem->work);
210 	}
211 }
212 
xdp_umem_pin_pages(struct xdp_umem * umem)213 static int xdp_umem_pin_pages(struct xdp_umem *umem)
214 {
215 	unsigned int gup_flags = FOLL_WRITE;
216 	long npgs;
217 	int err;
218 
219 	umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs),
220 			    GFP_KERNEL | __GFP_NOWARN);
221 	if (!umem->pgs)
222 		return -ENOMEM;
223 
224 	down_write(&current->mm->mmap_sem);
225 	npgs = get_user_pages(umem->address, umem->npgs,
226 			      gup_flags, &umem->pgs[0], NULL);
227 	up_write(&current->mm->mmap_sem);
228 
229 	if (npgs != umem->npgs) {
230 		if (npgs >= 0) {
231 			umem->npgs = npgs;
232 			err = -ENOMEM;
233 			goto out_pin;
234 		}
235 		err = npgs;
236 		goto out_pgs;
237 	}
238 	return 0;
239 
240 out_pin:
241 	xdp_umem_unpin_pages(umem);
242 out_pgs:
243 	kfree(umem->pgs);
244 	umem->pgs = NULL;
245 	return err;
246 }
247 
xdp_umem_account_pages(struct xdp_umem * umem)248 static int xdp_umem_account_pages(struct xdp_umem *umem)
249 {
250 	unsigned long lock_limit, new_npgs, old_npgs;
251 
252 	if (capable(CAP_IPC_LOCK))
253 		return 0;
254 
255 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
256 	umem->user = get_uid(current_user());
257 
258 	do {
259 		old_npgs = atomic_long_read(&umem->user->locked_vm);
260 		new_npgs = old_npgs + umem->npgs;
261 		if (new_npgs > lock_limit) {
262 			free_uid(umem->user);
263 			umem->user = NULL;
264 			return -ENOBUFS;
265 		}
266 	} while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
267 				     new_npgs) != old_npgs);
268 	return 0;
269 }
270 
xdp_umem_reg(struct xdp_umem * umem,struct xdp_umem_reg * mr)271 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
272 {
273 	u32 chunk_size = mr->chunk_size, headroom = mr->headroom;
274 	unsigned int chunks, chunks_per_page;
275 	u64 addr = mr->addr, size = mr->len;
276 	int size_chk, err, i;
277 
278 	if (chunk_size < XDP_UMEM_MIN_CHUNK_SIZE || chunk_size > PAGE_SIZE) {
279 		/* Strictly speaking we could support this, if:
280 		 * - huge pages, or*
281 		 * - using an IOMMU, or
282 		 * - making sure the memory area is consecutive
283 		 * but for now, we simply say "computer says no".
284 		 */
285 		return -EINVAL;
286 	}
287 
288 	if (!is_power_of_2(chunk_size))
289 		return -EINVAL;
290 
291 	if (!PAGE_ALIGNED(addr)) {
292 		/* Memory area has to be page size aligned. For
293 		 * simplicity, this might change.
294 		 */
295 		return -EINVAL;
296 	}
297 
298 	if ((addr + size) < addr)
299 		return -EINVAL;
300 
301 	chunks = (unsigned int)div_u64(size, chunk_size);
302 	if (chunks == 0)
303 		return -EINVAL;
304 
305 	chunks_per_page = PAGE_SIZE / chunk_size;
306 	if (chunks < chunks_per_page || chunks % chunks_per_page)
307 		return -EINVAL;
308 
309 	headroom = ALIGN(headroom, 64);
310 
311 	size_chk = chunk_size - headroom - XDP_PACKET_HEADROOM;
312 	if (size_chk < 0)
313 		return -EINVAL;
314 
315 	umem->pid = get_task_pid(current, PIDTYPE_PID);
316 	umem->address = (unsigned long)addr;
317 	umem->props.chunk_mask = ~((u64)chunk_size - 1);
318 	umem->props.size = size;
319 	umem->headroom = headroom;
320 	umem->chunk_size_nohr = chunk_size - headroom;
321 	umem->npgs = size / PAGE_SIZE;
322 	umem->pgs = NULL;
323 	umem->user = NULL;
324 	INIT_LIST_HEAD(&umem->xsk_list);
325 	spin_lock_init(&umem->xsk_list_lock);
326 
327 	refcount_set(&umem->users, 1);
328 
329 	err = xdp_umem_account_pages(umem);
330 	if (err)
331 		goto out;
332 
333 	err = xdp_umem_pin_pages(umem);
334 	if (err)
335 		goto out_account;
336 
337 	umem->pages = kcalloc(umem->npgs, sizeof(*umem->pages), GFP_KERNEL);
338 	if (!umem->pages) {
339 		err = -ENOMEM;
340 		goto out_account;
341 	}
342 
343 	for (i = 0; i < umem->npgs; i++)
344 		umem->pages[i].addr = page_address(umem->pgs[i]);
345 
346 	return 0;
347 
348 out_account:
349 	xdp_umem_unaccount_pages(umem);
350 out:
351 	put_pid(umem->pid);
352 	return err;
353 }
354 
xdp_umem_create(struct xdp_umem_reg * mr)355 struct xdp_umem *xdp_umem_create(struct xdp_umem_reg *mr)
356 {
357 	struct xdp_umem *umem;
358 	int err;
359 
360 	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
361 	if (!umem)
362 		return ERR_PTR(-ENOMEM);
363 
364 	err = xdp_umem_reg(umem, mr);
365 	if (err) {
366 		kfree(umem);
367 		return ERR_PTR(err);
368 	}
369 
370 	return umem;
371 }
372 
xdp_umem_validate_queues(struct xdp_umem * umem)373 bool xdp_umem_validate_queues(struct xdp_umem *umem)
374 {
375 	return umem->fq && umem->cq;
376 }
377