1 // SPDX-License-Identifier: GPL-2.0
2 #define pr_fmt(fmt) "kcov: " fmt
3
4 #define DISABLE_BRANCH_PROFILING
5 #include <linux/atomic.h>
6 #include <linux/compiler.h>
7 #include <linux/errno.h>
8 #include <linux/export.h>
9 #include <linux/types.h>
10 #include <linux/file.h>
11 #include <linux/fs.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/preempt.h>
15 #include <linux/printk.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/vmalloc.h>
20 #include <linux/debugfs.h>
21 #include <linux/uaccess.h>
22 #include <linux/kcov.h>
23 #include <linux/refcount.h>
24 #include <asm/setup.h>
25
26 /* Number of 64-bit words written per one comparison: */
27 #define KCOV_WORDS_PER_CMP 4
28
29 /*
30 * kcov descriptor (one per opened debugfs file).
31 * State transitions of the descriptor:
32 * - initial state after open()
33 * - then there must be a single ioctl(KCOV_INIT_TRACE) call
34 * - then, mmap() call (several calls are allowed but not useful)
35 * - then, ioctl(KCOV_ENABLE, arg), where arg is
36 * KCOV_TRACE_PC - to trace only the PCs
37 * or
38 * KCOV_TRACE_CMP - to trace only the comparison operands
39 * - then, ioctl(KCOV_DISABLE) to disable the task.
40 * Enabling/disabling ioctls can be repeated (only one task a time allowed).
41 */
42 struct kcov {
43 /*
44 * Reference counter. We keep one for:
45 * - opened file descriptor
46 * - task with enabled coverage (we can't unwire it from another task)
47 */
48 refcount_t refcount;
49 /* The lock protects mode, size, area and t. */
50 spinlock_t lock;
51 enum kcov_mode mode;
52 /* Size of arena (in long's for KCOV_MODE_TRACE). */
53 unsigned size;
54 /* Coverage buffer shared with user space. */
55 void *area;
56 /* Task for which we collect coverage, or NULL. */
57 struct task_struct *t;
58 };
59
check_kcov_mode(enum kcov_mode needed_mode,struct task_struct * t)60 static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
61 {
62 unsigned int mode;
63
64 /*
65 * We are interested in code coverage as a function of a syscall inputs,
66 * so we ignore code executed in interrupts.
67 */
68 if (!in_task())
69 return false;
70 mode = READ_ONCE(t->kcov_mode);
71 /*
72 * There is some code that runs in interrupts but for which
73 * in_interrupt() returns false (e.g. preempt_schedule_irq()).
74 * READ_ONCE()/barrier() effectively provides load-acquire wrt
75 * interrupts, there are paired barrier()/WRITE_ONCE() in
76 * kcov_ioctl_locked().
77 */
78 barrier();
79 return mode == needed_mode;
80 }
81
canonicalize_ip(unsigned long ip)82 static notrace unsigned long canonicalize_ip(unsigned long ip)
83 {
84 #ifdef CONFIG_RANDOMIZE_BASE
85 ip -= kaslr_offset();
86 #endif
87 return ip;
88 }
89
90 /*
91 * Entry point from instrumented code.
92 * This is called once per basic-block/edge.
93 */
__sanitizer_cov_trace_pc(void)94 void notrace __sanitizer_cov_trace_pc(void)
95 {
96 struct task_struct *t;
97 unsigned long *area;
98 unsigned long ip = canonicalize_ip(_RET_IP_);
99 unsigned long pos;
100
101 t = current;
102 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
103 return;
104
105 area = t->kcov_area;
106 /* The first 64-bit word is the number of subsequent PCs. */
107 pos = READ_ONCE(area[0]) + 1;
108 if (likely(pos < t->kcov_size)) {
109 area[pos] = ip;
110 WRITE_ONCE(area[0], pos);
111 }
112 }
113 EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
114
115 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
write_comp_data(u64 type,u64 arg1,u64 arg2,u64 ip)116 static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
117 {
118 struct task_struct *t;
119 u64 *area;
120 u64 count, start_index, end_pos, max_pos;
121
122 t = current;
123 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
124 return;
125
126 ip = canonicalize_ip(ip);
127
128 /*
129 * We write all comparison arguments and types as u64.
130 * The buffer was allocated for t->kcov_size unsigned longs.
131 */
132 area = (u64 *)t->kcov_area;
133 max_pos = t->kcov_size * sizeof(unsigned long);
134
135 count = READ_ONCE(area[0]);
136
137 /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
138 start_index = 1 + count * KCOV_WORDS_PER_CMP;
139 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
140 if (likely(end_pos <= max_pos)) {
141 area[start_index] = type;
142 area[start_index + 1] = arg1;
143 area[start_index + 2] = arg2;
144 area[start_index + 3] = ip;
145 WRITE_ONCE(area[0], count + 1);
146 }
147 }
148
__sanitizer_cov_trace_cmp1(u8 arg1,u8 arg2)149 void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
150 {
151 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
152 }
153 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
154
__sanitizer_cov_trace_cmp2(u16 arg1,u16 arg2)155 void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
156 {
157 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
158 }
159 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
160
__sanitizer_cov_trace_cmp4(u32 arg1,u32 arg2)161 void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
162 {
163 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
164 }
165 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
166
__sanitizer_cov_trace_cmp8(u64 arg1,u64 arg2)167 void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
168 {
169 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
170 }
171 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
172
__sanitizer_cov_trace_const_cmp1(u8 arg1,u8 arg2)173 void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
174 {
175 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
176 _RET_IP_);
177 }
178 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
179
__sanitizer_cov_trace_const_cmp2(u16 arg1,u16 arg2)180 void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
181 {
182 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
183 _RET_IP_);
184 }
185 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
186
__sanitizer_cov_trace_const_cmp4(u32 arg1,u32 arg2)187 void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
188 {
189 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
190 _RET_IP_);
191 }
192 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
193
__sanitizer_cov_trace_const_cmp8(u64 arg1,u64 arg2)194 void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
195 {
196 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
197 _RET_IP_);
198 }
199 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
200
__sanitizer_cov_trace_switch(u64 val,u64 * cases)201 void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
202 {
203 u64 i;
204 u64 count = cases[0];
205 u64 size = cases[1];
206 u64 type = KCOV_CMP_CONST;
207
208 switch (size) {
209 case 8:
210 type |= KCOV_CMP_SIZE(0);
211 break;
212 case 16:
213 type |= KCOV_CMP_SIZE(1);
214 break;
215 case 32:
216 type |= KCOV_CMP_SIZE(2);
217 break;
218 case 64:
219 type |= KCOV_CMP_SIZE(3);
220 break;
221 default:
222 return;
223 }
224 for (i = 0; i < count; i++)
225 write_comp_data(type, cases[i + 2], val, _RET_IP_);
226 }
227 EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
228 #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
229
kcov_get(struct kcov * kcov)230 static void kcov_get(struct kcov *kcov)
231 {
232 refcount_inc(&kcov->refcount);
233 }
234
kcov_put(struct kcov * kcov)235 static void kcov_put(struct kcov *kcov)
236 {
237 if (refcount_dec_and_test(&kcov->refcount)) {
238 vfree(kcov->area);
239 kfree(kcov);
240 }
241 }
242
kcov_task_init(struct task_struct * t)243 void kcov_task_init(struct task_struct *t)
244 {
245 WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
246 barrier();
247 t->kcov_size = 0;
248 t->kcov_area = NULL;
249 t->kcov = NULL;
250 }
251
kcov_task_exit(struct task_struct * t)252 void kcov_task_exit(struct task_struct *t)
253 {
254 struct kcov *kcov;
255
256 kcov = t->kcov;
257 if (kcov == NULL)
258 return;
259 spin_lock(&kcov->lock);
260 if (WARN_ON(kcov->t != t)) {
261 spin_unlock(&kcov->lock);
262 return;
263 }
264 /* Just to not leave dangling references behind. */
265 kcov_task_init(t);
266 kcov->t = NULL;
267 kcov->mode = KCOV_MODE_INIT;
268 spin_unlock(&kcov->lock);
269 kcov_put(kcov);
270 }
271
kcov_mmap(struct file * filep,struct vm_area_struct * vma)272 static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
273 {
274 int res = 0;
275 void *area;
276 struct kcov *kcov = vma->vm_file->private_data;
277 unsigned long size, off;
278 struct page *page;
279
280 area = vmalloc_user(vma->vm_end - vma->vm_start);
281 if (!area)
282 return -ENOMEM;
283
284 spin_lock(&kcov->lock);
285 size = kcov->size * sizeof(unsigned long);
286 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
287 vma->vm_end - vma->vm_start != size) {
288 res = -EINVAL;
289 goto exit;
290 }
291 if (!kcov->area) {
292 kcov->area = area;
293 vma->vm_flags |= VM_DONTEXPAND;
294 spin_unlock(&kcov->lock);
295 for (off = 0; off < size; off += PAGE_SIZE) {
296 page = vmalloc_to_page(kcov->area + off);
297 if (vm_insert_page(vma, vma->vm_start + off, page))
298 WARN_ONCE(1, "vm_insert_page() failed");
299 }
300 return 0;
301 }
302 exit:
303 spin_unlock(&kcov->lock);
304 vfree(area);
305 return res;
306 }
307
kcov_open(struct inode * inode,struct file * filep)308 static int kcov_open(struct inode *inode, struct file *filep)
309 {
310 struct kcov *kcov;
311
312 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
313 if (!kcov)
314 return -ENOMEM;
315 kcov->mode = KCOV_MODE_DISABLED;
316 refcount_set(&kcov->refcount, 1);
317 spin_lock_init(&kcov->lock);
318 filep->private_data = kcov;
319 return nonseekable_open(inode, filep);
320 }
321
kcov_close(struct inode * inode,struct file * filep)322 static int kcov_close(struct inode *inode, struct file *filep)
323 {
324 kcov_put(filep->private_data);
325 return 0;
326 }
327
328 /*
329 * Fault in a lazily-faulted vmalloc area before it can be used by
330 * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the
331 * vmalloc fault handling path is instrumented.
332 */
kcov_fault_in_area(struct kcov * kcov)333 static void kcov_fault_in_area(struct kcov *kcov)
334 {
335 unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
336 unsigned long *area = kcov->area;
337 unsigned long offset;
338
339 for (offset = 0; offset < kcov->size; offset += stride)
340 READ_ONCE(area[offset]);
341 }
342
kcov_ioctl_locked(struct kcov * kcov,unsigned int cmd,unsigned long arg)343 static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
344 unsigned long arg)
345 {
346 struct task_struct *t;
347 unsigned long size, unused;
348
349 switch (cmd) {
350 case KCOV_INIT_TRACE:
351 /*
352 * Enable kcov in trace mode and setup buffer size.
353 * Must happen before anything else.
354 */
355 if (kcov->mode != KCOV_MODE_DISABLED)
356 return -EBUSY;
357 /*
358 * Size must be at least 2 to hold current position and one PC.
359 * Later we allocate size * sizeof(unsigned long) memory,
360 * that must not overflow.
361 */
362 size = arg;
363 if (size < 2 || size > INT_MAX / sizeof(unsigned long))
364 return -EINVAL;
365 kcov->size = size;
366 kcov->mode = KCOV_MODE_INIT;
367 return 0;
368 case KCOV_ENABLE:
369 /*
370 * Enable coverage for the current task.
371 * At this point user must have been enabled trace mode,
372 * and mmapped the file. Coverage collection is disabled only
373 * at task exit or voluntary by KCOV_DISABLE. After that it can
374 * be enabled for another task.
375 */
376 if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
377 return -EINVAL;
378 t = current;
379 if (kcov->t != NULL || t->kcov != NULL)
380 return -EBUSY;
381 if (arg == KCOV_TRACE_PC)
382 kcov->mode = KCOV_MODE_TRACE_PC;
383 else if (arg == KCOV_TRACE_CMP)
384 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
385 kcov->mode = KCOV_MODE_TRACE_CMP;
386 #else
387 return -ENOTSUPP;
388 #endif
389 else
390 return -EINVAL;
391 kcov_fault_in_area(kcov);
392 /* Cache in task struct for performance. */
393 t->kcov_size = kcov->size;
394 t->kcov_area = kcov->area;
395 /* See comment in check_kcov_mode(). */
396 barrier();
397 WRITE_ONCE(t->kcov_mode, kcov->mode);
398 t->kcov = kcov;
399 kcov->t = t;
400 /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */
401 kcov_get(kcov);
402 return 0;
403 case KCOV_DISABLE:
404 /* Disable coverage for the current task. */
405 unused = arg;
406 if (unused != 0 || current->kcov != kcov)
407 return -EINVAL;
408 t = current;
409 if (WARN_ON(kcov->t != t))
410 return -EINVAL;
411 kcov_task_init(t);
412 kcov->t = NULL;
413 kcov->mode = KCOV_MODE_INIT;
414 kcov_put(kcov);
415 return 0;
416 default:
417 return -ENOTTY;
418 }
419 }
420
kcov_ioctl(struct file * filep,unsigned int cmd,unsigned long arg)421 static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
422 {
423 struct kcov *kcov;
424 int res;
425
426 kcov = filep->private_data;
427 spin_lock(&kcov->lock);
428 res = kcov_ioctl_locked(kcov, cmd, arg);
429 spin_unlock(&kcov->lock);
430 return res;
431 }
432
433 static const struct file_operations kcov_fops = {
434 .open = kcov_open,
435 .unlocked_ioctl = kcov_ioctl,
436 .compat_ioctl = kcov_ioctl,
437 .mmap = kcov_mmap,
438 .release = kcov_close,
439 };
440
kcov_init(void)441 static int __init kcov_init(void)
442 {
443 /*
444 * The kcov debugfs file won't ever get removed and thus,
445 * there is no need to protect it against removal races. The
446 * use of debugfs_create_file_unsafe() is actually safe here.
447 */
448 debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
449
450 return 0;
451 }
452
453 device_initcall(kcov_init);
454
