1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains common tag-based KASAN code.
4 *
5 * Copyright (c) 2018 Google, Inc.
6 * Copyright (c) 2020 Google, Inc.
7 */
8
9 #include <linux/atomic.h>
10 #include <linux/init.h>
11 #include <linux/kasan.h>
12 #include <linux/kernel.h>
13 #include <linux/memblock.h>
14 #include <linux/memory.h>
15 #include <linux/mm.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include "kasan.h"
21 #include "../slab.h"
22
23 #define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)
24
25 enum kasan_arg_stacktrace {
26 KASAN_ARG_STACKTRACE_DEFAULT,
27 KASAN_ARG_STACKTRACE_OFF,
28 KASAN_ARG_STACKTRACE_ON,
29 };
30
31 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
32
33 /* Whether to collect alloc/free stack traces. */
34 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
35
36 /* Non-zero, as initial pointer values are 0. */
37 #define STACK_RING_BUSY_PTR ((void *)1)
38
39 struct kasan_stack_ring stack_ring = {
40 .lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
41 };
42
43 /* kasan.stacktrace=off/on */
early_kasan_flag_stacktrace(char * arg)44 static int __init early_kasan_flag_stacktrace(char *arg)
45 {
46 if (!arg)
47 return -EINVAL;
48
49 if (!strcmp(arg, "off"))
50 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
51 else if (!strcmp(arg, "on"))
52 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
53 else
54 return -EINVAL;
55
56 return 0;
57 }
58 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
59
60 /* kasan.stack_ring_size=<number of entries> */
early_kasan_flag_stack_ring_size(char * arg)61 static int __init early_kasan_flag_stack_ring_size(char *arg)
62 {
63 if (!arg)
64 return -EINVAL;
65
66 return kstrtoul(arg, 0, &stack_ring.size);
67 }
68 early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);
69
kasan_init_tags(void)70 void __init kasan_init_tags(void)
71 {
72 switch (kasan_arg_stacktrace) {
73 case KASAN_ARG_STACKTRACE_DEFAULT:
74 /* Default is specified by kasan_flag_stacktrace definition. */
75 break;
76 case KASAN_ARG_STACKTRACE_OFF:
77 static_branch_disable(&kasan_flag_stacktrace);
78 break;
79 case KASAN_ARG_STACKTRACE_ON:
80 static_branch_enable(&kasan_flag_stacktrace);
81 break;
82 }
83
84 if (kasan_stack_collection_enabled()) {
85 if (!stack_ring.size)
86 stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
87 stack_ring.entries = memblock_alloc(
88 sizeof(stack_ring.entries[0]) * stack_ring.size,
89 SMP_CACHE_BYTES);
90 if (WARN_ON(!stack_ring.entries))
91 static_branch_disable(&kasan_flag_stacktrace);
92 }
93 }
94
save_stack_info(struct kmem_cache * cache,void * object,gfp_t gfp_flags,bool is_free)95 static void save_stack_info(struct kmem_cache *cache, void *object,
96 gfp_t gfp_flags, bool is_free)
97 {
98 unsigned long flags;
99 depot_stack_handle_t stack;
100 u64 pos;
101 struct kasan_stack_ring_entry *entry;
102 void *old_ptr;
103
104 stack = kasan_save_stack(gfp_flags, true);
105
106 /*
107 * Prevent save_stack_info() from modifying stack ring
108 * when kasan_complete_mode_report_info() is walking it.
109 */
110 read_lock_irqsave(&stack_ring.lock, flags);
111
112 next:
113 pos = atomic64_fetch_add(1, &stack_ring.pos);
114 entry = &stack_ring.entries[pos % stack_ring.size];
115
116 /* Detect stack ring entry slots that are being written to. */
117 old_ptr = READ_ONCE(entry->ptr);
118 if (old_ptr == STACK_RING_BUSY_PTR)
119 goto next; /* Busy slot. */
120 if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
121 goto next; /* Busy slot. */
122
123 WRITE_ONCE(entry->size, cache->object_size);
124 WRITE_ONCE(entry->pid, current->pid);
125 WRITE_ONCE(entry->stack, stack);
126 WRITE_ONCE(entry->is_free, is_free);
127
128 /*
129 * Paired with smp_load_acquire() in kasan_complete_mode_report_info().
130 */
131 smp_store_release(&entry->ptr, (s64)object);
132
133 read_unlock_irqrestore(&stack_ring.lock, flags);
134 }
135
kasan_save_alloc_info(struct kmem_cache * cache,void * object,gfp_t flags)136 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
137 {
138 save_stack_info(cache, object, flags, false);
139 }
140
kasan_save_free_info(struct kmem_cache * cache,void * object)141 void kasan_save_free_info(struct kmem_cache *cache, void *object)
142 {
143 save_stack_info(cache, object, 0, true);
144 }
145
