1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shadow Call Stack support.
4 *
5 * Copyright (C) 2019 Google LLC
6 */
7
8 #include <linux/kasan.h>
9 #include <linux/mm.h>
10 #include <linux/scs.h>
11 #include <linux/slab.h>
12 #include <linux/vmstat.h>
13
14 static struct kmem_cache *scs_cache;
15
__scs_account(void * s,int account)16 static void __scs_account(void *s, int account)
17 {
18 struct page *scs_page = virt_to_page(s);
19
20 mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
21 account * (SCS_SIZE / SZ_1K));
22 }
23
scs_alloc(int node)24 static void *scs_alloc(int node)
25 {
26 void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
27
28 if (!s)
29 return NULL;
30
31 *__scs_magic(s) = SCS_END_MAGIC;
32
33 /*
34 * Poison the allocation to catch unintentional accesses to
35 * the shadow stack when KASAN is enabled.
36 */
37 kasan_poison_object_data(scs_cache, s);
38 __scs_account(s, 1);
39 return s;
40 }
41
scs_free(void * s)42 static void scs_free(void *s)
43 {
44 __scs_account(s, -1);
45 kasan_unpoison_object_data(scs_cache, s);
46 kmem_cache_free(scs_cache, s);
47 }
48
scs_init(void)49 void __init scs_init(void)
50 {
51 scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
52 }
53
scs_prepare(struct task_struct * tsk,int node)54 int scs_prepare(struct task_struct *tsk, int node)
55 {
56 void *s = scs_alloc(node);
57
58 if (!s)
59 return -ENOMEM;
60
61 task_scs(tsk) = task_scs_sp(tsk) = s;
62 return 0;
63 }
64
scs_check_usage(struct task_struct * tsk)65 static void scs_check_usage(struct task_struct *tsk)
66 {
67 static unsigned long highest;
68
69 unsigned long *p, prev, curr = highest, used = 0;
70
71 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
72 return;
73
74 for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
75 if (!READ_ONCE_NOCHECK(*p))
76 break;
77 used += sizeof(*p);
78 }
79
80 while (used > curr) {
81 prev = cmpxchg_relaxed(&highest, curr, used);
82
83 if (prev == curr) {
84 pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
85 tsk->comm, task_pid_nr(tsk), used);
86 break;
87 }
88
89 curr = prev;
90 }
91 }
92
scs_release(struct task_struct * tsk)93 void scs_release(struct task_struct *tsk)
94 {
95 void *s = task_scs(tsk);
96
97 if (!s)
98 return;
99
100 WARN(task_scs_end_corrupted(tsk),
101 "corrupted shadow stack detected when freeing task\n");
102 scs_check_usage(tsk);
103 scs_free(s);
104 }
105
