1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * preemptoff and irqoff tracepoints
4 *
5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
6 */
7
8 #include <linux/kallsyms.h>
9 #include <linux/uaccess.h>
10 #include <linux/module.h>
11 #include <linux/ftrace.h>
12 #include <linux/kprobes.h>
13 #include "trace.h"
14
15 #define CREATE_TRACE_POINTS
16 #include <trace/events/preemptirq.h>
17
18 #ifdef CONFIG_TRACE_IRQFLAGS
19 /* Per-cpu variable to prevent redundant calls when IRQs already off */
20 static DEFINE_PER_CPU(int, tracing_irq_cpu);
21
22 /*
23 * Like trace_hardirqs_on() but without the lockdep invocation. This is
24 * used in the low level entry code where the ordering vs. RCU is important
25 * and lockdep uses a staged approach which splits the lockdep hardirq
26 * tracking into a RCU on and a RCU off section.
27 */
trace_hardirqs_on_prepare(void)28 void trace_hardirqs_on_prepare(void)
29 {
30 if (this_cpu_read(tracing_irq_cpu)) {
31 if (!in_nmi())
32 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
33 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
34 this_cpu_write(tracing_irq_cpu, 0);
35 }
36 }
37 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
38 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
39
trace_hardirqs_on(void)40 void trace_hardirqs_on(void)
41 {
42 if (this_cpu_read(tracing_irq_cpu)) {
43 if (!in_nmi())
44 trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
45 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
46 this_cpu_write(tracing_irq_cpu, 0);
47 }
48
49 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
50 lockdep_hardirqs_on(CALLER_ADDR0);
51 }
52 EXPORT_SYMBOL(trace_hardirqs_on);
53 NOKPROBE_SYMBOL(trace_hardirqs_on);
54
55 /*
56 * Like trace_hardirqs_off() but without the lockdep invocation. This is
57 * used in the low level entry code where the ordering vs. RCU is important
58 * and lockdep uses a staged approach which splits the lockdep hardirq
59 * tracking into a RCU on and a RCU off section.
60 */
trace_hardirqs_off_finish(void)61 void trace_hardirqs_off_finish(void)
62 {
63 if (!this_cpu_read(tracing_irq_cpu)) {
64 this_cpu_write(tracing_irq_cpu, 1);
65 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
66 if (!in_nmi())
67 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
68 }
69
70 }
71 EXPORT_SYMBOL(trace_hardirqs_off_finish);
72 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
73
trace_hardirqs_off(void)74 void trace_hardirqs_off(void)
75 {
76 lockdep_hardirqs_off(CALLER_ADDR0);
77
78 if (!this_cpu_read(tracing_irq_cpu)) {
79 this_cpu_write(tracing_irq_cpu, 1);
80 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
81 if (!in_nmi())
82 trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
83 }
84 }
85 EXPORT_SYMBOL(trace_hardirqs_off);
86 NOKPROBE_SYMBOL(trace_hardirqs_off);
87
trace_hardirqs_on_caller(unsigned long caller_addr)88 __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
89 {
90 if (this_cpu_read(tracing_irq_cpu)) {
91 if (!in_nmi())
92 trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
93 tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
94 this_cpu_write(tracing_irq_cpu, 0);
95 }
96
97 lockdep_hardirqs_on_prepare(CALLER_ADDR0);
98 lockdep_hardirqs_on(CALLER_ADDR0);
99 }
100 EXPORT_SYMBOL(trace_hardirqs_on_caller);
101 NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
102
trace_hardirqs_off_caller(unsigned long caller_addr)103 __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
104 {
105 lockdep_hardirqs_off(CALLER_ADDR0);
106
107 if (!this_cpu_read(tracing_irq_cpu)) {
108 this_cpu_write(tracing_irq_cpu, 1);
109 tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
110 if (!in_nmi())
111 trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
112 }
113 }
114 EXPORT_SYMBOL(trace_hardirqs_off_caller);
115 NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
116 #endif /* CONFIG_TRACE_IRQFLAGS */
117
118 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
119
trace_preempt_on(unsigned long a0,unsigned long a1)120 void trace_preempt_on(unsigned long a0, unsigned long a1)
121 {
122 if (!in_nmi())
123 trace_preempt_enable_rcuidle(a0, a1);
124 tracer_preempt_on(a0, a1);
125 }
126
trace_preempt_off(unsigned long a0,unsigned long a1)127 void trace_preempt_off(unsigned long a0, unsigned long a1)
128 {
129 if (!in_nmi())
130 trace_preempt_disable_rcuidle(a0, a1);
131 tracer_preempt_off(a0, a1);
132 }
133 #endif
134
