1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2015 Mentor Graphics Corporation.
4 */
5
6 #include <linux/compiler.h>
7 #include <linux/hrtimer.h>
8 #include <linux/time.h>
9 #include <asm/barrier.h>
10 #include <asm/bug.h>
11 #include <asm/cp15.h>
12 #include <asm/page.h>
13 #include <asm/unistd.h>
14 #include <asm/vdso_datapage.h>
15
16 #ifndef CONFIG_AEABI
17 #error This code depends on AEABI system call conventions
18 #endif
19
20 extern struct vdso_data *__get_datapage(void);
21
__vdso_read_begin(const struct vdso_data * vdata)22 static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)
23 {
24 u32 seq;
25 repeat:
26 seq = READ_ONCE(vdata->seq_count);
27 if (seq & 1) {
28 cpu_relax();
29 goto repeat;
30 }
31 return seq;
32 }
33
vdso_read_begin(const struct vdso_data * vdata)34 static notrace u32 vdso_read_begin(const struct vdso_data *vdata)
35 {
36 u32 seq;
37
38 seq = __vdso_read_begin(vdata);
39
40 smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */
41 return seq;
42 }
43
vdso_read_retry(const struct vdso_data * vdata,u32 start)44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)
45 {
46 smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */
47 return vdata->seq_count != start;
48 }
49
clock_gettime_fallback(clockid_t _clkid,struct timespec * _ts)50 static notrace long clock_gettime_fallback(clockid_t _clkid,
51 struct timespec *_ts)
52 {
53 register struct timespec *ts asm("r1") = _ts;
54 register clockid_t clkid asm("r0") = _clkid;
55 register long ret asm ("r0");
56 register long nr asm("r7") = __NR_clock_gettime;
57
58 asm volatile(
59 " swi #0\n"
60 : "=r" (ret)
61 : "r" (clkid), "r" (ts), "r" (nr)
62 : "memory");
63
64 return ret;
65 }
66
do_realtime_coarse(struct timespec * ts,struct vdso_data * vdata)67 static notrace int do_realtime_coarse(struct timespec *ts,
68 struct vdso_data *vdata)
69 {
70 u32 seq;
71
72 do {
73 seq = vdso_read_begin(vdata);
74
75 ts->tv_sec = vdata->xtime_coarse_sec;
76 ts->tv_nsec = vdata->xtime_coarse_nsec;
77
78 } while (vdso_read_retry(vdata, seq));
79
80 return 0;
81 }
82
do_monotonic_coarse(struct timespec * ts,struct vdso_data * vdata)83 static notrace int do_monotonic_coarse(struct timespec *ts,
84 struct vdso_data *vdata)
85 {
86 struct timespec tomono;
87 u32 seq;
88
89 do {
90 seq = vdso_read_begin(vdata);
91
92 ts->tv_sec = vdata->xtime_coarse_sec;
93 ts->tv_nsec = vdata->xtime_coarse_nsec;
94
95 tomono.tv_sec = vdata->wtm_clock_sec;
96 tomono.tv_nsec = vdata->wtm_clock_nsec;
97
98 } while (vdso_read_retry(vdata, seq));
99
100 ts->tv_sec += tomono.tv_sec;
101 timespec_add_ns(ts, tomono.tv_nsec);
102
103 return 0;
104 }
105
106 #ifdef CONFIG_ARM_ARCH_TIMER
107
get_ns(struct vdso_data * vdata)108 static notrace u64 get_ns(struct vdso_data *vdata)
109 {
110 u64 cycle_delta;
111 u64 cycle_now;
112 u64 nsec;
113
114 isb();
115 cycle_now = read_sysreg(CNTVCT);
116
117 cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;
118
119 nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;
120 nsec >>= vdata->cs_shift;
121
122 return nsec;
123 }
124
do_realtime(struct timespec * ts,struct vdso_data * vdata)125 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
126 {
127 u64 nsecs;
128 u32 seq;
129
130 do {
131 seq = vdso_read_begin(vdata);
132
133 if (!vdata->tk_is_cntvct)
134 return -1;
135
136 ts->tv_sec = vdata->xtime_clock_sec;
137 nsecs = get_ns(vdata);
138
139 } while (vdso_read_retry(vdata, seq));
140
141 ts->tv_nsec = 0;
142 timespec_add_ns(ts, nsecs);
143
144 return 0;
145 }
146
do_monotonic(struct timespec * ts,struct vdso_data * vdata)147 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
148 {
149 struct timespec tomono;
150 u64 nsecs;
151 u32 seq;
152
153 do {
154 seq = vdso_read_begin(vdata);
155
156 if (!vdata->tk_is_cntvct)
157 return -1;
158
159 ts->tv_sec = vdata->xtime_clock_sec;
160 nsecs = get_ns(vdata);
161
162 tomono.tv_sec = vdata->wtm_clock_sec;
163 tomono.tv_nsec = vdata->wtm_clock_nsec;
164
165 } while (vdso_read_retry(vdata, seq));
166
167 ts->tv_sec += tomono.tv_sec;
168 ts->tv_nsec = 0;
169 timespec_add_ns(ts, nsecs + tomono.tv_nsec);
170
171 return 0;
172 }
173
174 #else /* CONFIG_ARM_ARCH_TIMER */
175
do_realtime(struct timespec * ts,struct vdso_data * vdata)176 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)
177 {
178 return -1;
179 }
180
do_monotonic(struct timespec * ts,struct vdso_data * vdata)181 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)
182 {
183 return -1;
184 }
185
186 #endif /* CONFIG_ARM_ARCH_TIMER */
187
__vdso_clock_gettime(clockid_t clkid,struct timespec * ts)188 notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts)
189 {
190 struct vdso_data *vdata;
191 int ret = -1;
192
193 vdata = __get_datapage();
194
195 switch (clkid) {
196 case CLOCK_REALTIME_COARSE:
197 ret = do_realtime_coarse(ts, vdata);
198 break;
199 case CLOCK_MONOTONIC_COARSE:
200 ret = do_monotonic_coarse(ts, vdata);
201 break;
202 case CLOCK_REALTIME:
203 ret = do_realtime(ts, vdata);
204 break;
205 case CLOCK_MONOTONIC:
206 ret = do_monotonic(ts, vdata);
207 break;
208 default:
209 break;
210 }
211
212 if (ret)
213 ret = clock_gettime_fallback(clkid, ts);
214
215 return ret;
216 }
217
gettimeofday_fallback(struct timeval * _tv,struct timezone * _tz)218 static notrace long gettimeofday_fallback(struct timeval *_tv,
219 struct timezone *_tz)
220 {
221 register struct timezone *tz asm("r1") = _tz;
222 register struct timeval *tv asm("r0") = _tv;
223 register long ret asm ("r0");
224 register long nr asm("r7") = __NR_gettimeofday;
225
226 asm volatile(
227 " swi #0\n"
228 : "=r" (ret)
229 : "r" (tv), "r" (tz), "r" (nr)
230 : "memory");
231
232 return ret;
233 }
234
__vdso_gettimeofday(struct timeval * tv,struct timezone * tz)235 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
236 {
237 struct timespec ts;
238 struct vdso_data *vdata;
239 int ret;
240
241 vdata = __get_datapage();
242
243 ret = do_realtime(&ts, vdata);
244 if (ret)
245 return gettimeofday_fallback(tv, tz);
246
247 if (tv) {
248 tv->tv_sec = ts.tv_sec;
249 tv->tv_usec = ts.tv_nsec / 1000;
250 }
251 if (tz) {
252 tz->tz_minuteswest = vdata->tz_minuteswest;
253 tz->tz_dsttime = vdata->tz_dsttime;
254 }
255
256 return ret;
257 }
258
259 /* Avoid unresolved references emitted by GCC */
260
__aeabi_unwind_cpp_pr0(void)261 void __aeabi_unwind_cpp_pr0(void)
262 {
263 }
264
__aeabi_unwind_cpp_pr1(void)265 void __aeabi_unwind_cpp_pr1(void)
266 {
267 }
268
__aeabi_unwind_cpp_pr2(void)269 void __aeabi_unwind_cpp_pr2(void)
270 {
271 }
272
