22 static notrace u32 __vdso_read_begin(const struct vdso_data *vdata)  in __vdso_read_begin()  argument
26 	seq = READ_ONCE(vdata->seq_count);  in __vdso_read_begin()
34 static notrace u32 vdso_read_begin(const struct vdso_data *vdata)  in vdso_read_begin()  argument
38 	seq = __vdso_read_begin(vdata);  in vdso_read_begin()
44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)  in vdso_read_retry()  argument
47 	return vdata->seq_count != start;  in vdso_read_retry()
68 				      struct vdso_data *vdata)  in do_realtime_coarse()  argument
73 		seq = vdso_read_begin(vdata);  in do_realtime_coarse()
75 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_realtime_coarse()
76 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_realtime_coarse()
78 	} while (vdso_read_retry(vdata, seq));  in do_realtime_coarse()
84 				       struct vdso_data *vdata)  in do_monotonic_coarse()  argument
90 		seq = vdso_read_begin(vdata);  in do_monotonic_coarse()
92 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_monotonic_coarse()
93 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_monotonic_coarse()
95 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic_coarse()
96 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic_coarse()
98 	} while (vdso_read_retry(vdata, seq));  in do_monotonic_coarse()
108 static notrace u64 get_ns(struct vdso_data *vdata)  in get_ns()  argument
117 	cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask;  in get_ns()
119 	nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec;  in get_ns()
120 	nsec >>= vdata->cs_shift;  in get_ns()
125 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
131 		seq = vdso_read_begin(vdata);  in do_realtime()
133 		if (!vdata->tk_is_cntvct)  in do_realtime()
136 		ts->tv_sec = vdata->xtime_clock_sec;  in do_realtime()
137 		nsecs = get_ns(vdata);  in do_realtime()
139 	} while (vdso_read_retry(vdata, seq));  in do_realtime()
147 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
154 		seq = vdso_read_begin(vdata);  in do_monotonic()
156 		if (!vdata->tk_is_cntvct)  in do_monotonic()
159 		ts->tv_sec = vdata->xtime_clock_sec;  in do_monotonic()
160 		nsecs = get_ns(vdata);  in do_monotonic()
162 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic()
163 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic()
165 	} while (vdso_read_retry(vdata, seq));  in do_monotonic()
176 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
181 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
190 	struct vdso_data *vdata;  in __vdso_clock_gettime()  local
193 	vdata = __get_datapage();  in __vdso_clock_gettime()
197 		ret = do_realtime_coarse(ts, vdata);  in __vdso_clock_gettime()
200 		ret = do_monotonic_coarse(ts, vdata);  in __vdso_clock_gettime()
203 		ret = do_realtime(ts, vdata);  in __vdso_clock_gettime()
206 		ret = do_monotonic(ts, vdata);  in __vdso_clock_gettime()
238 	struct vdso_data *vdata;  in __vdso_gettimeofday()  local
241 	vdata = __get_datapage();  in __vdso_gettimeofday()
243 	ret = do_realtime(&ts, vdata);  in __vdso_gettimeofday()
252 		tz->tz_minuteswest = vdata->tz_minuteswest;  in __vdso_gettimeofday()
253 		tz->tz_dsttime = vdata->tz_dsttime;  in __vdso_gettimeofday()