22 static notrace unsigned int __vdso_read_begin(const struct vdso_data *vdata)  in __vdso_read_begin()  argument
26 	seq = READ_ONCE(vdata->seq_count);  in __vdso_read_begin()
34 static notrace unsigned int vdso_read_begin(const struct vdso_data *vdata)  in vdso_read_begin()  argument
38 	seq = __vdso_read_begin(vdata);  in vdso_read_begin()
44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)  in vdso_read_retry()  argument
47 	return vdata->seq_count != start;  in vdso_read_retry()
67 				      struct vdso_data *vdata)  in do_realtime_coarse()  argument
72 		seq = vdso_read_begin(vdata);  in do_realtime_coarse()
74 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_realtime_coarse()
75 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_realtime_coarse()
77 	} while (vdso_read_retry(vdata, seq));  in do_realtime_coarse()
82 				       struct vdso_data *vdata)  in do_monotonic_coarse()  argument
88 		seq = vdso_read_begin(vdata);  in do_monotonic_coarse()
90 		ts->tv_sec = vdata->xtime_coarse_sec + vdata->wtm_clock_sec;  in do_monotonic_coarse()
91 		ns = vdata->xtime_coarse_nsec + vdata->wtm_clock_nsec;  in do_monotonic_coarse()
93 	} while (vdso_read_retry(vdata, seq));  in do_monotonic_coarse()
116 static notrace int do_realtime(struct __kernel_old_timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
121 		count = vdso_read_begin(vdata);  in do_realtime()
122 		ts->tv_sec = vdata->xtime_clock_sec;  in do_realtime()
123 		ns = vdata->xtime_clock_nsec;  in do_realtime()
124 		ns += vgetsns(vdata);  in do_realtime()
125 		ns >>= vdata->cs_shift;  in do_realtime()
126 	} while (vdso_read_retry(vdata, count));  in do_realtime()
134 static notrace int do_monotonic(struct __kernel_old_timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
140 		seq = vdso_read_begin(vdata);  in do_monotonic()
142 		ts->tv_sec = vdata->xtime_clock_sec;  in do_monotonic()
143 		ns = vdata->xtime_clock_nsec;  in do_monotonic()
144 		ns += vgetsns(vdata);  in do_monotonic()
145 		ns >>= vdata->cs_shift;  in do_monotonic()
147 		ts->tv_sec += vdata->wtm_clock_sec;  in do_monotonic()
148 		ns += vdata->wtm_clock_nsec;  in do_monotonic()
150 	} while (vdso_read_retry(vdata, seq));  in do_monotonic()
160 	struct vdso_data *vdata;  in __vdso_clock_gettime()  local
163 	vdata = __get_datapage();  in __vdso_clock_gettime()
164 	if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)  in __vdso_clock_gettime()
169 		ret = do_realtime_coarse(ts, vdata);  in __vdso_clock_gettime()
172 		ret = do_monotonic_coarse(ts, vdata);  in __vdso_clock_gettime()
175 		ret = do_realtime(ts, vdata);  in __vdso_clock_gettime()
178 		ret = do_monotonic(ts, vdata);  in __vdso_clock_gettime()
208 	struct vdso_data *vdata = __get_datapage();  in __vdso_clock_getres()  local
217 		res->tv_nsec = vdata->hrtimer_res;  in __vdso_clock_getres()
249 	struct vdso_data *vdata;  in __vdso_gettimeofday()  local
252 	vdata = __get_datapage();  in __vdso_gettimeofday()
254 	if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)  in __vdso_gettimeofday()
257 	ret = do_realtime(&ts, vdata);  in __vdso_gettimeofday()
264 		tz->tz_minuteswest = vdata->tz_minuteswest;  in __vdso_gettimeofday()
265 		tz->tz_dsttime = vdata->tz_dsttime;  in __vdso_gettimeofday()