22 static notrace unsigned int __vdso_read_begin(const struct vdso_data *vdata)  in __vdso_read_begin()  argument
26 	seq = READ_ONCE(vdata->seq_count);  in __vdso_read_begin()
34 static notrace unsigned int vdso_read_begin(const struct vdso_data *vdata)  in vdso_read_begin()  argument
38 	seq = __vdso_read_begin(vdata);  in vdso_read_begin()
44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start)  in vdso_read_retry()  argument
47 	return vdata->seq_count != start;  in vdso_read_retry()
67 				      struct vdso_data *vdata)  in do_realtime_coarse()  argument
72 		seq = vdso_read_begin(vdata);  in do_realtime_coarse()
74 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_realtime_coarse()
75 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_realtime_coarse()
77 	} while (vdso_read_retry(vdata, seq));  in do_realtime_coarse()
82 				       struct vdso_data *vdata)  in do_monotonic_coarse()  argument
88 		seq = vdso_read_begin(vdata);  in do_monotonic_coarse()
90 		ts->tv_sec = vdata->xtime_coarse_sec;  in do_monotonic_coarse()
91 		ts->tv_nsec = vdata->xtime_coarse_nsec;  in do_monotonic_coarse()
93 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic_coarse()
94 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic_coarse()
96 	} while (vdso_read_retry(vdata, seq));  in do_monotonic_coarse()
118 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata)  in do_realtime()  argument
123 		count = vdso_read_begin(vdata);  in do_realtime()
124 		ts->tv_sec = vdata->xtime_clock_sec;  in do_realtime()
125 		ns = vdata->xtime_clock_nsec;  in do_realtime()
126 		ns += vgetsns(vdata);  in do_realtime()
127 		ns >>= vdata->cs_shift;  in do_realtime()
128 	} while (vdso_read_retry(vdata, count));  in do_realtime()
136 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata)  in do_monotonic()  argument
143 		seq = vdso_read_begin(vdata);  in do_monotonic()
145 		ts->tv_sec = vdata->xtime_clock_sec;  in do_monotonic()
146 		nsecs = vdata->xtime_clock_nsec;  in do_monotonic()
147 		nsecs += vgetsns(vdata);  in do_monotonic()
148 		nsecs >>= vdata->cs_shift;  in do_monotonic()
150 		tomono.tv_sec = vdata->wtm_clock_sec;  in do_monotonic()
151 		tomono.tv_nsec = vdata->wtm_clock_nsec;  in do_monotonic()
153 	} while (vdso_read_retry(vdata, seq));  in do_monotonic()
163 	struct vdso_data *vdata;  in __vdso_clock_gettime()  local
166 	vdata = __get_datapage();  in __vdso_clock_gettime()
167 	if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)  in __vdso_clock_gettime()
172 		ret = do_realtime_coarse(ts, vdata);  in __vdso_clock_gettime()
175 		ret = do_monotonic_coarse(ts, vdata);  in __vdso_clock_gettime()
178 		ret = do_realtime(ts, vdata);  in __vdso_clock_gettime()
181 		ret = do_monotonic(ts, vdata);  in __vdso_clock_gettime()
211 	struct vdso_data *vdata = __get_datapage();  in __vdso_clock_getres()  local
220 		res->tv_nsec = vdata->hrtimer_res;  in __vdso_clock_getres()
252 	struct vdso_data *vdata;  in __vdso_gettimeofday()  local
255 	vdata = __get_datapage();  in __vdso_gettimeofday()
257 	if (vdata->cycle_count_offset == EMPTY_REG_OFFSET)  in __vdso_gettimeofday()
260 	ret = do_realtime(&ts, vdata);  in __vdso_gettimeofday()
267 		tz->tz_minuteswest = vdata->tz_minuteswest;  in __vdso_gettimeofday()
268 		tz->tz_dsttime = vdata->tz_dsttime;  in __vdso_gettimeofday()