Lines Matching +full:no +full:- +full:tick +full:- +full:in +full:- +full:suspend
1 // SPDX-License-Identifier: GPL-2.0
4 * timer.c, moved in commit 8524070b7982.
21 #include <linux/tick.h>
28 #include "tick-internal.h"
37 /* Update timekeeper when a tick has passed */
63 * struct tk_fast - NMI safe timekeeper
76 /* Suspend-time cycles value for halted fast timekeeper. */
93 * returns nanoseconds already so no conversion is required, hence mult=1
119 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { in tk_normalize_xtime()
120 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in tk_normalize_xtime()
121 tk->xtime_sec++; in tk_normalize_xtime()
123 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { in tk_normalize_xtime()
124 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in tk_normalize_xtime()
125 tk->raw_sec++; in tk_normalize_xtime()
133 ts.tv_sec = tk->xtime_sec; in tk_xtime()
134 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_xtime()
140 tk->xtime_sec = ts->tv_sec; in tk_set_xtime()
141 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_set_xtime()
146 tk->xtime_sec += ts->tv_sec; in tk_xtime_add()
147 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_xtime_add()
156 * Verify consistency of: offset_real = -wall_to_monotonic in tk_set_wall_to_mono()
159 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, in tk_set_wall_to_mono()
160 -tk->wall_to_monotonic.tv_nsec); in tk_set_wall_to_mono()
161 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); in tk_set_wall_to_mono()
162 tk->wall_to_monotonic = wtm; in tk_set_wall_to_mono()
163 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); in tk_set_wall_to_mono()
164 tk->offs_real = timespec64_to_ktime(tmp); in tk_set_wall_to_mono()
165 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); in tk_set_wall_to_mono()
170 tk->offs_boot = ktime_add(tk->offs_boot, delta); in tk_update_sleep_time()
175 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); in tk_update_sleep_time()
179 * tk_clock_read - atomic clocksource read() helper
181 * This helper is necessary to use in the read paths because, while the
188 * a read of the fast-timekeeper tkrs (which is protected by its own locking
193 struct clocksource *clock = READ_ONCE(tkr->clock); in tk_clock_read()
195 return clock->read(clock); in tk_clock_read()
199 #define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
204 u64 max_cycles = tk->tkr_mono.clock->max_cycles; in timekeeping_check_update()
205 const char *name = tk->tkr_mono.clock->name; in timekeeping_check_update()
219 if (tk->underflow_seen) { in timekeeping_check_update()
220 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
221 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name); in timekeeping_check_update()
224 tk->last_warning = jiffies; in timekeeping_check_update()
226 tk->underflow_seen = 0; in timekeeping_check_update()
229 if (tk->overflow_seen) { in timekeeping_check_update()
230 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
231 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name); in timekeeping_check_update()
234 tk->last_warning = jiffies; in timekeeping_check_update()
236 tk->overflow_seen = 0; in timekeeping_check_update()
256 last = tkr->cycle_last; in timekeeping_get_delta()
257 mask = tkr->mask; in timekeeping_get_delta()
258 max = tkr->clock->max_cycles; in timekeeping_get_delta()
265 * mask-relative negative values. in timekeeping_get_delta()
268 tk->underflow_seen = 1; in timekeeping_get_delta()
274 tk->overflow_seen = 1; in timekeeping_get_delta()
275 delta = tkr->clock->max_cycles; in timekeeping_get_delta()
292 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); in timekeeping_get_delta()
299 * tk_setup_internals - Set up internals to use clocksource clock.
315 ++tk->cs_was_changed_seq; in tk_setup_internals()
316 old_clock = tk->tkr_mono.clock; in tk_setup_internals()
317 tk->tkr_mono.clock = clock; in tk_setup_internals()
318 tk->tkr_mono.mask = clock->mask; in tk_setup_internals()
319 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); in tk_setup_internals()
321 tk->tkr_raw.clock = clock; in tk_setup_internals()
322 tk->tkr_raw.mask = clock->mask; in tk_setup_internals()
323 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; in tk_setup_internals()
325 /* Do the ns -> cycle conversion first, using original mult */ in tk_setup_internals()
327 tmp <<= clock->shift; in tk_setup_internals()
329 tmp += clock->mult/2; in tk_setup_internals()
330 do_div(tmp, clock->mult); in tk_setup_internals()
335 tk->cycle_interval = interval; in tk_setup_internals()
337 /* Go back from cycles -> shifted ns */ in tk_setup_internals()
338 tk->xtime_interval = interval * clock->mult; in tk_setup_internals()
339 tk->xtime_remainder = ntpinterval - tk->xtime_interval; in tk_setup_internals()
340 tk->raw_interval = interval * clock->mult; in tk_setup_internals()
344 int shift_change = clock->shift - old_clock->shift; in tk_setup_internals()
346 tk->tkr_mono.xtime_nsec >>= -shift_change; in tk_setup_internals()
347 tk->tkr_raw.xtime_nsec >>= -shift_change; in tk_setup_internals()
349 tk->tkr_mono.xtime_nsec <<= shift_change; in tk_setup_internals()
350 tk->tkr_raw.xtime_nsec <<= shift_change; in tk_setup_internals()
354 tk->tkr_mono.shift = clock->shift; in tk_setup_internals()
355 tk->tkr_raw.shift = clock->shift; in tk_setup_internals()
357 tk->ntp_error = 0; in tk_setup_internals()
358 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; in tk_setup_internals()
359 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; in tk_setup_internals()
366 tk->tkr_mono.mult = clock->mult; in tk_setup_internals()
367 tk->tkr_raw.mult = clock->mult; in tk_setup_internals()
368 tk->ntp_err_mult = 0; in tk_setup_internals()
369 tk->skip_second_overflow = 0; in tk_setup_internals()
378 nsec = delta * tkr->mult + tkr->xtime_nsec; in timekeeping_delta_to_ns()
379 nsec >>= tkr->shift; in timekeeping_delta_to_ns()
397 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask); in timekeeping_cycles_to_ns()
402 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
412 * which is still consistent. In the worst case this can result is a
419 struct tk_read_base *base = tkf->base; in update_fast_timekeeper()
422 raw_write_seqcount_latch(&tkf->seq); in update_fast_timekeeper()
428 raw_write_seqcount_latch(&tkf->seq); in update_fast_timekeeper()
438 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask); in fast_tk_get_delta_ns()
449 seq = raw_read_seqcount_latch(&tkf->seq); in __ktime_get_fast_ns()
450 tkr = tkf->base + (seq & 0x01); in __ktime_get_fast_ns()
451 now = ktime_to_ns(tkr->base); in __ktime_get_fast_ns()
453 } while (read_seqcount_latch_retry(&tkf->seq, seq)); in __ktime_get_fast_ns()
459 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
476 * |12345678---> reader order
485 * for a CPU local observation is when an NMI hits in the middle of
497 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
509 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
525 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
526 * partially updated. Since the tk->offs_boot update is a rare event, this
536 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot))); in ktime_get_boot_fast_ns()
541 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
547 * with the possibility of wrong timestamps in post processing.
553 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai))); in ktime_get_tai_fast_ns()
564 seq = raw_read_seqcount_latch(&tkf->seq); in __ktime_get_real_fast()
565 tkr = tkf->base + (seq & 0x01); in __ktime_get_real_fast()
566 basem = ktime_to_ns(tkr->base); in __ktime_get_real_fast()
567 baser = ktime_to_ns(tkr->base_real); in __ktime_get_real_fast()
569 } while (read_seqcount_latch_retry(&tkf->seq, seq)); in __ktime_get_real_fast()
577 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
588 * ktime_get_fast_timestamps: - NMI safe timestamps
594 * happens late during resume and not in timekeeping_resume(). That could
612 * real time does not. On the next readout both are in sync again.
619 * steps of suspend/resume. The accessors still work, but the timestamps
622 * For regular suspend/resume there is no observable difference vs. sched
625 * OTOH, access to sched clock is not guaranteed across suspend/resume on
626 * all systems either so it depends on the hardware in use.
629 * using sched clock in a similar way as during early boot. But it's not as
637 snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono); in ktime_get_fast_timestamps()
638 snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot)); in ktime_get_fast_timestamps()
642 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
654 const struct tk_read_base *tkr = &tk->tkr_mono; in halt_fast_timekeeper()
659 tkr_dummy.base_real = tkr->base + tk->offs_real; in halt_fast_timekeeper()
662 tkr = &tk->tkr_raw; in halt_fast_timekeeper()
676 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
695 * pvclock_gtod_unregister_notifier - unregister a pvclock
713 * tk_update_leap_state - helper to update the next_leap_ktime
717 tk->next_leap_ktime = ntp_get_next_leap(); in tk_update_leap_state()
718 if (tk->next_leap_ktime != KTIME_MAX) in tk_update_leap_state()
720 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); in tk_update_leap_state()
738 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); in tk_update_ktime_data()
739 nsec = (u32) tk->wall_to_monotonic.tv_nsec; in tk_update_ktime_data()
740 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); in tk_update_ktime_data()
745 * this into account before updating tk->ktime_sec. in tk_update_ktime_data()
747 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_update_ktime_data()
750 tk->ktime_sec = seconds; in tk_update_ktime_data()
753 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); in tk_update_ktime_data()
760 tk->ntp_error = 0; in timekeeping_update()
770 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; in timekeeping_update()
771 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); in timekeeping_update()
772 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); in timekeeping_update()
775 tk->clock_was_set_seq++; in timekeeping_update()
777 * The mirroring of the data to the shadow-timekeeper needs in timekeeping_update()
778 * to happen last here to ensure we don't over-write the in timekeeping_update()
787 * timekeeping_forward_now - update clock to the current time
798 cycle_now = tk_clock_read(&tk->tkr_mono); in timekeeping_forward_now()
799 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in timekeeping_forward_now()
800 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_forward_now()
801 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_forward_now()
803 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult; in timekeeping_forward_now()
804 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult; in timekeeping_forward_now()
810 * ktime_get_real_ts64 - Returns the time of day in a timespec64.
813 * Returns the time of day in a timespec64 (WARN if suspended).
826 ts->tv_sec = tk->xtime_sec; in ktime_get_real_ts64()
827 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_real_ts64()
831 ts->tv_nsec = 0; in ktime_get_real_ts64()
847 base = tk->tkr_mono.base; in ktime_get()
848 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get()
866 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; in ktime_get_resolution_ns()
890 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_with_offset()
891 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_with_offset()
911 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_coarse_with_offset()
912 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; in ktime_get_coarse_with_offset()
921 * ktime_mono_to_any() - convert monotonic time to any other time
941 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
952 base = tk->tkr_raw.base; in ktime_get_raw()
953 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw()
962 * ktime_get_ts64 - get the monotonic clock in timespec64 format
967 * in normalized timespec64 format in the variable pointed to by @ts.
980 ts->tv_sec = tk->xtime_sec; in ktime_get_ts64()
981 nsec = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_ts64()
982 tomono = tk->wall_to_monotonic; in ktime_get_ts64()
986 ts->tv_sec += tomono.tv_sec; in ktime_get_ts64()
987 ts->tv_nsec = 0; in ktime_get_ts64()
993 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
996 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
1006 return tk->ktime_sec; in ktime_get_seconds()
1011 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
1015 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
1017 * counter to provide "atomic" access to the 64bit tk->xtime_sec
1027 return tk->xtime_sec; in ktime_get_real_seconds()
1031 seconds = tk->xtime_sec; in ktime_get_real_seconds()
1040 * __ktime_get_real_seconds - The same as ktime_get_real_seconds
1048 return tk->xtime_sec; in __ktime_get_real_seconds()
1052 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1069 now = tk_clock_read(&tk->tkr_mono); in ktime_get_snapshot()
1070 systime_snapshot->cs_id = tk->tkr_mono.clock->id; in ktime_get_snapshot()
1071 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; in ktime_get_snapshot()
1072 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; in ktime_get_snapshot()
1073 base_real = ktime_add(tk->tkr_mono.base, in ktime_get_snapshot()
1075 base_raw = tk->tkr_raw.base; in ktime_get_snapshot()
1076 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); in ktime_get_snapshot()
1077 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); in ktime_get_snapshot()
1080 systime_snapshot->cycles = now; in ktime_get_snapshot()
1081 systime_snapshot->real = ktime_add_ns(base_real, nsec_real); in ktime_get_snapshot()
1082 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw); in ktime_get_snapshot()
1093 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) || in scale64_check_overflow()
1094 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem))) in scale64_check_overflow()
1095 return -EOVERFLOW; in scale64_check_overflow()
1104 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1107 * @total_history_cycles: Total history length in cycles
1137 total_history_cycles - partial_history_cycles : in adjust_historical_crosststamp()
1145 ktime_sub(ts->sys_monoraw, history->raw)); in adjust_historical_crosststamp()
1152 * If there is a discontinuity in the history, scale monotonic raw in adjust_historical_crosststamp()
1160 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); in adjust_historical_crosststamp()
1163 ktime_sub(ts->sys_realtime, history->real)); in adjust_historical_crosststamp()
1172 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw); in adjust_historical_crosststamp()
1173 ts->sys_realtime = ktime_add_ns(history->real, corr_real); in adjust_historical_crosststamp()
1175 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); in adjust_historical_crosststamp()
1176 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); in adjust_historical_crosststamp()
1183 * cycle_between - true if test occurs chronologically between before and after
1195 * get_device_system_crosststamp - Synchronously capture system/device timestamp
1230 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx); in get_device_system_crosststamp()
1239 if (tk->tkr_mono.clock != system_counterval.cs) in get_device_system_crosststamp()
1240 return -ENODEV; in get_device_system_crosststamp()
1247 now = tk_clock_read(&tk->tkr_mono); in get_device_system_crosststamp()
1248 interval_start = tk->tkr_mono.cycle_last; in get_device_system_crosststamp()
1250 clock_was_set_seq = tk->clock_was_set_seq; in get_device_system_crosststamp()
1251 cs_was_changed_seq = tk->cs_was_changed_seq; in get_device_system_crosststamp()
1258 base_real = ktime_add(tk->tkr_mono.base, in get_device_system_crosststamp()
1260 base_raw = tk->tkr_raw.base; in get_device_system_crosststamp()
1262 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, in get_device_system_crosststamp()
1264 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, in get_device_system_crosststamp()
1268 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real); in get_device_system_crosststamp()
1269 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); in get_device_system_crosststamp()
1285 !cycle_between(history_begin->cycles, in get_device_system_crosststamp()
1287 history_begin->cs_was_changed_seq != cs_was_changed_seq) in get_device_system_crosststamp()
1288 return -EINVAL; in get_device_system_crosststamp()
1289 partial_history_cycles = cycles - system_counterval.cycles; in get_device_system_crosststamp()
1290 total_history_cycles = cycles - history_begin->cycles; in get_device_system_crosststamp()
1292 history_begin->clock_was_set_seq != clock_was_set_seq; in get_device_system_crosststamp()
1307 * do_settimeofday64 - Sets the time of day.
1320 return -EINVAL; in do_settimeofday64()
1330 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) { in do_settimeofday64()
1331 ret = -EINVAL; in do_settimeofday64()
1335 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta)); in do_settimeofday64()
1357 * timekeeping_inject_offset - Adds or subtracts from the current time.
1369 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC) in timekeeping_inject_offset()
1370 return -EINVAL; in timekeeping_inject_offset()
1379 if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 || in timekeeping_inject_offset()
1381 ret = -EINVAL; in timekeeping_inject_offset()
1386 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts)); in timekeeping_inject_offset()
1411 * would either need to write a program to do it in /etc/rc (and risk
1414 * compile in the timezone information into the kernel. Bad, bad....
1416 * - TYT, 1992-01-01
1418 * The best thing to do is to keep the CMOS clock in universal time (UTC)
1435 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1439 tk->tai_offset = tai_offset; in __timekeeping_set_tai_offset()
1440 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); in __timekeeping_set_tai_offset()
1444 * change_clocksource - Swaps clocksources if a new one is available
1458 * If the cs is in module, get a module reference. Succeeds in change_clocksource()
1459 * for built-in code (owner == NULL) as well. in change_clocksource()
1461 if (try_module_get(new->owner)) { in change_clocksource()
1462 if (!new->enable || new->enable(new) == 0) in change_clocksource()
1465 module_put(new->owner); in change_clocksource()
1474 old = tk->tkr_mono.clock; in change_clocksource()
1484 if (old->disable) in change_clocksource()
1485 old->disable(old); in change_clocksource()
1487 module_put(old->owner); in change_clocksource()
1494 * timekeeping_notify - Install a new clock source
1504 if (tk->tkr_mono.clock == clock) in timekeeping_notify()
1508 return tk->tkr_mono.clock == clock ? 0 : -1; in timekeeping_notify()
1512 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1515 * Returns the raw monotonic time (completely un-modified by ntp)
1525 ts->tv_sec = tk->raw_sec; in ktime_get_raw_ts64()
1526 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw_ts64()
1530 ts->tv_nsec = 0; in ktime_get_raw_ts64()
1537 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1548 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; in timekeeping_valid_for_hres()
1556 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1567 ret = tk->tkr_mono.clock->max_idle_ns; in timekeeping_max_deferment()
1575 * read_persistent_clock64 - Return time from the persistent clock.
1582 * XXX - Do be sure to remove it once all arches implement it.
1586 ts->tv_sec = 0; in read_persistent_clock64()
1587 ts->tv_nsec = 0; in read_persistent_clock64()
1591 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1595 * @wall_time: - current time as returned by persistent clock
1596 * @boot_offset: - offset that is defined as wall_time - boot_time
1614 * The flag starts of false and is only set when a suspend reaches
1616 * timekeeper clocksource is not stopping across suspend and has been
1621 * If a suspend fails before reaching timekeeping_resume() then the flag
1630 * timekeeping_init - Initializes the clocksource and common timekeeping values
1662 if (clock->enable) in timekeeping_init()
1663 clock->enable(clock); in timekeeping_init()
1667 tk->raw_sec = 0; in timekeeping_init()
1677 /* time in seconds when suspend began for persistent clock */
1681 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1683 * @delta: Pointer to the delta value in timespec64 format
1685 * Takes a timespec offset measuring a suspend interval and properly
1698 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); in __timekeeping_inject_sleeptime()
1707 * 1) non-stop clocksource
1716 * 3) becomes needless, so in this case we don't need to call
1730 * But if system has 2), 2) will definitely be used, so in this
1740 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1748 * a suspend offset to be injected into the timekeeping values.
1775 * timekeeping_resume - Resumes the generic timekeeping subsystem.
1780 struct clocksource *clock = tk->tkr_mono.clock; in timekeeping_resume()
1797 * used: Nonstop clocksource during suspend, persistent clock and rtc in timekeeping_resume()
1802 * suspend-nonstop clocksource -> persistent clock -> rtc in timekeeping_resume()
1803 * The less preferred source will only be tried if there is no better in timekeeping_resume()
1804 * usable source. The rtc part is handled separately in rtc core code. in timekeeping_resume()
1806 cycle_now = tk_clock_read(&tk->tkr_mono); in timekeeping_resume()
1821 /* Re-base the last cycle value */ in timekeeping_resume()
1822 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_resume()
1823 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_resume()
1825 tk->ntp_error = 0; in timekeeping_resume()
1868 * use in suspend timing. in timekeeping_suspend()
1870 curr_clock = tk->tkr_mono.clock; in timekeeping_suspend()
1871 cycle_now = tk->tkr_mono.cycle_last; in timekeeping_suspend()
1876 * To avoid drift caused by repeated suspend/resumes, in timekeeping_suspend()
1878 * try to compensate so the difference in system time in timekeeping_suspend()
1908 /* sysfs resume/suspend bits for timekeeping */
1911 .suspend = timekeeping_suspend,
1928 s64 interval = tk->cycle_interval; in timekeeping_apply_adjustment()
1932 } else if (mult_adj == -1) { in timekeeping_apply_adjustment()
1933 interval = -interval; in timekeeping_apply_adjustment()
1934 offset = -offset; in timekeeping_apply_adjustment()
1959 * So offset stores the non-accumulated cycles. Thus the current in timekeeping_apply_adjustment()
1960 * time (in shifted nanoseconds) is: in timekeeping_apply_adjustment()
1963 * to keep time consistent. In other words, we can't jump back in timekeeping_apply_adjustment()
1964 * in time, and we also want to avoid jumping forward in time. in timekeeping_apply_adjustment()
1983 * xtime_nsec_2 = xtime_nsec_1 - offset in timekeeping_apply_adjustment()
1985 * xtime_nsec -= offset in timekeeping_apply_adjustment()
1987 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { in timekeeping_apply_adjustment()
1993 tk->tkr_mono.mult += mult_adj; in timekeeping_apply_adjustment()
1994 tk->xtime_interval += interval; in timekeeping_apply_adjustment()
1995 tk->tkr_mono.xtime_nsec -= offset; in timekeeping_apply_adjustment()
2007 * Determine the multiplier from the current NTP tick length. in timekeeping_adjust()
2008 * Avoid expensive division when the tick length doesn't change. in timekeeping_adjust()
2010 if (likely(tk->ntp_tick == ntp_tick_length())) { in timekeeping_adjust()
2011 mult = tk->tkr_mono.mult - tk->ntp_err_mult; in timekeeping_adjust()
2013 tk->ntp_tick = ntp_tick_length(); in timekeeping_adjust()
2014 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - in timekeeping_adjust()
2015 tk->xtime_remainder, tk->cycle_interval); in timekeeping_adjust()
2020 * to catch up with it. If it's ahead and there was a remainder in the in timekeeping_adjust()
2021 * tick division, the clock will slow down. Otherwise it will stay in timekeeping_adjust()
2022 * ahead until the tick length changes to a non-divisible value. in timekeeping_adjust()
2024 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; in timekeeping_adjust()
2025 mult += tk->ntp_err_mult; in timekeeping_adjust()
2027 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); in timekeeping_adjust()
2029 if (unlikely(tk->tkr_mono.clock->maxadj && in timekeeping_adjust()
2030 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) in timekeeping_adjust()
2031 > tk->tkr_mono.clock->maxadj))) { in timekeeping_adjust()
2034 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, in timekeeping_adjust()
2035 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); in timekeeping_adjust()
2041 * in the code above, its possible the required corrective factor to in timekeeping_adjust()
2048 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { in timekeeping_adjust()
2049 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << in timekeeping_adjust()
2050 tk->tkr_mono.shift; in timekeeping_adjust()
2051 tk->xtime_sec--; in timekeeping_adjust()
2052 tk->skip_second_overflow = 1; in timekeeping_adjust()
2057 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2065 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in accumulate_nsecs_to_secs()
2068 while (tk->tkr_mono.xtime_nsec >= nsecps) { in accumulate_nsecs_to_secs()
2071 tk->tkr_mono.xtime_nsec -= nsecps; in accumulate_nsecs_to_secs()
2072 tk->xtime_sec++; in accumulate_nsecs_to_secs()
2076 * i.e. xtime_nsec underflowed in timekeeping_adjust() in accumulate_nsecs_to_secs()
2078 if (unlikely(tk->skip_second_overflow)) { in accumulate_nsecs_to_secs()
2079 tk->skip_second_overflow = 0; in accumulate_nsecs_to_secs()
2084 leap = second_overflow(tk->xtime_sec); in accumulate_nsecs_to_secs()
2088 tk->xtime_sec += leap; in accumulate_nsecs_to_secs()
2093 timespec64_sub(tk->wall_to_monotonic, ts)); in accumulate_nsecs_to_secs()
2095 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); in accumulate_nsecs_to_secs()
2104 * logarithmic_accumulation - shifted accumulation of cycles
2115 u64 interval = tk->cycle_interval << shift; in logarithmic_accumulation()
2123 offset -= interval; in logarithmic_accumulation()
2124 tk->tkr_mono.cycle_last += interval; in logarithmic_accumulation()
2125 tk->tkr_raw.cycle_last += interval; in logarithmic_accumulation()
2127 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; in logarithmic_accumulation()
2131 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; in logarithmic_accumulation()
2132 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in logarithmic_accumulation()
2133 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { in logarithmic_accumulation()
2134 tk->tkr_raw.xtime_nsec -= snsec_per_sec; in logarithmic_accumulation()
2135 tk->raw_sec++; in logarithmic_accumulation()
2139 tk->ntp_error += tk->ntp_tick << shift; in logarithmic_accumulation()
2140 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << in logarithmic_accumulation()
2141 (tk->ntp_error_shift + shift); in logarithmic_accumulation()
2147 * timekeeping_advance - Updates the timekeeper to the current time and
2148 * current NTP tick length
2165 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), in timekeeping_advance()
2166 tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in timekeeping_advance()
2169 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK) in timekeeping_advance()
2180 * chunk in one go, and then try to consume the next smaller in timekeeping_advance()
2183 shift = ilog2(offset) - ilog2(tk->cycle_interval); in timekeeping_advance()
2186 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; in timekeeping_advance()
2188 while (offset >= tk->cycle_interval) { in timekeeping_advance()
2191 if (offset < tk->cycle_interval<<shift) in timekeeping_advance()
2192 shift--; in timekeeping_advance()
2226 * update_wall_time - Uses the current clocksource to increment the wall time
2236 * getboottime64 - Return the real time of system boot.
2239 * Returns the wall-time of boot in a timespec64.
2241 * This is based on the wall_to_monotonic offset and the total suspend
2249 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); in getboottime64()
2278 mono = tk->wall_to_monotonic; in ktime_get_coarse_ts64()
2296 * ktime_get_update_offsets_now - hrtimer helper
2298 * @offs_real: pointer to storage for monotonic -> realtime offset
2299 * @offs_boot: pointer to storage for monotonic -> boottime offset
2300 * @offs_tai: pointer to storage for monotonic -> clock tai offset
2303 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2319 base = tk->tkr_mono.base; in ktime_get_update_offsets_now()
2320 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_update_offsets_now()
2323 if (*cwsseq != tk->clock_was_set_seq) { in ktime_get_update_offsets_now()
2324 *cwsseq = tk->clock_was_set_seq; in ktime_get_update_offsets_now()
2325 *offs_real = tk->offs_real; in ktime_get_update_offsets_now()
2326 *offs_boot = tk->offs_boot; in ktime_get_update_offsets_now()
2327 *offs_tai = tk->offs_tai; in ktime_get_update_offsets_now()
2331 if (unlikely(base >= tk->next_leap_ktime)) in ktime_get_update_offsets_now()
2332 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); in ktime_get_update_offsets_now()
2340 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2344 if (txc->modes & ADJ_ADJTIME) { in timekeeping_validate_timex()
2346 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT)) in timekeeping_validate_timex()
2347 return -EINVAL; in timekeeping_validate_timex()
2348 if (!(txc->modes & ADJ_OFFSET_READONLY) && in timekeeping_validate_timex()
2350 return -EPERM; in timekeeping_validate_timex()
2352 /* In order to modify anything, you gotta be super-user! */ in timekeeping_validate_timex()
2353 if (txc->modes && !capable(CAP_SYS_TIME)) in timekeeping_validate_timex()
2354 return -EPERM; in timekeeping_validate_timex()
2359 if (txc->modes & ADJ_TICK && in timekeeping_validate_timex()
2360 (txc->tick < 900000/USER_HZ || in timekeeping_validate_timex()
2361 txc->tick > 1100000/USER_HZ)) in timekeeping_validate_timex()
2362 return -EINVAL; in timekeeping_validate_timex()
2365 if (txc->modes & ADJ_SETOFFSET) { in timekeeping_validate_timex()
2366 /* In order to inject time, you gotta be super-user! */ in timekeeping_validate_timex()
2368 return -EPERM; in timekeeping_validate_timex()
2375 * The field tv_usec/tv_nsec must always be non-negative and in timekeeping_validate_timex()
2378 if (txc->time.tv_usec < 0) in timekeeping_validate_timex()
2379 return -EINVAL; in timekeeping_validate_timex()
2381 if (txc->modes & ADJ_NANO) { in timekeeping_validate_timex()
2382 if (txc->time.tv_usec >= NSEC_PER_SEC) in timekeeping_validate_timex()
2383 return -EINVAL; in timekeeping_validate_timex()
2385 if (txc->time.tv_usec >= USEC_PER_SEC) in timekeeping_validate_timex()
2386 return -EINVAL; in timekeeping_validate_timex()
2392 * only happen on 64-bit systems: in timekeeping_validate_timex()
2394 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) { in timekeeping_validate_timex()
2395 if (LLONG_MIN / PPM_SCALE > txc->freq) in timekeeping_validate_timex()
2396 return -EINVAL; in timekeeping_validate_timex()
2397 if (LLONG_MAX / PPM_SCALE < txc->freq) in timekeeping_validate_timex()
2398 return -EINVAL; in timekeeping_validate_timex()
2405 * random_get_entropy_fallback - Returns the raw clock source value,
2406 * used by random.c for platforms with no valid random_get_entropy().
2411 struct clocksource *clock = READ_ONCE(tkr->clock); in random_get_entropy_fallback()
2415 return clock->read(clock); in random_get_entropy_fallback()
2420 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2438 if (txc->modes & ADJ_SETOFFSET) { in do_adjtimex()
2440 delta.tv_sec = txc->time.tv_sec; in do_adjtimex()
2441 delta.tv_nsec = txc->time.tv_usec; in do_adjtimex()
2442 if (!(txc->modes & ADJ_NANO)) in do_adjtimex()
2459 orig_tai = tai = tk->tai_offset; in do_adjtimex()
2475 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK)) in do_adjtimex()
2488 * hardpps() - Accessor function to NTP __hardpps function