13 #include <trace/events/timer.h>
21 static void posix_cpu_timer_rearm(struct k_itimer *timer);
33  * Called after updating RLIMIT_CPU to run cpu timer and update
60 	 * If the encoded PID is 0, then the timer is targeted at current  in pid_for_clock()
106 static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer)  in cpu_timer_task_rcu()  argument
108 	return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock));  in cpu_timer_task_rcu()
115 static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)  in bump_cpu_timer()  argument
117 	u64 delta, incr, expires = timer->it.cpu.node.expires;  in bump_cpu_timer()
120 	if (!timer->it_interval)  in bump_cpu_timer()
126 	incr = timer->it_interval;  in bump_cpu_timer()
137 		timer->it.cpu.node.expires += incr;  in bump_cpu_timer()
138 		timer->it_overrun += 1LL << i;  in bump_cpu_timer()
141 	return timer->it.cpu.node.expires;  in bump_cpu_timer()
262  * timer. That means group cputime accounting is already active. Called
283  * CPU timers armed. Before starting a timer it's required to check whether
301 		 * The POSIX timer interface allows for absolute time expiry  in thread_group_start_cputime()
303 		 * to synchronize the timer to the clock every time we start it.  in thread_group_start_cputime()
376  * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
378  * new timer already all-zeros initialized.
393 	 * If posix timer expiry is handled in task work context then  in posix_cpu_timer_create()
394 	 * timer::it_lock can be taken without disabling interrupts as all  in posix_cpu_timer_create()
396 	 * lock class key otherwise regular posix timer expiry would record  in posix_cpu_timer_create()
410 static struct posix_cputimer_base *timer_base(struct k_itimer *timer,  in timer_base()  argument
413 	int clkidx = CPUCLOCK_WHICH(timer->it_clock);  in timer_base()
415 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in timer_base()
427 static void trigger_base_recalc_expires(struct k_itimer *timer,  in trigger_base_recalc_expires()  argument
430 	struct posix_cputimer_base *base = timer_base(timer, tsk);  in trigger_base_recalc_expires()
436  * Dequeue the timer and reset the base if it was its earliest expiration.
441  * If another timer gets queued between this and the next tick, its
445 static void disarm_timer(struct k_itimer *timer, struct task_struct *p)  in disarm_timer()  argument
447 	struct cpu_timer *ctmr = &timer->it.cpu;  in disarm_timer()
453 	base = timer_base(timer, p);  in disarm_timer()
455 		trigger_base_recalc_expires(timer, p);  in disarm_timer()
460  * Clean up a CPU-clock timer that is about to be destroyed.
461  * This is called from timer deletion with the timer already locked.
462  * If we return TIMER_RETRY, it's necessary to release the timer's lock
463  * and try again.  (This happens when the timer is in the middle of firing.)
465 static int posix_cpu_timer_del(struct k_itimer *timer)  in posix_cpu_timer_del()  argument
467 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_del()
474 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_del()
480 	 * thread timer list entry concurrent read/writes.  in posix_cpu_timer_del()
486 		 * should have removed this timer from the timer queue.  in posix_cpu_timer_del()
490 		if (timer->it.cpu.firing)  in posix_cpu_timer_del()
493 			disarm_timer(timer, p);  in posix_cpu_timer_del()
547  * Insert the timer on the appropriate list before any timers that
550 static void arm_timer(struct k_itimer *timer, struct task_struct *p)  in arm_timer()  argument
552 	struct posix_cputimer_base *base = timer_base(timer, p);  in arm_timer()
553 	struct cpu_timer *ctmr = &timer->it.cpu;  in arm_timer()
560 	 * We are the new earliest-expiring POSIX 1.b timer, hence  in arm_timer()
568 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in arm_timer()
575  * The timer is locked, fire it and arrange for its reload.
577 static void cpu_timer_fire(struct k_itimer *timer)  in cpu_timer_fire()  argument
579 	struct cpu_timer *ctmr = &timer->it.cpu;  in cpu_timer_fire()
581 	if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {  in cpu_timer_fire()
586 	} else if (unlikely(timer->sigq == NULL)) {  in cpu_timer_fire()
589 		 * not a normal timer from sys_timer_create.  in cpu_timer_fire()
591 		wake_up_process(timer->it_process);  in cpu_timer_fire()
593 	} else if (!timer->it_interval) {  in cpu_timer_fire()
595 		 * One-shot timer.  Clear it as soon as it's fired.  in cpu_timer_fire()
597 		posix_timer_event(timer, 0);  in cpu_timer_fire()
599 	} else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {  in cpu_timer_fire()
603 		 * reload the timer.  But we need to keep it  in cpu_timer_fire()
606 		posix_cpu_timer_rearm(timer);  in cpu_timer_fire()
607 		++timer->it_requeue_pending;  in cpu_timer_fire()
613  * This is called with the timer locked and interrupts disabled.
614  * If we return TIMER_RETRY, it's necessary to release the timer's lock
615  * and try again.  (This happens when the timer is in the middle of firing.)
617 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,  in posix_cpu_timer_set()  argument
620 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_set()
622 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_set()
629 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_set()
660 	 * Disarm any old timer after extracting its expiry time.  in posix_cpu_timer_set()
662 	old_incr = timer->it_interval;  in posix_cpu_timer_set()
665 	if (unlikely(timer->it.cpu.firing)) {  in posix_cpu_timer_set()
666 		timer->it.cpu.firing = -1;  in posix_cpu_timer_set()
676 	 * timer, we need a sample to balance the thread expiry  in posix_cpu_timer_set()
680 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_set()
691 			 * Update the timer in case it has overrun already.  in posix_cpu_timer_set()
693 			 * with the next reloaded timer already ticking,  in posix_cpu_timer_set()
697 			u64 exp = bump_cpu_timer(timer, val);  in posix_cpu_timer_set()
711 		 * We are colliding with the timer actually firing.  in posix_cpu_timer_set()
712 		 * Punt after filling in the timer's old value, and  in posix_cpu_timer_set()
726 	 * For a timer with no notification action, we don't actually  in posix_cpu_timer_set()
727 	 * arm the timer (we'll just fake it for timer_gettime).  in posix_cpu_timer_set()
731 		arm_timer(timer, p);  in posix_cpu_timer_set()
739 	timer->it_interval = timespec64_to_ktime(new->it_interval);  in posix_cpu_timer_set()
742 	 * This acts as a modification timestamp for the timer,  in posix_cpu_timer_set()
744 	 * that we have reset the timer manually.  in posix_cpu_timer_set()
746 	timer->it_requeue_pending = (timer->it_requeue_pending + 2) &  in posix_cpu_timer_set()
748 	timer->it_overrun_last = 0;  in posix_cpu_timer_set()
749 	timer->it_overrun = -1;  in posix_cpu_timer_set()
758 			cpu_timer_fire(timer);  in posix_cpu_timer_set()
770 			trigger_base_recalc_expires(timer, p);  in posix_cpu_timer_set()
782 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)  in posix_cpu_timer_get()  argument
784 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_get()
785 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_get()
790 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_get()
797 	itp->it_interval = ktime_to_timespec64(timer->it_interval);  in posix_cpu_timer_get()
805 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_get()
814 		 * The timer should have expired already, but the firing  in posix_cpu_timer_get()
976 	 * RLIMIT_CPU) nothing to check. Also skip the process wide timer  in check_process_timers()
1037  * when the last timer signal was delivered and we have to reload the timer.
1039 static void posix_cpu_timer_rearm(struct k_itimer *timer)  in posix_cpu_timer_rearm()  argument
1041 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_rearm()
1048 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_rearm()
1052 	/* Protect timer list r/w in arm_timer() */  in posix_cpu_timer_rearm()
1058 	 * Fetch the current sample and update the timer's expiry time.  in posix_cpu_timer_rearm()
1060 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_rearm()
1065 	bump_cpu_timer(timer, now);  in posix_cpu_timer_rearm()
1070 	arm_timer(timer, p);  in posix_cpu_timer_rearm()
1105  * true if a timer has expired, else return false.
1128 	 * acquire the sighand lock to handle timer expiry.  in fastpath_timer_check()
1132 	 * the change yet, the timer checks are delayed until the next  in fastpath_timer_check()
1134 	 * timer. This isn't an issue in practice because these types of  in fastpath_timer_check()
1216 	 * newly expired timers, reenable the fast path check in the timer  in posix_cpu_timers_enable_work()
1251 	struct k_itimer *timer, *next;  in handle_posix_cpu_timers()  local
1277 		 * The above timer checks have updated the expiry cache and  in handle_posix_cpu_timers()
1280 		 * consistent. So the next timer interrupt fastpath check  in handle_posix_cpu_timers()
1283 		 * If timer expiry runs in the timer interrupt context then  in handle_posix_cpu_timers()
1289 		 * If timer expiry is deferred to task work context then  in handle_posix_cpu_timers()
1298 		 *   work ignored posix CPU timer handling because the  in handle_posix_cpu_timers()
1301 		 *   ticks have happened since the start of the timer  in handle_posix_cpu_timers()
1309 	 * We must release sighand lock before taking any timer's lock.  in handle_posix_cpu_timers()
1310 	 * There is a potential race with timer deletion here, as the  in handle_posix_cpu_timers()
1312 	 * the firing flag in each timer, so that a deletion attempt  in handle_posix_cpu_timers()
1313 	 * that gets the timer lock before we do will give it up and  in handle_posix_cpu_timers()
1314 	 * spin until we've taken care of that timer below.  in handle_posix_cpu_timers()
1321 	 * each timer's lock before clearing its firing flag, so no  in handle_posix_cpu_timers()
1322 	 * timer call will interfere.  in handle_posix_cpu_timers()
1324 	list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {  in handle_posix_cpu_timers()
1331 		 * because all other operations on timer::it_lock happen in  in handle_posix_cpu_timers()
1334 		spin_lock(&timer->it_lock);  in handle_posix_cpu_timers()
1335 		list_del_init(&timer->it.cpu.elist);  in handle_posix_cpu_timers()
1336 		cpu_firing = timer->it.cpu.firing;  in handle_posix_cpu_timers()
1337 		timer->it.cpu.firing = 0;  in handle_posix_cpu_timers()
1340 		 * of the timer, which already reported this  in handle_posix_cpu_timers()
1344 			cpu_timer_fire(timer);  in handle_posix_cpu_timers()
1345 		spin_unlock(&timer->it_lock);  in handle_posix_cpu_timers()
1350  * This is called from the timer interrupt handler.  The irq handler has
1412 	 * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF  in set_process_cpu_timer()
1425 	struct k_itimer timer;  in do_cpu_nanosleep()  local
1430 	 * Set up a temporary timer and then wait for it to go off.  in do_cpu_nanosleep()
1432 	memset(&timer, 0, sizeof timer);  in do_cpu_nanosleep()
1433 	spin_lock_init(&timer.it_lock);  in do_cpu_nanosleep()
1434 	timer.it_clock = which_clock;  in do_cpu_nanosleep()
1435 	timer.it_overrun = -1;  in do_cpu_nanosleep()
1436 	error = posix_cpu_timer_create(&timer);  in do_cpu_nanosleep()
1437 	timer.it_process = current;  in do_cpu_nanosleep()
1446 		spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1447 		error = posix_cpu_timer_set(&timer, flags, &it, NULL);  in do_cpu_nanosleep()
1449 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1454 			if (!cpu_timer_getexpires(&timer.it.cpu)) {  in do_cpu_nanosleep()
1456 				 * Our timer fired and was reset, below  in do_cpu_nanosleep()
1459 				posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1460 				spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1468 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1470 			spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1476 		expires = cpu_timer_getexpires(&timer.it.cpu);  in do_cpu_nanosleep()
1477 		error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);  in do_cpu_nanosleep()
1480 			 * Timer is now unarmed, deletion can not fail.  in do_cpu_nanosleep()
1482 			posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1484 		spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1488 			 * We need to handle case when timer was or is in the  in do_cpu_nanosleep()
1492 			spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1493 			error = posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1494 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1569 static int process_cpu_timer_create(struct k_itimer *timer)  in process_cpu_timer_create()  argument
1571 	timer->it_clock = PROCESS_CLOCK;  in process_cpu_timer_create()
1572 	return posix_cpu_timer_create(timer);  in process_cpu_timer_create()
1589 static int thread_cpu_timer_create(struct k_itimer *timer)  in thread_cpu_timer_create()  argument
1591 	timer->it_clock = THREAD_CLOCK;  in thread_cpu_timer_create()
1592 	return posix_cpu_timer_create(timer);  in thread_cpu_timer_create()