635 static u64 vtime_delta(struct vtime *vtime)  in vtime_delta()  argument
640 	if (clock < vtime->starttime)  in vtime_delta()
643 	return clock - vtime->starttime;  in vtime_delta()
646 static u64 get_vtime_delta(struct vtime *vtime)  in get_vtime_delta()  argument
648 	u64 delta = vtime_delta(vtime);  in get_vtime_delta()
659 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);  in get_vtime_delta()
660 	vtime->starttime += delta;  in get_vtime_delta()
666 				 struct vtime *vtime)  in vtime_account_system()  argument
668 	vtime->stime += get_vtime_delta(vtime);  in vtime_account_system()
669 	if (vtime->stime >= TICK_NSEC) {  in vtime_account_system()
670 		account_system_time(tsk, irq_count(), vtime->stime);  in vtime_account_system()
671 		vtime->stime = 0;  in vtime_account_system()
676 				struct vtime *vtime)  in vtime_account_guest()  argument
678 	vtime->gtime += get_vtime_delta(vtime);  in vtime_account_guest()
679 	if (vtime->gtime >= TICK_NSEC) {  in vtime_account_guest()
680 		account_guest_time(tsk, vtime->gtime);  in vtime_account_guest()
681 		vtime->gtime = 0;  in vtime_account_guest()
686 				   struct vtime *vtime)  in __vtime_account_kernel()  argument
689 	if (vtime->state == VTIME_GUEST)  in __vtime_account_kernel()
690 		vtime_account_guest(tsk, vtime);  in __vtime_account_kernel()
692 		vtime_account_system(tsk, vtime);  in __vtime_account_kernel()
697 	struct vtime *vtime = &tsk->vtime;  in vtime_account_kernel()  local
699 	if (!vtime_delta(vtime))  in vtime_account_kernel()
702 	write_seqcount_begin(&vtime->seqcount);  in vtime_account_kernel()
703 	__vtime_account_kernel(tsk, vtime);  in vtime_account_kernel()
704 	write_seqcount_end(&vtime->seqcount);  in vtime_account_kernel()
709 	struct vtime *vtime = &tsk->vtime;  in vtime_user_enter()  local
711 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_enter()
712 	vtime_account_system(tsk, vtime);  in vtime_user_enter()
713 	vtime->state = VTIME_USER;  in vtime_user_enter()
714 	write_seqcount_end(&vtime->seqcount);  in vtime_user_enter()
719 	struct vtime *vtime = &tsk->vtime;  in vtime_user_exit()  local
721 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_exit()
722 	vtime->utime += get_vtime_delta(vtime);  in vtime_user_exit()
723 	if (vtime->utime >= TICK_NSEC) {  in vtime_user_exit()
724 		account_user_time(tsk, vtime->utime);  in vtime_user_exit()
725 		vtime->utime = 0;  in vtime_user_exit()
727 	vtime->state = VTIME_SYS;  in vtime_user_exit()
728 	write_seqcount_end(&vtime->seqcount);  in vtime_user_exit()
733 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_enter()  local
741 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_enter()
742 	vtime_account_system(tsk, vtime);  in vtime_guest_enter()
744 	vtime->state = VTIME_GUEST;  in vtime_guest_enter()
745 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_enter()
751 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_exit()  local
753 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_exit()
754 	vtime_account_guest(tsk, vtime);  in vtime_guest_exit()
756 	vtime->state = VTIME_SYS;  in vtime_guest_exit()
757 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_exit()
763 	account_idle_time(get_vtime_delta(&tsk->vtime));  in vtime_account_idle()
768 	struct vtime *vtime = &prev->vtime;  in vtime_task_switch_generic()  local
770 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
771 	if (vtime->state == VTIME_IDLE)  in vtime_task_switch_generic()
774 		__vtime_account_kernel(prev, vtime);  in vtime_task_switch_generic()
775 	vtime->state = VTIME_INACTIVE;  in vtime_task_switch_generic()
776 	vtime->cpu = -1;  in vtime_task_switch_generic()
777 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
779 	vtime = &current->vtime;  in vtime_task_switch_generic()
781 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
783 		vtime->state = VTIME_IDLE;  in vtime_task_switch_generic()
785 		vtime->state = VTIME_GUEST;  in vtime_task_switch_generic()
787 		vtime->state = VTIME_SYS;  in vtime_task_switch_generic()
788 	vtime->starttime = sched_clock();  in vtime_task_switch_generic()
789 	vtime->cpu = smp_processor_id();  in vtime_task_switch_generic()
790 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
795 	struct vtime *vtime = &t->vtime;  in vtime_init_idle()  local
799 	write_seqcount_begin(&vtime->seqcount);  in vtime_init_idle()
800 	vtime->state = VTIME_IDLE;  in vtime_init_idle()
801 	vtime->starttime = sched_clock();  in vtime_init_idle()
802 	vtime->cpu = cpu;  in vtime_init_idle()
803 	write_seqcount_end(&vtime->seqcount);  in vtime_init_idle()
809 	struct vtime *vtime = &t->vtime;  in task_gtime()  local
817 		seq = read_seqcount_begin(&vtime->seqcount);  in task_gtime()
820 		if (vtime->state == VTIME_GUEST)  in task_gtime()
821 			gtime += vtime->gtime + vtime_delta(vtime);  in task_gtime()
823 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_gtime()
835 	struct vtime *vtime = &t->vtime;  in task_cputime()  local
846 		seq = read_seqcount_begin(&vtime->seqcount);  in task_cputime()
852 		if (vtime->state < VTIME_SYS)  in task_cputime()
855 		delta = vtime_delta(vtime);  in task_cputime()
861 		if (vtime->state == VTIME_SYS)  in task_cputime()
862 			*stime += vtime->stime + delta;  in task_cputime()
864 			*utime += vtime->utime + delta;  in task_cputime()
865 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_cputime()
868 static int vtime_state_fetch(struct vtime *vtime, int cpu)  in vtime_state_fetch()  argument
870 	int state = READ_ONCE(vtime->state);  in vtime_state_fetch()
876 	if (vtime->cpu != cpu && vtime->cpu != -1)  in vtime_state_fetch()
894 static u64 kcpustat_user_vtime(struct vtime *vtime)  in kcpustat_user_vtime()  argument
896 	if (vtime->state == VTIME_USER)  in kcpustat_user_vtime()
897 		return vtime->utime + vtime_delta(vtime);  in kcpustat_user_vtime()
898 	else if (vtime->state == VTIME_GUEST)  in kcpustat_user_vtime()
899 		return vtime->gtime + vtime_delta(vtime);  in kcpustat_user_vtime()
908 	struct vtime *vtime = &tsk->vtime;  in kcpustat_field_vtime()  local
914 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_field_vtime()
916 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_field_vtime()
932 				*val += vtime->stime + vtime_delta(vtime);  in kcpustat_field_vtime()
936 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
940 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
944 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
948 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
953 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_field_vtime()
996 	struct vtime *vtime = &tsk->vtime;  in kcpustat_cpu_fetch_vtime()  local
1004 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_cpu_fetch_vtime()
1006 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_cpu_fetch_vtime()
1017 		delta = vtime_delta(vtime);  in kcpustat_cpu_fetch_vtime()
1024 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;  in kcpustat_cpu_fetch_vtime()
1027 				cpustat[CPUTIME_NICE] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1029 				cpustat[CPUTIME_USER] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1033 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1034 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1036 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1037 				cpustat[CPUTIME_USER] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1040 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_cpu_fetch_vtime()