633 static u64 vtime_delta(struct vtime *vtime)  in vtime_delta()  argument
638 	if (clock < vtime->starttime)  in vtime_delta()
641 	return clock - vtime->starttime;  in vtime_delta()
644 static u64 get_vtime_delta(struct vtime *vtime)  in get_vtime_delta()  argument
646 	u64 delta = vtime_delta(vtime);  in get_vtime_delta()
657 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);  in get_vtime_delta()
658 	vtime->starttime += delta;  in get_vtime_delta()
664 				 struct vtime *vtime)  in vtime_account_system()  argument
666 	vtime->stime += get_vtime_delta(vtime);  in vtime_account_system()
667 	if (vtime->stime >= TICK_NSEC) {  in vtime_account_system()
668 		account_system_time(tsk, irq_count(), vtime->stime);  in vtime_account_system()
669 		vtime->stime = 0;  in vtime_account_system()
674 				struct vtime *vtime)  in vtime_account_guest()  argument
676 	vtime->gtime += get_vtime_delta(vtime);  in vtime_account_guest()
677 	if (vtime->gtime >= TICK_NSEC) {  in vtime_account_guest()
678 		account_guest_time(tsk, vtime->gtime);  in vtime_account_guest()
679 		vtime->gtime = 0;  in vtime_account_guest()
684 				   struct vtime *vtime)  in __vtime_account_kernel()  argument
687 	if (vtime->state == VTIME_GUEST)  in __vtime_account_kernel()
688 		vtime_account_guest(tsk, vtime);  in __vtime_account_kernel()
690 		vtime_account_system(tsk, vtime);  in __vtime_account_kernel()
695 	struct vtime *vtime = &tsk->vtime;  in vtime_account_kernel()  local
697 	if (!vtime_delta(vtime))  in vtime_account_kernel()
700 	write_seqcount_begin(&vtime->seqcount);  in vtime_account_kernel()
701 	__vtime_account_kernel(tsk, vtime);  in vtime_account_kernel()
702 	write_seqcount_end(&vtime->seqcount);  in vtime_account_kernel()
707 	struct vtime *vtime = &tsk->vtime;  in vtime_user_enter()  local
709 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_enter()
710 	vtime_account_system(tsk, vtime);  in vtime_user_enter()
711 	vtime->state = VTIME_USER;  in vtime_user_enter()
712 	write_seqcount_end(&vtime->seqcount);  in vtime_user_enter()
717 	struct vtime *vtime = &tsk->vtime;  in vtime_user_exit()  local
719 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_exit()
720 	vtime->utime += get_vtime_delta(vtime);  in vtime_user_exit()
721 	if (vtime->utime >= TICK_NSEC) {  in vtime_user_exit()
722 		account_user_time(tsk, vtime->utime);  in vtime_user_exit()
723 		vtime->utime = 0;  in vtime_user_exit()
725 	vtime->state = VTIME_SYS;  in vtime_user_exit()
726 	write_seqcount_end(&vtime->seqcount);  in vtime_user_exit()
731 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_enter()  local
739 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_enter()
740 	vtime_account_system(tsk, vtime);  in vtime_guest_enter()
742 	vtime->state = VTIME_GUEST;  in vtime_guest_enter()
743 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_enter()
749 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_exit()  local
751 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_exit()
752 	vtime_account_guest(tsk, vtime);  in vtime_guest_exit()
754 	vtime->state = VTIME_SYS;  in vtime_guest_exit()
755 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_exit()
761 	account_idle_time(get_vtime_delta(&tsk->vtime));  in vtime_account_idle()
766 	struct vtime *vtime = &prev->vtime;  in vtime_task_switch_generic()  local
768 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
769 	if (vtime->state == VTIME_IDLE)  in vtime_task_switch_generic()
772 		__vtime_account_kernel(prev, vtime);  in vtime_task_switch_generic()
773 	vtime->state = VTIME_INACTIVE;  in vtime_task_switch_generic()
774 	vtime->cpu = -1;  in vtime_task_switch_generic()
775 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
777 	vtime = &current->vtime;  in vtime_task_switch_generic()
779 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
781 		vtime->state = VTIME_IDLE;  in vtime_task_switch_generic()
783 		vtime->state = VTIME_GUEST;  in vtime_task_switch_generic()
785 		vtime->state = VTIME_SYS;  in vtime_task_switch_generic()
786 	vtime->starttime = sched_clock();  in vtime_task_switch_generic()
787 	vtime->cpu = smp_processor_id();  in vtime_task_switch_generic()
788 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
793 	struct vtime *vtime = &t->vtime;  in vtime_init_idle()  local
797 	write_seqcount_begin(&vtime->seqcount);  in vtime_init_idle()
798 	vtime->state = VTIME_IDLE;  in vtime_init_idle()
799 	vtime->starttime = sched_clock();  in vtime_init_idle()
800 	vtime->cpu = cpu;  in vtime_init_idle()
801 	write_seqcount_end(&vtime->seqcount);  in vtime_init_idle()
807 	struct vtime *vtime = &t->vtime;  in task_gtime()  local
815 		seq = read_seqcount_begin(&vtime->seqcount);  in task_gtime()
818 		if (vtime->state == VTIME_GUEST)  in task_gtime()
819 			gtime += vtime->gtime + vtime_delta(vtime);  in task_gtime()
821 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_gtime()
833 	struct vtime *vtime = &t->vtime;  in task_cputime()  local
844 		seq = read_seqcount_begin(&vtime->seqcount);  in task_cputime()
850 		if (vtime->state < VTIME_SYS)  in task_cputime()
853 		delta = vtime_delta(vtime);  in task_cputime()
859 		if (vtime->state == VTIME_SYS)  in task_cputime()
860 			*stime += vtime->stime + delta;  in task_cputime()
862 			*utime += vtime->utime + delta;  in task_cputime()
863 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_cputime()
866 static int vtime_state_fetch(struct vtime *vtime, int cpu)  in vtime_state_fetch()  argument
868 	int state = READ_ONCE(vtime->state);  in vtime_state_fetch()
874 	if (vtime->cpu != cpu && vtime->cpu != -1)  in vtime_state_fetch()
892 static u64 kcpustat_user_vtime(struct vtime *vtime)  in kcpustat_user_vtime()  argument
894 	if (vtime->state == VTIME_USER)  in kcpustat_user_vtime()
895 		return vtime->utime + vtime_delta(vtime);  in kcpustat_user_vtime()
896 	else if (vtime->state == VTIME_GUEST)  in kcpustat_user_vtime()
897 		return vtime->gtime + vtime_delta(vtime);  in kcpustat_user_vtime()
906 	struct vtime *vtime = &tsk->vtime;  in kcpustat_field_vtime()  local
912 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_field_vtime()
914 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_field_vtime()
930 				*val += vtime->stime + vtime_delta(vtime);  in kcpustat_field_vtime()
934 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
938 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
942 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
946 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
951 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_field_vtime()
994 	struct vtime *vtime = &tsk->vtime;  in kcpustat_cpu_fetch_vtime()  local
1002 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_cpu_fetch_vtime()
1004 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_cpu_fetch_vtime()
1015 		delta = vtime_delta(vtime);  in kcpustat_cpu_fetch_vtime()
1022 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;  in kcpustat_cpu_fetch_vtime()
1025 				cpustat[CPUTIME_NICE] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1027 				cpustat[CPUTIME_USER] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1031 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1032 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1034 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1035 				cpustat[CPUTIME_USER] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1038 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_cpu_fetch_vtime()