648 static u64 vtime_delta(struct vtime *vtime)  in vtime_delta()  argument
653 	if (clock < vtime->starttime)  in vtime_delta()
656 	return clock - vtime->starttime;  in vtime_delta()
659 static u64 get_vtime_delta(struct vtime *vtime)  in get_vtime_delta()  argument
661 	u64 delta = vtime_delta(vtime);  in get_vtime_delta()
672 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);  in get_vtime_delta()
673 	vtime->starttime += delta;  in get_vtime_delta()
679 				 struct vtime *vtime)  in vtime_account_system()  argument
681 	vtime->stime += get_vtime_delta(vtime);  in vtime_account_system()
682 	if (vtime->stime >= TICK_NSEC) {  in vtime_account_system()
683 		account_system_time(tsk, irq_count(), vtime->stime);  in vtime_account_system()
684 		vtime->stime = 0;  in vtime_account_system()
689 				struct vtime *vtime)  in vtime_account_guest()  argument
691 	vtime->gtime += get_vtime_delta(vtime);  in vtime_account_guest()
692 	if (vtime->gtime >= TICK_NSEC) {  in vtime_account_guest()
693 		account_guest_time(tsk, vtime->gtime);  in vtime_account_guest()
694 		vtime->gtime = 0;  in vtime_account_guest()
699 				   struct vtime *vtime)  in __vtime_account_kernel()  argument
702 	if (vtime->state == VTIME_GUEST)  in __vtime_account_kernel()
703 		vtime_account_guest(tsk, vtime);  in __vtime_account_kernel()
705 		vtime_account_system(tsk, vtime);  in __vtime_account_kernel()
710 	struct vtime *vtime = &tsk->vtime;  in vtime_account_kernel()  local
712 	if (!vtime_delta(vtime))  in vtime_account_kernel()
715 	write_seqcount_begin(&vtime->seqcount);  in vtime_account_kernel()
716 	__vtime_account_kernel(tsk, vtime);  in vtime_account_kernel()
717 	write_seqcount_end(&vtime->seqcount);  in vtime_account_kernel()
722 	struct vtime *vtime = &tsk->vtime;  in vtime_user_enter()  local
724 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_enter()
725 	vtime_account_system(tsk, vtime);  in vtime_user_enter()
726 	vtime->state = VTIME_USER;  in vtime_user_enter()
727 	write_seqcount_end(&vtime->seqcount);  in vtime_user_enter()
732 	struct vtime *vtime = &tsk->vtime;  in vtime_user_exit()  local
734 	write_seqcount_begin(&vtime->seqcount);  in vtime_user_exit()
735 	vtime->utime += get_vtime_delta(vtime);  in vtime_user_exit()
736 	if (vtime->utime >= TICK_NSEC) {  in vtime_user_exit()
737 		account_user_time(tsk, vtime->utime);  in vtime_user_exit()
738 		vtime->utime = 0;  in vtime_user_exit()
740 	vtime->state = VTIME_SYS;  in vtime_user_exit()
741 	write_seqcount_end(&vtime->seqcount);  in vtime_user_exit()
746 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_enter()  local
754 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_enter()
755 	vtime_account_system(tsk, vtime);  in vtime_guest_enter()
757 	vtime->state = VTIME_GUEST;  in vtime_guest_enter()
758 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_enter()
764 	struct vtime *vtime = &tsk->vtime;  in vtime_guest_exit()  local
766 	write_seqcount_begin(&vtime->seqcount);  in vtime_guest_exit()
767 	vtime_account_guest(tsk, vtime);  in vtime_guest_exit()
769 	vtime->state = VTIME_SYS;  in vtime_guest_exit()
770 	write_seqcount_end(&vtime->seqcount);  in vtime_guest_exit()
776 	account_idle_time(get_vtime_delta(&tsk->vtime));  in vtime_account_idle()
781 	struct vtime *vtime = &prev->vtime;  in vtime_task_switch_generic()  local
783 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
784 	if (vtime->state == VTIME_IDLE)  in vtime_task_switch_generic()
787 		__vtime_account_kernel(prev, vtime);  in vtime_task_switch_generic()
788 	vtime->state = VTIME_INACTIVE;  in vtime_task_switch_generic()
789 	vtime->cpu = -1;  in vtime_task_switch_generic()
790 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
792 	vtime = &current->vtime;  in vtime_task_switch_generic()
794 	write_seqcount_begin(&vtime->seqcount);  in vtime_task_switch_generic()
796 		vtime->state = VTIME_IDLE;  in vtime_task_switch_generic()
798 		vtime->state = VTIME_GUEST;  in vtime_task_switch_generic()
800 		vtime->state = VTIME_SYS;  in vtime_task_switch_generic()
801 	vtime->starttime = sched_clock();  in vtime_task_switch_generic()
802 	vtime->cpu = smp_processor_id();  in vtime_task_switch_generic()
803 	write_seqcount_end(&vtime->seqcount);  in vtime_task_switch_generic()
808 	struct vtime *vtime = &t->vtime;  in vtime_init_idle()  local
812 	write_seqcount_begin(&vtime->seqcount);  in vtime_init_idle()
813 	vtime->state = VTIME_IDLE;  in vtime_init_idle()
814 	vtime->starttime = sched_clock();  in vtime_init_idle()
815 	vtime->cpu = cpu;  in vtime_init_idle()
816 	write_seqcount_end(&vtime->seqcount);  in vtime_init_idle()
822 	struct vtime *vtime = &t->vtime;  in task_gtime()  local
830 		seq = read_seqcount_begin(&vtime->seqcount);  in task_gtime()
833 		if (vtime->state == VTIME_GUEST)  in task_gtime()
834 			gtime += vtime->gtime + vtime_delta(vtime);  in task_gtime()
836 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_gtime()
848 	struct vtime *vtime = &t->vtime;  in task_cputime()  local
861 		seq = read_seqcount_begin(&vtime->seqcount);  in task_cputime()
867 		if (vtime->state < VTIME_SYS)  in task_cputime()
871 		delta = vtime_delta(vtime);  in task_cputime()
877 		if (vtime->state == VTIME_SYS)  in task_cputime()
878 			*stime += vtime->stime + delta;  in task_cputime()
880 			*utime += vtime->utime + delta;  in task_cputime()
881 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in task_cputime()
886 static int vtime_state_fetch(struct vtime *vtime, int cpu)  in vtime_state_fetch()  argument
888 	int state = READ_ONCE(vtime->state);  in vtime_state_fetch()
894 	if (vtime->cpu != cpu && vtime->cpu != -1)  in vtime_state_fetch()
912 static u64 kcpustat_user_vtime(struct vtime *vtime)  in kcpustat_user_vtime()  argument
914 	if (vtime->state == VTIME_USER)  in kcpustat_user_vtime()
915 		return vtime->utime + vtime_delta(vtime);  in kcpustat_user_vtime()
916 	else if (vtime->state == VTIME_GUEST)  in kcpustat_user_vtime()
917 		return vtime->gtime + vtime_delta(vtime);  in kcpustat_user_vtime()
926 	struct vtime *vtime = &tsk->vtime;  in kcpustat_field_vtime()  local
932 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_field_vtime()
934 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_field_vtime()
950 				*val += vtime->stime + vtime_delta(vtime);  in kcpustat_field_vtime()
954 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
958 				*val += kcpustat_user_vtime(vtime);  in kcpustat_field_vtime()
962 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
966 				*val += vtime->gtime + vtime_delta(vtime);  in kcpustat_field_vtime()
971 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_field_vtime()
1014 	struct vtime *vtime = &tsk->vtime;  in kcpustat_cpu_fetch_vtime()  local
1022 		seq = read_seqcount_begin(&vtime->seqcount);  in kcpustat_cpu_fetch_vtime()
1024 		state = vtime_state_fetch(vtime, cpu);  in kcpustat_cpu_fetch_vtime()
1035 		delta = vtime_delta(vtime);  in kcpustat_cpu_fetch_vtime()
1042 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;  in kcpustat_cpu_fetch_vtime()
1045 				cpustat[CPUTIME_NICE] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1047 				cpustat[CPUTIME_USER] += vtime->utime + delta;  in kcpustat_cpu_fetch_vtime()
1051 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1052 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1054 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1055 				cpustat[CPUTIME_USER] += vtime->gtime + delta;  in kcpustat_cpu_fetch_vtime()
1058 	} while (read_seqcount_retry(&vtime->seqcount, seq));  in kcpustat_cpu_fetch_vtime()