146 	int (*switch_event)(struct perf_sched *sched, struct evsel *evsel,
149 	int (*runtime_event)(struct perf_sched *sched, struct evsel *evsel,
152 	int (*wakeup_event)(struct perf_sched *sched, struct evsel *evsel,
156 	int (*fork_event)(struct perf_sched *sched, union perf_event *event,
159 	int (*migrate_task_event)(struct perf_sched *sched,
304 static void burn_nsecs(struct perf_sched *sched, u64 nsecs)  in burn_nsecs()  argument
310 	} while (T1 + sched->run_measurement_overhead < T0 + nsecs);  in burn_nsecs()
323 static void calibrate_run_measurement_overhead(struct perf_sched *sched)  in calibrate_run_measurement_overhead()  argument
330 		burn_nsecs(sched, 0);  in calibrate_run_measurement_overhead()
335 	sched->run_measurement_overhead = min_delta;  in calibrate_run_measurement_overhead()
340 static void calibrate_sleep_measurement_overhead(struct perf_sched *sched)  in calibrate_sleep_measurement_overhead()  argument
353 	sched->sleep_measurement_overhead = min_delta;  in calibrate_sleep_measurement_overhead()
386 static void add_sched_event_run(struct perf_sched *sched, struct task_desc *task,  in add_sched_event_run()  argument
396 		sched->nr_run_events_optimized++;  in add_sched_event_run()
406 	sched->nr_run_events++;  in add_sched_event_run()
409 static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *task,  in add_sched_event_wakeup()  argument
420 		sched->targetless_wakeups++;  in add_sched_event_wakeup()
424 		sched->multitarget_wakeups++;  in add_sched_event_wakeup()
433 	sched->nr_wakeup_events++;  in add_sched_event_wakeup()
436 static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *task,  in add_sched_event_sleep()  argument
443 	sched->nr_sleep_events++;  in add_sched_event_sleep()
446 static struct task_desc *register_pid(struct perf_sched *sched,  in register_pid()  argument
452 	if (sched->pid_to_task == NULL) {  in register_pid()
455 		BUG_ON((sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *))) == NULL);  in register_pid()
458 		BUG_ON((sched->pid_to_task = realloc(sched->pid_to_task, (pid + 1) *  in register_pid()
461 			sched->pid_to_task[pid_max++] = NULL;  in register_pid()
464 	task = sched->pid_to_task[pid];  in register_pid()
471 	task->nr = sched->nr_tasks;  in register_pid()
477 	add_sched_event_sleep(sched, task, 0, 0);  in register_pid()
479 	sched->pid_to_task[pid] = task;  in register_pid()
480 	sched->nr_tasks++;  in register_pid()
481 	sched->tasks = realloc(sched->tasks, sched->nr_tasks * sizeof(struct task_desc *));  in register_pid()
482 	BUG_ON(!sched->tasks);  in register_pid()
483 	sched->tasks[task->nr] = task;  in register_pid()
486 		printf("registered task #%ld, PID %ld (%s)\n", sched->nr_tasks, pid, comm);  in register_pid()
492 static void print_task_traces(struct perf_sched *sched)  in print_task_traces()  argument
497 	for (i = 0; i < sched->nr_tasks; i++) {  in print_task_traces()
498 		task = sched->tasks[i];  in print_task_traces()
504 static void add_cross_task_wakeups(struct perf_sched *sched)  in add_cross_task_wakeups()  argument
509 	for (i = 0; i < sched->nr_tasks; i++) {  in add_cross_task_wakeups()
510 		task1 = sched->tasks[i];  in add_cross_task_wakeups()
512 		if (j == sched->nr_tasks)  in add_cross_task_wakeups()
514 		task2 = sched->tasks[j];  in add_cross_task_wakeups()
515 		add_sched_event_wakeup(sched, task1, 0, task2);  in add_cross_task_wakeups()
519 static void perf_sched__process_event(struct perf_sched *sched,  in perf_sched__process_event()  argument
526 			burn_nsecs(sched, atom->duration);  in perf_sched__process_event()
560 static int self_open_counters(struct perf_sched *sched, unsigned long cur_task)  in self_open_counters()  argument
579 			if (sched->force) {  in self_open_counters()
581 				limit.rlim_cur += sched->nr_tasks - cur_task;  in self_open_counters()
615 	struct perf_sched *sched;  member
623 	struct perf_sched *sched = parms->sched;  in thread_func()  local
638 	ret = pthread_mutex_lock(&sched->start_work_mutex);  in thread_func()
640 	ret = pthread_mutex_unlock(&sched->start_work_mutex);  in thread_func()
647 		perf_sched__process_event(sched, this_task->atoms[i]);  in thread_func()
655 	ret = pthread_mutex_lock(&sched->work_done_wait_mutex);  in thread_func()
657 	ret = pthread_mutex_unlock(&sched->work_done_wait_mutex);  in thread_func()
663 static void create_tasks(struct perf_sched *sched)  in create_tasks()  argument
675 	err = pthread_mutex_lock(&sched->start_work_mutex);  in create_tasks()
677 	err = pthread_mutex_lock(&sched->work_done_wait_mutex);  in create_tasks()
679 	for (i = 0; i < sched->nr_tasks; i++) {  in create_tasks()
682 		parms->task = task = sched->tasks[i];  in create_tasks()
683 		parms->sched = sched;  in create_tasks()
684 		parms->fd = self_open_counters(sched, i);  in create_tasks()
694 static void wait_for_tasks(struct perf_sched *sched)  in wait_for_tasks()  argument
700 	sched->start_time = get_nsecs();  in wait_for_tasks()
701 	sched->cpu_usage = 0;  in wait_for_tasks()
702 	pthread_mutex_unlock(&sched->work_done_wait_mutex);  in wait_for_tasks()
704 	for (i = 0; i < sched->nr_tasks; i++) {  in wait_for_tasks()
705 		task = sched->tasks[i];  in wait_for_tasks()
710 	ret = pthread_mutex_lock(&sched->work_done_wait_mutex);  in wait_for_tasks()
715 	pthread_mutex_unlock(&sched->start_work_mutex);  in wait_for_tasks()
717 	for (i = 0; i < sched->nr_tasks; i++) {  in wait_for_tasks()
718 		task = sched->tasks[i];  in wait_for_tasks()
722 		sched->cpu_usage += task->cpu_usage;  in wait_for_tasks()
727 	if (!sched->runavg_cpu_usage)  in wait_for_tasks()
728 		sched->runavg_cpu_usage = sched->cpu_usage;  in wait_for_tasks()
729 …sched->runavg_cpu_usage = (sched->runavg_cpu_usage * (sched->replay_repeat - 1) + sched->cpu_usage…  in wait_for_tasks()
731 	sched->parent_cpu_usage = cpu_usage_1 - cpu_usage_0;  in wait_for_tasks()
732 	if (!sched->runavg_parent_cpu_usage)  in wait_for_tasks()
733 		sched->runavg_parent_cpu_usage = sched->parent_cpu_usage;  in wait_for_tasks()
734 	sched->runavg_parent_cpu_usage = (sched->runavg_parent_cpu_usage * (sched->replay_repeat - 1) +  in wait_for_tasks()
735 					 sched->parent_cpu_usage)/sched->replay_repeat;  in wait_for_tasks()
737 	ret = pthread_mutex_lock(&sched->start_work_mutex);  in wait_for_tasks()
740 	for (i = 0; i < sched->nr_tasks; i++) {  in wait_for_tasks()
741 		task = sched->tasks[i];  in wait_for_tasks()
747 static void run_one_test(struct perf_sched *sched)  in run_one_test()  argument
752 	wait_for_tasks(sched);  in run_one_test()
756 	sched->sum_runtime += delta;  in run_one_test()
757 	sched->nr_runs++;  in run_one_test()
759 	avg_delta = sched->sum_runtime / sched->nr_runs;  in run_one_test()
764 	sched->sum_fluct += fluct;  in run_one_test()
765 	if (!sched->run_avg)  in run_one_test()
766 		sched->run_avg = delta;  in run_one_test()
767 	sched->run_avg = (sched->run_avg * (sched->replay_repeat - 1) + delta) / sched->replay_repeat;  in run_one_test()
769 	printf("#%-3ld: %0.3f, ", sched->nr_runs, (double)delta / NSEC_PER_MSEC);  in run_one_test()
771 	printf("ravg: %0.2f, ", (double)sched->run_avg / NSEC_PER_MSEC);  in run_one_test()
774 		(double)sched->cpu_usage / NSEC_PER_MSEC, (double)sched->runavg_cpu_usage / NSEC_PER_MSEC);  in run_one_test()
782 		(double)sched->parent_cpu_usage / NSEC_PER_MSEC,  in run_one_test()
783 		(double)sched->runavg_parent_cpu_usage / NSEC_PER_MSEC);  in run_one_test()
788 	if (sched->nr_sleep_corrections)  in run_one_test()
789 		printf(" (%ld sleep corrections)\n", sched->nr_sleep_corrections);  in run_one_test()
790 	sched->nr_sleep_corrections = 0;  in run_one_test()
793 static void test_calibrations(struct perf_sched *sched)  in test_calibrations()  argument
798 	burn_nsecs(sched, NSEC_PER_MSEC);  in test_calibrations()
811 replay_wakeup_event(struct perf_sched *sched,  in replay_wakeup_event()  argument
825 	waker = register_pid(sched, sample->tid, "<unknown>");  in replay_wakeup_event()
826 	wakee = register_pid(sched, pid, comm);  in replay_wakeup_event()
828 	add_sched_event_wakeup(sched, waker, sample->time, wakee);  in replay_wakeup_event()
832 static int replay_switch_event(struct perf_sched *sched,  in replay_switch_event()  argument
853 	timestamp0 = sched->cpu_last_switched[cpu];  in replay_switch_event()
867 	prev = register_pid(sched, prev_pid, prev_comm);  in replay_switch_event()
868 	next = register_pid(sched, next_pid, next_comm);  in replay_switch_event()
870 	sched->cpu_last_switched[cpu] = timestamp;  in replay_switch_event()
872 	add_sched_event_run(sched, prev, timestamp, delta);  in replay_switch_event()
873 	add_sched_event_sleep(sched, prev, timestamp, prev_state);  in replay_switch_event()
878 static int replay_fork_event(struct perf_sched *sched,  in replay_fork_event()  argument
901 	register_pid(sched, parent->tid, thread__comm_str(parent));  in replay_fork_event()
902 	register_pid(sched, child->tid, thread__comm_str(child));  in replay_fork_event()
1017 static int thread_atoms_insert(struct perf_sched *sched, struct thread *thread)  in thread_atoms_insert()  argument
1027 	__thread_latency_insert(&sched->atom_root, atoms, &sched->cmp_pid);  in thread_atoms_insert()
1106 static int latency_switch_event(struct perf_sched *sched,  in latency_switch_event()  argument
1122 	timestamp0 = sched->cpu_last_switched[cpu];  in latency_switch_event()
1123 	sched->cpu_last_switched[cpu] = timestamp;  in latency_switch_event()
1139 	out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);  in latency_switch_event()
1141 		if (thread_atoms_insert(sched, sched_out))  in latency_switch_event()
1143 		out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);  in latency_switch_event()
1152 	in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);  in latency_switch_event()
1154 		if (thread_atoms_insert(sched, sched_in))  in latency_switch_event()
1156 		in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);  in latency_switch_event()
1176 static int latency_runtime_event(struct perf_sched *sched,  in latency_runtime_event()  argument
1184 	struct work_atoms *atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);  in latency_runtime_event()
1193 		if (thread_atoms_insert(sched, thread))  in latency_runtime_event()
1195 		atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);  in latency_runtime_event()
1211 static int latency_wakeup_event(struct perf_sched *sched,  in latency_wakeup_event()  argument
1226 	atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);  in latency_wakeup_event()
1228 		if (thread_atoms_insert(sched, wakee))  in latency_wakeup_event()
1230 		atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);  in latency_wakeup_event()
1254 	if (sched->profile_cpu == -1 && atom->state != THREAD_SLEEPING)  in latency_wakeup_event()
1257 	sched->nr_timestamps++;  in latency_wakeup_event()
1259 		sched->nr_unordered_timestamps++;  in latency_wakeup_event()
1272 static int latency_migrate_task_event(struct perf_sched *sched,  in latency_migrate_task_event()  argument
1287 	if (sched->profile_cpu == -1)  in latency_migrate_task_event()
1293 	atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);  in latency_migrate_task_event()
1295 		if (thread_atoms_insert(sched, migrant))  in latency_migrate_task_event()
1297 		register_pid(sched, migrant->tid, thread__comm_str(migrant));  in latency_migrate_task_event()
1298 		atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);  in latency_migrate_task_event()
1312 	sched->nr_timestamps++;  in latency_migrate_task_event()
1315 		sched->nr_unordered_timestamps++;  in latency_migrate_task_event()
1322 static void output_lat_thread(struct perf_sched *sched, struct work_atoms *work_list)  in output_lat_thread()  argument
1337 	sched->all_runtime += work_list->total_runtime;  in output_lat_thread()
1338 	sched->all_count   += work_list->nb_atoms;  in output_lat_thread()
1463 static void perf_sched__sort_lat(struct perf_sched *sched)  in perf_sched__sort_lat()  argument
1466 	struct rb_root_cached *root = &sched->atom_root;  in perf_sched__sort_lat()
1476 		__thread_latency_insert(&sched->sorted_atom_root, data, &sched->sort_list);  in perf_sched__sort_lat()
1478 	if (root == &sched->atom_root) {  in perf_sched__sort_lat()
1479 		root = &sched->merged_atom_root;  in perf_sched__sort_lat()
1489 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in process_sched_wakeup_event()  local
1491 	if (sched->tp_handler->wakeup_event)  in process_sched_wakeup_event()
1492 		return sched->tp_handler->wakeup_event(sched, evsel, sample, machine);  in process_sched_wakeup_event()
1512 map__findnew_thread(struct perf_sched *sched, struct machine *machine, pid_t pid, pid_t tid)  in map__findnew_thread()  argument
1519 	if (!sched->map.color_pids || !thread || thread__priv(thread))  in map__findnew_thread()
1522 	if (thread_map__has(sched->map.color_pids, tid))  in map__findnew_thread()
1529 static int map_switch_event(struct perf_sched *sched, struct evsel *evsel,  in map_switch_event()  argument
1549 	if (this_cpu.cpu > sched->max_cpu.cpu)  in map_switch_event()
1550 		sched->max_cpu = this_cpu;  in map_switch_event()
1552 	if (sched->map.comp) {  in map_switch_event()
1553 		cpus_nr = bitmap_weight(sched->map.comp_cpus_mask, MAX_CPUS);  in map_switch_event()
1554 		if (!test_and_set_bit(this_cpu.cpu, sched->map.comp_cpus_mask)) {  in map_switch_event()
1555 			sched->map.comp_cpus[cpus_nr++] = this_cpu;  in map_switch_event()
1559 		cpus_nr = sched->max_cpu.cpu;  in map_switch_event()
1561 	timestamp0 = sched->cpu_last_switched[this_cpu.cpu];  in map_switch_event()
1562 	sched->cpu_last_switched[this_cpu.cpu] = timestamp;  in map_switch_event()
1573 	sched_in = map__findnew_thread(sched, machine, -1, next_pid);  in map_switch_event()
1583 	sched->curr_thread[this_cpu.cpu] = thread__get(sched_in);  in map_switch_event()
1597 			tr->shortname[0] = sched->next_shortname1;  in map_switch_event()
1598 			tr->shortname[1] = sched->next_shortname2;  in map_switch_event()
1600 			if (sched->next_shortname1 < 'Z') {  in map_switch_event()
1601 				sched->next_shortname1++;  in map_switch_event()
1603 				sched->next_shortname1 = 'A';  in map_switch_event()
1604 				if (sched->next_shortname2 < '9')  in map_switch_event()
1605 					sched->next_shortname2++;  in map_switch_event()
1607 					sched->next_shortname2 = '0';  in map_switch_event()
1615 			.cpu = sched->map.comp ? sched->map.comp_cpus[i].cpu : i,  in map_switch_event()
1617 		struct thread *curr_thread = sched->curr_thread[cpu.cpu];  in map_switch_event()
1625 		if (sched->map.cpus && !perf_cpu_map__has(sched->map.cpus, cpu))  in map_switch_event()
1628 		if (sched->map.color_cpus && perf_cpu_map__has(sched->map.color_cpus, cpu))  in map_switch_event()
1636 		if (sched->curr_thread[cpu.cpu]) {  in map_switch_event()
1637 			curr_tr = thread__get_runtime(sched->curr_thread[cpu.cpu]);  in map_switch_event()
1647 	if (sched->map.cpus && !perf_cpu_map__has(sched->map.cpus, this_cpu))  in map_switch_event()
1663 	if (sched->map.comp && new_cpu)  in map_switch_event()
1679 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in process_sched_switch_event()  local
1684 	if (sched->curr_pid[this_cpu] != (u32)-1) {  in process_sched_switch_event()
1689 		if (sched->curr_pid[this_cpu] != prev_pid)  in process_sched_switch_event()
1690 			sched->nr_context_switch_bugs++;  in process_sched_switch_event()
1693 	if (sched->tp_handler->switch_event)  in process_sched_switch_event()
1694 		err = sched->tp_handler->switch_event(sched, evsel, sample, machine);  in process_sched_switch_event()
1696 	sched->curr_pid[this_cpu] = next_pid;  in process_sched_switch_event()
1705 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in process_sched_runtime_event()  local
1707 	if (sched->tp_handler->runtime_event)  in process_sched_runtime_event()
1708 		return sched->tp_handler->runtime_event(sched, evsel, sample, machine);  in process_sched_runtime_event()
1718 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in perf_sched__process_fork_event()  local
1724 	if (sched->tp_handler->fork_event)  in perf_sched__process_fork_event()
1725 		return sched->tp_handler->fork_event(sched, event, machine);  in perf_sched__process_fork_event()
1735 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in process_sched_migrate_task_event()  local
1737 	if (sched->tp_handler->migrate_task_event)  in process_sched_migrate_task_event()
1738 		return sched->tp_handler->migrate_task_event(sched, evsel, sample, machine);  in process_sched_migrate_task_event()
1795 static int perf_sched__read_events(struct perf_sched *sched)  in perf_sched__read_events()  argument
1808 		.force = sched->force,  in perf_sched__read_events()
1812 	session = perf_session__new(&data, &sched->tool);  in perf_sched__read_events()
1830 		sched->nr_events      = session->evlist->stats.nr_events[0];  in perf_sched__read_events()
1831 		sched->nr_lost_events = session->evlist->stats.total_lost;  in perf_sched__read_events()
1832 		sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];  in perf_sched__read_events()
1935 static void timehist_header(struct perf_sched *sched)  in timehist_header()  argument
1937 	u32 ncpus = sched->max_cpu.cpu + 1;  in timehist_header()
1942 	if (sched->show_cpu_visual) {  in timehist_header()
1955 	if (sched->show_state)  in timehist_header()
1965 	if (sched->show_cpu_visual)  in timehist_header()
1971 	if (sched->show_state)  in timehist_header()
1981 	if (sched->show_cpu_visual)  in timehist_header()
1988 	if (sched->show_state)  in timehist_header()
2006 static void timehist_print_sample(struct perf_sched *sched,  in timehist_print_sample()  argument
2016 	u32 max_cpus = sched->max_cpu.cpu + 1;  in timehist_print_sample()
2027 	if (sched->show_cpu_visual) {  in timehist_print_sample()
2051 	if (sched->show_state)  in timehist_print_sample()
2054 	if (sched->show_next) {  in timehist_print_sample()
2059 	if (sched->show_wakeups && !sched->show_next)  in timehist_print_sample()
2065 	if (sched->show_callchain)  in timehist_print_sample()
2152 static void save_task_callchain(struct perf_sched *sched,  in save_task_callchain()  argument
2167 	if (!sched->show_callchain || sample->callchain == NULL)  in save_task_callchain()
2171 				      NULL, NULL, sched->max_stack + 2) != 0) {  in save_task_callchain()
2295 static void save_idle_callchain(struct perf_sched *sched,  in save_idle_callchain()  argument
2299 	if (!sched->show_callchain || sample->callchain == NULL)  in save_idle_callchain()
2305 static struct thread *timehist_get_thread(struct perf_sched *sched,  in timehist_get_thread()  argument
2326 		save_task_callchain(sched, sample, evsel, machine);  in timehist_get_thread()
2327 		if (sched->idle_hist) {  in timehist_get_thread()
2345 				save_idle_callchain(sched, itr, sample);  in timehist_get_thread()
2352 static bool timehist_skip_sample(struct perf_sched *sched,  in timehist_skip_sample()  argument
2361 		sched->skipped_samples++;  in timehist_skip_sample()
2364 	if (sched->idle_hist) {  in timehist_skip_sample()
2375 static void timehist_print_wakeup_event(struct perf_sched *sched,  in timehist_print_wakeup_event()  argument
2389 	if (timehist_skip_sample(sched, thread, evsel, sample) &&  in timehist_print_wakeup_event()
2390 	    timehist_skip_sample(sched, awakened, evsel, sample)) {  in timehist_print_wakeup_event()
2396 	if (sched->show_cpu_visual)  in timehist_print_wakeup_event()
2397 		printf(" %*s ", sched->max_cpu.cpu + 1, "");  in timehist_print_wakeup_event()
2424 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in timehist_sched_wakeup_event()  local
2442 	if (sched->show_wakeups &&  in timehist_sched_wakeup_event()
2443 	    !perf_time__skip_sample(&sched->ptime, sample->time))  in timehist_sched_wakeup_event()
2444 		timehist_print_wakeup_event(sched, evsel, sample, machine, thread);  in timehist_sched_wakeup_event()
2449 static void timehist_print_migration_event(struct perf_sched *sched,  in timehist_print_migration_event()  argument
2460 	if (sched->summary_only)  in timehist_print_migration_event()
2463 	max_cpus = sched->max_cpu.cpu + 1;  in timehist_print_migration_event()
2471 	if (timehist_skip_sample(sched, thread, evsel, sample) &&  in timehist_print_migration_event()
2472 	    timehist_skip_sample(sched, migrated, evsel, sample)) {  in timehist_print_migration_event()
2479 	if (sched->show_cpu_visual) {  in timehist_print_migration_event()
2508 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in timehist_migrate_task_event()  local
2525 	timehist_print_migration_event(sched, evsel, sample, machine, thread);  in timehist_migrate_task_event()
2536 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in timehist_sched_change_event()  local
2537 	struct perf_time_interval *ptime = &sched->ptime;  in timehist_sched_change_event()
2552 	thread = timehist_get_thread(sched, sample, machine, evsel);  in timehist_sched_change_event()
2558 	if (timehist_skip_sample(sched, thread, evsel, sample))  in timehist_sched_change_event()
2594 	if (!sched->idle_hist || thread->tid == 0) {  in timehist_sched_change_event()
2598 		if (sched->idle_hist) {  in timehist_sched_change_event()
2631 	if (!sched->summary_only)  in timehist_sched_change_event()
2632 		timehist_print_sample(sched, evsel, sample, &al, thread, t, state);  in timehist_sched_change_event()
2635 	if (sched->hist_time.start == 0 && t >= ptime->start)  in timehist_sched_change_event()
2636 		sched->hist_time.start = t;  in timehist_sched_change_event()
2638 		sched->hist_time.end = t;  in timehist_sched_change_event()
2722 	struct perf_sched *sched;  member
2742 		if (stats->sched->show_state)  in __show_thread_runtime()
2821 static void timehist_print_summary(struct perf_sched *sched,  in timehist_print_summary()  argument
2830 	u64 hist_time = sched->hist_time.end - sched->hist_time.start;  in timehist_print_summary()
2833 	totals.sched = sched;  in timehist_print_summary()
2835 	if (sched->idle_hist) {  in timehist_print_summary()
2839 	} else if (sched->show_state) {  in timehist_print_summary()
2850 	       sched->show_state ? "(msec)" : "%");  in timehist_print_summary()
2864 	if (sched->skipped_samples && !sched->idle_hist)  in timehist_print_summary()
2886 	if (sched->idle_hist && sched->show_callchain) {  in timehist_print_summary()
2926 	printf(" (x %d)\n", sched->max_cpu.cpu);  in timehist_print_summary()
2941 	struct perf_sched *sched = container_of(tool, struct perf_sched, tool);  in perf_timehist__process_sample()  local
2947 	if (this_cpu.cpu > sched->max_cpu.cpu)  in perf_timehist__process_sample()
2948 		sched->max_cpu = this_cpu;  in perf_timehist__process_sample()
2959 static int timehist_check_attr(struct perf_sched *sched,  in timehist_check_attr()  argument
2972 		if (sched->show_callchain && !evsel__has_callchain(evsel)) {  in timehist_check_attr()
2974 			sched->show_callchain = 0;  in timehist_check_attr()
2982 static int perf_sched__timehist(struct perf_sched *sched)  in perf_sched__timehist()  argument
2996 		.force = sched->force,  in perf_sched__timehist()
3006 	sched->tool.sample	 = perf_timehist__process_sample;  in perf_sched__timehist()
3007 	sched->tool.mmap	 = perf_event__process_mmap;  in perf_sched__timehist()
3008 	sched->tool.comm	 = perf_event__process_comm;  in perf_sched__timehist()
3009 	sched->tool.exit	 = perf_event__process_exit;  in perf_sched__timehist()
3010 	sched->tool.fork	 = perf_event__process_fork;  in perf_sched__timehist()
3011 	sched->tool.lost	 = process_lost;  in perf_sched__timehist()
3012 	sched->tool.attr	 = perf_event__process_attr;  in perf_sched__timehist()
3013 	sched->tool.tracing_data = perf_event__process_tracing_data;  in perf_sched__timehist()
3014 	sched->tool.build_id	 = perf_event__process_build_id;  in perf_sched__timehist()
3016 	sched->tool.ordered_events = true;  in perf_sched__timehist()
3017 	sched->tool.ordering_requires_timestamps = true;  in perf_sched__timehist()
3019 	symbol_conf.use_callchain = sched->show_callchain;  in perf_sched__timehist()
3021 	session = perf_session__new(&data, &sched->tool);  in perf_sched__timehist()
3035 	if (perf_time__parse_str(&sched->ptime, sched->time_str) != 0) {  in perf_sched__timehist()
3040 	if (timehist_check_attr(sched, evlist) != 0)  in perf_sched__timehist()
3059 	if (sched->show_migrations &&  in perf_sched__timehist()
3064 	sched->max_cpu.cpu = session->header.env.nr_cpus_online;  in perf_sched__timehist()
3065 	if (sched->max_cpu.cpu == 0)  in perf_sched__timehist()
3066 		sched->max_cpu.cpu = 4;  in perf_sched__timehist()
3067 	if (init_idle_threads(sched->max_cpu.cpu))  in perf_sched__timehist()
3071 	if (sched->summary_only)  in perf_sched__timehist()
3072 		sched->summary = sched->summary_only;  in perf_sched__timehist()
3074 	if (!sched->summary_only)  in perf_sched__timehist()
3075 		timehist_header(sched);  in perf_sched__timehist()
3083 	sched->nr_events      = evlist->stats.nr_events[0];  in perf_sched__timehist()
3084 	sched->nr_lost_events = evlist->stats.total_lost;  in perf_sched__timehist()
3085 	sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST];  in perf_sched__timehist()
3087 	if (sched->summary)  in perf_sched__timehist()
3088 		timehist_print_summary(sched, session);  in perf_sched__timehist()
3098 static void print_bad_events(struct perf_sched *sched)  in print_bad_events()  argument
3100 	if (sched->nr_unordered_timestamps && sched->nr_timestamps) {  in print_bad_events()
3102 			(double)sched->nr_unordered_timestamps/(double)sched->nr_timestamps*100.0,  in print_bad_events()
3103 			sched->nr_unordered_timestamps, sched->nr_timestamps);  in print_bad_events()
3105 	if (sched->nr_lost_events && sched->nr_events) {  in print_bad_events()
3107 			(double)sched->nr_lost_events/(double)sched->nr_events * 100.0,  in print_bad_events()
3108 			sched->nr_lost_events, sched->nr_events, sched->nr_lost_chunks);  in print_bad_events()
3110 	if (sched->nr_context_switch_bugs && sched->nr_timestamps) {  in print_bad_events()
3112 			(double)sched->nr_context_switch_bugs/(double)sched->nr_timestamps*100.0,  in print_bad_events()
3113 			sched->nr_context_switch_bugs, sched->nr_timestamps);  in print_bad_events()
3114 		if (sched->nr_lost_events)  in print_bad_events()
3161 static void perf_sched__merge_lat(struct perf_sched *sched)  in perf_sched__merge_lat()  argument
3166 	if (sched->skip_merge)  in perf_sched__merge_lat()
3169 	while ((node = rb_first_cached(&sched->atom_root))) {  in perf_sched__merge_lat()
3170 		rb_erase_cached(node, &sched->atom_root);  in perf_sched__merge_lat()
3172 		__merge_work_atoms(&sched->merged_atom_root, data);  in perf_sched__merge_lat()
3176 static int perf_sched__lat(struct perf_sched *sched)  in perf_sched__lat()  argument
3182 	if (perf_sched__read_events(sched))  in perf_sched__lat()
3185 	perf_sched__merge_lat(sched);  in perf_sched__lat()
3186 	perf_sched__sort_lat(sched);  in perf_sched__lat()
3192 	next = rb_first_cached(&sched->sorted_atom_root);  in perf_sched__lat()
3198 		output_lat_thread(sched, work_list);  in perf_sched__lat()
3205 		(double)sched->all_runtime / NSEC_PER_MSEC, sched->all_count);  in perf_sched__lat()
3209 	print_bad_events(sched);  in perf_sched__lat()
3215 static int setup_map_cpus(struct perf_sched *sched)  in setup_map_cpus()  argument
3219 	sched->max_cpu.cpu  = sysconf(_SC_NPROCESSORS_CONF);  in setup_map_cpus()
3221 	if (sched->map.comp) {  in setup_map_cpus()
3222 		sched->map.comp_cpus = zalloc(sched->max_cpu.cpu * sizeof(int));  in setup_map_cpus()
3223 		if (!sched->map.comp_cpus)  in setup_map_cpus()
3227 	if (!sched->map.cpus_str)  in setup_map_cpus()
3230 	map = perf_cpu_map__new(sched->map.cpus_str);  in setup_map_cpus()
3232 		pr_err("failed to get cpus map from %s\n", sched->map.cpus_str);  in setup_map_cpus()
3236 	sched->map.cpus = map;  in setup_map_cpus()
3240 static int setup_color_pids(struct perf_sched *sched)  in setup_color_pids()  argument
3244 	if (!sched->map.color_pids_str)  in setup_color_pids()
3247 	map = thread_map__new_by_tid_str(sched->map.color_pids_str);  in setup_color_pids()
3249 		pr_err("failed to get thread map from %s\n", sched->map.color_pids_str);  in setup_color_pids()
3253 	sched->map.color_pids = map;  in setup_color_pids()
3257 static int setup_color_cpus(struct perf_sched *sched)  in setup_color_cpus()  argument
3261 	if (!sched->map.color_cpus_str)  in setup_color_cpus()
3264 	map = perf_cpu_map__new(sched->map.color_cpus_str);  in setup_color_cpus()
3266 		pr_err("failed to get thread map from %s\n", sched->map.color_cpus_str);  in setup_color_cpus()
3270 	sched->map.color_cpus = map;  in setup_color_cpus()
3274 static int perf_sched__map(struct perf_sched *sched)  in perf_sched__map()  argument
3276 	if (setup_map_cpus(sched))  in perf_sched__map()
3279 	if (setup_color_pids(sched))  in perf_sched__map()
3282 	if (setup_color_cpus(sched))  in perf_sched__map()
3286 	if (perf_sched__read_events(sched))  in perf_sched__map()
3288 	print_bad_events(sched);  in perf_sched__map()
3292 static int perf_sched__replay(struct perf_sched *sched)  in perf_sched__replay()  argument
3296 	calibrate_run_measurement_overhead(sched);  in perf_sched__replay()
3297 	calibrate_sleep_measurement_overhead(sched);  in perf_sched__replay()
3299 	test_calibrations(sched);  in perf_sched__replay()
3301 	if (perf_sched__read_events(sched))  in perf_sched__replay()
3304 	printf("nr_run_events:        %ld\n", sched->nr_run_events);  in perf_sched__replay()
3305 	printf("nr_sleep_events:      %ld\n", sched->nr_sleep_events);  in perf_sched__replay()
3306 	printf("nr_wakeup_events:     %ld\n", sched->nr_wakeup_events);  in perf_sched__replay()
3308 	if (sched->targetless_wakeups)  in perf_sched__replay()
3309 		printf("target-less wakeups:  %ld\n", sched->targetless_wakeups);  in perf_sched__replay()
3310 	if (sched->multitarget_wakeups)  in perf_sched__replay()
3311 		printf("multi-target wakeups: %ld\n", sched->multitarget_wakeups);  in perf_sched__replay()
3312 	if (sched->nr_run_events_optimized)  in perf_sched__replay()
3314 			sched->nr_run_events_optimized);  in perf_sched__replay()
3316 	print_task_traces(sched);  in perf_sched__replay()
3317 	add_cross_task_wakeups(sched);  in perf_sched__replay()
3319 	create_tasks(sched);  in perf_sched__replay()
3321 	for (i = 0; i < sched->replay_repeat; i++)  in perf_sched__replay()
3322 		run_one_test(sched);  in perf_sched__replay()
3327 static void setup_sorting(struct perf_sched *sched, const struct option *options,  in setup_sorting()  argument
3330 	char *tmp, *tok, *str = strdup(sched->sort_order);  in setup_sorting()
3334 		if (sort_dimension__add(tok, &sched->sort_list) < 0) {  in setup_sorting()
3342 	sort_dimension__add("pid", &sched->cmp_pid);  in setup_sorting()
3422 	struct perf_sched sched = {  in cmd_sched()  local
3431 		.cmp_pid	      = LIST_HEAD_INIT(sched.cmp_pid),  in cmd_sched()
3432 		.sort_list	      = LIST_HEAD_INIT(sched.sort_list),  in cmd_sched()
3451 	OPT_BOOLEAN('f', "force", &sched.force, "don't complain, do it"),  in cmd_sched()
3455 	OPT_STRING('s', "sort", &sched.sort_order, "key[,key2...]",  in cmd_sched()
3457 	OPT_INTEGER('C', "CPU", &sched.profile_cpu,  in cmd_sched()
3459 	OPT_BOOLEAN('p', "pids", &sched.skip_merge,  in cmd_sched()
3464 	OPT_UINTEGER('r', "repeat", &sched.replay_repeat,  in cmd_sched()
3469 	OPT_BOOLEAN(0, "compact", &sched.map.comp,  in cmd_sched()
3471 	OPT_STRING(0, "color-pids", &sched.map.color_pids_str, "pids",  in cmd_sched()
3473 	OPT_STRING(0, "color-cpus", &sched.map.color_cpus_str, "cpus",  in cmd_sched()
3475 	OPT_STRING(0, "cpus", &sched.map.cpus_str, "cpus",  in cmd_sched()
3484 	OPT_BOOLEAN('g', "call-graph", &sched.show_callchain,  in cmd_sched()
3486 	OPT_UINTEGER(0, "max-stack", &sched.max_stack,  in cmd_sched()
3490 	OPT_BOOLEAN('s', "summary", &sched.summary_only,  in cmd_sched()
3492 	OPT_BOOLEAN('S', "with-summary", &sched.summary,  in cmd_sched()
3494 	OPT_BOOLEAN('w', "wakeups", &sched.show_wakeups, "Show wakeup events"),  in cmd_sched()
3495 	OPT_BOOLEAN('n', "next", &sched.show_next, "Show next task"),  in cmd_sched()
3496 	OPT_BOOLEAN('M', "migrations", &sched.show_migrations, "Show migration events"),  in cmd_sched()
3497 	OPT_BOOLEAN('V', "cpu-visual", &sched.show_cpu_visual, "Add CPU visual"),  in cmd_sched()
3498 	OPT_BOOLEAN('I', "idle-hist", &sched.idle_hist, "Show idle events only"),  in cmd_sched()
3499 	OPT_STRING(0, "time", &sched.time_str, "str",  in cmd_sched()
3501 	OPT_BOOLEAN(0, "state", &sched.show_state, "Show task state when sched-out"),  in cmd_sched()
3550 	for (i = 0; i < ARRAY_SIZE(sched.curr_pid); i++)  in cmd_sched()
3551 		sched.curr_pid[i] = -1;  in cmd_sched()
3567 		sched.tp_handler = &lat_ops;  in cmd_sched()
3573 		setup_sorting(&sched, latency_options, latency_usage);  in cmd_sched()
3574 		return perf_sched__lat(&sched);  in cmd_sched()
3581 		sched.tp_handler = &map_ops;  in cmd_sched()
3582 		setup_sorting(&sched, latency_options, latency_usage);  in cmd_sched()
3583 		return perf_sched__map(&sched);  in cmd_sched()
3585 		sched.tp_handler = &replay_ops;  in cmd_sched()
3591 		return perf_sched__replay(&sched);  in cmd_sched()
3599 		if ((sched.show_wakeups || sched.show_next) &&  in cmd_sched()
3600 		    sched.summary_only) {  in cmd_sched()
3603 			if (sched.show_wakeups)  in cmd_sched()
3605 			if (sched.show_next)  in cmd_sched()
3613 		return perf_sched__timehist(&sched);  in cmd_sched()