134 static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused,  in sort_dimension__add()
166 	if (kwork->report == KWORK_REPORT_LATENCY)  in sort_dimension__add()
179 static void setup_sorting(struct perf_kwork *kwork,  in setup_sorting()  argument
183 	char *tmp, *tok, *str = strdup(kwork->sort_order);  in setup_sorting()
187 		if (sort_dimension__add(kwork, tok, &kwork->sort_list) < 0)  in setup_sorting()
192 	pr_debug("Sort order: %s\n", kwork->sort_order);  in setup_sorting()
196 static struct kwork_atom *atom_new(struct perf_kwork *kwork,  in atom_new()  argument
203 	list_for_each_entry(page, &kwork->atom_page_list, list) {  in atom_new()
223 	list_add_tail(&page->list, &kwork->atom_page_list);  in atom_new()
351 static void profile_update_timespan(struct perf_kwork *kwork,  in profile_update_timespan()  argument
354 	if (!kwork->summary)  in profile_update_timespan()
357 	if ((kwork->timestart == 0) || (kwork->timestart > sample->time))  in profile_update_timespan()
358 		kwork->timestart = sample->time;  in profile_update_timespan()
360 	if (kwork->timeend < sample->time)  in profile_update_timespan()
361 		kwork->timeend = sample->time;  in profile_update_timespan()
364 static bool profile_event_match(struct perf_kwork *kwork,  in profile_event_match()  argument
370 	struct perf_time_interval *ptime = &kwork->ptime;  in profile_event_match()
372 	if ((kwork->cpu_list != NULL) && !test_bit(cpu, kwork->cpu_bitmap))  in profile_event_match()
379 	if ((kwork->profile_name != NULL) &&  in profile_event_match()
381 	    (strcmp(work->name, kwork->profile_name) != 0))  in profile_event_match()
384 	profile_update_timespan(kwork, sample);  in profile_event_match()
388 static int work_push_atom(struct perf_kwork *kwork,  in work_push_atom()  argument
403 	atom = atom_new(kwork, sample);  in work_push_atom()
407 	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);  in work_push_atom()
413 	if (!profile_event_match(kwork, work, sample))  in work_push_atom()
433 static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork,  in work_pop_atom()  argument
448 	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);  in work_pop_atom()
455 	if (!profile_event_match(kwork, work, sample))  in work_pop_atom()
463 	src_atom = atom_new(kwork, sample);  in work_pop_atom()
495 static int report_entry_event(struct perf_kwork *kwork,  in report_entry_event()  argument
501 	return work_push_atom(kwork, class, KWORK_TRACE_ENTRY,  in report_entry_event()
506 static int report_exit_event(struct perf_kwork *kwork,  in report_exit_event()  argument
515 	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,  in report_exit_event()
550 static int latency_raise_event(struct perf_kwork *kwork,  in latency_raise_event()  argument
556 	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,  in latency_raise_event()
561 static int latency_entry_event(struct perf_kwork *kwork,  in latency_entry_event()  argument
570 	atom = work_pop_atom(kwork, class, KWORK_TRACE_ENTRY,  in latency_entry_event()
584 static void timehist_save_callchain(struct perf_kwork *kwork,  in timehist_save_callchain()  argument
594 	if (!kwork->show_callchain || sample->callchain == NULL)  in timehist_save_callchain()
607 				      NULL, NULL, kwork->max_stack + 2) != 0) {  in timehist_save_callchain()
633 static void timehist_print_event(struct perf_kwork *kwork,  in timehist_print_event()  argument
690 	if (kwork->show_callchain) {  in timehist_print_event()
709 static int timehist_raise_event(struct perf_kwork *kwork,  in timehist_raise_event()  argument
715 	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,  in timehist_raise_event()
720 static int timehist_entry_event(struct perf_kwork *kwork,  in timehist_entry_event()  argument
729 	ret = work_push_atom(kwork, class, KWORK_TRACE_ENTRY,  in timehist_entry_event()
736 		timehist_save_callchain(kwork, sample, evsel, machine);  in timehist_entry_event()
741 static int timehist_exit_event(struct perf_kwork *kwork,  in timehist_exit_event()  argument
759 	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,  in timehist_exit_event()
769 		timehist_print_event(kwork, work, atom, sample, &al);  in timehist_exit_event()
784 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_irq_handler_entry_event()  local
786 	if (kwork->tp_handler->entry_event)  in process_irq_handler_entry_event()
787 		return kwork->tp_handler->entry_event(kwork, &kwork_irq,  in process_irq_handler_entry_event()
797 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_irq_handler_exit_event()  local
799 	if (kwork->tp_handler->exit_event)  in process_irq_handler_exit_event()
800 		return kwork->tp_handler->exit_event(kwork, &kwork_irq,  in process_irq_handler_exit_event()
855 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_raise_event()  local
857 	if (kwork->tp_handler->raise_event)  in process_softirq_raise_event()
858 		return kwork->tp_handler->raise_event(kwork, &kwork_softirq,  in process_softirq_raise_event()
869 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_entry_event()  local
871 	if (kwork->tp_handler->entry_event)  in process_softirq_entry_event()
872 		return kwork->tp_handler->entry_event(kwork, &kwork_softirq,  in process_softirq_entry_event()
883 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_exit_event()  local
885 	if (kwork->tp_handler->exit_event)  in process_softirq_exit_event()
886 		return kwork->tp_handler->exit_event(kwork, &kwork_softirq,  in process_softirq_exit_event()
976 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_activate_work_event()  local
978 	if (kwork->tp_handler->raise_event)  in process_workqueue_activate_work_event()
979 		return kwork->tp_handler->raise_event(kwork, &kwork_workqueue,  in process_workqueue_activate_work_event()
990 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_execute_start_event()  local
992 	if (kwork->tp_handler->entry_event)  in process_workqueue_execute_start_event()
993 		return kwork->tp_handler->entry_event(kwork, &kwork_workqueue,  in process_workqueue_execute_start_event()
1004 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_execute_end_event()  local
1006 	if (kwork->tp_handler->exit_event)  in process_workqueue_execute_end_event()
1007 		return kwork->tp_handler->exit_event(kwork, &kwork_workqueue,  in process_workqueue_execute_end_event()
1078 static int report_print_work(struct perf_kwork *kwork, struct kwork_work *work)  in report_print_work()  argument
1106 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_work()
1110 	} else if (kwork->report == KWORK_REPORT_LATENCY) { // avg delay  in report_print_work()
1125 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_work()
1141 	else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_work()
1159 static int report_print_header(struct perf_kwork *kwork)  in report_print_header()  argument
1168 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_header()
1171 	} else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_header()
1178 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_header()
1183 	} else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_header()
1231 static void print_summary(struct perf_kwork *kwork)  in print_summary()  argument
1233 	u64 time = kwork->timeend - kwork->timestart;  in print_summary()
1235 	printf("  Total count            : %9" PRIu64 "\n", kwork->all_count);  in print_summary()
1237 	       (double)kwork->all_runtime / NSEC_PER_MSEC,  in print_summary()
1238 	       time == 0 ? 0 : (double)kwork->all_runtime / time);  in print_summary()
1254 static void print_skipped_events(struct perf_kwork *kwork)  in print_skipped_events()  argument
1263 	if ((kwork->nr_skipped_events[KWORK_TRACE_MAX] != 0) &&  in print_skipped_events()
1264 	    (kwork->nr_events != 0)) {  in print_skipped_events()
1266 		       (double)kwork->nr_skipped_events[KWORK_TRACE_MAX] /  in print_skipped_events()
1267 		       (double)kwork->nr_events * 100.0,  in print_skipped_events()
1268 		       kwork->nr_skipped_events[KWORK_TRACE_MAX]);  in print_skipped_events()
1272 			       kwork->nr_skipped_events[i],  in print_skipped_events()
1280 		       nr_list_entry(&kwork->atom_page_list));  in print_skipped_events()
1283 static void print_bad_events(struct perf_kwork *kwork)  in print_bad_events()  argument
1285 	if ((kwork->nr_lost_events != 0) && (kwork->nr_events != 0)) {  in print_bad_events()
1287 		       (double)kwork->nr_lost_events /  in print_bad_events()
1288 		       (double)kwork->nr_events * 100.0,  in print_bad_events()
1289 		       kwork->nr_lost_events, kwork->nr_events,  in print_bad_events()
1290 		       kwork->nr_lost_chunks);  in print_bad_events()
1294 static void work_sort(struct perf_kwork *kwork, struct kwork_class *class)  in work_sort()  argument
1308 		work_insert(&kwork->sorted_work_root,  in work_sort()
1309 			       data, &kwork->sort_list);  in work_sort()
1313 static void perf_kwork__sort(struct perf_kwork *kwork)  in perf_kwork__sort()  argument
1317 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__sort()
1318 		work_sort(kwork, class);  in perf_kwork__sort()
1321 static int perf_kwork__check_config(struct perf_kwork *kwork,  in perf_kwork__check_config()  argument
1342 	switch (kwork->report) {  in perf_kwork__check_config()
1344 		kwork->tp_handler = &report_ops;  in perf_kwork__check_config()
1347 		kwork->tp_handler = &latency_ops;  in perf_kwork__check_config()
1350 		kwork->tp_handler = &timehist_ops;  in perf_kwork__check_config()
1353 		pr_debug("Invalid report type %d\n", kwork->report);  in perf_kwork__check_config()
1357 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__check_config()
1362 	if (kwork->cpu_list != NULL) {  in perf_kwork__check_config()
1364 					       kwork->cpu_list,  in perf_kwork__check_config()
1365 					       kwork->cpu_bitmap);  in perf_kwork__check_config()
1372 	if (kwork->time_str != NULL) {  in perf_kwork__check_config()
1373 		ret = perf_time__parse_str(&kwork->ptime, kwork->time_str);  in perf_kwork__check_config()
1381 		if (kwork->show_callchain && !evsel__has_callchain(evsel)) {  in perf_kwork__check_config()
1383 			kwork->show_callchain = 0;  in perf_kwork__check_config()
1391 static int perf_kwork__read_events(struct perf_kwork *kwork)  in perf_kwork__read_events()  argument
1399 		.force = kwork->force,  in perf_kwork__read_events()
1402 	session = perf_session__new(&data, &kwork->tool);  in perf_kwork__read_events()
1410 	if (perf_kwork__check_config(kwork, session) != 0)  in perf_kwork__read_events()
1421 	if (kwork->report == KWORK_REPORT_TIMEHIST)  in perf_kwork__read_events()
1430 	kwork->nr_events      = session->evlist->stats.nr_events[0];  in perf_kwork__read_events()
1431 	kwork->nr_lost_events = session->evlist->stats.total_lost;  in perf_kwork__read_events()
1432 	kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];  in perf_kwork__read_events()
1439 static void process_skipped_events(struct perf_kwork *kwork,  in process_skipped_events()  argument
1447 		kwork->nr_skipped_events[i] += count;  in process_skipped_events()
1448 		kwork->nr_skipped_events[KWORK_TRACE_MAX] += count;  in process_skipped_events()
1452 struct kwork_work *perf_kwork_add_work(struct perf_kwork *kwork,  in perf_kwork_add_work()  argument
1462 	work_insert(&class->work_root, work, &kwork->cmp_id);  in perf_kwork_add_work()
1475 static int perf_kwork__report_bpf(struct perf_kwork *kwork)  in perf_kwork__report_bpf()  argument
1482 	ret = perf_kwork__trace_prepare_bpf(kwork);  in perf_kwork__report_bpf()
1497 	perf_kwork__report_read_bpf(kwork);  in perf_kwork__report_bpf()
1504 static int perf_kwork__report(struct perf_kwork *kwork)  in perf_kwork__report()  argument
1510 	if (kwork->use_bpf)  in perf_kwork__report()
1511 		ret = perf_kwork__report_bpf(kwork);  in perf_kwork__report()
1513 		ret = perf_kwork__read_events(kwork);  in perf_kwork__report()
1518 	perf_kwork__sort(kwork);  in perf_kwork__report()
1522 	ret = report_print_header(kwork);  in perf_kwork__report()
1523 	next = rb_first_cached(&kwork->sorted_work_root);  in perf_kwork__report()
1526 		process_skipped_events(kwork, work);  in perf_kwork__report()
1529 			report_print_work(kwork, work);  in perf_kwork__report()
1530 			if (kwork->summary) {  in perf_kwork__report()
1531 				kwork->all_runtime += work->total_runtime;  in perf_kwork__report()
1532 				kwork->all_count += work->nr_atoms;  in perf_kwork__report()
1539 	if (kwork->summary) {  in perf_kwork__report()
1540 		print_summary(kwork);  in perf_kwork__report()
1544 	print_bad_events(kwork);  in perf_kwork__report()
1545 	print_skipped_events(kwork);  in perf_kwork__report()
1573 static int perf_kwork__timehist(struct perf_kwork *kwork)  in perf_kwork__timehist()  argument
1578 	kwork->tool.comm	 = perf_event__process_comm;  in perf_kwork__timehist()
1579 	kwork->tool.exit	 = perf_event__process_exit;  in perf_kwork__timehist()
1580 	kwork->tool.fork	 = perf_event__process_fork;  in perf_kwork__timehist()
1581 	kwork->tool.attr	 = perf_event__process_attr;  in perf_kwork__timehist()
1582 	kwork->tool.tracing_data = perf_event__process_tracing_data;  in perf_kwork__timehist()
1583 	kwork->tool.build_id	 = perf_event__process_build_id;  in perf_kwork__timehist()
1584 	kwork->tool.ordered_events = true;  in perf_kwork__timehist()
1585 	kwork->tool.ordering_requires_timestamps = true;  in perf_kwork__timehist()
1586 	symbol_conf.use_callchain = kwork->show_callchain;  in perf_kwork__timehist()
1595 	return perf_kwork__read_events(kwork);  in perf_kwork__timehist()
1598 static void setup_event_list(struct perf_kwork *kwork,  in setup_event_list()  argument
1606 	if (kwork->event_list_str == NULL)  in setup_event_list()
1609 	str = strdup(kwork->event_list_str);  in setup_event_list()
1615 				list_add_tail(&class->list, &kwork->class_list);  in setup_event_list()
1630 	if (list_empty(&kwork->class_list)) {  in setup_event_list()
1633 				      &kwork->class_list);  in setup_event_list()
1638 	list_for_each_entry(class, &kwork->class_list, list)  in setup_event_list()
1643 static int perf_kwork__record(struct perf_kwork *kwork,  in perf_kwork__record()  argument
1660 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__record()
1670 	list_for_each_entry(class, &kwork->class_list, list) {  in perf_kwork__record()
1692 	static struct perf_kwork kwork = {  in cmd_kwork()  local
1693 		.class_list          = LIST_HEAD_INIT(kwork.class_list),  in cmd_kwork()
1699 		.atom_page_list      = LIST_HEAD_INIT(kwork.atom_page_list),  in cmd_kwork()
1700 		.sort_list           = LIST_HEAD_INIT(kwork.sort_list),  in cmd_kwork()
1701 		.cmp_id              = LIST_HEAD_INIT(kwork.cmp_id),  in cmd_kwork()
1729 	OPT_STRING('k', "kwork", &kwork.event_list_str, "kwork",  in cmd_kwork()
1731 	OPT_BOOLEAN('f', "force", &kwork.force, "don't complain, do it"),  in cmd_kwork()
1735 	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",  in cmd_kwork()
1737 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1739 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1741 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1745 	OPT_BOOLEAN('S', "with-summary", &kwork.summary,  in cmd_kwork()
1748 	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,  in cmd_kwork()
1754 	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",  in cmd_kwork()
1756 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1758 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1760 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1765 	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,  in cmd_kwork()
1775 	OPT_BOOLEAN('g', "call-graph", &kwork.show_callchain,  in cmd_kwork()
1777 	OPT_UINTEGER(0, "max-stack", &kwork.max_stack,  in cmd_kwork()
1781 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1783 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1785 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1817 	setup_event_list(&kwork, kwork_options, kwork_usage);  in cmd_kwork()
1818 	sort_dimension__add(&kwork, "id", &kwork.cmp_id);  in cmd_kwork()
1821 		return perf_kwork__record(&kwork, argc, argv);  in cmd_kwork()
1823 		kwork.sort_order = default_report_sort_order;  in cmd_kwork()
1829 		kwork.report = KWORK_REPORT_RUNTIME;  in cmd_kwork()
1830 		setup_sorting(&kwork, report_options, report_usage);  in cmd_kwork()
1831 		return perf_kwork__report(&kwork);  in cmd_kwork()
1833 		kwork.sort_order = default_latency_sort_order;  in cmd_kwork()
1839 		kwork.report = KWORK_REPORT_LATENCY;  in cmd_kwork()
1840 		setup_sorting(&kwork, latency_options, latency_usage);  in cmd_kwork()
1841 		return perf_kwork__report(&kwork);  in cmd_kwork()
1848 		kwork.report = KWORK_REPORT_TIMEHIST;  in cmd_kwork()
1849 		return perf_kwork__timehist(&kwork);  in cmd_kwork()