127 static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused,  in sort_dimension__add()
159 	if (kwork->report == KWORK_REPORT_LATENCY)  in sort_dimension__add()
172 static void setup_sorting(struct perf_kwork *kwork,  in setup_sorting()  argument
176 	char *tmp, *tok, *str = strdup(kwork->sort_order);  in setup_sorting()
180 		if (sort_dimension__add(kwork, tok, &kwork->sort_list) < 0)  in setup_sorting()
185 	pr_debug("Sort order: %s\n", kwork->sort_order);  in setup_sorting()
189 static struct kwork_atom *atom_new(struct perf_kwork *kwork,  in atom_new()  argument
196 	list_for_each_entry(page, &kwork->atom_page_list, list) {  in atom_new()
216 	list_add_tail(&page->list, &kwork->atom_page_list);  in atom_new()
344 static void profile_update_timespan(struct perf_kwork *kwork,  in profile_update_timespan()  argument
347 	if (!kwork->summary)  in profile_update_timespan()
350 	if ((kwork->timestart == 0) || (kwork->timestart > sample->time))  in profile_update_timespan()
351 		kwork->timestart = sample->time;  in profile_update_timespan()
353 	if (kwork->timeend < sample->time)  in profile_update_timespan()
354 		kwork->timeend = sample->time;  in profile_update_timespan()
357 static bool profile_event_match(struct perf_kwork *kwork,  in profile_event_match()  argument
363 	struct perf_time_interval *ptime = &kwork->ptime;  in profile_event_match()
365 	if ((kwork->cpu_list != NULL) && !test_bit(cpu, kwork->cpu_bitmap))  in profile_event_match()
372 	if ((kwork->profile_name != NULL) &&  in profile_event_match()
374 	    (strcmp(work->name, kwork->profile_name) != 0))  in profile_event_match()
377 	profile_update_timespan(kwork, sample);  in profile_event_match()
381 static int work_push_atom(struct perf_kwork *kwork,  in work_push_atom()  argument
396 	atom = atom_new(kwork, sample);  in work_push_atom()
400 	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);  in work_push_atom()
406 	if (!profile_event_match(kwork, work, sample))  in work_push_atom()
426 static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork,  in work_pop_atom()  argument
441 	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);  in work_pop_atom()
448 	if (!profile_event_match(kwork, work, sample))  in work_pop_atom()
456 	src_atom = atom_new(kwork, sample);  in work_pop_atom()
488 static int report_entry_event(struct perf_kwork *kwork,  in report_entry_event()  argument
494 	return work_push_atom(kwork, class, KWORK_TRACE_ENTRY,  in report_entry_event()
499 static int report_exit_event(struct perf_kwork *kwork,  in report_exit_event()  argument
508 	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,  in report_exit_event()
543 static int latency_raise_event(struct perf_kwork *kwork,  in latency_raise_event()  argument
549 	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,  in latency_raise_event()
554 static int latency_entry_event(struct perf_kwork *kwork,  in latency_entry_event()  argument
563 	atom = work_pop_atom(kwork, class, KWORK_TRACE_ENTRY,  in latency_entry_event()
577 static void timehist_save_callchain(struct perf_kwork *kwork,  in timehist_save_callchain()  argument
587 	if (!kwork->show_callchain || sample->callchain == NULL)  in timehist_save_callchain()
598 				      NULL, NULL, kwork->max_stack + 2) != 0) {  in timehist_save_callchain()
624 static void timehist_print_event(struct perf_kwork *kwork,  in timehist_print_event()  argument
681 	if (kwork->show_callchain) {  in timehist_print_event()
694 static int timehist_raise_event(struct perf_kwork *kwork,  in timehist_raise_event()  argument
700 	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,  in timehist_raise_event()
705 static int timehist_entry_event(struct perf_kwork *kwork,  in timehist_entry_event()  argument
714 	ret = work_push_atom(kwork, class, KWORK_TRACE_ENTRY,  in timehist_entry_event()
721 		timehist_save_callchain(kwork, sample, evsel, machine);  in timehist_entry_event()
726 static int timehist_exit_event(struct perf_kwork *kwork,  in timehist_exit_event()  argument
741 	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,  in timehist_exit_event()
749 		timehist_print_event(kwork, work, atom, sample, &al);  in timehist_exit_event()
762 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_irq_handler_entry_event()  local
764 	if (kwork->tp_handler->entry_event)  in process_irq_handler_entry_event()
765 		return kwork->tp_handler->entry_event(kwork, &kwork_irq,  in process_irq_handler_entry_event()
775 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_irq_handler_exit_event()  local
777 	if (kwork->tp_handler->exit_event)  in process_irq_handler_exit_event()
778 		return kwork->tp_handler->exit_event(kwork, &kwork_irq,  in process_irq_handler_exit_event()
833 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_raise_event()  local
835 	if (kwork->tp_handler->raise_event)  in process_softirq_raise_event()
836 		return kwork->tp_handler->raise_event(kwork, &kwork_softirq,  in process_softirq_raise_event()
847 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_entry_event()  local
849 	if (kwork->tp_handler->entry_event)  in process_softirq_entry_event()
850 		return kwork->tp_handler->entry_event(kwork, &kwork_softirq,  in process_softirq_entry_event()
861 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_softirq_exit_event()  local
863 	if (kwork->tp_handler->exit_event)  in process_softirq_exit_event()
864 		return kwork->tp_handler->exit_event(kwork, &kwork_softirq,  in process_softirq_exit_event()
954 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_activate_work_event()  local
956 	if (kwork->tp_handler->raise_event)  in process_workqueue_activate_work_event()
957 		return kwork->tp_handler->raise_event(kwork, &kwork_workqueue,  in process_workqueue_activate_work_event()
968 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_execute_start_event()  local
970 	if (kwork->tp_handler->entry_event)  in process_workqueue_execute_start_event()
971 		return kwork->tp_handler->entry_event(kwork, &kwork_workqueue,  in process_workqueue_execute_start_event()
982 	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);  in process_workqueue_execute_end_event()  local
984 	if (kwork->tp_handler->exit_event)  in process_workqueue_execute_end_event()
985 		return kwork->tp_handler->exit_event(kwork, &kwork_workqueue,  in process_workqueue_execute_end_event()
1056 static int report_print_work(struct perf_kwork *kwork, struct kwork_work *work)  in report_print_work()  argument
1084 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_work()
1088 	} else if (kwork->report == KWORK_REPORT_LATENCY) { // avg delay  in report_print_work()
1103 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_work()
1119 	else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_work()
1137 static int report_print_header(struct perf_kwork *kwork)  in report_print_header()  argument
1146 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_header()
1149 	} else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_header()
1156 	if (kwork->report == KWORK_REPORT_RUNTIME) {  in report_print_header()
1161 	} else if (kwork->report == KWORK_REPORT_LATENCY) {  in report_print_header()
1209 static void print_summary(struct perf_kwork *kwork)  in print_summary()  argument
1211 	u64 time = kwork->timeend - kwork->timestart;  in print_summary()
1213 	printf("  Total count            : %9" PRIu64 "\n", kwork->all_count);  in print_summary()
1215 	       (double)kwork->all_runtime / NSEC_PER_MSEC,  in print_summary()
1216 	       time == 0 ? 0 : (double)kwork->all_runtime / time);  in print_summary()
1232 static void print_skipped_events(struct perf_kwork *kwork)  in print_skipped_events()  argument
1241 	if ((kwork->nr_skipped_events[KWORK_TRACE_MAX] != 0) &&  in print_skipped_events()
1242 	    (kwork->nr_events != 0)) {  in print_skipped_events()
1244 		       (double)kwork->nr_skipped_events[KWORK_TRACE_MAX] /  in print_skipped_events()
1245 		       (double)kwork->nr_events * 100.0,  in print_skipped_events()
1246 		       kwork->nr_skipped_events[KWORK_TRACE_MAX]);  in print_skipped_events()
1250 			       kwork->nr_skipped_events[i],  in print_skipped_events()
1258 		       nr_list_entry(&kwork->atom_page_list));  in print_skipped_events()
1261 static void print_bad_events(struct perf_kwork *kwork)  in print_bad_events()  argument
1263 	if ((kwork->nr_lost_events != 0) && (kwork->nr_events != 0)) {  in print_bad_events()
1265 		       (double)kwork->nr_lost_events /  in print_bad_events()
1266 		       (double)kwork->nr_events * 100.0,  in print_bad_events()
1267 		       kwork->nr_lost_events, kwork->nr_events,  in print_bad_events()
1268 		       kwork->nr_lost_chunks);  in print_bad_events()
1272 static void work_sort(struct perf_kwork *kwork, struct kwork_class *class)  in work_sort()  argument
1286 		work_insert(&kwork->sorted_work_root,  in work_sort()
1287 			       data, &kwork->sort_list);  in work_sort()
1291 static void perf_kwork__sort(struct perf_kwork *kwork)  in perf_kwork__sort()  argument
1295 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__sort()
1296 		work_sort(kwork, class);  in perf_kwork__sort()
1299 static int perf_kwork__check_config(struct perf_kwork *kwork,  in perf_kwork__check_config()  argument
1320 	switch (kwork->report) {  in perf_kwork__check_config()
1322 		kwork->tp_handler = &report_ops;  in perf_kwork__check_config()
1325 		kwork->tp_handler = &latency_ops;  in perf_kwork__check_config()
1328 		kwork->tp_handler = &timehist_ops;  in perf_kwork__check_config()
1331 		pr_debug("Invalid report type %d\n", kwork->report);  in perf_kwork__check_config()
1335 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__check_config()
1340 	if (kwork->cpu_list != NULL) {  in perf_kwork__check_config()
1342 					       kwork->cpu_list,  in perf_kwork__check_config()
1343 					       kwork->cpu_bitmap);  in perf_kwork__check_config()
1350 	if (kwork->time_str != NULL) {  in perf_kwork__check_config()
1351 		ret = perf_time__parse_str(&kwork->ptime, kwork->time_str);  in perf_kwork__check_config()
1359 		if (kwork->show_callchain && !evsel__has_callchain(evsel)) {  in perf_kwork__check_config()
1361 			kwork->show_callchain = 0;  in perf_kwork__check_config()
1369 static int perf_kwork__read_events(struct perf_kwork *kwork)  in perf_kwork__read_events()  argument
1377 		.force = kwork->force,  in perf_kwork__read_events()
1380 	session = perf_session__new(&data, &kwork->tool);  in perf_kwork__read_events()
1388 	if (perf_kwork__check_config(kwork, session) != 0)  in perf_kwork__read_events()
1399 	if (kwork->report == KWORK_REPORT_TIMEHIST)  in perf_kwork__read_events()
1408 	kwork->nr_events      = session->evlist->stats.nr_events[0];  in perf_kwork__read_events()
1409 	kwork->nr_lost_events = session->evlist->stats.total_lost;  in perf_kwork__read_events()
1410 	kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];  in perf_kwork__read_events()
1417 static void process_skipped_events(struct perf_kwork *kwork,  in process_skipped_events()  argument
1425 		kwork->nr_skipped_events[i] += count;  in process_skipped_events()
1426 		kwork->nr_skipped_events[KWORK_TRACE_MAX] += count;  in process_skipped_events()
1430 struct kwork_work *perf_kwork_add_work(struct perf_kwork *kwork,  in perf_kwork_add_work()  argument
1440 	work_insert(&class->work_root, work, &kwork->cmp_id);  in perf_kwork_add_work()
1453 static int perf_kwork__report_bpf(struct perf_kwork *kwork)  in perf_kwork__report_bpf()  argument
1460 	ret = perf_kwork__trace_prepare_bpf(kwork);  in perf_kwork__report_bpf()
1475 	perf_kwork__report_read_bpf(kwork);  in perf_kwork__report_bpf()
1482 static int perf_kwork__report(struct perf_kwork *kwork)  in perf_kwork__report()  argument
1488 	if (kwork->use_bpf)  in perf_kwork__report()
1489 		ret = perf_kwork__report_bpf(kwork);  in perf_kwork__report()
1491 		ret = perf_kwork__read_events(kwork);  in perf_kwork__report()
1496 	perf_kwork__sort(kwork);  in perf_kwork__report()
1500 	ret = report_print_header(kwork);  in perf_kwork__report()
1501 	next = rb_first_cached(&kwork->sorted_work_root);  in perf_kwork__report()
1504 		process_skipped_events(kwork, work);  in perf_kwork__report()
1507 			report_print_work(kwork, work);  in perf_kwork__report()
1508 			if (kwork->summary) {  in perf_kwork__report()
1509 				kwork->all_runtime += work->total_runtime;  in perf_kwork__report()
1510 				kwork->all_count += work->nr_atoms;  in perf_kwork__report()
1517 	if (kwork->summary) {  in perf_kwork__report()
1518 		print_summary(kwork);  in perf_kwork__report()
1522 	print_bad_events(kwork);  in perf_kwork__report()
1523 	print_skipped_events(kwork);  in perf_kwork__report()
1551 static int perf_kwork__timehist(struct perf_kwork *kwork)  in perf_kwork__timehist()  argument
1556 	kwork->tool.comm	 = perf_event__process_comm;  in perf_kwork__timehist()
1557 	kwork->tool.exit	 = perf_event__process_exit;  in perf_kwork__timehist()
1558 	kwork->tool.fork	 = perf_event__process_fork;  in perf_kwork__timehist()
1559 	kwork->tool.attr	 = perf_event__process_attr;  in perf_kwork__timehist()
1560 	kwork->tool.tracing_data = perf_event__process_tracing_data;  in perf_kwork__timehist()
1561 	kwork->tool.build_id	 = perf_event__process_build_id;  in perf_kwork__timehist()
1562 	kwork->tool.ordered_events = true;  in perf_kwork__timehist()
1563 	kwork->tool.ordering_requires_timestamps = true;  in perf_kwork__timehist()
1564 	symbol_conf.use_callchain = kwork->show_callchain;  in perf_kwork__timehist()
1573 	return perf_kwork__read_events(kwork);  in perf_kwork__timehist()
1576 static void setup_event_list(struct perf_kwork *kwork,  in setup_event_list()  argument
1584 	if (kwork->event_list_str == NULL)  in setup_event_list()
1587 	str = strdup(kwork->event_list_str);  in setup_event_list()
1593 				list_add_tail(&class->list, &kwork->class_list);  in setup_event_list()
1608 	if (list_empty(&kwork->class_list)) {  in setup_event_list()
1611 				      &kwork->class_list);  in setup_event_list()
1616 	list_for_each_entry(class, &kwork->class_list, list)  in setup_event_list()
1621 static int perf_kwork__record(struct perf_kwork *kwork,  in perf_kwork__record()  argument
1638 	list_for_each_entry(class, &kwork->class_list, list)  in perf_kwork__record()
1648 	list_for_each_entry(class, &kwork->class_list, list) {  in perf_kwork__record()
1670 	static struct perf_kwork kwork = {  in cmd_kwork()  local
1671 		.class_list          = LIST_HEAD_INIT(kwork.class_list),  in cmd_kwork()
1677 		.atom_page_list      = LIST_HEAD_INIT(kwork.atom_page_list),  in cmd_kwork()
1678 		.sort_list           = LIST_HEAD_INIT(kwork.sort_list),  in cmd_kwork()
1679 		.cmp_id              = LIST_HEAD_INIT(kwork.cmp_id),  in cmd_kwork()
1707 	OPT_STRING('k', "kwork", &kwork.event_list_str, "kwork",  in cmd_kwork()
1709 	OPT_BOOLEAN('f', "force", &kwork.force, "don't complain, do it"),  in cmd_kwork()
1713 	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",  in cmd_kwork()
1715 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1717 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1719 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1723 	OPT_BOOLEAN('S', "with-summary", &kwork.summary,  in cmd_kwork()
1726 	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,  in cmd_kwork()
1732 	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",  in cmd_kwork()
1734 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1736 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1738 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1743 	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,  in cmd_kwork()
1753 	OPT_BOOLEAN('g', "call-graph", &kwork.show_callchain,  in cmd_kwork()
1755 	OPT_UINTEGER(0, "max-stack", &kwork.max_stack,  in cmd_kwork()
1759 	OPT_STRING(0, "time", &kwork.time_str, "str",  in cmd_kwork()
1761 	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",  in cmd_kwork()
1763 	OPT_STRING('n', "name", &kwork.profile_name, "name",  in cmd_kwork()
1795 	setup_event_list(&kwork, kwork_options, kwork_usage);  in cmd_kwork()
1796 	sort_dimension__add(&kwork, "id", &kwork.cmp_id);  in cmd_kwork()
1799 		return perf_kwork__record(&kwork, argc, argv);  in cmd_kwork()
1801 		kwork.sort_order = default_report_sort_order;  in cmd_kwork()
1807 		kwork.report = KWORK_REPORT_RUNTIME;  in cmd_kwork()
1808 		setup_sorting(&kwork, report_options, report_usage);  in cmd_kwork()
1809 		return perf_kwork__report(&kwork);  in cmd_kwork()
1811 		kwork.sort_order = default_latency_sort_order;  in cmd_kwork()
1817 		kwork.report = KWORK_REPORT_LATENCY;  in cmd_kwork()
1818 		setup_sorting(&kwork, latency_options, latency_usage);  in cmd_kwork()
1819 		return perf_kwork__report(&kwork);  in cmd_kwork()
1826 		kwork.report = KWORK_REPORT_TIMEHIST;  in cmd_kwork()
1827 		return perf_kwork__timehist(&kwork);  in cmd_kwork()