48 static int bpf_program_profiler__destroy(struct evsel *evsel)  in bpf_program_profiler__destroy()  argument
53 				 &evsel->bpf_counter_list, list) {  in bpf_program_profiler__destroy()
58 	assert(list_empty(&evsel->bpf_counter_list));  in bpf_program_profiler__destroy()
102 static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id)  in bpf_program_profiler_load_one()  argument
128 	skel->rodata->num_cpu = evsel__nr_cpus(evsel);  in bpf_program_profiler_load_one()
130 	bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel));  in bpf_program_profiler_load_one()
157 	list_add(&counter->list, &evsel->bpf_counter_list);  in bpf_program_profiler_load_one()
167 static int bpf_program_profiler__load(struct evsel *evsel, struct target *target)  in bpf_program_profiler__load()  argument
186 		ret = bpf_program_profiler_load_one(evsel, prog_id);  in bpf_program_profiler__load()
188 			bpf_program_profiler__destroy(evsel);  in bpf_program_profiler__load()
198 static int bpf_program_profiler__enable(struct evsel *evsel)  in bpf_program_profiler__enable()  argument
203 	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {  in bpf_program_profiler__enable()
207 			bpf_program_profiler__destroy(evsel);  in bpf_program_profiler__enable()
214 static int bpf_program_profiler__disable(struct evsel *evsel)  in bpf_program_profiler__disable()  argument
218 	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {  in bpf_program_profiler__disable()
225 static int bpf_program_profiler__read(struct evsel *evsel)  in bpf_program_profiler__read()  argument
238 	if (list_empty(&evsel->bpf_counter_list))  in bpf_program_profiler__read()
241 	perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) {  in bpf_program_profiler__read()
242 		counts = perf_counts(evsel->counts, idx, 0);  in bpf_program_profiler__read()
247 	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {  in bpf_program_profiler__read()
260 			idx = perf_cpu_map__idx(evsel__cpus(evsel),  in bpf_program_profiler__read()
264 			counts = perf_counts(evsel->counts, idx, 0);  in bpf_program_profiler__read()
273 static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx,  in bpf_program_profiler__install_pe()  argument
280 	list_for_each_entry(counter, &evsel->bpf_counter_list, list) {  in bpf_program_profiler__install_pe()
362 static int bperf_check_target(struct evsel *evsel,  in bperf_check_target()  argument
367 	if (evsel->core.leader->nr_members > 1) {  in bperf_check_target()
378 		*filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel));  in bperf_check_target()
381 		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);  in bperf_check_target()
382 	} else if (target->pid || evsel->evlist->workload.pid != -1) {  in bperf_check_target()
384 		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);  in bperf_check_target()
395 static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd,  in bperf_reload_leader_program()  argument
425 	err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY);  in bperf_reload_leader_program()
428 	evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id);  in bperf_reload_leader_program()
429 	assert(evsel->bperf_leader_link_fd >= 0);  in bperf_reload_leader_program()
435 	evsel->leader_skel = skel;  in bperf_reload_leader_program()
436 	evsel__open_per_cpu(evsel, all_cpu_map, -1);  in bperf_reload_leader_program()
444 static int bperf__load(struct evsel *evsel, struct target *target)  in bperf__load()  argument
451 	if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt))  in bperf__load()
460 	evsel->bperf_leader_prog_fd = -1;  in bperf__load()
461 	evsel->bperf_leader_link_fd = -1;  in bperf__load()
475 	err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry);  in bperf__load()
477 		err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY);  in bperf__load()
482 	evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id);  in bperf__load()
483 	if (evsel->bperf_leader_link_fd < 0 &&  in bperf__load()
484 	    bperf_reload_leader_program(evsel, attr_map_fd, &entry)) {  in bperf__load()
493 	evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id(  in bperf__load()
494 		bpf_link_get_prog_id(evsel->bperf_leader_link_fd));  in bperf__load()
495 	assert(evsel->bperf_leader_prog_fd >= 0);  in bperf__load()
504 	err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0);  in bperf__load()
512 	evsel->follower_skel = bperf_follower_bpf__open();  in bperf__load()
513 	if (!evsel->follower_skel) {  in bperf__load()
520 	bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX,  in bperf__load()
521 				       evsel->bperf_leader_prog_fd, "on_switch");  in bperf__load()
524 	bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd);  in bperf__load()
527 	bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings,  in bperf__load()
530 	bpf_map__set_max_entries(evsel->follower_skel->maps.filter,  in bperf__load()
532 	err = bperf_follower_bpf__load(evsel->follower_skel);  in bperf__load()
535 		bperf_follower_bpf__destroy(evsel->follower_skel);  in bperf__load()
536 		evsel->follower_skel = NULL;  in bperf__load()
546 			key = perf_thread_map__pid(evsel->core.threads, i);  in bperf__load()
548 			key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu;  in bperf__load()
552 		filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter);  in bperf__load()
556 	evsel->follower_skel->bss->type = filter_type;  in bperf__load()
558 	err = bperf_follower_bpf__attach(evsel->follower_skel);  in bperf__load()
561 	if (err && evsel->bperf_leader_link_fd >= 0)  in bperf__load()
562 		close(evsel->bperf_leader_link_fd);  in bperf__load()
563 	if (err && evsel->bperf_leader_prog_fd >= 0)  in bperf__load()
564 		close(evsel->bperf_leader_prog_fd);  in bperf__load()
574 static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)  in bperf__install_pe()  argument
576 	struct bperf_leader_bpf *skel = evsel->leader_skel;  in bperf__install_pe()
586 static int bperf_sync_counters(struct evsel *evsel)  in bperf_sync_counters()  argument
593 		bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu);  in bperf_sync_counters()
598 static int bperf__enable(struct evsel *evsel)  in bperf__enable()  argument
600 	evsel->follower_skel->bss->enabled = 1;  in bperf__enable()
604 static int bperf__disable(struct evsel *evsel)  in bperf__disable()  argument
606 	evsel->follower_skel->bss->enabled = 0;  in bperf__disable()
610 static int bperf__read(struct evsel *evsel)  in bperf__read()  argument
612 	struct bperf_follower_bpf *skel = evsel->follower_skel;  in bperf__read()
620 	bperf_sync_counters(evsel);  in bperf__read()
630 		switch (evsel->follower_skel->bss->type) {  in bperf__read()
634 			perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) {  in bperf__read()
635 				counts = perf_counts(evsel->counts, j, 0);  in bperf__read()
642 			cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu;  in bperf__read()
644 			counts = perf_counts(evsel->counts, i, 0);  in bperf__read()
651 			counts = perf_counts(evsel->counts, 0, i);  in bperf__read()
670 static int bperf__destroy(struct evsel *evsel)  in bperf__destroy()  argument
672 	bperf_follower_bpf__destroy(evsel->follower_skel);  in bperf__destroy()
673 	close(evsel->bperf_leader_prog_fd);  in bperf__destroy()
674 	close(evsel->bperf_leader_link_fd);  in bperf__destroy()
764 static inline bool bpf_counter_skip(struct evsel *evsel)  in bpf_counter_skip()  argument
766 	return evsel->bpf_counter_ops == NULL;  in bpf_counter_skip()
769 int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)  in bpf_counter__install_pe()  argument
771 	if (bpf_counter_skip(evsel))  in bpf_counter__install_pe()
773 	return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd);  in bpf_counter__install_pe()
776 int bpf_counter__load(struct evsel *evsel, struct target *target)  in bpf_counter__load()  argument
779 		evsel->bpf_counter_ops = &bpf_program_profiler_ops;  in bpf_counter__load()
781 		evsel->bpf_counter_ops = &bperf_cgrp_ops;  in bpf_counter__load()
782 	else if (target->use_bpf || evsel->bpf_counter ||  in bpf_counter__load()
783 		 evsel__match_bpf_counter_events(evsel->name))  in bpf_counter__load()
784 		evsel->bpf_counter_ops = &bperf_ops;  in bpf_counter__load()
786 	if (evsel->bpf_counter_ops)  in bpf_counter__load()
787 		return evsel->bpf_counter_ops->load(evsel, target);  in bpf_counter__load()
791 int bpf_counter__enable(struct evsel *evsel)  in bpf_counter__enable()  argument
793 	if (bpf_counter_skip(evsel))  in bpf_counter__enable()
795 	return evsel->bpf_counter_ops->enable(evsel);  in bpf_counter__enable()
798 int bpf_counter__disable(struct evsel *evsel)  in bpf_counter__disable()  argument
800 	if (bpf_counter_skip(evsel))  in bpf_counter__disable()
802 	return evsel->bpf_counter_ops->disable(evsel);  in bpf_counter__disable()
805 int bpf_counter__read(struct evsel *evsel)  in bpf_counter__read()  argument
807 	if (bpf_counter_skip(evsel))  in bpf_counter__read()
809 	return evsel->bpf_counter_ops->read(evsel);  in bpf_counter__read()
812 void bpf_counter__destroy(struct evsel *evsel)  in bpf_counter__destroy()  argument
814 	if (bpf_counter_skip(evsel))  in bpf_counter__destroy()
816 	evsel->bpf_counter_ops->destroy(evsel);  in bpf_counter__destroy()
817 	evsel->bpf_counter_ops = NULL;  in bpf_counter__destroy()
818 	evsel->bpf_skel = NULL;  in bpf_counter__destroy()