5306 	struct perf_buffer *pb;  member
5330 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,  in perf_buffer__free_cpu_buf()  argument
5336 	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))  in perf_buffer__free_cpu_buf()
5346 void perf_buffer__free(struct perf_buffer *pb)  in perf_buffer__free()  argument
5350 	if (!pb)  in perf_buffer__free()
5352 	if (pb->cpu_bufs) {  in perf_buffer__free()
5353 		for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) {  in perf_buffer__free()
5354 			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__free()
5356 			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);  in perf_buffer__free()
5357 			perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__free()
5359 		free(pb->cpu_bufs);  in perf_buffer__free()
5361 	if (pb->epoll_fd >= 0)  in perf_buffer__free()
5362 		close(pb->epoll_fd);  in perf_buffer__free()
5363 	free(pb->events);  in perf_buffer__free()
5364 	free(pb);  in perf_buffer__free()
5368 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,  in perf_buffer__open_cpu_buf()  argument
5379 	cpu_buf->pb = pb;  in perf_buffer__open_cpu_buf()
5392 	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,  in perf_buffer__open_cpu_buf()
5413 	perf_buffer__free_cpu_buf(pb, cpu_buf);  in perf_buffer__open_cpu_buf()
5461 	struct perf_buffer *pb;  in __perf_buffer__new()  local
5486 	pb = calloc(1, sizeof(*pb));  in __perf_buffer__new()
5487 	if (!pb)  in __perf_buffer__new()
5490 	pb->event_cb = p->event_cb;  in __perf_buffer__new()
5491 	pb->sample_cb = p->sample_cb;  in __perf_buffer__new()
5492 	pb->lost_cb = p->lost_cb;  in __perf_buffer__new()
5493 	pb->ctx = p->ctx;  in __perf_buffer__new()
5495 	pb->page_size = getpagesize();  in __perf_buffer__new()
5496 	pb->mmap_size = pb->page_size * page_cnt;  in __perf_buffer__new()
5497 	pb->map_fd = map_fd;  in __perf_buffer__new()
5499 	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in __perf_buffer__new()
5500 	if (pb->epoll_fd < 0) {  in __perf_buffer__new()
5508 		pb->cpu_cnt = p->cpu_cnt;  in __perf_buffer__new()
5510 		pb->cpu_cnt = libbpf_num_possible_cpus();  in __perf_buffer__new()
5511 		if (pb->cpu_cnt < 0) {  in __perf_buffer__new()
5512 			err = pb->cpu_cnt;  in __perf_buffer__new()
5515 		if (map.max_entries < pb->cpu_cnt)  in __perf_buffer__new()
5516 			pb->cpu_cnt = map.max_entries;  in __perf_buffer__new()
5519 	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));  in __perf_buffer__new()
5520 	if (!pb->events) {  in __perf_buffer__new()
5525 	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));  in __perf_buffer__new()
5526 	if (!pb->cpu_bufs) {  in __perf_buffer__new()
5532 	for (i = 0; i < pb->cpu_cnt; i++) {  in __perf_buffer__new()
5539 		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);  in __perf_buffer__new()
5545 		pb->cpu_bufs[i] = cpu_buf;  in __perf_buffer__new()
5547 		err = bpf_map_update_elem(pb->map_fd, &map_key,  in __perf_buffer__new()
5557 		pb->events[i].events = EPOLLIN;  in __perf_buffer__new()
5558 		pb->events[i].data.ptr = cpu_buf;  in __perf_buffer__new()
5559 		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,  in __perf_buffer__new()
5560 			      &pb->events[i]) < 0) {  in __perf_buffer__new()
5569 	return pb;  in __perf_buffer__new()
5572 	if (pb)  in __perf_buffer__new()
5573 		perf_buffer__free(pb);  in __perf_buffer__new()
5594 	struct perf_buffer *pb = cpu_buf->pb;  in perf_buffer__process_record()  local
5598 	if (pb->event_cb)  in perf_buffer__process_record()
5599 		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);  in perf_buffer__process_record()
5605 		if (pb->sample_cb)  in perf_buffer__process_record()
5606 			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);  in perf_buffer__process_record()
5612 		if (pb->lost_cb)  in perf_buffer__process_record()
5613 			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);  in perf_buffer__process_record()
5623 static int perf_buffer__process_records(struct perf_buffer *pb,  in perf_buffer__process_records()  argument
5628 	ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,  in perf_buffer__process_records()
5629 					 pb->page_size, &cpu_buf->buf,  in perf_buffer__process_records()
5637 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)  in perf_buffer__poll()  argument
5641 	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);  in perf_buffer__poll()
5643 		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;  in perf_buffer__poll()
5645 		err = perf_buffer__process_records(pb, cpu_buf);  in perf_buffer__poll()