84 __weak int hw_breakpoint_weight(struct perf_event *bp)  in hw_breakpoint_weight()  argument
118 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)  in task_bp_pinned()  argument
120 	struct task_struct *tsk = bp->hw.target;  in task_bp_pinned()
134 static const struct cpumask *cpumask_of_bp(struct perf_event *bp)  in cpumask_of_bp()  argument
136 	if (bp->cpu >= 0)  in cpumask_of_bp()
137 		return cpumask_of(bp->cpu);  in cpumask_of_bp()
146 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,  in fetch_bp_busy_slots()  argument
149 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in fetch_bp_busy_slots()
157 		if (!bp->hw.target)  in fetch_bp_busy_slots()
160 			nr += task_bp_pinned(cpu, bp, type);  in fetch_bp_busy_slots()
185 static void toggle_bp_task_slot(struct perf_event *bp, int cpu,  in toggle_bp_task_slot()  argument
191 	old_idx = task_bp_pinned(cpu, bp, type) - 1;  in toggle_bp_task_slot()
204 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,  in toggle_bp_slot()  argument
207 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in toggle_bp_slot()
214 	if (!bp->hw.target) {  in toggle_bp_slot()
215 		get_bp_info(bp->cpu, type)->cpu_pinned += weight;  in toggle_bp_slot()
221 		toggle_bp_task_slot(bp, cpu, type, weight);  in toggle_bp_slot()
224 		list_add_tail(&bp->hw.bp_list, &bp_task_head);  in toggle_bp_slot()
226 		list_del(&bp->hw.bp_list);  in toggle_bp_slot()
232 __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)  in arch_unregister_hw_breakpoint()  argument
281 static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)  in __reserve_bp_slot()  argument
297 	weight = hw_breakpoint_weight(bp);  in __reserve_bp_slot()
299 	fetch_bp_busy_slots(&slots, bp, type);  in __reserve_bp_slot()
310 	toggle_bp_slot(bp, true, type, weight);  in __reserve_bp_slot()
315 int reserve_bp_slot(struct perf_event *bp)  in reserve_bp_slot()  argument
321 	ret = __reserve_bp_slot(bp, bp->attr.bp_type);  in reserve_bp_slot()
328 static void __release_bp_slot(struct perf_event *bp, u64 bp_type)  in __release_bp_slot()  argument
334 	weight = hw_breakpoint_weight(bp);  in __release_bp_slot()
335 	toggle_bp_slot(bp, false, type, weight);  in __release_bp_slot()
338 void release_bp_slot(struct perf_event *bp)  in release_bp_slot()  argument
342 	arch_unregister_hw_breakpoint(bp);  in release_bp_slot()
343 	__release_bp_slot(bp, bp->attr.bp_type);  in release_bp_slot()
348 static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in __modify_bp_slot()  argument
352 	__release_bp_slot(bp, old_type);  in __modify_bp_slot()
354 	err = __reserve_bp_slot(bp, new_type);  in __modify_bp_slot()
364 		WARN_ON(__reserve_bp_slot(bp, old_type));  in __modify_bp_slot()
370 static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in modify_bp_slot()  argument
375 	ret = __modify_bp_slot(bp, old_type, new_type);  in modify_bp_slot()
385 int dbg_reserve_bp_slot(struct perf_event *bp)  in dbg_reserve_bp_slot()  argument
390 	return __reserve_bp_slot(bp, bp->attr.bp_type);  in dbg_reserve_bp_slot()
393 int dbg_release_bp_slot(struct perf_event *bp)  in dbg_release_bp_slot()  argument
398 	__release_bp_slot(bp, bp->attr.bp_type);  in dbg_release_bp_slot()
403 static int hw_breakpoint_parse(struct perf_event *bp,  in hw_breakpoint_parse()  argument
409 	err = hw_breakpoint_arch_parse(bp, attr, hw);  in hw_breakpoint_parse()
427 int register_perf_hw_breakpoint(struct perf_event *bp)  in register_perf_hw_breakpoint()  argument
432 	err = reserve_bp_slot(bp);  in register_perf_hw_breakpoint()
436 	err = hw_breakpoint_parse(bp, &bp->attr, &hw);  in register_perf_hw_breakpoint()
438 		release_bp_slot(bp);  in register_perf_hw_breakpoint()
442 	bp->hw.info = hw;  in register_perf_hw_breakpoint()
474 modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,  in modify_user_hw_breakpoint_check()  argument
480 	err = hw_breakpoint_parse(bp, attr, &hw);  in modify_user_hw_breakpoint_check()
487 		old_attr = bp->attr;  in modify_user_hw_breakpoint_check()
493 	if (bp->attr.bp_type != attr->bp_type) {  in modify_user_hw_breakpoint_check()
494 		err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);  in modify_user_hw_breakpoint_check()
499 	hw_breakpoint_copy_attr(&bp->attr, attr);  in modify_user_hw_breakpoint_check()
500 	bp->hw.info = hw;  in modify_user_hw_breakpoint_check()
510 int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)  in modify_user_hw_breakpoint()  argument
520 	if (irqs_disabled() && bp->ctx && bp->ctx->task == current)  in modify_user_hw_breakpoint()
521 		perf_event_disable_local(bp);  in modify_user_hw_breakpoint()
523 		perf_event_disable(bp);  in modify_user_hw_breakpoint()
525 	err = modify_user_hw_breakpoint_check(bp, attr, false);  in modify_user_hw_breakpoint()
527 	if (!bp->attr.disabled)  in modify_user_hw_breakpoint()
528 		perf_event_enable(bp);  in modify_user_hw_breakpoint()
538 void unregister_hw_breakpoint(struct perf_event *bp)  in unregister_hw_breakpoint()  argument
540 	if (!bp)  in unregister_hw_breakpoint()
542 	perf_event_release_kernel(bp);  in unregister_hw_breakpoint()
558 	struct perf_event * __percpu *cpu_events, *bp;  in register_wide_hw_breakpoint()  local
568 		bp = perf_event_create_kernel_counter(attr, cpu, NULL,  in register_wide_hw_breakpoint()
570 		if (IS_ERR(bp)) {  in register_wide_hw_breakpoint()
571 			err = PTR_ERR(bp);  in register_wide_hw_breakpoint()
575 		per_cpu(*cpu_events, cpu) = bp;  in register_wide_hw_breakpoint()
613 static int hw_breakpoint_event_init(struct perf_event *bp)  in hw_breakpoint_event_init()  argument
617 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)  in hw_breakpoint_event_init()
623 	if (has_branch_stack(bp))  in hw_breakpoint_event_init()
626 	err = register_perf_hw_breakpoint(bp);  in hw_breakpoint_event_init()
630 	bp->destroy = bp_perf_event_destroy;  in hw_breakpoint_event_init()
635 static int hw_breakpoint_add(struct perf_event *bp, int flags)  in hw_breakpoint_add()  argument
638 		bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_add()
640 	if (is_sampling_event(bp)) {  in hw_breakpoint_add()
641 		bp->hw.last_period = bp->hw.sample_period;  in hw_breakpoint_add()
642 		perf_swevent_set_period(bp);  in hw_breakpoint_add()
645 	return arch_install_hw_breakpoint(bp);  in hw_breakpoint_add()
648 static void hw_breakpoint_del(struct perf_event *bp, int flags)  in hw_breakpoint_del()  argument
650 	arch_uninstall_hw_breakpoint(bp);  in hw_breakpoint_del()
653 static void hw_breakpoint_start(struct perf_event *bp, int flags)  in hw_breakpoint_start()  argument
655 	bp->hw.state = 0;  in hw_breakpoint_start()
658 static void hw_breakpoint_stop(struct perf_event *bp, int flags)  in hw_breakpoint_stop()  argument
660 	bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_stop()