71 __weak int hw_breakpoint_weight(struct perf_event *bp)  in hw_breakpoint_weight()  argument
105 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)  in task_bp_pinned()  argument
107 	struct task_struct *tsk = bp->hw.target;  in task_bp_pinned()
121 static const struct cpumask *cpumask_of_bp(struct perf_event *bp)  in cpumask_of_bp()  argument
123 	if (bp->cpu >= 0)  in cpumask_of_bp()
124 		return cpumask_of(bp->cpu);  in cpumask_of_bp()
133 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,  in fetch_bp_busy_slots()  argument
136 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in fetch_bp_busy_slots()
144 		if (!bp->hw.target)  in fetch_bp_busy_slots()
147 			nr += task_bp_pinned(cpu, bp, type);  in fetch_bp_busy_slots()
172 static void toggle_bp_task_slot(struct perf_event *bp, int cpu,  in toggle_bp_task_slot()  argument
178 	old_idx = task_bp_pinned(cpu, bp, type) - 1;  in toggle_bp_task_slot()
191 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,  in toggle_bp_slot()  argument
194 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in toggle_bp_slot()
201 	if (!bp->hw.target) {  in toggle_bp_slot()
202 		get_bp_info(bp->cpu, type)->cpu_pinned += weight;  in toggle_bp_slot()
208 		toggle_bp_task_slot(bp, cpu, type, weight);  in toggle_bp_slot()
211 		list_add_tail(&bp->hw.bp_list, &bp_task_head);  in toggle_bp_slot()
213 		list_del(&bp->hw.bp_list);  in toggle_bp_slot()
216 __weak int arch_reserve_bp_slot(struct perf_event *bp)  in arch_reserve_bp_slot()  argument
221 __weak void arch_release_bp_slot(struct perf_event *bp)  in arch_release_bp_slot()  argument
228 __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)  in arch_unregister_hw_breakpoint()  argument
277 static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)  in __reserve_bp_slot()  argument
294 	weight = hw_breakpoint_weight(bp);  in __reserve_bp_slot()
296 	fetch_bp_busy_slots(&slots, bp, type);  in __reserve_bp_slot()
307 	ret = arch_reserve_bp_slot(bp);  in __reserve_bp_slot()
311 	toggle_bp_slot(bp, true, type, weight);  in __reserve_bp_slot()
316 int reserve_bp_slot(struct perf_event *bp)  in reserve_bp_slot()  argument
322 	ret = __reserve_bp_slot(bp, bp->attr.bp_type);  in reserve_bp_slot()
329 static void __release_bp_slot(struct perf_event *bp, u64 bp_type)  in __release_bp_slot()  argument
334 	arch_release_bp_slot(bp);  in __release_bp_slot()
337 	weight = hw_breakpoint_weight(bp);  in __release_bp_slot()
338 	toggle_bp_slot(bp, false, type, weight);  in __release_bp_slot()
341 void release_bp_slot(struct perf_event *bp)  in release_bp_slot()  argument
345 	arch_unregister_hw_breakpoint(bp);  in release_bp_slot()
346 	__release_bp_slot(bp, bp->attr.bp_type);  in release_bp_slot()
351 static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in __modify_bp_slot()  argument
355 	__release_bp_slot(bp, old_type);  in __modify_bp_slot()
357 	err = __reserve_bp_slot(bp, new_type);  in __modify_bp_slot()
367 		WARN_ON(__reserve_bp_slot(bp, old_type));  in __modify_bp_slot()
373 static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in modify_bp_slot()  argument
378 	ret = __modify_bp_slot(bp, old_type, new_type);  in modify_bp_slot()
388 int dbg_reserve_bp_slot(struct perf_event *bp)  in dbg_reserve_bp_slot()  argument
393 	return __reserve_bp_slot(bp, bp->attr.bp_type);  in dbg_reserve_bp_slot()
396 int dbg_release_bp_slot(struct perf_event *bp)  in dbg_release_bp_slot()  argument
401 	__release_bp_slot(bp, bp->attr.bp_type);  in dbg_release_bp_slot()
406 static int hw_breakpoint_parse(struct perf_event *bp,  in hw_breakpoint_parse()  argument
412 	err = hw_breakpoint_arch_parse(bp, attr, hw);  in hw_breakpoint_parse()
430 int register_perf_hw_breakpoint(struct perf_event *bp)  in register_perf_hw_breakpoint()  argument
435 	err = reserve_bp_slot(bp);  in register_perf_hw_breakpoint()
439 	err = hw_breakpoint_parse(bp, &bp->attr, &hw);  in register_perf_hw_breakpoint()
441 		release_bp_slot(bp);  in register_perf_hw_breakpoint()
445 	bp->hw.info = hw;  in register_perf_hw_breakpoint()
477 modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,  in modify_user_hw_breakpoint_check()  argument
483 	err = hw_breakpoint_parse(bp, attr, &hw);  in modify_user_hw_breakpoint_check()
490 		old_attr = bp->attr;  in modify_user_hw_breakpoint_check()
496 	if (bp->attr.bp_type != attr->bp_type) {  in modify_user_hw_breakpoint_check()
497 		err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);  in modify_user_hw_breakpoint_check()
502 	hw_breakpoint_copy_attr(&bp->attr, attr);  in modify_user_hw_breakpoint_check()
503 	bp->hw.info = hw;  in modify_user_hw_breakpoint_check()
513 int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)  in modify_user_hw_breakpoint()  argument
523 	if (irqs_disabled() && bp->ctx && bp->ctx->task == current)  in modify_user_hw_breakpoint()
524 		perf_event_disable_local(bp);  in modify_user_hw_breakpoint()
526 		perf_event_disable(bp);  in modify_user_hw_breakpoint()
528 	err = modify_user_hw_breakpoint_check(bp, attr, false);  in modify_user_hw_breakpoint()
530 	if (!bp->attr.disabled)  in modify_user_hw_breakpoint()
531 		perf_event_enable(bp);  in modify_user_hw_breakpoint()
541 void unregister_hw_breakpoint(struct perf_event *bp)  in unregister_hw_breakpoint()  argument
543 	if (!bp)  in unregister_hw_breakpoint()
545 	perf_event_release_kernel(bp);  in unregister_hw_breakpoint()
561 	struct perf_event * __percpu *cpu_events, *bp;  in register_wide_hw_breakpoint()  local
571 		bp = perf_event_create_kernel_counter(attr, cpu, NULL,  in register_wide_hw_breakpoint()
573 		if (IS_ERR(bp)) {  in register_wide_hw_breakpoint()
574 			err = PTR_ERR(bp);  in register_wide_hw_breakpoint()
578 		per_cpu(*cpu_events, cpu) = bp;  in register_wide_hw_breakpoint()
616 static int hw_breakpoint_event_init(struct perf_event *bp)  in hw_breakpoint_event_init()  argument
620 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)  in hw_breakpoint_event_init()
626 	if (has_branch_stack(bp))  in hw_breakpoint_event_init()
629 	err = register_perf_hw_breakpoint(bp);  in hw_breakpoint_event_init()
633 	bp->destroy = bp_perf_event_destroy;  in hw_breakpoint_event_init()
638 static int hw_breakpoint_add(struct perf_event *bp, int flags)  in hw_breakpoint_add()  argument
641 		bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_add()
643 	if (is_sampling_event(bp)) {  in hw_breakpoint_add()
644 		bp->hw.last_period = bp->hw.sample_period;  in hw_breakpoint_add()
645 		perf_swevent_set_period(bp);  in hw_breakpoint_add()
648 	return arch_install_hw_breakpoint(bp);  in hw_breakpoint_add()
651 static void hw_breakpoint_del(struct perf_event *bp, int flags)  in hw_breakpoint_del()  argument
653 	arch_uninstall_hw_breakpoint(bp);  in hw_breakpoint_del()
656 static void hw_breakpoint_start(struct perf_event *bp, int flags)  in hw_breakpoint_start()  argument
658 	bp->hw.state = 0;  in hw_breakpoint_start()
661 static void hw_breakpoint_stop(struct perf_event *bp, int flags)  in hw_breakpoint_stop()  argument
663 	bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_stop()