71 __weak int hw_breakpoint_weight(struct perf_event *bp)  in hw_breakpoint_weight()  argument
105 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)  in task_bp_pinned()  argument
107 	struct task_struct *tsk = bp->hw.target;  in task_bp_pinned()
121 static const struct cpumask *cpumask_of_bp(struct perf_event *bp)  in cpumask_of_bp()  argument
123 	if (bp->cpu >= 0)  in cpumask_of_bp()
124 		return cpumask_of(bp->cpu);  in cpumask_of_bp()
133 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,  in fetch_bp_busy_slots()  argument
136 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in fetch_bp_busy_slots()
144 		if (!bp->hw.target)  in fetch_bp_busy_slots()
147 			nr += task_bp_pinned(cpu, bp, type);  in fetch_bp_busy_slots()
172 static void toggle_bp_task_slot(struct perf_event *bp, int cpu,  in toggle_bp_task_slot()  argument
178 	old_idx = task_bp_pinned(cpu, bp, type) - 1;  in toggle_bp_task_slot()
191 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,  in toggle_bp_slot()  argument
194 	const struct cpumask *cpumask = cpumask_of_bp(bp);  in toggle_bp_slot()
201 	if (!bp->hw.target) {  in toggle_bp_slot()
202 		get_bp_info(bp->cpu, type)->cpu_pinned += weight;  in toggle_bp_slot()
208 		toggle_bp_task_slot(bp, cpu, type, weight);  in toggle_bp_slot()
211 		list_add_tail(&bp->hw.bp_list, &bp_task_head);  in toggle_bp_slot()
213 		list_del(&bp->hw.bp_list);  in toggle_bp_slot()
219 __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)  in arch_unregister_hw_breakpoint()  argument
268 static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)  in __reserve_bp_slot()  argument
284 	weight = hw_breakpoint_weight(bp);  in __reserve_bp_slot()
286 	fetch_bp_busy_slots(&slots, bp, type);  in __reserve_bp_slot()
297 	toggle_bp_slot(bp, true, type, weight);  in __reserve_bp_slot()
302 int reserve_bp_slot(struct perf_event *bp)  in reserve_bp_slot()  argument
308 	ret = __reserve_bp_slot(bp, bp->attr.bp_type);  in reserve_bp_slot()
315 static void __release_bp_slot(struct perf_event *bp, u64 bp_type)  in __release_bp_slot()  argument
321 	weight = hw_breakpoint_weight(bp);  in __release_bp_slot()
322 	toggle_bp_slot(bp, false, type, weight);  in __release_bp_slot()
325 void release_bp_slot(struct perf_event *bp)  in release_bp_slot()  argument
329 	arch_unregister_hw_breakpoint(bp);  in release_bp_slot()
330 	__release_bp_slot(bp, bp->attr.bp_type);  in release_bp_slot()
335 static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in __modify_bp_slot()  argument
339 	__release_bp_slot(bp, old_type);  in __modify_bp_slot()
341 	err = __reserve_bp_slot(bp, new_type);  in __modify_bp_slot()
351 		WARN_ON(__reserve_bp_slot(bp, old_type));  in __modify_bp_slot()
357 static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type)  in modify_bp_slot()  argument
362 	ret = __modify_bp_slot(bp, old_type, new_type);  in modify_bp_slot()
372 int dbg_reserve_bp_slot(struct perf_event *bp)  in dbg_reserve_bp_slot()  argument
377 	return __reserve_bp_slot(bp, bp->attr.bp_type);  in dbg_reserve_bp_slot()
380 int dbg_release_bp_slot(struct perf_event *bp)  in dbg_release_bp_slot()  argument
385 	__release_bp_slot(bp, bp->attr.bp_type);  in dbg_release_bp_slot()
390 static int hw_breakpoint_parse(struct perf_event *bp,  in hw_breakpoint_parse()  argument
396 	err = hw_breakpoint_arch_parse(bp, attr, hw);  in hw_breakpoint_parse()
414 int register_perf_hw_breakpoint(struct perf_event *bp)  in register_perf_hw_breakpoint()  argument
419 	err = reserve_bp_slot(bp);  in register_perf_hw_breakpoint()
423 	err = hw_breakpoint_parse(bp, &bp->attr, &hw);  in register_perf_hw_breakpoint()
425 		release_bp_slot(bp);  in register_perf_hw_breakpoint()
429 	bp->hw.info = hw;  in register_perf_hw_breakpoint()
461 modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,  in modify_user_hw_breakpoint_check()  argument
467 	err = hw_breakpoint_parse(bp, attr, &hw);  in modify_user_hw_breakpoint_check()
474 		old_attr = bp->attr;  in modify_user_hw_breakpoint_check()
480 	if (bp->attr.bp_type != attr->bp_type) {  in modify_user_hw_breakpoint_check()
481 		err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);  in modify_user_hw_breakpoint_check()
486 	hw_breakpoint_copy_attr(&bp->attr, attr);  in modify_user_hw_breakpoint_check()
487 	bp->hw.info = hw;  in modify_user_hw_breakpoint_check()
497 int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)  in modify_user_hw_breakpoint()  argument
507 	if (irqs_disabled() && bp->ctx && bp->ctx->task == current)  in modify_user_hw_breakpoint()
508 		perf_event_disable_local(bp);  in modify_user_hw_breakpoint()
510 		perf_event_disable(bp);  in modify_user_hw_breakpoint()
512 	err = modify_user_hw_breakpoint_check(bp, attr, false);  in modify_user_hw_breakpoint()
514 	if (!bp->attr.disabled)  in modify_user_hw_breakpoint()
515 		perf_event_enable(bp);  in modify_user_hw_breakpoint()
525 void unregister_hw_breakpoint(struct perf_event *bp)  in unregister_hw_breakpoint()  argument
527 	if (!bp)  in unregister_hw_breakpoint()
529 	perf_event_release_kernel(bp);  in unregister_hw_breakpoint()
545 	struct perf_event * __percpu *cpu_events, *bp;  in register_wide_hw_breakpoint()  local
555 		bp = perf_event_create_kernel_counter(attr, cpu, NULL,  in register_wide_hw_breakpoint()
557 		if (IS_ERR(bp)) {  in register_wide_hw_breakpoint()
558 			err = PTR_ERR(bp);  in register_wide_hw_breakpoint()
562 		per_cpu(*cpu_events, cpu) = bp;  in register_wide_hw_breakpoint()
600 static int hw_breakpoint_event_init(struct perf_event *bp)  in hw_breakpoint_event_init()  argument
604 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)  in hw_breakpoint_event_init()
610 	if (has_branch_stack(bp))  in hw_breakpoint_event_init()
613 	err = register_perf_hw_breakpoint(bp);  in hw_breakpoint_event_init()
617 	bp->destroy = bp_perf_event_destroy;  in hw_breakpoint_event_init()
622 static int hw_breakpoint_add(struct perf_event *bp, int flags)  in hw_breakpoint_add()  argument
625 		bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_add()
627 	if (is_sampling_event(bp)) {  in hw_breakpoint_add()
628 		bp->hw.last_period = bp->hw.sample_period;  in hw_breakpoint_add()
629 		perf_swevent_set_period(bp);  in hw_breakpoint_add()
632 	return arch_install_hw_breakpoint(bp);  in hw_breakpoint_add()
635 static void hw_breakpoint_del(struct perf_event *bp, int flags)  in hw_breakpoint_del()  argument
637 	arch_uninstall_hw_breakpoint(bp);  in hw_breakpoint_del()
640 static void hw_breakpoint_start(struct perf_event *bp, int flags)  in hw_breakpoint_start()  argument
642 	bp->hw.state = 0;  in hw_breakpoint_start()
645 static void hw_breakpoint_stop(struct perf_event *bp, int flags)  in hw_breakpoint_stop()  argument
647 	bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_stop()