1 // SPDX-License-Identifier: GPL-2.0+
15  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
16  * using the CPU's debug registers.
17  * This file contains the arch-independent routines.
24 #include <linux/cpu.h>
32 #include <linux/percpu-rwsem.h>
51  * Per-CPU constraints data.
54 	/* Number of pinned CPU breakpoints in a CPU. */
56 	/* Histogram of pinned task breakpoints in a CPU. */
62 static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)  in get_bp_info()  argument
64 	return per_cpu_ptr(bp_cpuinfo + type, cpu);  in get_bp_info()
67 /* Number of pinned CPU breakpoints globally. */
69 /* Number of pinned CPU-independent task breakpoints. */
84  * Synchronizes accesses to the per-CPU constraints; the locking rules are:
86  *  1. Atomic updates to bp_cpuinfo::tsk_pinned only require a held read-lock
89  *  2. Holding a write-lock is required for computations that require a
92  *  3. In all other cases, non-atomic accesses require the appropriately held
93  *     lock (read-lock for read-only accesses; write-lock for reads/writes).
98  * Return mutex to serialize accesses to per-task lists in task_bps_ht. Since
100  * insert/delete concurrently; therefore, a mutex per task is sufficient.
103  * hw_breakpoint-only mutex, which may be infrequently used. The caveat here is
104  * that hw_breakpoint may contend with per-task perf event list management. The
110 	struct task_struct *tsk = bp->hw.target;  in get_task_bps_mutex()
112 	return tsk ? &tsk->perf_event_mutex : NULL;  in get_task_bps_mutex()
123 		 * this child->perf_event_mutex cannot ever deadlock against  in bp_constraints_lock()
124 		 * the parent->perf_event_mutex usage from  in bp_constraints_lock()
128 		 * ->perf_event_list.  in bp_constraints_lock()
188 	hist->count = kcalloc(hw_breakpoint_slots_cached(type), sizeof(*hist->count), GFP_KERNEL);  in bp_slots_histogram_alloc()
189 	return hist->count;  in bp_slots_histogram_alloc()
194 	kfree(hist->count);  in bp_slots_histogram_free()
199 	int i, cpu, err_cpu;  in init_breakpoint_slots()  local
204 	for_each_possible_cpu(cpu) {  in init_breakpoint_slots()
206 			struct bp_cpuinfo *info = get_bp_info(cpu, i);  in init_breakpoint_slots()
208 			if (!bp_slots_histogram_alloc(&info->tsk_pinned, i))  in init_breakpoint_slots()
223 			bp_slots_histogram_free(&get_bp_info(err_cpu, i)->tsk_pinned);  in init_breakpoint_slots()
224 		if (err_cpu == cpu)  in init_breakpoint_slots()
232 	return -ENOMEM;  in init_breakpoint_slots()
239 	const int old_idx = old - 1;  in bp_slots_histogram_add()
243 		WARN_ON(atomic_dec_return_relaxed(&hist->count[old_idx]) < 0);  in bp_slots_histogram_add()
245 		WARN_ON(atomic_inc_return_relaxed(&hist->count[new_idx]) < 0);  in bp_slots_histogram_add()
251 	for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) {  in bp_slots_histogram_max()
252 		const int count = atomic_read(&hist->count[i]);  in bp_slots_histogram_max()
255 		ASSERT_EXCLUSIVE_WRITER(hist->count[i]);  in bp_slots_histogram_max()
268 	for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) {  in bp_slots_histogram_max_merge()
269 		const int count1 = atomic_read(&hist1->count[i]);  in bp_slots_histogram_max_merge()
270 		const int count2 = atomic_read(&hist2->count[i]);  in bp_slots_histogram_max_merge()
273 		ASSERT_EXCLUSIVE_WRITER(hist1->count[i]);  in bp_slots_histogram_max_merge()
274 		ASSERT_EXCLUSIVE_WRITER(hist2->count[i]);  in bp_slots_histogram_max_merge()
300  * Return the maximum number of pinned breakpoints a task has in this CPU.
302 static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)  in max_task_bp_pinned()  argument
304 	struct bp_slots_histogram *tsk_pinned = &get_bp_info(cpu, type)->tsk_pinned;  in max_task_bp_pinned()
319  * If @cpu is -1, but the result of task_bp_pinned() is not CPU-independent,
322 static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)  in task_bp_pinned()  argument
329 	 * We need a stable snapshot of the per-task breakpoint list.  in task_bp_pinned()
334 	head = rhltable_lookup(&task_bps_ht, &bp->hw.target, task_bps_ht_params);  in task_bp_pinned()
339 		if (find_slot_idx(iter->attr.bp_type) != type)  in task_bp_pinned()
342 		if (iter->cpu >= 0) {  in task_bp_pinned()
343 			if (cpu == -1) {  in task_bp_pinned()
344 				count = -1;  in task_bp_pinned()
346 			} else if (cpu != iter->cpu)  in task_bp_pinned()
360 	if (bp->cpu >= 0)  in cpumask_of_bp()
361 		return cpumask_of(bp->cpu);  in cpumask_of_bp()
367  * CPU (cpu > -1) or across all of them (cpu = -1).
374 	int cpu;  in max_bp_pinned_slots()  local
376 	if (bp->hw.target && bp->cpu < 0) {  in max_bp_pinned_slots()
377 		int max_pinned = task_bp_pinned(-1, bp, type);  in max_bp_pinned_slots()
381 			 * Fast path: task_bp_pinned() is CPU-independent and  in max_bp_pinned_slots()
382 			 * returns the same value for any CPU.  in max_bp_pinned_slots()
389 	for_each_cpu(cpu, cpumask) {  in max_bp_pinned_slots()
390 		struct bp_cpuinfo *info = get_bp_info(cpu, type);  in max_bp_pinned_slots()
393 		nr = info->cpu_pinned;  in max_bp_pinned_slots()
394 		if (!bp->hw.target)  in max_bp_pinned_slots()
395 			nr += max_task_bp_pinned(cpu, type);  in max_bp_pinned_slots()
397 			nr += task_bp_pinned(cpu, bp, type);  in max_bp_pinned_slots()
411 	int cpu, next_tsk_pinned;  in toggle_bp_slot()  local
414 		weight = -weight;  in toggle_bp_slot()
416 	if (!bp->hw.target) {  in toggle_bp_slot()
418 		 * Update the pinned CPU slots, in per-CPU bp_cpuinfo and in the  in toggle_bp_slot()
421 		struct bp_cpuinfo *info = get_bp_info(bp->cpu, type);  in toggle_bp_slot()
424 		bp_slots_histogram_add(&cpu_pinned[type], info->cpu_pinned, weight);  in toggle_bp_slot()
425 		info->cpu_pinned += weight;  in toggle_bp_slot()
430 	 * If bp->hw.target, tsk_pinned is only modified, but not used  in toggle_bp_slot()
439 	 * Update the pinned task slots, in per-CPU bp_cpuinfo and in the global  in toggle_bp_slot()
442 	 *  1. This breakpoint targets all CPUs (cpu < 0), and there may only  in toggle_bp_slot()
446 	 *  2. This breakpoint targets a specific CPU (cpu >= 0), but there may  in toggle_bp_slot()
450 	 *        slots histogram and use the per-CPU histogram.  in toggle_bp_slot()
452 	 *     b. On disable: re-insert the existing breakpoints into the global  in toggle_bp_slot()
453 	 *        slots histogram and remove from per-CPU histogram.  in toggle_bp_slot()
456 	 *     update the per-CPU slots histogram.  in toggle_bp_slot()
462 		 * was the last task breakpoint for a specific CPU.  in toggle_bp_slot()
464 		int ret = rhltable_remove(&task_bps_ht, &bp->hw.bp_list, task_bps_ht_params);  in toggle_bp_slot()
470 	 * Note: If !enable, next_tsk_pinned will not count the to-be-removed breakpoint.  in toggle_bp_slot()
472 	next_tsk_pinned = task_bp_pinned(-1, bp, type);  in toggle_bp_slot()
475 		if (bp->cpu < 0) { /* Case 1: fast path */  in toggle_bp_slot()
480 			/* Add existing to per-CPU histograms. */  in toggle_bp_slot()
481 			for_each_possible_cpu(cpu) {  in toggle_bp_slot()
482 				bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned,  in toggle_bp_slot()
485 			/* Add this first CPU-pinned task breakpoint. */  in toggle_bp_slot()
486 			bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned,  in toggle_bp_slot()
490 					       -next_tsk_pinned);  in toggle_bp_slot()
492 			/* Remove this last CPU-pinned task breakpoint. */  in toggle_bp_slot()
493 			bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned,  in toggle_bp_slot()
495 			/* Remove all from per-CPU histograms. */  in toggle_bp_slot()
496 			for_each_possible_cpu(cpu) {  in toggle_bp_slot()
497 				bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned,  in toggle_bp_slot()
498 						       next_tsk_pinned, -next_tsk_pinned);  in toggle_bp_slot()
506 		for_each_cpu(cpu, cpumask) {  in toggle_bp_slot()
507 			next_tsk_pinned = task_bp_pinned(cpu, bp, type);  in toggle_bp_slot()
510 			bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned,  in toggle_bp_slot()
516 	 * Readers want a stable snapshot of the per-task breakpoint list.  in toggle_bp_slot()
521 		return rhltable_insert(&task_bps_ht, &bp->hw.bp_list, task_bps_ht_params);  in toggle_bp_slot()
536  * Function to perform processor-specific cleanup during unregistration
552  * before a pinned event in a same CPU.
554  *  == Non-pinned counter == (Considered as pinned for now)
556  *   - If attached to a single cpu, check:
558  *       (per_cpu(info->flexible, cpu) || (per_cpu(info->cpu_pinned, cpu)
559  *           + max(per_cpu(info->tsk_pinned, cpu)))) < HBP_NUM
561  *       -> If there are already non-pinned counters in this cpu, it means
563  *          Otherwise, we check that the maximum number of per task
564  *          breakpoints (for this cpu) plus the number of per cpu breakpoint
565  *          (for this cpu) doesn't cover every registers.
567  *   - If attached to every cpus, check:
569  *       (per_cpu(info->flexible, *) || (max(per_cpu(info->cpu_pinned, *))
570  *           + max(per_cpu(info->tsk_pinned, *)))) < HBP_NUM
572  *       -> This is roughly the same, except we check the number of per cpu
573  *          bp for every cpu and we keep the max one. Same for the per tasks
579  *   - If attached to a single cpu, check:
581  *       ((per_cpu(info->flexible, cpu) > 1) + per_cpu(info->cpu_pinned, cpu)
582  *            + max(per_cpu(info->tsk_pinned, cpu))) < HBP_NUM
584  *       -> Same checks as before. But now the info->flexible, if any, must keep
587  *   - If attached to every cpus, check:
589  *       ((per_cpu(info->flexible, *) > 1) + max(per_cpu(info->cpu_pinned, *))
590  *            + max(per_cpu(info->tsk_pinned, *))) < HBP_NUM
601 		return -ENOMEM;  in __reserve_bp_slot()
606 		return -EINVAL;  in __reserve_bp_slot()
614 		return -ENOSPC;  in __reserve_bp_slot()
626 	int ret = __reserve_bp_slot(bp, bp->attr.bp_type);  in reserve_bp_slot()
649 	__release_bp_slot(bp, bp->attr.bp_type);  in release_bp_slot()
694 		return -1;  in dbg_reserve_bp_slot()
698 	ret = __reserve_bp_slot(bp, bp->attr.bp_type);  in dbg_reserve_bp_slot()
707 		return -1;  in dbg_release_bp_slot()
711 	__release_bp_slot(bp, bp->attr.bp_type);  in dbg_release_bp_slot()
728 		if (attr->exclude_kernel)  in hw_breakpoint_parse()
729 			return -EINVAL;  in hw_breakpoint_parse()
735 			return -EPERM;  in hw_breakpoint_parse()
750 	err = hw_breakpoint_parse(bp, &bp->attr, &hw);  in register_perf_hw_breakpoint()
756 	bp->hw.info = hw;  in register_perf_hw_breakpoint()
762  * register_user_hw_breakpoint - register a hardware breakpoint for user space
774 	return perf_event_create_kernel_counter(attr, -1, tsk, triggered,  in register_user_hw_breakpoint()
782 	to->bp_addr = from->bp_addr;  in hw_breakpoint_copy_attr()
783 	to->bp_type = from->bp_type;  in hw_breakpoint_copy_attr()
784 	to->bp_len  = from->bp_len;  in hw_breakpoint_copy_attr()
785 	to->disabled = from->disabled;  in hw_breakpoint_copy_attr()
802 		old_attr = bp->attr;  in modify_user_hw_breakpoint_check()
805 			return -EINVAL;  in modify_user_hw_breakpoint_check()
808 	if (bp->attr.bp_type != attr->bp_type) {  in modify_user_hw_breakpoint_check()
809 		err = modify_bp_slot(bp, bp->attr.bp_type, attr->bp_type);  in modify_user_hw_breakpoint_check()
814 	hw_breakpoint_copy_attr(&bp->attr, attr);  in modify_user_hw_breakpoint_check()
815 	bp->hw.info = hw;  in modify_user_hw_breakpoint_check()
821  * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
835 	if (irqs_disabled() && bp->ctx && bp->ctx->task == current)  in modify_user_hw_breakpoint()
842 	if (!bp->attr.disabled)  in modify_user_hw_breakpoint()
850  * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
862  * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
876 	int cpu;  in register_wide_hw_breakpoint()  local
880 		return (void __percpu __force *)ERR_PTR(-ENOMEM);  in register_wide_hw_breakpoint()
883 	for_each_online_cpu(cpu) {  in register_wide_hw_breakpoint()
884 		bp = perf_event_create_kernel_counter(attr, cpu, NULL,  in register_wide_hw_breakpoint()
891 		per_cpu(*cpu_events, cpu) = bp;  in register_wide_hw_breakpoint()
904  * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
905  * @cpu_events: the per cpu set of events to unregister
909 	int cpu;  in unregister_wide_hw_breakpoint()  local
911 	for_each_possible_cpu(cpu)  in unregister_wide_hw_breakpoint()
912 		unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));  in unregister_wide_hw_breakpoint()
919  * hw_breakpoint_is_used - check if breakpoints are currently used
925 	int cpu;  in hw_breakpoint_is_used()  local
930 	for_each_possible_cpu(cpu) {  in hw_breakpoint_is_used()
932 			struct bp_cpuinfo *info = get_bp_info(cpu, type);  in hw_breakpoint_is_used()
934 			if (info->cpu_pinned)  in hw_breakpoint_is_used()
938 				if (atomic_read(&info->tsk_pinned.count[slot]))  in hw_breakpoint_is_used()
947 			 * Warn, because if there are CPU pinned counters,  in hw_breakpoint_is_used()
977 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)  in hw_breakpoint_event_init()
978 		return -ENOENT;  in hw_breakpoint_event_init()
984 		return -EOPNOTSUPP;  in hw_breakpoint_event_init()
990 	bp->destroy = bp_perf_event_destroy;  in hw_breakpoint_event_init()
998 		bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_add()
1001 		bp->hw.last_period = bp->hw.sample_period;  in hw_breakpoint_add()
1015 	bp->hw.state = 0;  in hw_breakpoint_start()
1020 	bp->hw.state = PERF_HES_STOPPED;  in hw_breakpoint_stop()