| /Linux-v5.15/include/linux/ |
| D | local_lock_internal.h | 68 local_lock_acquire(this_cpu_ptr(lock)); \
74 local_lock_acquire(this_cpu_ptr(lock)); \
80 local_lock_acquire(this_cpu_ptr(lock)); \
85 local_lock_release(this_cpu_ptr(lock)); \
91 local_lock_release(this_cpu_ptr(lock)); \
97 local_lock_release(this_cpu_ptr(lock)); \
119 spin_lock(this_cpu_ptr((__lock))); \
133 spin_unlock(this_cpu_ptr((__lock))); \
|
| /Linux-v5.15/kernel/ |
| D | irq_work.c | 57 if (llist_add(&work->node.llist, this_cpu_ptr(&lazy_list)) && in __irq_work_queue_local()
61 if (llist_add(&work->node.llist, this_cpu_ptr(&raised_list))) in __irq_work_queue_local()
122 raised = this_cpu_ptr(&raised_list); in irq_work_needs_cpu()
123 lazy = this_cpu_ptr(&lazy_list); in irq_work_needs_cpu()
186 irq_work_run_list(this_cpu_ptr(&raised_list)); in irq_work_run()
187 irq_work_run_list(this_cpu_ptr(&lazy_list)); in irq_work_run()
193 struct llist_head *raised = this_cpu_ptr(&raised_list); in irq_work_tick()
197 irq_work_run_list(this_cpu_ptr(&lazy_list)); in irq_work_tick()
|
| D | watchdog.c | 348 complete(this_cpu_ptr(&softlockup_completion)); in softlockup_fn()
368 if (completion_done(this_cpu_ptr(&softlockup_completion))) { in watchdog_timer_fn()
369 reinit_completion(this_cpu_ptr(&softlockup_completion)); in watchdog_timer_fn()
372 this_cpu_ptr(&softlockup_stop_work)); in watchdog_timer_fn()
394 period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts)); in watchdog_timer_fn()
452 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); in watchdog_enable()
453 struct completion *done = this_cpu_ptr(&softlockup_completion); in watchdog_enable()
478 struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); in watchdog_disable()
489 wait_for_completion(this_cpu_ptr(&softlockup_completion)); in watchdog_disable()
|
| D | smp.c | 417 struct cfd_seq_local *seq = this_cpu_ptr(&cfd_seq_local); in __smp_call_single_queue_debug()
418 struct call_function_data *cfd = this_cpu_ptr(&cfd_data); in __smp_call_single_queue_debug()
542 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU, in generic_smp_call_function_single_interrupt()
570 head = this_cpu_ptr(&call_single_queue); in flush_smp_call_function_queue()
571 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->handle, CFD_SEQ_NOCPU, in flush_smp_call_function_queue()
574 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->dequeue, in flush_smp_call_function_queue()
637 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->hdlend, in flush_smp_call_function_queue()
680 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->hdlend, CFD_SEQ_NOCPU, in flush_smp_call_function_queue()
688 if (llist_empty(this_cpu_ptr(&call_single_queue))) in flush_smp_call_function_from_idle()
691 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU, in flush_smp_call_function_from_idle()
[all …]
|
| D | user-return-notifier.c | 18 hlist_add_head(&urn->link, this_cpu_ptr(&return_notifier_list)); in user_return_notifier_register()
29 if (hlist_empty(this_cpu_ptr(&return_notifier_list))) in user_return_notifier_unregister()
|
| /Linux-v5.15/arch/powerpc/include/asm/ |
| D | idle.h | 14 *this_cpu_ptr(&idle_entry_purr_snap) = mfspr(SPRN_PURR); in snapshot_purr_idle_entry()
19 *this_cpu_ptr(&idle_entry_spurr_snap) = mfspr(SPRN_SPURR); in snapshot_spurr_idle_entry()
25 u64 in_purr = *this_cpu_ptr(&idle_entry_purr_snap); in update_idle_purr_accounting()
34 u64 *idle_spurr_cycles_ptr = this_cpu_ptr(&idle_spurr_cycles); in update_idle_spurr_accounting()
35 u64 in_spurr = *this_cpu_ptr(&idle_entry_spurr_snap); in update_idle_spurr_accounting()
89 return *this_cpu_ptr(&idle_spurr_cycles); in read_this_idle_spurr()
|
| D | xics.h | 115 struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); in xics_push_cppr()
128 struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); in xics_pop_cppr()
138 struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); in xics_set_base_cppr()
150 struct xics_cppr *os_cppr = this_cpu_ptr(&xics_cppr); in xics_cppr_top()
|
| /Linux-v5.15/arch/x86/kernel/cpu/mce/ |
| D | intel.c | 139 if (machine_check_poll(0, this_cpu_ptr(&mce_banks_owned))) in mce_intel_cmci_poll()
162 owned = this_cpu_ptr(mce_banks_owned); in cmci_toggle_interrupt_mode()
259 machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned)); in intel_threshold_interrupt()
269 unsigned long *owned = (void *)this_cpu_ptr(&mce_banks_owned); in cmci_discover()
291 __clear_bit(i, this_cpu_ptr(mce_poll_banks)); in cmci_discover()
315 __clear_bit(i, this_cpu_ptr(mce_poll_banks)); in cmci_discover()
326 WARN_ON(!test_bit(i, this_cpu_ptr(mce_poll_banks))); in cmci_discover()
351 machine_check_poll(0, this_cpu_ptr(&mce_banks_owned)); in cmci_recheck()
360 if (!test_bit(bank, this_cpu_ptr(mce_banks_owned))) in __cmci_disable_bank()
365 __clear_bit(bank, this_cpu_ptr(mce_banks_owned)); in __cmci_disable_bank()
|
| /Linux-v5.15/net/core/ |
| D | dst_cache.c | 68 return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); in dst_cache_get()
80 idst = this_cpu_ptr(dst_cache->cache); in dst_cache_get_ip4()
98 idst = this_cpu_ptr(dst_cache->cache); in dst_cache_set_ip4()
113 idst = this_cpu_ptr(dst_cache->cache); in dst_cache_set_ip6()
114 dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst, in dst_cache_set_ip6()
129 idst = this_cpu_ptr(dst_cache->cache); in dst_cache_get_ip6()
|
| /Linux-v5.15/arch/arm64/kvm/hyp/nvhe/ |
| D | psci-relay.c | 160 boot_args = this_cpu_ptr(&suspend_args); in psci_cpu_suspend()
161 init_params = this_cpu_ptr(&kvm_init_params); in psci_cpu_suspend()
187 boot_args = this_cpu_ptr(&suspend_args); in psci_system_suspend()
188 init_params = this_cpu_ptr(&kvm_init_params); in psci_system_suspend()
208 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; in kvm_host_psci_cpu_entry()
211 boot_args = this_cpu_ptr(&cpu_on_args); in kvm_host_psci_cpu_entry()
213 boot_args = this_cpu_ptr(&suspend_args); in kvm_host_psci_cpu_entry()
|
| /Linux-v5.15/arch/xtensa/kernel/ |
| D | hw_breakpoint.c | 177 i = alloc_slot(this_cpu_ptr(bp_on_reg), XCHAL_NUM_IBREAK, bp); in arch_install_hw_breakpoint()
184 i = alloc_slot(this_cpu_ptr(wp_on_reg), XCHAL_NUM_DBREAK, bp); in arch_install_hw_breakpoint()
215 i = free_slot(this_cpu_ptr(bp_on_reg), XCHAL_NUM_IBREAK, bp); in arch_uninstall_hw_breakpoint()
223 i = free_slot(this_cpu_ptr(wp_on_reg), XCHAL_NUM_DBREAK, bp); in arch_uninstall_hw_breakpoint()
268 struct perf_event *bp = this_cpu_ptr(wp_on_reg)[i]; in restore_dbreak()
280 struct perf_event **bp = this_cpu_ptr(bp_on_reg); in check_hw_breakpoint()
289 struct perf_event **bp = this_cpu_ptr(wp_on_reg); in check_hw_breakpoint()
|
| /Linux-v5.15/kernel/time/ |
| D | tick-sched.c | 311 irq_work_queue(this_cpu_ptr(&nohz_full_kick_work)); in tick_nohz_full_kick()
496 ts = this_cpu_ptr(&tick_cpu_sched); in __tick_nohz_task_switch()
589 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_tick_stopped()
1083 __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched)); in tick_nohz_idle_stop_tick()
1088 tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched)); in tick_nohz_idle_retain_tick()
1109 ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_enter()
1129 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_exit()
1142 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_got_tick()
1176 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_sleep_length()
1226 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_idle_calls()
[all …]
|
| /Linux-v5.15/arch/x86/events/intel/ |
| D | bts.c | 228 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in __bts_event_start()
256 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in bts_event_start()
257 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in bts_event_start()
287 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in __bts_event_stop()
301 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in bts_event_stop()
302 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in bts_event_stop()
335 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in intel_bts_enable_local()
355 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in intel_bts_disable_local()
446 struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds; in intel_bts_interrupt()
447 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); in intel_bts_interrupt()
[all …]
|
| /Linux-v5.15/net/netfilter/ |
| D | nft_counter.c | 37 this_cpu = this_cpu_ptr(priv->counter); in nft_counter_do_eval()
38 myseq = this_cpu_ptr(&nft_counter_seq); in nft_counter_do_eval()
69 this_cpu = this_cpu_ptr(cpu_stats); in nft_counter_do_init()
111 this_cpu = this_cpu_ptr(priv->counter); in nft_counter_reset()
243 this_cpu = this_cpu_ptr(cpu_stats); in nft_counter_clone()
268 this_cpu = this_cpu_ptr(priv->counter); in nft_counter_offload_stats()
269 myseq = this_cpu_ptr(&nft_counter_seq); in nft_counter_offload_stats()
|
| /Linux-v5.15/drivers/hv/ |
| D | hv_common.c | 129 inputarg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); in hv_common_cpu_init()
135 outputarg = (void **)this_cpu_ptr(hyperv_pcpu_output_arg); in hv_common_cpu_init()
157 inputarg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); in hv_common_cpu_die()
162 outputarg = (void **)this_cpu_ptr(hyperv_pcpu_output_arg); in hv_common_cpu_die()
|
| /Linux-v5.15/arch/arm/kernel/ |
| D | perf_event_xscale.c | 149 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); in xscale1pmu_handle_irq()
209 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale1pmu_enable_event()
245 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale1pmu_disable_event()
304 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale1pmu_start()
316 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale1pmu_stop()
501 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); in xscale2pmu_handle_irq()
555 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale2pmu_enable_event()
601 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale2pmu_disable_event()
667 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale2pmu_start()
679 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); in xscale2pmu_stop()
|
| /Linux-v5.15/arch/arm64/kernel/ |
| D | hw_breakpoint.c | 236 slots = this_cpu_ptr(bp_on_reg); in hw_breakpoint_control()
243 slots = this_cpu_ptr(wp_on_reg); in hw_breakpoint_control()
588 slots = this_cpu_ptr(bp_on_reg); in toggle_bp_registers()
592 slots = this_cpu_ptr(wp_on_reg); in toggle_bp_registers()
630 slots = this_cpu_ptr(bp_on_reg); in breakpoint_handler()
680 kernel_step = this_cpu_ptr(&stepping_kernel_bp); in breakpoint_handler()
765 slots = this_cpu_ptr(wp_on_reg); in watchpoint_handler()
831 kernel_step = this_cpu_ptr(&stepping_kernel_bp); in watchpoint_handler()
856 kernel_step = this_cpu_ptr(&stepping_kernel_bp); in reinstall_suspended_bps()
955 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { in hw_breakpoint_reset()
[all …]
|
| D | fpsimd.c | 305 this_cpu_ptr(&fpsimd_last_state); in fpsimd_save()
1118 this_cpu_ptr(&fpsimd_last_state); in fpsimd_bind_task_to_cpu()
1141 this_cpu_ptr(&fpsimd_last_state); in fpsimd_bind_state_to_cpu()
1352 char *sve_state = this_cpu_ptr(efi_sve_state); in __efi_fpsimd_begin()
1357 &this_cpu_ptr(&efi_fpsimd_state)->fpsr); in __efi_fpsimd_begin()
1359 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); in __efi_fpsimd_begin()
1379 char const *sve_state = this_cpu_ptr(efi_sve_state); in __efi_fpsimd_end()
1382 &this_cpu_ptr(&efi_fpsimd_state)->fpsr, in __efi_fpsimd_end()
1387 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); in __efi_fpsimd_end()
|
| /Linux-v5.15/kernel/rcu/ |
| D | tree_plugin.h | 31 (rdp == this_cpu_ptr(&rcu_data) && in rcu_rdp_is_offloaded()
311 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch()
479 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
636 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special()
820 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
852 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); in rcu_qs()
870 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_all_qs()
893 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) in rcu_note_context_switch()
1271 return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) && in rcu_needs_cpu()
1272 !rcu_rdp_is_offloaded(this_cpu_ptr(&rcu_data)); in rcu_needs_cpu()
[all …]
|
| /Linux-v5.15/arch/x86/xen/ |
| D | multicalls.c | 58 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); in xen_mc_flush()
140 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); in __xen_mc_entry()
171 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); in xen_mc_extend_args()
201 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); in xen_mc_callback()
|
| /Linux-v5.15/arch/x86/hyperv/ |
| D | hv_proc.c | 82 input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); in hv_call_deposit_pages()
134 input = *this_cpu_ptr(hyperv_pcpu_input_arg); in hv_call_add_logical_proc()
136 output = *this_cpu_ptr(hyperv_pcpu_output_arg); in hv_call_add_logical_proc()
182 input = *this_cpu_ptr(hyperv_pcpu_input_arg); in hv_call_create_vp()
|
| /Linux-v5.15/arch/sh/kernel/ |
| D | kprobes.c | 102 struct kprobe *saved = this_cpu_ptr(&saved_next_opcode); in arch_remove_kprobe()
111 saved = this_cpu_ptr(&saved_next_opcode2); in arch_remove_kprobe()
153 op1 = this_cpu_ptr(&saved_next_opcode); in prepare_singlestep()
154 op2 = this_cpu_ptr(&saved_next_opcode2); in prepare_singlestep()
326 p = this_cpu_ptr(&saved_next_opcode); in post_kprobe_handler()
338 p = this_cpu_ptr(&saved_next_opcode2); in post_kprobe_handler()
|
| /Linux-v5.15/arch/powerpc/platforms/powernv/ |
| D | opal-tracepoints.c | 55 depth = this_cpu_ptr(&opal_trace_depth); in __trace_opal_entry()
76 depth = this_cpu_ptr(&opal_trace_depth); in __trace_opal_exit()
|
| /Linux-v5.15/lib/ |
| D | irq_poll.c | 37 list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); in irq_poll_sched()
80 struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); in irq_poll_softirq()
196 this_cpu_ptr(&blk_cpu_iopoll)); in irq_poll_cpu_dead()
|
| /Linux-v5.15/arch/arm64/include/asm/ |
| D | hardirq.h | 36 ___ctx = this_cpu_ptr(&nmi_contexts); \
69 ___ctx = this_cpu_ptr(&nmi_contexts); \
|
