Home
last modified time | relevance | path

Searched refs:pcpu_hot (Results 1 – 23 of 23) sorted by relevance

/Linux-v6.6/arch/x86/include/asm/
Dpreempt.h27 return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED; in preempt_count()
35 old = raw_cpu_read_4(pcpu_hot.preempt_count); in preempt_count_set()
38 } while (raw_cpu_cmpxchg_4(pcpu_hot.preempt_count, old, new) != old); in preempt_count_set()
47 per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \
61 raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED); in set_preempt_need_resched()
66 raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED); in clear_preempt_need_resched()
71 return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED); in test_preempt_need_resched()
80 raw_cpu_add_4(pcpu_hot.preempt_count, val); in __preempt_count_add()
85 raw_cpu_add_4(pcpu_hot.preempt_count, -val); in __preempt_count_sub()
95 return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e, in __preempt_count_dec_and_test()
[all …]
Dcurrent.h14 struct pcpu_hot { struct
35 static_assert(sizeof(struct pcpu_hot) == 64); argument
37 DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
41 return this_cpu_read_stable(pcpu_hot.current_task); in get_current()
Dirq_stack.h119 call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \
138 if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
149 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
151 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
215 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
217 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
Dnospec-branch.h81 movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
84 movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
89 movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
94 movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
98 sarq $5, %gs:pcpu_hot + X86_call_depth; \
102 sarq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
Dsmp.h146 #define raw_smp_processor_id() this_cpu_read(pcpu_hot.cpu_number)
147 #define __smp_processor_id() __this_cpu_read(pcpu_hot.cpu_number)
Dhardirq.h63 #define local_softirq_pending_ref pcpu_hot.softirq_pending
Dprocessor.h521 return this_cpu_read_stable(pcpu_hot.top_of_stack); in current_top_of_stack()
/Linux-v6.6/arch/x86/kernel/
Dirq_32.c77 irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr); in execute_on_irq_stack()
115 if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu)) in irq_init_percpu_irqstack()
127 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph); in irq_init_percpu_irqstack()
128 per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps); in irq_init_percpu_irqstack()
138 irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr); in do_softirq_own_stack()
Dirq_64.c53 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; in map_irq_stack()
66 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; in map_irq_stack()
73 if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu)) in irq_init_percpu_irqstack()
Dasm-offsets.c117 OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack); in common()
118 OFFSET(X86_current_task, pcpu_hot, current_task); in common()
120 OFFSET(X86_call_depth, pcpu_hot, call_depth); in common()
Ddumpstack_32.c40 unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr); in in_hardirq_stack()
65 unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr); in in_softirq_stack()
Dprocess_32.c200 this_cpu_write(pcpu_hot.top_of_stack, in __switch_to()
210 raw_cpu_write(pcpu_hot.current_task, next_p); in __switch_to()
Dprocess_64.c569 this_cpu_read(pcpu_hot.hardirq_stack_inuse)); in __switch_to()
623 raw_cpu_write(pcpu_hot.current_task, next_p); in __switch_to()
624 raw_cpu_write(pcpu_hot.top_of_stack, task_top_of_stack(next_p)); in __switch_to()
Dsetup_percpu.c172 per_cpu(pcpu_hot.cpu_number, cpu) = cpu; in setup_per_cpu_areas()
Ddumpstack_64.c137 unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr); in in_irq_stack()
Dhead_64.S307 movq pcpu_hot + X86_current_task(%rdx), %rax
452 movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
Dtraps.c775 struct pt_regs *regs = (struct pt_regs *)this_cpu_read(pcpu_hot.top_of_stack) - 1; in sync_regs()
793 sp = this_cpu_read(pcpu_hot.top_of_stack); in vc_switch_off_ist()
Dsmpboot.c964 per_cpu(pcpu_hot.current_task, cpu) = idle; in common_cpu_up()
974 per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle); in common_cpu_up()
/Linux-v6.6/arch/x86/entry/
Dentry_64_compat.S61 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
193 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
334 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
Dentry_64.S95 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
1218 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
1534 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
Dentry_32.S1157 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp
1219 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
/Linux-v6.6/arch/x86/lib/
Dretpoline.S339 shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth)
/Linux-v6.6/arch/x86/kernel/cpu/
Dcommon.c2048 DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
2053 EXPORT_PER_CPU_SYMBOL(pcpu_hot);