1 /*
2 * Housekeeping management. Manage the targets for routine code that can run on
3 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
4 *
5 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
6 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
7 *
8 */
9 #include "sched.h"
10
11 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
12 EXPORT_SYMBOL_GPL(housekeeping_overriden);
13 static cpumask_var_t housekeeping_mask;
14 static unsigned int housekeeping_flags;
15
housekeeping_any_cpu(enum hk_flags flags)16 int housekeeping_any_cpu(enum hk_flags flags)
17 {
18 if (static_branch_unlikely(&housekeeping_overriden))
19 if (housekeeping_flags & flags)
20 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
21 return smp_processor_id();
22 }
23 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
24
housekeeping_cpumask(enum hk_flags flags)25 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
26 {
27 if (static_branch_unlikely(&housekeeping_overriden))
28 if (housekeeping_flags & flags)
29 return housekeeping_mask;
30 return cpu_possible_mask;
31 }
32 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
33
housekeeping_affine(struct task_struct * t,enum hk_flags flags)34 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
35 {
36 if (static_branch_unlikely(&housekeeping_overriden))
37 if (housekeeping_flags & flags)
38 set_cpus_allowed_ptr(t, housekeeping_mask);
39 }
40 EXPORT_SYMBOL_GPL(housekeeping_affine);
41
housekeeping_test_cpu(int cpu,enum hk_flags flags)42 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
43 {
44 if (static_branch_unlikely(&housekeeping_overriden))
45 if (housekeeping_flags & flags)
46 return cpumask_test_cpu(cpu, housekeeping_mask);
47 return true;
48 }
49 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
50
housekeeping_init(void)51 void __init housekeeping_init(void)
52 {
53 if (!housekeeping_flags)
54 return;
55
56 static_branch_enable(&housekeeping_overriden);
57
58 if (housekeeping_flags & HK_FLAG_TICK)
59 sched_tick_offload_init();
60
61 /* We need at least one CPU to handle housekeeping work */
62 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
63 }
64
housekeeping_setup(char * str,enum hk_flags flags)65 static int __init housekeeping_setup(char *str, enum hk_flags flags)
66 {
67 cpumask_var_t non_housekeeping_mask;
68 int err;
69
70 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
71 err = cpulist_parse(str, non_housekeeping_mask);
72 if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
73 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
74 free_bootmem_cpumask_var(non_housekeeping_mask);
75 return 0;
76 }
77
78 if (!housekeeping_flags) {
79 alloc_bootmem_cpumask_var(&housekeeping_mask);
80 cpumask_andnot(housekeeping_mask,
81 cpu_possible_mask, non_housekeeping_mask);
82 if (cpumask_empty(housekeeping_mask))
83 cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
84 } else {
85 cpumask_var_t tmp;
86
87 alloc_bootmem_cpumask_var(&tmp);
88 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
89 if (!cpumask_equal(tmp, housekeeping_mask)) {
90 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
91 free_bootmem_cpumask_var(tmp);
92 free_bootmem_cpumask_var(non_housekeeping_mask);
93 return 0;
94 }
95 free_bootmem_cpumask_var(tmp);
96 }
97
98 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
99 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
100 tick_nohz_full_setup(non_housekeeping_mask);
101 } else {
102 pr_warn("Housekeeping: nohz unsupported."
103 " Build with CONFIG_NO_HZ_FULL\n");
104 free_bootmem_cpumask_var(non_housekeeping_mask);
105 return 0;
106 }
107 }
108
109 housekeeping_flags |= flags;
110
111 free_bootmem_cpumask_var(non_housekeeping_mask);
112
113 return 1;
114 }
115
housekeeping_nohz_full_setup(char * str)116 static int __init housekeeping_nohz_full_setup(char *str)
117 {
118 unsigned int flags;
119
120 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
121
122 return housekeeping_setup(str, flags);
123 }
124 __setup("nohz_full=", housekeeping_nohz_full_setup);
125
housekeeping_isolcpus_setup(char * str)126 static int __init housekeeping_isolcpus_setup(char *str)
127 {
128 unsigned int flags = 0;
129
130 while (isalpha(*str)) {
131 if (!strncmp(str, "nohz,", 5)) {
132 str += 5;
133 flags |= HK_FLAG_TICK;
134 continue;
135 }
136
137 if (!strncmp(str, "domain,", 7)) {
138 str += 7;
139 flags |= HK_FLAG_DOMAIN;
140 continue;
141 }
142
143 pr_warn("isolcpus: Error, unknown flag\n");
144 return 0;
145 }
146
147 /* Default behaviour for isolcpus without flags */
148 if (!flags)
149 flags |= HK_FLAG_DOMAIN;
150
151 return housekeeping_setup(str, flags);
152 }
153 __setup("isolcpus=", housekeeping_isolcpus_setup);
154
