1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_INTEL_RDT_H
3 #define _ASM_X86_INTEL_RDT_H
4
5 #include <linux/sched.h>
6 #include <linux/kernfs.h>
7 #include <linux/jump_label.h>
8
9 #define IA32_L3_QOS_CFG 0xc81
10 #define IA32_L2_QOS_CFG 0xc82
11 #define IA32_L3_CBM_BASE 0xc90
12 #define IA32_L2_CBM_BASE 0xd10
13 #define IA32_MBA_THRTL_BASE 0xd50
14
15 #define L3_QOS_CDP_ENABLE 0x01ULL
16
17 #define L2_QOS_CDP_ENABLE 0x01ULL
18
19 /*
20 * Event IDs are used to program IA32_QM_EVTSEL before reading event
21 * counter from IA32_QM_CTR
22 */
23 #define QOS_L3_OCCUP_EVENT_ID 0x01
24 #define QOS_L3_MBM_TOTAL_EVENT_ID 0x02
25 #define QOS_L3_MBM_LOCAL_EVENT_ID 0x03
26
27 #define CQM_LIMBOCHECK_INTERVAL 1000
28
29 #define MBM_CNTR_WIDTH 24
30 #define MBM_OVERFLOW_INTERVAL 1000
31 #define MAX_MBA_BW 100u
32
33 #define RMID_VAL_ERROR BIT_ULL(63)
34 #define RMID_VAL_UNAVAIL BIT_ULL(62)
35
36 DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
37
38 /**
39 * struct mon_evt - Entry in the event list of a resource
40 * @evtid: event id
41 * @name: name of the event
42 */
43 struct mon_evt {
44 u32 evtid;
45 char *name;
46 struct list_head list;
47 };
48
49 /**
50 * struct mon_data_bits - Monitoring details for each event file
51 * @rid: Resource id associated with the event file.
52 * @evtid: Event id associated with the event file
53 * @domid: The domain to which the event file belongs
54 */
55 union mon_data_bits {
56 void *priv;
57 struct {
58 unsigned int rid : 10;
59 unsigned int evtid : 8;
60 unsigned int domid : 14;
61 } u;
62 };
63
64 struct rmid_read {
65 struct rdtgroup *rgrp;
66 struct rdt_domain *d;
67 int evtid;
68 bool first;
69 u64 val;
70 };
71
72 extern unsigned int intel_cqm_threshold;
73 extern bool rdt_alloc_capable;
74 extern bool rdt_mon_capable;
75 extern unsigned int rdt_mon_features;
76
77 enum rdt_group_type {
78 RDTCTRL_GROUP = 0,
79 RDTMON_GROUP,
80 RDT_NUM_GROUP,
81 };
82
83 /**
84 * enum rdtgrp_mode - Mode of a RDT resource group
85 * @RDT_MODE_SHAREABLE: This resource group allows sharing of its allocations
86 * @RDT_MODE_EXCLUSIVE: No sharing of this resource group's allocations allowed
87 * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking
88 * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations
89 * allowed AND the allocations are Cache Pseudo-Locked
90 *
91 * The mode of a resource group enables control over the allowed overlap
92 * between allocations associated with different resource groups (classes
93 * of service). User is able to modify the mode of a resource group by
94 * writing to the "mode" resctrl file associated with the resource group.
95 *
96 * The "shareable", "exclusive", and "pseudo-locksetup" modes are set by
97 * writing the appropriate text to the "mode" file. A resource group enters
98 * "pseudo-locked" mode after the schemata is written while the resource
99 * group is in "pseudo-locksetup" mode.
100 */
101 enum rdtgrp_mode {
102 RDT_MODE_SHAREABLE = 0,
103 RDT_MODE_EXCLUSIVE,
104 RDT_MODE_PSEUDO_LOCKSETUP,
105 RDT_MODE_PSEUDO_LOCKED,
106
107 /* Must be last */
108 RDT_NUM_MODES,
109 };
110
111 /**
112 * struct mongroup - store mon group's data in resctrl fs.
113 * @mon_data_kn kernlfs node for the mon_data directory
114 * @parent: parent rdtgrp
115 * @crdtgrp_list: child rdtgroup node list
116 * @rmid: rmid for this rdtgroup
117 */
118 struct mongroup {
119 struct kernfs_node *mon_data_kn;
120 struct rdtgroup *parent;
121 struct list_head crdtgrp_list;
122 u32 rmid;
123 };
124
125 /**
126 * struct pseudo_lock_region - pseudo-lock region information
127 * @r: RDT resource to which this pseudo-locked region
128 * belongs
129 * @d: RDT domain to which this pseudo-locked region
130 * belongs
131 * @cbm: bitmask of the pseudo-locked region
132 * @lock_thread_wq: waitqueue used to wait on the pseudo-locking thread
133 * completion
134 * @thread_done: variable used by waitqueue to test if pseudo-locking
135 * thread completed
136 * @cpu: core associated with the cache on which the setup code
137 * will be run
138 * @line_size: size of the cache lines
139 * @size: size of pseudo-locked region in bytes
140 * @kmem: the kernel memory associated with pseudo-locked region
141 * @minor: minor number of character device associated with this
142 * region
143 * @debugfs_dir: pointer to this region's directory in the debugfs
144 * filesystem
145 * @pm_reqs: Power management QoS requests related to this region
146 */
147 struct pseudo_lock_region {
148 struct rdt_resource *r;
149 struct rdt_domain *d;
150 u32 cbm;
151 wait_queue_head_t lock_thread_wq;
152 int thread_done;
153 int cpu;
154 unsigned int line_size;
155 unsigned int size;
156 void *kmem;
157 unsigned int minor;
158 struct dentry *debugfs_dir;
159 struct list_head pm_reqs;
160 };
161
162 /**
163 * struct rdtgroup - store rdtgroup's data in resctrl file system.
164 * @kn: kernfs node
165 * @rdtgroup_list: linked list for all rdtgroups
166 * @closid: closid for this rdtgroup
167 * @cpu_mask: CPUs assigned to this rdtgroup
168 * @flags: status bits
169 * @waitcount: how many cpus expect to find this
170 * group when they acquire rdtgroup_mutex
171 * @type: indicates type of this rdtgroup - either
172 * monitor only or ctrl_mon group
173 * @mon: mongroup related data
174 * @mode: mode of resource group
175 * @plr: pseudo-locked region
176 */
177 struct rdtgroup {
178 struct kernfs_node *kn;
179 struct list_head rdtgroup_list;
180 u32 closid;
181 struct cpumask cpu_mask;
182 int flags;
183 atomic_t waitcount;
184 enum rdt_group_type type;
185 struct mongroup mon;
186 enum rdtgrp_mode mode;
187 struct pseudo_lock_region *plr;
188 };
189
190 /* rdtgroup.flags */
191 #define RDT_DELETED 1
192
193 /* rftype.flags */
194 #define RFTYPE_FLAGS_CPUS_LIST 1
195
196 /*
197 * Define the file type flags for base and info directories.
198 */
199 #define RFTYPE_INFO BIT(0)
200 #define RFTYPE_BASE BIT(1)
201 #define RF_CTRLSHIFT 4
202 #define RF_MONSHIFT 5
203 #define RF_TOPSHIFT 6
204 #define RFTYPE_CTRL BIT(RF_CTRLSHIFT)
205 #define RFTYPE_MON BIT(RF_MONSHIFT)
206 #define RFTYPE_TOP BIT(RF_TOPSHIFT)
207 #define RFTYPE_RES_CACHE BIT(8)
208 #define RFTYPE_RES_MB BIT(9)
209 #define RF_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL)
210 #define RF_MON_INFO (RFTYPE_INFO | RFTYPE_MON)
211 #define RF_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP)
212 #define RF_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL)
213
214 /* List of all resource groups */
215 extern struct list_head rdt_all_groups;
216
217 extern int max_name_width, max_data_width;
218
219 int __init rdtgroup_init(void);
220 void __exit rdtgroup_exit(void);
221
222 /**
223 * struct rftype - describe each file in the resctrl file system
224 * @name: File name
225 * @mode: Access mode
226 * @kf_ops: File operations
227 * @flags: File specific RFTYPE_FLAGS_* flags
228 * @fflags: File specific RF_* or RFTYPE_* flags
229 * @seq_show: Show content of the file
230 * @write: Write to the file
231 */
232 struct rftype {
233 char *name;
234 umode_t mode;
235 struct kernfs_ops *kf_ops;
236 unsigned long flags;
237 unsigned long fflags;
238
239 int (*seq_show)(struct kernfs_open_file *of,
240 struct seq_file *sf, void *v);
241 /*
242 * write() is the generic write callback which maps directly to
243 * kernfs write operation and overrides all other operations.
244 * Maximum write size is determined by ->max_write_len.
245 */
246 ssize_t (*write)(struct kernfs_open_file *of,
247 char *buf, size_t nbytes, loff_t off);
248 };
249
250 /**
251 * struct mbm_state - status for each MBM counter in each domain
252 * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes)
253 * @prev_msr Value of IA32_QM_CTR for this RMID last time we read it
254 * @chunks_bw Total local data moved. Used for bandwidth calculation
255 * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
256 * @prev_bw The most recent bandwidth in MBps
257 * @delta_bw Difference between the current and previous bandwidth
258 * @delta_comp Indicates whether to compute the delta_bw
259 */
260 struct mbm_state {
261 u64 chunks;
262 u64 prev_msr;
263 u64 chunks_bw;
264 u64 prev_bw_msr;
265 u32 prev_bw;
266 u32 delta_bw;
267 bool delta_comp;
268 };
269
270 /**
271 * struct rdt_domain - group of cpus sharing an RDT resource
272 * @list: all instances of this resource
273 * @id: unique id for this instance
274 * @cpu_mask: which cpus share this resource
275 * @rmid_busy_llc:
276 * bitmap of which limbo RMIDs are above threshold
277 * @mbm_total: saved state for MBM total bandwidth
278 * @mbm_local: saved state for MBM local bandwidth
279 * @mbm_over: worker to periodically read MBM h/w counters
280 * @cqm_limbo: worker to periodically read CQM h/w counters
281 * @mbm_work_cpu:
282 * worker cpu for MBM h/w counters
283 * @cqm_work_cpu:
284 * worker cpu for CQM h/w counters
285 * @ctrl_val: array of cache or mem ctrl values (indexed by CLOSID)
286 * @mbps_val: When mba_sc is enabled, this holds the bandwidth in MBps
287 * @new_ctrl: new ctrl value to be loaded
288 * @have_new_ctrl: did user provide new_ctrl for this domain
289 * @plr: pseudo-locked region (if any) associated with domain
290 */
291 struct rdt_domain {
292 struct list_head list;
293 int id;
294 struct cpumask cpu_mask;
295 unsigned long *rmid_busy_llc;
296 struct mbm_state *mbm_total;
297 struct mbm_state *mbm_local;
298 struct delayed_work mbm_over;
299 struct delayed_work cqm_limbo;
300 int mbm_work_cpu;
301 int cqm_work_cpu;
302 u32 *ctrl_val;
303 u32 *mbps_val;
304 u32 new_ctrl;
305 bool have_new_ctrl;
306 struct pseudo_lock_region *plr;
307 };
308
309 /**
310 * struct msr_param - set a range of MSRs from a domain
311 * @res: The resource to use
312 * @low: Beginning index from base MSR
313 * @high: End index
314 */
315 struct msr_param {
316 struct rdt_resource *res;
317 int low;
318 int high;
319 };
320
321 /**
322 * struct rdt_cache - Cache allocation related data
323 * @cbm_len: Length of the cache bit mask
324 * @min_cbm_bits: Minimum number of consecutive bits to be set
325 * @cbm_idx_mult: Multiplier of CBM index
326 * @cbm_idx_offset: Offset of CBM index. CBM index is computed by:
327 * closid * cbm_idx_multi + cbm_idx_offset
328 * in a cache bit mask
329 * @shareable_bits: Bitmask of shareable resource with other
330 * executing entities
331 */
332 struct rdt_cache {
333 unsigned int cbm_len;
334 unsigned int min_cbm_bits;
335 unsigned int cbm_idx_mult;
336 unsigned int cbm_idx_offset;
337 unsigned int shareable_bits;
338 };
339
340 /**
341 * struct rdt_membw - Memory bandwidth allocation related data
342 * @max_delay: Max throttle delay. Delay is the hardware
343 * representation for memory bandwidth.
344 * @min_bw: Minimum memory bandwidth percentage user can request
345 * @bw_gran: Granularity at which the memory bandwidth is allocated
346 * @delay_linear: True if memory B/W delay is in linear scale
347 * @mba_sc: True if MBA software controller(mba_sc) is enabled
348 * @mb_map: Mapping of memory B/W percentage to memory B/W delay
349 */
350 struct rdt_membw {
351 u32 max_delay;
352 u32 min_bw;
353 u32 bw_gran;
354 u32 delay_linear;
355 bool mba_sc;
356 u32 *mb_map;
357 };
358
is_llc_occupancy_enabled(void)359 static inline bool is_llc_occupancy_enabled(void)
360 {
361 return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
362 }
363
is_mbm_total_enabled(void)364 static inline bool is_mbm_total_enabled(void)
365 {
366 return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
367 }
368
is_mbm_local_enabled(void)369 static inline bool is_mbm_local_enabled(void)
370 {
371 return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
372 }
373
is_mbm_enabled(void)374 static inline bool is_mbm_enabled(void)
375 {
376 return (is_mbm_total_enabled() || is_mbm_local_enabled());
377 }
378
is_mbm_event(int e)379 static inline bool is_mbm_event(int e)
380 {
381 return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
382 e <= QOS_L3_MBM_LOCAL_EVENT_ID);
383 }
384
385 struct rdt_parse_data {
386 struct rdtgroup *rdtgrp;
387 char *buf;
388 };
389
390 /**
391 * struct rdt_resource - attributes of an RDT resource
392 * @rid: The index of the resource
393 * @alloc_enabled: Is allocation enabled on this machine
394 * @mon_enabled: Is monitoring enabled for this feature
395 * @alloc_capable: Is allocation available on this machine
396 * @mon_capable: Is monitor feature available on this machine
397 * @name: Name to use in "schemata" file
398 * @num_closid: Number of CLOSIDs available
399 * @cache_level: Which cache level defines scope of this resource
400 * @default_ctrl: Specifies default cache cbm or memory B/W percent.
401 * @msr_base: Base MSR address for CBMs
402 * @msr_update: Function pointer to update QOS MSRs
403 * @data_width: Character width of data when displaying
404 * @domains: All domains for this resource
405 * @cache: Cache allocation related data
406 * @format_str: Per resource format string to show domain value
407 * @parse_ctrlval: Per resource function pointer to parse control values
408 * @evt_list: List of monitoring events
409 * @num_rmid: Number of RMIDs available
410 * @mon_scale: cqm counter * mon_scale = occupancy in bytes
411 * @fflags: flags to choose base and info files
412 */
413 struct rdt_resource {
414 int rid;
415 bool alloc_enabled;
416 bool mon_enabled;
417 bool alloc_capable;
418 bool mon_capable;
419 char *name;
420 int num_closid;
421 int cache_level;
422 u32 default_ctrl;
423 unsigned int msr_base;
424 void (*msr_update) (struct rdt_domain *d, struct msr_param *m,
425 struct rdt_resource *r);
426 int data_width;
427 struct list_head domains;
428 struct rdt_cache cache;
429 struct rdt_membw membw;
430 const char *format_str;
431 int (*parse_ctrlval)(struct rdt_parse_data *data,
432 struct rdt_resource *r,
433 struct rdt_domain *d);
434 struct list_head evt_list;
435 int num_rmid;
436 unsigned int mon_scale;
437 unsigned long fflags;
438 };
439
440 int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
441 struct rdt_domain *d);
442 int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
443 struct rdt_domain *d);
444
445 extern struct mutex rdtgroup_mutex;
446
447 extern struct rdt_resource rdt_resources_all[];
448 extern struct rdtgroup rdtgroup_default;
449 DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
450
451 extern struct dentry *debugfs_resctrl;
452
453 enum {
454 RDT_RESOURCE_L3,
455 RDT_RESOURCE_L3DATA,
456 RDT_RESOURCE_L3CODE,
457 RDT_RESOURCE_L2,
458 RDT_RESOURCE_L2DATA,
459 RDT_RESOURCE_L2CODE,
460 RDT_RESOURCE_MBA,
461
462 /* Must be the last */
463 RDT_NUM_RESOURCES,
464 };
465
466 #define for_each_capable_rdt_resource(r) \
467 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
468 r++) \
469 if (r->alloc_capable || r->mon_capable)
470
471 #define for_each_alloc_capable_rdt_resource(r) \
472 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
473 r++) \
474 if (r->alloc_capable)
475
476 #define for_each_mon_capable_rdt_resource(r) \
477 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
478 r++) \
479 if (r->mon_capable)
480
481 #define for_each_alloc_enabled_rdt_resource(r) \
482 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
483 r++) \
484 if (r->alloc_enabled)
485
486 #define for_each_mon_enabled_rdt_resource(r) \
487 for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
488 r++) \
489 if (r->mon_enabled)
490
491 /* CPUID.(EAX=10H, ECX=ResID=1).EAX */
492 union cpuid_0x10_1_eax {
493 struct {
494 unsigned int cbm_len:5;
495 } split;
496 unsigned int full;
497 };
498
499 /* CPUID.(EAX=10H, ECX=ResID=3).EAX */
500 union cpuid_0x10_3_eax {
501 struct {
502 unsigned int max_delay:12;
503 } split;
504 unsigned int full;
505 };
506
507 /* CPUID.(EAX=10H, ECX=ResID).EDX */
508 union cpuid_0x10_x_edx {
509 struct {
510 unsigned int cos_max:16;
511 } split;
512 unsigned int full;
513 };
514
515 void rdt_last_cmd_clear(void);
516 void rdt_last_cmd_puts(const char *s);
517 void rdt_last_cmd_printf(const char *fmt, ...);
518
519 void rdt_ctrl_update(void *arg);
520 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
521 void rdtgroup_kn_unlock(struct kernfs_node *kn);
522 int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name);
523 int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name,
524 umode_t mask);
525 struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
526 struct list_head **pos);
527 ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
528 char *buf, size_t nbytes, loff_t off);
529 int rdtgroup_schemata_show(struct kernfs_open_file *of,
530 struct seq_file *s, void *v);
531 bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
532 unsigned long cbm, int closid, bool exclusive);
533 unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_domain *d,
534 unsigned long cbm);
535 enum rdtgrp_mode rdtgroup_mode_by_closid(int closid);
536 int rdtgroup_tasks_assigned(struct rdtgroup *r);
537 int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp);
538 int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp);
539 bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, unsigned long cbm);
540 bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d);
541 int rdt_pseudo_lock_init(void);
542 void rdt_pseudo_lock_release(void);
543 int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);
544 void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);
545 struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
546 int update_domains(struct rdt_resource *r, int closid);
547 int closids_supported(void);
548 void closid_free(int closid);
549 int alloc_rmid(void);
550 void free_rmid(u32 rmid);
551 int rdt_get_mon_l3_config(struct rdt_resource *r);
552 void mon_event_count(void *info);
553 int rdtgroup_mondata_show(struct seq_file *m, void *arg);
554 void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
555 unsigned int dom_id);
556 void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
557 struct rdt_domain *d);
558 void mon_event_read(struct rmid_read *rr, struct rdt_domain *d,
559 struct rdtgroup *rdtgrp, int evtid, int first);
560 void mbm_setup_overflow_handler(struct rdt_domain *dom,
561 unsigned long delay_ms);
562 void mbm_handle_overflow(struct work_struct *work);
563 bool is_mba_sc(struct rdt_resource *r);
564 void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
565 u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
566 void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
567 void cqm_handle_limbo(struct work_struct *work);
568 bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
569 void __check_limbo(struct rdt_domain *d, bool force_free);
570
571 #endif /* _ASM_X86_INTEL_RDT_H */
572
