1  /* SPDX-License-Identifier: GPL-2.0-only */
2  /* Copyright(c) 2020 Intel Corporation. */
3  
4  #ifndef __CXL_H__
5  #define __CXL_H__
6  
7  #include <linux/libnvdimm.h>
8  #include <linux/bitfield.h>
9  #include <linux/bitops.h>
10  #include <linux/log2.h>
11  #include <linux/io.h>
12  
13  /**
14   * DOC: cxl objects
15   *
16   * The CXL core objects like ports, decoders, and regions are shared
17   * between the subsystem drivers cxl_acpi, cxl_pci, and core drivers
18   * (port-driver, region-driver, nvdimm object-drivers... etc).
19   */
20  
21  /* CXL 2.0 8.2.4 CXL Component Register Layout and Definition */
22  #define CXL_COMPONENT_REG_BLOCK_SIZE SZ_64K
23  
24  /* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/
25  #define CXL_CM_OFFSET 0x1000
26  #define CXL_CM_CAP_HDR_OFFSET 0x0
27  #define   CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0)
28  #define     CM_CAP_HDR_CAP_ID 1
29  #define   CXL_CM_CAP_HDR_VERSION_MASK GENMASK(19, 16)
30  #define     CM_CAP_HDR_CAP_VERSION 1
31  #define   CXL_CM_CAP_HDR_CACHE_MEM_VERSION_MASK GENMASK(23, 20)
32  #define     CM_CAP_HDR_CACHE_MEM_VERSION 1
33  #define   CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24)
34  #define CXL_CM_CAP_PTR_MASK GENMASK(31, 20)
35  
36  #define   CXL_CM_CAP_CAP_ID_RAS 0x2
37  #define   CXL_CM_CAP_CAP_ID_HDM 0x5
38  #define   CXL_CM_CAP_CAP_HDM_VERSION 1
39  
40  /* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */
41  #define CXL_HDM_DECODER_CAP_OFFSET 0x0
42  #define   CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0)
43  #define   CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4)
44  #define   CXL_HDM_DECODER_INTERLEAVE_11_8 BIT(8)
45  #define   CXL_HDM_DECODER_INTERLEAVE_14_12 BIT(9)
46  #define CXL_HDM_DECODER_CTRL_OFFSET 0x4
47  #define   CXL_HDM_DECODER_ENABLE BIT(1)
48  #define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
49  #define CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)
50  #define CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)
51  #define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)
52  #define CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)
53  #define   CXL_HDM_DECODER0_CTRL_IG_MASK GENMASK(3, 0)
54  #define   CXL_HDM_DECODER0_CTRL_IW_MASK GENMASK(7, 4)
55  #define   CXL_HDM_DECODER0_CTRL_LOCK BIT(8)
56  #define   CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
57  #define   CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
58  #define   CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)
59  #define   CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)
60  #define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
61  #define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
62  #define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
63  #define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
64  
65  /* HDM decoder control register constants CXL 3.0 8.2.5.19.7 */
66  #define CXL_DECODER_MIN_GRANULARITY 256
67  #define CXL_DECODER_MAX_ENCODED_IG 6
68  
cxl_hdm_decoder_count(u32 cap_hdr)69  static inline int cxl_hdm_decoder_count(u32 cap_hdr)
70  {
71  	int val = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, cap_hdr);
72  
73  	return val ? val * 2 : 1;
74  }
75  
76  /* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
eig_to_granularity(u16 eig,unsigned int * granularity)77  static inline int eig_to_granularity(u16 eig, unsigned int *granularity)
78  {
79  	if (eig > CXL_DECODER_MAX_ENCODED_IG)
80  		return -EINVAL;
81  	*granularity = CXL_DECODER_MIN_GRANULARITY << eig;
82  	return 0;
83  }
84  
85  /* Encode defined in CXL ECN "3, 6, 12 and 16-way memory Interleaving" */
eiw_to_ways(u8 eiw,unsigned int * ways)86  static inline int eiw_to_ways(u8 eiw, unsigned int *ways)
87  {
88  	switch (eiw) {
89  	case 0 ... 4:
90  		*ways = 1 << eiw;
91  		break;
92  	case 8 ... 10:
93  		*ways = 3 << (eiw - 8);
94  		break;
95  	default:
96  		return -EINVAL;
97  	}
98  
99  	return 0;
100  }
101  
granularity_to_eig(int granularity,u16 * eig)102  static inline int granularity_to_eig(int granularity, u16 *eig)
103  {
104  	if (granularity > SZ_16K || granularity < CXL_DECODER_MIN_GRANULARITY ||
105  	    !is_power_of_2(granularity))
106  		return -EINVAL;
107  	*eig = ilog2(granularity) - 8;
108  	return 0;
109  }
110  
ways_to_eiw(unsigned int ways,u8 * eiw)111  static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
112  {
113  	if (ways > 16)
114  		return -EINVAL;
115  	if (is_power_of_2(ways)) {
116  		*eiw = ilog2(ways);
117  		return 0;
118  	}
119  	if (ways % 3)
120  		return -EINVAL;
121  	ways /= 3;
122  	if (!is_power_of_2(ways))
123  		return -EINVAL;
124  	*eiw = ilog2(ways) + 8;
125  	return 0;
126  }
127  
128  /* RAS Registers CXL 2.0 8.2.5.9 CXL RAS Capability Structure */
129  #define CXL_RAS_UNCORRECTABLE_STATUS_OFFSET 0x0
130  #define   CXL_RAS_UNCORRECTABLE_STATUS_MASK (GENMASK(16, 14) | GENMASK(11, 0))
131  #define CXL_RAS_UNCORRECTABLE_MASK_OFFSET 0x4
132  #define   CXL_RAS_UNCORRECTABLE_MASK_MASK (GENMASK(16, 14) | GENMASK(11, 0))
133  #define   CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK BIT(8)
134  #define CXL_RAS_UNCORRECTABLE_SEVERITY_OFFSET 0x8
135  #define   CXL_RAS_UNCORRECTABLE_SEVERITY_MASK (GENMASK(16, 14) | GENMASK(11, 0))
136  #define CXL_RAS_CORRECTABLE_STATUS_OFFSET 0xC
137  #define   CXL_RAS_CORRECTABLE_STATUS_MASK GENMASK(6, 0)
138  #define CXL_RAS_CORRECTABLE_MASK_OFFSET 0x10
139  #define   CXL_RAS_CORRECTABLE_MASK_MASK GENMASK(6, 0)
140  #define CXL_RAS_CAP_CONTROL_OFFSET 0x14
141  #define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0)
142  #define CXL_RAS_HEADER_LOG_OFFSET 0x18
143  #define CXL_RAS_CAPABILITY_LENGTH 0x58
144  #define CXL_HEADERLOG_SIZE SZ_512
145  #define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32)
146  
147  /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */
148  #define CXLDEV_CAP_ARRAY_OFFSET 0x0
149  #define   CXLDEV_CAP_ARRAY_CAP_ID 0
150  #define   CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0)
151  #define   CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32)
152  /* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */
153  #define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0)
154  /* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */
155  #define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1
156  #define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2
157  #define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3
158  #define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000
159  
160  /* CXL 3.0 8.2.8.3.1 Event Status Register */
161  #define CXLDEV_DEV_EVENT_STATUS_OFFSET		0x00
162  #define CXLDEV_EVENT_STATUS_INFO		BIT(0)
163  #define CXLDEV_EVENT_STATUS_WARN		BIT(1)
164  #define CXLDEV_EVENT_STATUS_FAIL		BIT(2)
165  #define CXLDEV_EVENT_STATUS_FATAL		BIT(3)
166  
167  #define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO |	\
168  				 CXLDEV_EVENT_STATUS_WARN |	\
169  				 CXLDEV_EVENT_STATUS_FAIL |	\
170  				 CXLDEV_EVENT_STATUS_FATAL)
171  
172  /* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */
173  #define CXLDEV_EVENT_INT_MODE_MASK	GENMASK(1, 0)
174  #define CXLDEV_EVENT_INT_MSGNUM_MASK	GENMASK(7, 4)
175  
176  /* CXL 2.0 8.2.8.4 Mailbox Registers */
177  #define CXLDEV_MBOX_CAPS_OFFSET 0x00
178  #define   CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0)
179  #define   CXLDEV_MBOX_CAP_BG_CMD_IRQ BIT(6)
180  #define   CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK GENMASK(10, 7)
181  #define CXLDEV_MBOX_CTRL_OFFSET 0x04
182  #define   CXLDEV_MBOX_CTRL_DOORBELL BIT(0)
183  #define   CXLDEV_MBOX_CTRL_BG_CMD_IRQ BIT(2)
184  #define CXLDEV_MBOX_CMD_OFFSET 0x08
185  #define   CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
186  #define   CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16)
187  #define CXLDEV_MBOX_STATUS_OFFSET 0x10
188  #define   CXLDEV_MBOX_STATUS_BG_CMD BIT(0)
189  #define   CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32)
190  #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18
191  #define   CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
192  #define   CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK GENMASK_ULL(22, 16)
193  #define   CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK GENMASK_ULL(47, 32)
194  #define   CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)
195  #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20
196  
197  /*
198   * Using struct_group() allows for per register-block-type helper routines,
199   * without requiring block-type agnostic code to include the prefix.
200   */
201  struct cxl_regs {
202  	/*
203  	 * Common set of CXL Component register block base pointers
204  	 * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure
205  	 * @ras: CXL 2.0 8.2.5.9 CXL RAS Capability Structure
206  	 */
207  	struct_group_tagged(cxl_component_regs, component,
208  		void __iomem *hdm_decoder;
209  		void __iomem *ras;
210  	);
211  	/*
212  	 * Common set of CXL Device register block base pointers
213  	 * @status: CXL 2.0 8.2.8.3 Device Status Registers
214  	 * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers
215  	 * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers
216  	 */
217  	struct_group_tagged(cxl_device_regs, device_regs,
218  		void __iomem *status, *mbox, *memdev;
219  	);
220  
221  	struct_group_tagged(cxl_pmu_regs, pmu_regs,
222  		void __iomem *pmu;
223  	);
224  };
225  
226  struct cxl_reg_map {
227  	bool valid;
228  	int id;
229  	unsigned long offset;
230  	unsigned long size;
231  };
232  
233  struct cxl_component_reg_map {
234  	struct cxl_reg_map hdm_decoder;
235  	struct cxl_reg_map ras;
236  };
237  
238  struct cxl_device_reg_map {
239  	struct cxl_reg_map status;
240  	struct cxl_reg_map mbox;
241  	struct cxl_reg_map memdev;
242  };
243  
244  struct cxl_pmu_reg_map {
245  	struct cxl_reg_map pmu;
246  };
247  
248  /**
249   * struct cxl_register_map - DVSEC harvested register block mapping parameters
250   * @dev: device for devm operations and logging
251   * @base: virtual base of the register-block-BAR + @block_offset
252   * @resource: physical resource base of the register block
253   * @max_size: maximum mapping size to perform register search
254   * @reg_type: see enum cxl_regloc_type
255   * @component_map: cxl_reg_map for component registers
256   * @device_map: cxl_reg_maps for device registers
257   * @pmu_map: cxl_reg_maps for CXL Performance Monitoring Units
258   */
259  struct cxl_register_map {
260  	struct device *dev;
261  	void __iomem *base;
262  	resource_size_t resource;
263  	resource_size_t max_size;
264  	u8 reg_type;
265  	union {
266  		struct cxl_component_reg_map component_map;
267  		struct cxl_device_reg_map device_map;
268  		struct cxl_pmu_reg_map pmu_map;
269  	};
270  };
271  
272  void cxl_probe_component_regs(struct device *dev, void __iomem *base,
273  			      struct cxl_component_reg_map *map);
274  void cxl_probe_device_regs(struct device *dev, void __iomem *base,
275  			   struct cxl_device_reg_map *map);
276  int cxl_map_component_regs(const struct cxl_register_map *map,
277  			   struct cxl_component_regs *regs,
278  			   unsigned long map_mask);
279  int cxl_map_device_regs(const struct cxl_register_map *map,
280  			struct cxl_device_regs *regs);
281  int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs,
282  		     struct cxl_register_map *map);
283  
284  enum cxl_regloc_type;
285  int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type);
286  int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
287  			       struct cxl_register_map *map, int index);
288  int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
289  		      struct cxl_register_map *map);
290  int cxl_setup_regs(struct cxl_register_map *map);
291  struct cxl_dport;
292  resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
293  					   struct cxl_dport *dport);
294  
295  #define CXL_RESOURCE_NONE ((resource_size_t) -1)
296  #define CXL_TARGET_STRLEN 20
297  
298  /*
299   * cxl_decoder flags that define the type of memory / devices this
300   * decoder supports as well as configuration lock status See "CXL 2.0
301   * 8.2.5.12.7 CXL HDM Decoder 0 Control Register" for details.
302   * Additionally indicate whether decoder settings were autodetected,
303   * user customized.
304   */
305  #define CXL_DECODER_F_RAM   BIT(0)
306  #define CXL_DECODER_F_PMEM  BIT(1)
307  #define CXL_DECODER_F_TYPE2 BIT(2)
308  #define CXL_DECODER_F_TYPE3 BIT(3)
309  #define CXL_DECODER_F_LOCK  BIT(4)
310  #define CXL_DECODER_F_ENABLE    BIT(5)
311  #define CXL_DECODER_F_MASK  GENMASK(5, 0)
312  
313  enum cxl_decoder_type {
314  	CXL_DECODER_DEVMEM = 2,
315  	CXL_DECODER_HOSTONLYMEM = 3,
316  };
317  
318  /*
319   * Current specification goes up to 8, double that seems a reasonable
320   * software max for the foreseeable future
321   */
322  #define CXL_DECODER_MAX_INTERLEAVE 16
323  
324  
325  /**
326   * struct cxl_decoder - Common CXL HDM Decoder Attributes
327   * @dev: this decoder's device
328   * @id: kernel device name id
329   * @hpa_range: Host physical address range mapped by this decoder
330   * @interleave_ways: number of cxl_dports in this decode
331   * @interleave_granularity: data stride per dport
332   * @target_type: accelerator vs expander (type2 vs type3) selector
333   * @region: currently assigned region for this decoder
334   * @flags: memory type capabilities and locking
335   * @commit: device/decoder-type specific callback to commit settings to hw
336   * @reset: device/decoder-type specific callback to reset hw settings
337  */
338  struct cxl_decoder {
339  	struct device dev;
340  	int id;
341  	struct range hpa_range;
342  	int interleave_ways;
343  	int interleave_granularity;
344  	enum cxl_decoder_type target_type;
345  	struct cxl_region *region;
346  	unsigned long flags;
347  	int (*commit)(struct cxl_decoder *cxld);
348  	int (*reset)(struct cxl_decoder *cxld);
349  };
350  
351  /*
352   * CXL_DECODER_DEAD prevents endpoints from being reattached to regions
353   * while cxld_unregister() is running
354   */
355  enum cxl_decoder_mode {
356  	CXL_DECODER_NONE,
357  	CXL_DECODER_RAM,
358  	CXL_DECODER_PMEM,
359  	CXL_DECODER_MIXED,
360  	CXL_DECODER_DEAD,
361  };
362  
cxl_decoder_mode_name(enum cxl_decoder_mode mode)363  static inline const char *cxl_decoder_mode_name(enum cxl_decoder_mode mode)
364  {
365  	static const char * const names[] = {
366  		[CXL_DECODER_NONE] = "none",
367  		[CXL_DECODER_RAM] = "ram",
368  		[CXL_DECODER_PMEM] = "pmem",
369  		[CXL_DECODER_MIXED] = "mixed",
370  	};
371  
372  	if (mode >= CXL_DECODER_NONE && mode <= CXL_DECODER_MIXED)
373  		return names[mode];
374  	return "mixed";
375  }
376  
377  /*
378   * Track whether this decoder is reserved for region autodiscovery, or
379   * free for userspace provisioning.
380   */
381  enum cxl_decoder_state {
382  	CXL_DECODER_STATE_MANUAL,
383  	CXL_DECODER_STATE_AUTO,
384  };
385  
386  /**
387   * struct cxl_endpoint_decoder - Endpoint  / SPA to DPA decoder
388   * @cxld: base cxl_decoder_object
389   * @dpa_res: actively claimed DPA span of this decoder
390   * @skip: offset into @dpa_res where @cxld.hpa_range maps
391   * @mode: which memory type / access-mode-partition this decoder targets
392   * @state: autodiscovery state
393   * @pos: interleave position in @cxld.region
394   */
395  struct cxl_endpoint_decoder {
396  	struct cxl_decoder cxld;
397  	struct resource *dpa_res;
398  	resource_size_t skip;
399  	enum cxl_decoder_mode mode;
400  	enum cxl_decoder_state state;
401  	int pos;
402  };
403  
404  /**
405   * struct cxl_switch_decoder - Switch specific CXL HDM Decoder
406   * @cxld: base cxl_decoder object
407   * @target_lock: coordinate coherent reads of the target list
408   * @nr_targets: number of elements in @target
409   * @target: active ordered target list in current decoder configuration
410   *
411   * The 'switch' decoder type represents the decoder instances of cxl_port's that
412   * route from the root of a CXL memory decode topology to the endpoints. They
413   * come in two flavors, root-level decoders, statically defined by platform
414   * firmware, and mid-level decoders, where interleave-granularity,
415   * interleave-width, and the target list are mutable.
416   */
417  struct cxl_switch_decoder {
418  	struct cxl_decoder cxld;
419  	seqlock_t target_lock;
420  	int nr_targets;
421  	struct cxl_dport *target[];
422  };
423  
424  struct cxl_root_decoder;
425  typedef struct cxl_dport *(*cxl_calc_hb_fn)(struct cxl_root_decoder *cxlrd,
426  					    int pos);
427  
428  /**
429   * struct cxl_root_decoder - Static platform CXL address decoder
430   * @res: host / parent resource for region allocations
431   * @region_id: region id for next region provisioning event
432   * @calc_hb: which host bridge covers the n'th position by granularity
433   * @platform_data: platform specific configuration data
434   * @range_lock: sync region autodiscovery by address range
435   * @cxlsd: base cxl switch decoder
436   */
437  struct cxl_root_decoder {
438  	struct resource *res;
439  	atomic_t region_id;
440  	cxl_calc_hb_fn calc_hb;
441  	void *platform_data;
442  	struct mutex range_lock;
443  	struct cxl_switch_decoder cxlsd;
444  };
445  
446  /*
447   * enum cxl_config_state - State machine for region configuration
448   * @CXL_CONFIG_IDLE: Any sysfs attribute can be written freely
449   * @CXL_CONFIG_INTERLEAVE_ACTIVE: region size has been set, no more
450   * changes to interleave_ways or interleave_granularity
451   * @CXL_CONFIG_ACTIVE: All targets have been added the region is now
452   * active
453   * @CXL_CONFIG_RESET_PENDING: see commit_store()
454   * @CXL_CONFIG_COMMIT: Soft-config has been committed to hardware
455   */
456  enum cxl_config_state {
457  	CXL_CONFIG_IDLE,
458  	CXL_CONFIG_INTERLEAVE_ACTIVE,
459  	CXL_CONFIG_ACTIVE,
460  	CXL_CONFIG_RESET_PENDING,
461  	CXL_CONFIG_COMMIT,
462  };
463  
464  /**
465   * struct cxl_region_params - region settings
466   * @state: allow the driver to lockdown further parameter changes
467   * @uuid: unique id for persistent regions
468   * @interleave_ways: number of endpoints in the region
469   * @interleave_granularity: capacity each endpoint contributes to a stripe
470   * @res: allocated iomem capacity for this region
471   * @targets: active ordered targets in current decoder configuration
472   * @nr_targets: number of targets
473   *
474   * State transitions are protected by the cxl_region_rwsem
475   */
476  struct cxl_region_params {
477  	enum cxl_config_state state;
478  	uuid_t uuid;
479  	int interleave_ways;
480  	int interleave_granularity;
481  	struct resource *res;
482  	struct cxl_endpoint_decoder *targets[CXL_DECODER_MAX_INTERLEAVE];
483  	int nr_targets;
484  };
485  
486  /*
487   * Indicate whether this region has been assembled by autodetection or
488   * userspace assembly. Prevent endpoint decoders outside of automatic
489   * detection from being added to the region.
490   */
491  #define CXL_REGION_F_AUTO 0
492  
493  /*
494   * Require that a committed region successfully complete a teardown once
495   * any of its associated decoders have been torn down. This maintains
496   * the commit state for the region since there are committed decoders,
497   * but blocks cxl_region_probe().
498   */
499  #define CXL_REGION_F_NEEDS_RESET 1
500  
501  /**
502   * struct cxl_region - CXL region
503   * @dev: This region's device
504   * @id: This region's id. Id is globally unique across all regions
505   * @mode: Endpoint decoder allocation / access mode
506   * @type: Endpoint decoder target type
507   * @cxl_nvb: nvdimm bridge for coordinating @cxlr_pmem setup / shutdown
508   * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge
509   * @flags: Region state flags
510   * @params: active + config params for the region
511   */
512  struct cxl_region {
513  	struct device dev;
514  	int id;
515  	enum cxl_decoder_mode mode;
516  	enum cxl_decoder_type type;
517  	struct cxl_nvdimm_bridge *cxl_nvb;
518  	struct cxl_pmem_region *cxlr_pmem;
519  	unsigned long flags;
520  	struct cxl_region_params params;
521  };
522  
523  struct cxl_nvdimm_bridge {
524  	int id;
525  	struct device dev;
526  	struct cxl_port *port;
527  	struct nvdimm_bus *nvdimm_bus;
528  	struct nvdimm_bus_descriptor nd_desc;
529  };
530  
531  #define CXL_DEV_ID_LEN 19
532  
533  struct cxl_nvdimm {
534  	struct device dev;
535  	struct cxl_memdev *cxlmd;
536  	u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */
537  };
538  
539  struct cxl_pmem_region_mapping {
540  	struct cxl_memdev *cxlmd;
541  	struct cxl_nvdimm *cxl_nvd;
542  	u64 start;
543  	u64 size;
544  	int position;
545  };
546  
547  struct cxl_pmem_region {
548  	struct device dev;
549  	struct cxl_region *cxlr;
550  	struct nd_region *nd_region;
551  	struct range hpa_range;
552  	int nr_mappings;
553  	struct cxl_pmem_region_mapping mapping[];
554  };
555  
556  struct cxl_dax_region {
557  	struct device dev;
558  	struct cxl_region *cxlr;
559  	struct range hpa_range;
560  };
561  
562  /**
563   * struct cxl_port - logical collection of upstream port devices and
564   *		     downstream port devices to construct a CXL memory
565   *		     decode hierarchy.
566   * @dev: this port's device
567   * @uport_dev: PCI or platform device implementing the upstream port capability
568   * @host_bridge: Shortcut to the platform attach point for this port
569   * @id: id for port device-name
570   * @dports: cxl_dport instances referenced by decoders
571   * @endpoints: cxl_ep instances, endpoints that are a descendant of this port
572   * @regions: cxl_region_ref instances, regions mapped by this port
573   * @parent_dport: dport that points to this port in the parent
574   * @decoder_ida: allocator for decoder ids
575   * @comp_map: component register capability mappings
576   * @nr_dports: number of entries in @dports
577   * @hdm_end: track last allocated HDM decoder instance for allocation ordering
578   * @commit_end: cursor to track highest committed decoder for commit ordering
579   * @component_reg_phys: component register capability base address (optional)
580   * @dead: last ep has been removed, force port re-creation
581   * @depth: How deep this port is relative to the root. depth 0 is the root.
582   * @cdat: Cached CDAT data
583   * @cdat_available: Should a CDAT attribute be available in sysfs
584   */
585  struct cxl_port {
586  	struct device dev;
587  	struct device *uport_dev;
588  	struct device *host_bridge;
589  	int id;
590  	struct xarray dports;
591  	struct xarray endpoints;
592  	struct xarray regions;
593  	struct cxl_dport *parent_dport;
594  	struct ida decoder_ida;
595  	struct cxl_register_map comp_map;
596  	int nr_dports;
597  	int hdm_end;
598  	int commit_end;
599  	resource_size_t component_reg_phys;
600  	bool dead;
601  	unsigned int depth;
602  	struct cxl_cdat {
603  		void *table;
604  		size_t length;
605  	} cdat;
606  	bool cdat_available;
607  };
608  
609  static inline struct cxl_dport *
cxl_find_dport_by_dev(struct cxl_port * port,const struct device * dport_dev)610  cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev)
611  {
612  	return xa_load(&port->dports, (unsigned long)dport_dev);
613  }
614  
615  struct cxl_rcrb_info {
616  	resource_size_t base;
617  	u16 aer_cap;
618  };
619  
620  /**
621   * struct cxl_dport - CXL downstream port
622   * @dport_dev: PCI bridge or firmware device representing the downstream link
623   * @comp_map: component register capability mappings
624   * @port_id: unique hardware identifier for dport in decoder target list
625   * @rcrb: Data about the Root Complex Register Block layout
626   * @rch: Indicate whether this dport was enumerated in RCH or VH mode
627   * @port: reference to cxl_port that contains this downstream port
628   */
629  struct cxl_dport {
630  	struct device *dport_dev;
631  	struct cxl_register_map comp_map;
632  	int port_id;
633  	struct cxl_rcrb_info rcrb;
634  	bool rch;
635  	struct cxl_port *port;
636  };
637  
638  /**
639   * struct cxl_ep - track an endpoint's interest in a port
640   * @ep: device that hosts a generic CXL endpoint (expander or accelerator)
641   * @dport: which dport routes to this endpoint on @port
642   * @next: cxl switch port across the link attached to @dport NULL if
643   *	  attached to an endpoint
644   */
645  struct cxl_ep {
646  	struct device *ep;
647  	struct cxl_dport *dport;
648  	struct cxl_port *next;
649  };
650  
651  /**
652   * struct cxl_region_ref - track a region's interest in a port
653   * @port: point in topology to install this reference
654   * @decoder: decoder assigned for @region in @port
655   * @region: region for this reference
656   * @endpoints: cxl_ep references for region members beneath @port
657   * @nr_targets_set: track how many targets have been programmed during setup
658   * @nr_eps: number of endpoints beneath @port
659   * @nr_targets: number of distinct targets needed to reach @nr_eps
660   */
661  struct cxl_region_ref {
662  	struct cxl_port *port;
663  	struct cxl_decoder *decoder;
664  	struct cxl_region *region;
665  	struct xarray endpoints;
666  	int nr_targets_set;
667  	int nr_eps;
668  	int nr_targets;
669  };
670  
671  /*
672   * The platform firmware device hosting the root is also the top of the
673   * CXL port topology. All other CXL ports have another CXL port as their
674   * parent and their ->uport_dev / host device is out-of-line of the port
675   * ancestry.
676   */
is_cxl_root(struct cxl_port * port)677  static inline bool is_cxl_root(struct cxl_port *port)
678  {
679  	return port->uport_dev == port->dev.parent;
680  }
681  
682  bool is_cxl_port(const struct device *dev);
683  struct cxl_port *to_cxl_port(const struct device *dev);
684  struct pci_bus;
685  int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
686  			      struct pci_bus *bus);
687  struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);
688  struct cxl_port *devm_cxl_add_port(struct device *host,
689  				   struct device *uport_dev,
690  				   resource_size_t component_reg_phys,
691  				   struct cxl_dport *parent_dport);
692  struct cxl_port *find_cxl_root(struct cxl_port *port);
693  int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
694  void cxl_bus_rescan(void);
695  void cxl_bus_drain(void);
696  struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
697  				   struct cxl_dport **dport);
698  struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
699  				   struct cxl_dport **dport);
700  bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
701  
702  struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
703  				     struct device *dport, int port_id,
704  				     resource_size_t component_reg_phys);
705  struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
706  					 struct device *dport_dev, int port_id,
707  					 resource_size_t rcrb);
708  
709  struct cxl_decoder *to_cxl_decoder(struct device *dev);
710  struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
711  struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
712  struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);
713  bool is_root_decoder(struct device *dev);
714  bool is_switch_decoder(struct device *dev);
715  bool is_endpoint_decoder(struct device *dev);
716  struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
717  						unsigned int nr_targets,
718  						cxl_calc_hb_fn calc_hb);
719  struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos);
720  struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
721  						    unsigned int nr_targets);
722  int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);
723  struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);
724  int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map);
725  int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
726  int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
727  
728  /**
729   * struct cxl_endpoint_dvsec_info - Cached DVSEC info
730   * @mem_enabled: cached value of mem_enabled in the DVSEC at init time
731   * @ranges: Number of active HDM ranges this device uses.
732   * @port: endpoint port associated with this info instance
733   * @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE
734   */
735  struct cxl_endpoint_dvsec_info {
736  	bool mem_enabled;
737  	int ranges;
738  	struct cxl_port *port;
739  	struct range dvsec_range[2];
740  };
741  
742  struct cxl_hdm;
743  struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
744  				   struct cxl_endpoint_dvsec_info *info);
745  int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
746  				struct cxl_endpoint_dvsec_info *info);
747  int devm_cxl_add_passthrough_decoder(struct cxl_port *port);
748  int cxl_dvsec_rr_decode(struct device *dev, int dvsec,
749  			struct cxl_endpoint_dvsec_info *info);
750  
751  bool is_cxl_region(struct device *dev);
752  
753  extern struct bus_type cxl_bus_type;
754  
755  struct cxl_driver {
756  	const char *name;
757  	int (*probe)(struct device *dev);
758  	void (*remove)(struct device *dev);
759  	struct device_driver drv;
760  	int id;
761  };
762  
to_cxl_drv(struct device_driver * drv)763  static inline struct cxl_driver *to_cxl_drv(struct device_driver *drv)
764  {
765  	return container_of(drv, struct cxl_driver, drv);
766  }
767  
768  int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
769  			  const char *modname);
770  #define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME)
771  void cxl_driver_unregister(struct cxl_driver *cxl_drv);
772  
773  #define module_cxl_driver(__cxl_driver) \
774  	module_driver(__cxl_driver, cxl_driver_register, cxl_driver_unregister)
775  
776  #define CXL_DEVICE_NVDIMM_BRIDGE	1
777  #define CXL_DEVICE_NVDIMM		2
778  #define CXL_DEVICE_PORT			3
779  #define CXL_DEVICE_ROOT			4
780  #define CXL_DEVICE_MEMORY_EXPANDER	5
781  #define CXL_DEVICE_REGION		6
782  #define CXL_DEVICE_PMEM_REGION		7
783  #define CXL_DEVICE_DAX_REGION		8
784  #define CXL_DEVICE_PMU			9
785  
786  #define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
787  #define CXL_MODALIAS_FMT "cxl:t%d"
788  
789  struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev);
790  struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
791  						     struct cxl_port *port);
792  struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev);
793  bool is_cxl_nvdimm(struct device *dev);
794  bool is_cxl_nvdimm_bridge(struct device *dev);
795  int devm_cxl_add_nvdimm(struct cxl_memdev *cxlmd);
796  struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd);
797  
798  #ifdef CONFIG_CXL_REGION
799  bool is_cxl_pmem_region(struct device *dev);
800  struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev);
801  int cxl_add_to_region(struct cxl_port *root,
802  		      struct cxl_endpoint_decoder *cxled);
803  struct cxl_dax_region *to_cxl_dax_region(struct device *dev);
804  #else
is_cxl_pmem_region(struct device * dev)805  static inline bool is_cxl_pmem_region(struct device *dev)
806  {
807  	return false;
808  }
to_cxl_pmem_region(struct device * dev)809  static inline struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
810  {
811  	return NULL;
812  }
cxl_add_to_region(struct cxl_port * root,struct cxl_endpoint_decoder * cxled)813  static inline int cxl_add_to_region(struct cxl_port *root,
814  				    struct cxl_endpoint_decoder *cxled)
815  {
816  	return 0;
817  }
to_cxl_dax_region(struct device * dev)818  static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
819  {
820  	return NULL;
821  }
822  #endif
823  
824  /*
825   * Unit test builds overrides this to __weak, find the 'strong' version
826   * of these symbols in tools/testing/cxl/.
827   */
828  #ifndef __mock
829  #define __mock static
830  #endif
831  
832  #endif /* __CXL_H__ */
833