1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include "registers.h"
12
13 #define IDXD_DRIVER_VERSION "1.00"
14
15 extern struct kmem_cache *idxd_desc_pool;
16
17 #define IDXD_REG_TIMEOUT 50
18 #define IDXD_DRAIN_TIMEOUT 5000
19
20 enum idxd_type {
21 IDXD_TYPE_UNKNOWN = -1,
22 IDXD_TYPE_DSA = 0,
23 IDXD_TYPE_MAX
24 };
25
26 #define IDXD_NAME_SIZE 128
27
28 struct idxd_device_driver {
29 struct device_driver drv;
30 };
31
32 struct idxd_irq_entry {
33 struct idxd_device *idxd;
34 int id;
35 struct llist_head pending_llist;
36 struct list_head work_list;
37 };
38
39 struct idxd_group {
40 struct device conf_dev;
41 struct idxd_device *idxd;
42 struct grpcfg grpcfg;
43 int id;
44 int num_engines;
45 int num_wqs;
46 bool use_token_limit;
47 u8 tokens_allowed;
48 u8 tokens_reserved;
49 int tc_a;
50 int tc_b;
51 };
52
53 #define IDXD_MAX_PRIORITY 0xf
54
55 enum idxd_wq_state {
56 IDXD_WQ_DISABLED = 0,
57 IDXD_WQ_ENABLED,
58 };
59
60 enum idxd_wq_flag {
61 WQ_FLAG_DEDICATED = 0,
62 };
63
64 enum idxd_wq_type {
65 IDXD_WQT_NONE = 0,
66 IDXD_WQT_KERNEL,
67 IDXD_WQT_USER,
68 };
69
70 struct idxd_cdev {
71 struct cdev cdev;
72 struct device *dev;
73 int minor;
74 struct wait_queue_head err_queue;
75 };
76
77 #define IDXD_ALLOCATED_BATCH_SIZE 128U
78 #define WQ_NAME_SIZE 1024
79 #define WQ_TYPE_SIZE 10
80
81 enum idxd_op_type {
82 IDXD_OP_BLOCK = 0,
83 IDXD_OP_NONBLOCK = 1,
84 };
85
86 enum idxd_complete_type {
87 IDXD_COMPLETE_NORMAL = 0,
88 IDXD_COMPLETE_ABORT,
89 };
90
91 struct idxd_wq {
92 void __iomem *dportal;
93 struct device conf_dev;
94 struct idxd_cdev idxd_cdev;
95 struct idxd_device *idxd;
96 int id;
97 enum idxd_wq_type type;
98 struct idxd_group *group;
99 int client_count;
100 struct mutex wq_lock; /* mutex for workqueue */
101 u32 size;
102 u32 threshold;
103 u32 priority;
104 enum idxd_wq_state state;
105 unsigned long flags;
106 union wqcfg *wqcfg;
107 u32 vec_ptr; /* interrupt steering */
108 struct dsa_hw_desc **hw_descs;
109 int num_descs;
110 struct dsa_completion_record *compls;
111 dma_addr_t compls_addr;
112 int compls_size;
113 struct idxd_desc **descs;
114 struct sbitmap_queue sbq;
115 struct dma_chan dma_chan;
116 char name[WQ_NAME_SIZE + 1];
117 u64 max_xfer_bytes;
118 u32 max_batch_size;
119 };
120
121 struct idxd_engine {
122 struct device conf_dev;
123 int id;
124 struct idxd_group *group;
125 struct idxd_device *idxd;
126 };
127
128 /* shadow registers */
129 struct idxd_hw {
130 u32 version;
131 union gen_cap_reg gen_cap;
132 union wq_cap_reg wq_cap;
133 union group_cap_reg group_cap;
134 union engine_cap_reg engine_cap;
135 struct opcap opcap;
136 };
137
138 enum idxd_device_state {
139 IDXD_DEV_HALTED = -1,
140 IDXD_DEV_DISABLED = 0,
141 IDXD_DEV_CONF_READY,
142 IDXD_DEV_ENABLED,
143 };
144
145 enum idxd_device_flag {
146 IDXD_FLAG_CONFIGURABLE = 0,
147 IDXD_FLAG_CMD_RUNNING,
148 };
149
150 struct idxd_device {
151 enum idxd_type type;
152 struct device conf_dev;
153 struct list_head list;
154 struct idxd_hw hw;
155 enum idxd_device_state state;
156 unsigned long flags;
157 int id;
158 int major;
159 u8 cmd_status;
160
161 struct pci_dev *pdev;
162 void __iomem *reg_base;
163
164 spinlock_t dev_lock; /* spinlock for device */
165 struct completion *cmd_done;
166 struct idxd_group *groups;
167 struct idxd_wq *wqs;
168 struct idxd_engine *engines;
169
170 int num_groups;
171
172 u32 msix_perm_offset;
173 u32 wqcfg_offset;
174 u32 grpcfg_offset;
175 u32 perfmon_offset;
176
177 u64 max_xfer_bytes;
178 u32 max_batch_size;
179 int max_groups;
180 int max_engines;
181 int max_tokens;
182 int max_wqs;
183 int max_wq_size;
184 int token_limit;
185 int nr_tokens; /* non-reserved tokens */
186 unsigned int wqcfg_size;
187
188 union sw_err_reg sw_err;
189 wait_queue_head_t cmd_waitq;
190 struct msix_entry *msix_entries;
191 int num_wq_irqs;
192 struct idxd_irq_entry *irq_entries;
193
194 struct dma_device dma_dev;
195 struct workqueue_struct *wq;
196 struct work_struct work;
197 };
198
199 /* IDXD software descriptor */
200 struct idxd_desc {
201 struct dsa_hw_desc *hw;
202 dma_addr_t desc_dma;
203 struct dsa_completion_record *completion;
204 dma_addr_t compl_dma;
205 struct dma_async_tx_descriptor txd;
206 struct llist_node llnode;
207 struct list_head list;
208 int id;
209 int cpu;
210 struct idxd_wq *wq;
211 };
212
213 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
214 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
215
216 extern struct bus_type dsa_bus_type;
217
wq_dedicated(struct idxd_wq * wq)218 static inline bool wq_dedicated(struct idxd_wq *wq)
219 {
220 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
221 }
222
223 enum idxd_portal_prot {
224 IDXD_PORTAL_UNLIMITED = 0,
225 IDXD_PORTAL_LIMITED,
226 };
227
idxd_get_wq_portal_offset(enum idxd_portal_prot prot)228 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
229 {
230 return prot * 0x1000;
231 }
232
idxd_get_wq_portal_full_offset(int wq_id,enum idxd_portal_prot prot)233 static inline int idxd_get_wq_portal_full_offset(int wq_id,
234 enum idxd_portal_prot prot)
235 {
236 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
237 }
238
idxd_set_type(struct idxd_device * idxd)239 static inline void idxd_set_type(struct idxd_device *idxd)
240 {
241 struct pci_dev *pdev = idxd->pdev;
242
243 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
244 idxd->type = IDXD_TYPE_DSA;
245 else
246 idxd->type = IDXD_TYPE_UNKNOWN;
247 }
248
idxd_wq_get(struct idxd_wq * wq)249 static inline void idxd_wq_get(struct idxd_wq *wq)
250 {
251 wq->client_count++;
252 }
253
idxd_wq_put(struct idxd_wq * wq)254 static inline void idxd_wq_put(struct idxd_wq *wq)
255 {
256 wq->client_count--;
257 }
258
idxd_wq_refcount(struct idxd_wq * wq)259 static inline int idxd_wq_refcount(struct idxd_wq *wq)
260 {
261 return wq->client_count;
262 };
263
264 const char *idxd_get_dev_name(struct idxd_device *idxd);
265 int idxd_register_bus_type(void);
266 void idxd_unregister_bus_type(void);
267 int idxd_setup_sysfs(struct idxd_device *idxd);
268 void idxd_cleanup_sysfs(struct idxd_device *idxd);
269 int idxd_register_driver(void);
270 void idxd_unregister_driver(void);
271 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
272
273 /* device interrupt control */
274 irqreturn_t idxd_irq_handler(int vec, void *data);
275 irqreturn_t idxd_misc_thread(int vec, void *data);
276 irqreturn_t idxd_wq_thread(int irq, void *data);
277 void idxd_mask_error_interrupts(struct idxd_device *idxd);
278 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
279 void idxd_mask_msix_vectors(struct idxd_device *idxd);
280 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
281 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
282
283 /* device control */
284 void idxd_device_init_reset(struct idxd_device *idxd);
285 int idxd_device_enable(struct idxd_device *idxd);
286 int idxd_device_disable(struct idxd_device *idxd);
287 void idxd_device_reset(struct idxd_device *idxd);
288 void idxd_device_cleanup(struct idxd_device *idxd);
289 int idxd_device_config(struct idxd_device *idxd);
290 void idxd_device_wqs_clear_state(struct idxd_device *idxd);
291
292 /* work queue control */
293 int idxd_wq_alloc_resources(struct idxd_wq *wq);
294 void idxd_wq_free_resources(struct idxd_wq *wq);
295 int idxd_wq_enable(struct idxd_wq *wq);
296 int idxd_wq_disable(struct idxd_wq *wq);
297 void idxd_wq_drain(struct idxd_wq *wq);
298 int idxd_wq_map_portal(struct idxd_wq *wq);
299 void idxd_wq_unmap_portal(struct idxd_wq *wq);
300 void idxd_wq_disable_cleanup(struct idxd_wq *wq);
301
302 /* submission */
303 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
304 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
305 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
306
307 /* dmaengine */
308 int idxd_register_dma_device(struct idxd_device *idxd);
309 void idxd_unregister_dma_device(struct idxd_device *idxd);
310 int idxd_register_dma_channel(struct idxd_wq *wq);
311 void idxd_unregister_dma_channel(struct idxd_wq *wq);
312 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
313 void idxd_dma_complete_txd(struct idxd_desc *desc,
314 enum idxd_complete_type comp_type);
315 dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
316
317 /* cdev */
318 int idxd_cdev_register(void);
319 void idxd_cdev_remove(void);
320 int idxd_cdev_get_major(struct idxd_device *idxd);
321 int idxd_wq_add_cdev(struct idxd_wq *wq);
322 void idxd_wq_del_cdev(struct idxd_wq *wq);
323
324 #endif
325
