1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2014 Intel Corporation.
6 *
7 * Intel MIC X100 DMA Driver.
8 *
9 * Adapted from IOAT dma driver.
10 */
11 #ifndef _MIC_X100_DMA_H_
12 #define _MIC_X100_DMA_H_
13
14 #include <linux/kernel.h>
15 #include <linux/delay.h>
16 #include <linux/sched.h>
17 #include <linux/debugfs.h>
18 #include <linux/slab.h>
19 #include <linux/interrupt.h>
20 #include <linux/mic_bus.h>
21
22 #include "dmaengine.h"
23
24 /*
25 * MIC has a total of 8 dma channels.
26 * Four channels are assigned for host SW use & the remaining for MIC SW.
27 * MIC DMA transfer size & addresses need to be 64 byte aligned.
28 */
29 #define MIC_DMA_MAX_NUM_CHAN 8
30 #define MIC_DMA_NUM_CHAN 4
31 #define MIC_DMA_ALIGN_SHIFT DMAENGINE_ALIGN_64_BYTES
32 #define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT)
33 #define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4)
34
35 /*
36 * Register descriptions
37 * All the registers are 32 bit registers.
38 * DCR is a global register and all others are per-channel.
39 * DCR - bits 0, 2, 4, 6, 8, 10, 12, 14 - enable bits for channels 0 to 7
40 * bits 1, 3, 5, 7, 9, 11, 13, 15 - owner bits for channels 0 to 7
41 * DCAR - bit 24 & 25 interrupt masks for mic owned & host owned channels
42 * DHPR - head of the descriptor ring updated by s/w
43 * DTPR - tail of the descriptor ring updated by h/w
44 * DRAR_LO - lower 32 bits of descriptor ring's mic address
45 * DRAR_HI - 3:0 - remaining 4 bits of descriptor ring's mic address
46 * 20:4 descriptor ring size
47 * 25:21 mic smpt entry number
48 * DSTAT - 16:0 h/w completion count; 31:28 dma engine status
49 * DCHERR - this register is non-zero on error
50 * DCHERRMSK - interrupt mask register
51 */
52 #define MIC_DMA_HW_CMP_CNT_MASK 0x1ffff
53 #define MIC_DMA_CHAN_QUIESCE 0x20000000
54 #define MIC_DMA_SBOX_BASE 0x00010000
55 #define MIC_DMA_SBOX_DCR 0x0000A280
56 #define MIC_DMA_SBOX_CH_BASE 0x0001A000
57 #define MIC_DMA_SBOX_CHAN_OFF 0x40
58 #define MIC_DMA_SBOX_DCAR_IM0 (0x1 << 24)
59 #define MIC_DMA_SBOX_DCAR_IM1 (0x1 << 25)
60 #define MIC_DMA_SBOX_DRARHI_SYS_MASK (0x1 << 26)
61 #define MIC_DMA_REG_DCAR 0
62 #define MIC_DMA_REG_DHPR 4
63 #define MIC_DMA_REG_DTPR 8
64 #define MIC_DMA_REG_DRAR_LO 20
65 #define MIC_DMA_REG_DRAR_HI 24
66 #define MIC_DMA_REG_DSTAT 32
67 #define MIC_DMA_REG_DCHERR 44
68 #define MIC_DMA_REG_DCHERRMSK 48
69
70 /* HW dma desc */
71 struct mic_dma_desc {
72 u64 qw0;
73 u64 qw1;
74 };
75
76 enum mic_dma_chan_owner {
77 MIC_DMA_CHAN_MIC = 0,
78 MIC_DMA_CHAN_HOST
79 };
80
81 /*
82 * mic_dma_chan - channel specific information
83 * @ch_num: channel number
84 * @owner: owner of this channel
85 * @last_tail: cached value of descriptor ring tail
86 * @head: index of next descriptor in desc_ring
87 * @issued: hardware notification point
88 * @submitted: index that will be used to submit descriptors to h/w
89 * @api_ch: dma engine api channel
90 * @desc_ring: dma descriptor ring
91 * @desc_ring_micpa: mic physical address of desc_ring
92 * @status_dest: destination for status (fence) descriptor
93 * @status_dest_micpa: mic address for status_dest,
94 * DMA controller uses this address
95 * @tx_array: array of async_tx
96 * @cleanup_lock: lock held when processing completed tx
97 * @prep_lock: lock held in prep_memcpy & released in tx_submit
98 * @issue_lock: lock used to synchronize writes to head
99 * @cookie: mic_irq cookie used with mic irq request
100 */
101 struct mic_dma_chan {
102 int ch_num;
103 enum mic_dma_chan_owner owner;
104 u32 last_tail;
105 u32 head;
106 u32 issued;
107 u32 submitted;
108 struct dma_chan api_ch;
109 struct mic_dma_desc *desc_ring;
110 dma_addr_t desc_ring_micpa;
111 u64 *status_dest;
112 dma_addr_t status_dest_micpa;
113 struct dma_async_tx_descriptor *tx_array;
114 spinlock_t cleanup_lock;
115 spinlock_t prep_lock;
116 spinlock_t issue_lock;
117 struct mic_irq *cookie;
118 };
119
120 /*
121 * struct mic_dma_device - per mic device
122 * @mic_ch: dma channels
123 * @dma_dev: underlying dma device
124 * @mbdev: mic bus dma device
125 * @mmio: virtual address of the mmio space
126 * @dbg_dir: debugfs directory
127 * @start_ch: first channel number that can be used
128 * @max_xfer_size: maximum transfer size per dma descriptor
129 */
130 struct mic_dma_device {
131 struct mic_dma_chan mic_ch[MIC_DMA_MAX_NUM_CHAN];
132 struct dma_device dma_dev;
133 struct mbus_device *mbdev;
134 void __iomem *mmio;
135 struct dentry *dbg_dir;
136 int start_ch;
137 size_t max_xfer_size;
138 };
139
to_mic_dma_chan(struct dma_chan * ch)140 static inline struct mic_dma_chan *to_mic_dma_chan(struct dma_chan *ch)
141 {
142 return container_of(ch, struct mic_dma_chan, api_ch);
143 }
144
to_mic_dma_dev(struct mic_dma_chan * ch)145 static inline struct mic_dma_device *to_mic_dma_dev(struct mic_dma_chan *ch)
146 {
147 return
148 container_of((const typeof(((struct mic_dma_device *)0)->mic_ch)*)
149 (ch - ch->ch_num), struct mic_dma_device, mic_ch);
150 }
151
to_mbus_device(struct mic_dma_chan * ch)152 static inline struct mbus_device *to_mbus_device(struct mic_dma_chan *ch)
153 {
154 return to_mic_dma_dev(ch)->mbdev;
155 }
156
to_mbus_hw_ops(struct mic_dma_chan * ch)157 static inline struct mbus_hw_ops *to_mbus_hw_ops(struct mic_dma_chan *ch)
158 {
159 return to_mbus_device(ch)->hw_ops;
160 }
161
mic_dma_ch_to_device(struct mic_dma_chan * ch)162 static inline struct device *mic_dma_ch_to_device(struct mic_dma_chan *ch)
163 {
164 return to_mic_dma_dev(ch)->dma_dev.dev;
165 }
166
mic_dma_chan_to_mmio(struct mic_dma_chan * ch)167 static inline void __iomem *mic_dma_chan_to_mmio(struct mic_dma_chan *ch)
168 {
169 return to_mic_dma_dev(ch)->mmio;
170 }
171
mic_dma_read_reg(struct mic_dma_chan * ch,u32 reg)172 static inline u32 mic_dma_read_reg(struct mic_dma_chan *ch, u32 reg)
173 {
174 return ioread32(mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
175 ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
176 }
177
mic_dma_write_reg(struct mic_dma_chan * ch,u32 reg,u32 val)178 static inline void mic_dma_write_reg(struct mic_dma_chan *ch, u32 reg, u32 val)
179 {
180 iowrite32(val, mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
181 ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
182 }
183
mic_dma_mmio_read(struct mic_dma_chan * ch,u32 offset)184 static inline u32 mic_dma_mmio_read(struct mic_dma_chan *ch, u32 offset)
185 {
186 return ioread32(mic_dma_chan_to_mmio(ch) + offset);
187 }
188
mic_dma_mmio_write(struct mic_dma_chan * ch,u32 val,u32 offset)189 static inline void mic_dma_mmio_write(struct mic_dma_chan *ch, u32 val,
190 u32 offset)
191 {
192 iowrite32(val, mic_dma_chan_to_mmio(ch) + offset);
193 }
194
mic_dma_read_cmp_cnt(struct mic_dma_chan * ch)195 static inline u32 mic_dma_read_cmp_cnt(struct mic_dma_chan *ch)
196 {
197 return mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) &
198 MIC_DMA_HW_CMP_CNT_MASK;
199 }
200
mic_dma_chan_set_owner(struct mic_dma_chan * ch)201 static inline void mic_dma_chan_set_owner(struct mic_dma_chan *ch)
202 {
203 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
204 u32 chan_num = ch->ch_num;
205
206 dcr = (dcr & ~(0x1 << (chan_num * 2))) | (ch->owner << (chan_num * 2));
207 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
208 }
209
mic_dma_enable_chan(struct mic_dma_chan * ch)210 static inline void mic_dma_enable_chan(struct mic_dma_chan *ch)
211 {
212 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
213
214 dcr |= 2 << (ch->ch_num << 1);
215 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
216 }
217
mic_dma_disable_chan(struct mic_dma_chan * ch)218 static inline void mic_dma_disable_chan(struct mic_dma_chan *ch)
219 {
220 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
221
222 dcr &= ~(2 << (ch->ch_num << 1));
223 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
224 }
225
mic_dma_chan_set_desc_ring(struct mic_dma_chan * ch)226 static void mic_dma_chan_set_desc_ring(struct mic_dma_chan *ch)
227 {
228 u32 drar_hi;
229 dma_addr_t desc_ring_micpa = ch->desc_ring_micpa;
230
231 drar_hi = (MIC_DMA_DESC_RX_SIZE & 0x1ffff) << 4;
232 if (MIC_DMA_CHAN_MIC == ch->owner) {
233 drar_hi |= (desc_ring_micpa >> 32) & 0xf;
234 } else {
235 drar_hi |= MIC_DMA_SBOX_DRARHI_SYS_MASK;
236 drar_hi |= ((desc_ring_micpa >> 34)
237 & 0x1f) << 21;
238 drar_hi |= (desc_ring_micpa >> 32) & 0x3;
239 }
240 mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_LO, (u32) desc_ring_micpa);
241 mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_HI, drar_hi);
242 }
243
mic_dma_chan_mask_intr(struct mic_dma_chan * ch)244 static inline void mic_dma_chan_mask_intr(struct mic_dma_chan *ch)
245 {
246 u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
247
248 if (MIC_DMA_CHAN_MIC == ch->owner)
249 dcar |= MIC_DMA_SBOX_DCAR_IM0;
250 else
251 dcar |= MIC_DMA_SBOX_DCAR_IM1;
252 mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
253 }
254
mic_dma_chan_unmask_intr(struct mic_dma_chan * ch)255 static inline void mic_dma_chan_unmask_intr(struct mic_dma_chan *ch)
256 {
257 u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
258
259 if (MIC_DMA_CHAN_MIC == ch->owner)
260 dcar &= ~MIC_DMA_SBOX_DCAR_IM0;
261 else
262 dcar &= ~MIC_DMA_SBOX_DCAR_IM1;
263 mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
264 }
265
mic_dma_ack_interrupt(struct mic_dma_chan * ch)266 static void mic_dma_ack_interrupt(struct mic_dma_chan *ch)
267 {
268 if (MIC_DMA_CHAN_MIC == ch->owner) {
269 /* HW errata */
270 mic_dma_chan_mask_intr(ch);
271 mic_dma_chan_unmask_intr(ch);
272 }
273 to_mbus_hw_ops(ch)->ack_interrupt(to_mbus_device(ch), ch->ch_num);
274 }
275 #endif
276
