1 /*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2014 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * The full GNU General Public License is included in this distribution in
16 * the file called "COPYING".
17 *
18 * Intel MIC X100 DMA Driver.
19 *
20 * Adapted from IOAT dma driver.
21 */
22 #ifndef _MIC_X100_DMA_H_
23 #define _MIC_X100_DMA_H_
24
25 #include <linux/kernel.h>
26 #include <linux/delay.h>
27 #include <linux/sched.h>
28 #include <linux/debugfs.h>
29 #include <linux/slab.h>
30 #include <linux/interrupt.h>
31 #include <linux/mic_bus.h>
32
33 #include "dmaengine.h"
34
35 /*
36 * MIC has a total of 8 dma channels.
37 * Four channels are assigned for host SW use & the remaining for MIC SW.
38 * MIC DMA transfer size & addresses need to be 64 byte aligned.
39 */
40 #define MIC_DMA_MAX_NUM_CHAN 8
41 #define MIC_DMA_NUM_CHAN 4
42 #define MIC_DMA_ALIGN_SHIFT DMAENGINE_ALIGN_64_BYTES
43 #define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT)
44 #define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4)
45
46 /*
47 * Register descriptions
48 * All the registers are 32 bit registers.
49 * DCR is a global register and all others are per-channel.
50 * DCR - bits 0, 2, 4, 6, 8, 10, 12, 14 - enable bits for channels 0 to 7
51 * bits 1, 3, 5, 7, 9, 11, 13, 15 - owner bits for channels 0 to 7
52 * DCAR - bit 24 & 25 interrupt masks for mic owned & host owned channels
53 * DHPR - head of the descriptor ring updated by s/w
54 * DTPR - tail of the descriptor ring updated by h/w
55 * DRAR_LO - lower 32 bits of descriptor ring's mic address
56 * DRAR_HI - 3:0 - remaining 4 bits of descriptor ring's mic address
57 * 20:4 descriptor ring size
58 * 25:21 mic smpt entry number
59 * DSTAT - 16:0 h/w completion count; 31:28 dma engine status
60 * DCHERR - this register is non-zero on error
61 * DCHERRMSK - interrupt mask register
62 */
63 #define MIC_DMA_HW_CMP_CNT_MASK 0x1ffff
64 #define MIC_DMA_CHAN_QUIESCE 0x20000000
65 #define MIC_DMA_SBOX_BASE 0x00010000
66 #define MIC_DMA_SBOX_DCR 0x0000A280
67 #define MIC_DMA_SBOX_CH_BASE 0x0001A000
68 #define MIC_DMA_SBOX_CHAN_OFF 0x40
69 #define MIC_DMA_SBOX_DCAR_IM0 (0x1 << 24)
70 #define MIC_DMA_SBOX_DCAR_IM1 (0x1 << 25)
71 #define MIC_DMA_SBOX_DRARHI_SYS_MASK (0x1 << 26)
72 #define MIC_DMA_REG_DCAR 0
73 #define MIC_DMA_REG_DHPR 4
74 #define MIC_DMA_REG_DTPR 8
75 #define MIC_DMA_REG_DRAR_LO 20
76 #define MIC_DMA_REG_DRAR_HI 24
77 #define MIC_DMA_REG_DSTAT 32
78 #define MIC_DMA_REG_DCHERR 44
79 #define MIC_DMA_REG_DCHERRMSK 48
80
81 /* HW dma desc */
82 struct mic_dma_desc {
83 u64 qw0;
84 u64 qw1;
85 };
86
87 enum mic_dma_chan_owner {
88 MIC_DMA_CHAN_MIC = 0,
89 MIC_DMA_CHAN_HOST
90 };
91
92 /*
93 * mic_dma_chan - channel specific information
94 * @ch_num: channel number
95 * @owner: owner of this channel
96 * @last_tail: cached value of descriptor ring tail
97 * @head: index of next descriptor in desc_ring
98 * @issued: hardware notification point
99 * @submitted: index that will be used to submit descriptors to h/w
100 * @api_ch: dma engine api channel
101 * @desc_ring: dma descriptor ring
102 * @desc_ring_micpa: mic physical address of desc_ring
103 * @status_dest: destination for status (fence) descriptor
104 * @status_dest_micpa: mic address for status_dest,
105 * DMA controller uses this address
106 * @tx_array: array of async_tx
107 * @cleanup_lock: lock held when processing completed tx
108 * @prep_lock: lock held in prep_memcpy & released in tx_submit
109 * @issue_lock: lock used to synchronize writes to head
110 * @cookie: mic_irq cookie used with mic irq request
111 */
112 struct mic_dma_chan {
113 int ch_num;
114 enum mic_dma_chan_owner owner;
115 u32 last_tail;
116 u32 head;
117 u32 issued;
118 u32 submitted;
119 struct dma_chan api_ch;
120 struct mic_dma_desc *desc_ring;
121 dma_addr_t desc_ring_micpa;
122 u64 *status_dest;
123 dma_addr_t status_dest_micpa;
124 struct dma_async_tx_descriptor *tx_array;
125 spinlock_t cleanup_lock;
126 spinlock_t prep_lock;
127 spinlock_t issue_lock;
128 struct mic_irq *cookie;
129 };
130
131 /*
132 * struct mic_dma_device - per mic device
133 * @mic_ch: dma channels
134 * @dma_dev: underlying dma device
135 * @mbdev: mic bus dma device
136 * @mmio: virtual address of the mmio space
137 * @dbg_dir: debugfs directory
138 * @start_ch: first channel number that can be used
139 * @max_xfer_size: maximum transfer size per dma descriptor
140 */
141 struct mic_dma_device {
142 struct mic_dma_chan mic_ch[MIC_DMA_MAX_NUM_CHAN];
143 struct dma_device dma_dev;
144 struct mbus_device *mbdev;
145 void __iomem *mmio;
146 struct dentry *dbg_dir;
147 int start_ch;
148 size_t max_xfer_size;
149 };
150
to_mic_dma_chan(struct dma_chan * ch)151 static inline struct mic_dma_chan *to_mic_dma_chan(struct dma_chan *ch)
152 {
153 return container_of(ch, struct mic_dma_chan, api_ch);
154 }
155
to_mic_dma_dev(struct mic_dma_chan * ch)156 static inline struct mic_dma_device *to_mic_dma_dev(struct mic_dma_chan *ch)
157 {
158 return
159 container_of((const typeof(((struct mic_dma_device *)0)->mic_ch)*)
160 (ch - ch->ch_num), struct mic_dma_device, mic_ch);
161 }
162
to_mbus_device(struct mic_dma_chan * ch)163 static inline struct mbus_device *to_mbus_device(struct mic_dma_chan *ch)
164 {
165 return to_mic_dma_dev(ch)->mbdev;
166 }
167
to_mbus_hw_ops(struct mic_dma_chan * ch)168 static inline struct mbus_hw_ops *to_mbus_hw_ops(struct mic_dma_chan *ch)
169 {
170 return to_mbus_device(ch)->hw_ops;
171 }
172
mic_dma_ch_to_device(struct mic_dma_chan * ch)173 static inline struct device *mic_dma_ch_to_device(struct mic_dma_chan *ch)
174 {
175 return to_mic_dma_dev(ch)->dma_dev.dev;
176 }
177
mic_dma_chan_to_mmio(struct mic_dma_chan * ch)178 static inline void __iomem *mic_dma_chan_to_mmio(struct mic_dma_chan *ch)
179 {
180 return to_mic_dma_dev(ch)->mmio;
181 }
182
mic_dma_read_reg(struct mic_dma_chan * ch,u32 reg)183 static inline u32 mic_dma_read_reg(struct mic_dma_chan *ch, u32 reg)
184 {
185 return ioread32(mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
186 ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
187 }
188
mic_dma_write_reg(struct mic_dma_chan * ch,u32 reg,u32 val)189 static inline void mic_dma_write_reg(struct mic_dma_chan *ch, u32 reg, u32 val)
190 {
191 iowrite32(val, mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
192 ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
193 }
194
mic_dma_mmio_read(struct mic_dma_chan * ch,u32 offset)195 static inline u32 mic_dma_mmio_read(struct mic_dma_chan *ch, u32 offset)
196 {
197 return ioread32(mic_dma_chan_to_mmio(ch) + offset);
198 }
199
mic_dma_mmio_write(struct mic_dma_chan * ch,u32 val,u32 offset)200 static inline void mic_dma_mmio_write(struct mic_dma_chan *ch, u32 val,
201 u32 offset)
202 {
203 iowrite32(val, mic_dma_chan_to_mmio(ch) + offset);
204 }
205
mic_dma_read_cmp_cnt(struct mic_dma_chan * ch)206 static inline u32 mic_dma_read_cmp_cnt(struct mic_dma_chan *ch)
207 {
208 return mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) &
209 MIC_DMA_HW_CMP_CNT_MASK;
210 }
211
mic_dma_chan_set_owner(struct mic_dma_chan * ch)212 static inline void mic_dma_chan_set_owner(struct mic_dma_chan *ch)
213 {
214 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
215 u32 chan_num = ch->ch_num;
216
217 dcr = (dcr & ~(0x1 << (chan_num * 2))) | (ch->owner << (chan_num * 2));
218 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
219 }
220
mic_dma_enable_chan(struct mic_dma_chan * ch)221 static inline void mic_dma_enable_chan(struct mic_dma_chan *ch)
222 {
223 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
224
225 dcr |= 2 << (ch->ch_num << 1);
226 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
227 }
228
mic_dma_disable_chan(struct mic_dma_chan * ch)229 static inline void mic_dma_disable_chan(struct mic_dma_chan *ch)
230 {
231 u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
232
233 dcr &= ~(2 << (ch->ch_num << 1));
234 mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
235 }
236
mic_dma_chan_set_desc_ring(struct mic_dma_chan * ch)237 static void mic_dma_chan_set_desc_ring(struct mic_dma_chan *ch)
238 {
239 u32 drar_hi;
240 dma_addr_t desc_ring_micpa = ch->desc_ring_micpa;
241
242 drar_hi = (MIC_DMA_DESC_RX_SIZE & 0x1ffff) << 4;
243 if (MIC_DMA_CHAN_MIC == ch->owner) {
244 drar_hi |= (desc_ring_micpa >> 32) & 0xf;
245 } else {
246 drar_hi |= MIC_DMA_SBOX_DRARHI_SYS_MASK;
247 drar_hi |= ((desc_ring_micpa >> 34)
248 & 0x1f) << 21;
249 drar_hi |= (desc_ring_micpa >> 32) & 0x3;
250 }
251 mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_LO, (u32) desc_ring_micpa);
252 mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_HI, drar_hi);
253 }
254
mic_dma_chan_mask_intr(struct mic_dma_chan * ch)255 static inline void mic_dma_chan_mask_intr(struct mic_dma_chan *ch)
256 {
257 u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
258
259 if (MIC_DMA_CHAN_MIC == ch->owner)
260 dcar |= MIC_DMA_SBOX_DCAR_IM0;
261 else
262 dcar |= MIC_DMA_SBOX_DCAR_IM1;
263 mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
264 }
265
mic_dma_chan_unmask_intr(struct mic_dma_chan * ch)266 static inline void mic_dma_chan_unmask_intr(struct mic_dma_chan *ch)
267 {
268 u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
269
270 if (MIC_DMA_CHAN_MIC == ch->owner)
271 dcar &= ~MIC_DMA_SBOX_DCAR_IM0;
272 else
273 dcar &= ~MIC_DMA_SBOX_DCAR_IM1;
274 mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
275 }
276
mic_dma_ack_interrupt(struct mic_dma_chan * ch)277 static void mic_dma_ack_interrupt(struct mic_dma_chan *ch)
278 {
279 if (MIC_DMA_CHAN_MIC == ch->owner) {
280 /* HW errata */
281 mic_dma_chan_mask_intr(ch);
282 mic_dma_chan_unmask_intr(ch);
283 }
284 to_mbus_hw_ops(ch)->ack_interrupt(to_mbus_device(ch), ch->ch_num);
285 }
286 #endif
287
