1 /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
2 /* Copyright 2017-2019 NXP */
3
4 #include <linux/timer.h>
5 #include <linux/pci.h>
6 #include <linux/netdevice.h>
7 #include <linux/etherdevice.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/skbuff.h>
10 #include <linux/ethtool.h>
11 #include <linux/if_vlan.h>
12 #include <linux/phylink.h>
13 #include <linux/dim.h>
14
15 #include "enetc_hw.h"
16
17 #define ENETC_MAC_MAXFRM_SIZE 9600
18 #define ENETC_MAX_MTU (ENETC_MAC_MAXFRM_SIZE - \
19 (ETH_FCS_LEN + ETH_HLEN + VLAN_HLEN))
20
21 struct enetc_tx_swbd {
22 struct sk_buff *skb;
23 dma_addr_t dma;
24 u16 len;
25 u8 is_dma_page:1;
26 u8 check_wb:1;
27 u8 do_tstamp:1;
28 };
29
30 #define ENETC_RX_MAXFRM_SIZE ENETC_MAC_MAXFRM_SIZE
31 #define ENETC_RXB_TRUESIZE 2048 /* PAGE_SIZE >> 1 */
32 #define ENETC_RXB_PAD NET_SKB_PAD /* add extra space if needed */
33 #define ENETC_RXB_DMA_SIZE \
34 (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD)
35
36 struct enetc_rx_swbd {
37 dma_addr_t dma;
38 struct page *page;
39 u16 page_offset;
40 };
41
42 struct enetc_ring_stats {
43 unsigned int packets;
44 unsigned int bytes;
45 unsigned int rx_alloc_errs;
46 };
47
48 #define ENETC_RX_RING_DEFAULT_SIZE 512
49 #define ENETC_TX_RING_DEFAULT_SIZE 256
50 #define ENETC_DEFAULT_TX_WORK (ENETC_TX_RING_DEFAULT_SIZE / 2)
51
52 struct enetc_bdr {
53 struct device *dev; /* for DMA mapping */
54 struct net_device *ndev;
55 void *bd_base; /* points to Rx or Tx BD ring */
56 union {
57 void __iomem *tpir;
58 void __iomem *rcir;
59 };
60 u16 index;
61 int bd_count; /* # of BDs */
62 int next_to_use;
63 int next_to_clean;
64 union {
65 struct enetc_tx_swbd *tx_swbd;
66 struct enetc_rx_swbd *rx_swbd;
67 };
68 union {
69 void __iomem *tcir; /* Tx */
70 int next_to_alloc; /* Rx */
71 };
72 void __iomem *idr; /* Interrupt Detect Register pointer */
73
74 struct enetc_ring_stats stats;
75
76 dma_addr_t bd_dma_base;
77 u8 tsd_enable; /* Time specific departure */
78 bool ext_en; /* enable h/w descriptor extensions */
79 } ____cacheline_aligned_in_smp;
80
enetc_bdr_idx_inc(struct enetc_bdr * bdr,int * i)81 static inline void enetc_bdr_idx_inc(struct enetc_bdr *bdr, int *i)
82 {
83 if (unlikely(++*i == bdr->bd_count))
84 *i = 0;
85 }
86
enetc_bd_unused(struct enetc_bdr * bdr)87 static inline int enetc_bd_unused(struct enetc_bdr *bdr)
88 {
89 if (bdr->next_to_clean > bdr->next_to_use)
90 return bdr->next_to_clean - bdr->next_to_use - 1;
91
92 return bdr->bd_count + bdr->next_to_clean - bdr->next_to_use - 1;
93 }
94
95 /* Control BD ring */
96 #define ENETC_CBDR_DEFAULT_SIZE 64
97 struct enetc_cbdr {
98 void *bd_base; /* points to Rx or Tx BD ring */
99 void __iomem *pir;
100 void __iomem *cir;
101
102 int bd_count; /* # of BDs */
103 int next_to_use;
104 int next_to_clean;
105
106 dma_addr_t bd_dma_base;
107 };
108
109 #define ENETC_TXBD(BDR, i) (&(((union enetc_tx_bd *)((BDR).bd_base))[i]))
110
enetc_rxbd(struct enetc_bdr * rx_ring,int i)111 static inline union enetc_rx_bd *enetc_rxbd(struct enetc_bdr *rx_ring, int i)
112 {
113 int hw_idx = i;
114
115 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
116 if (rx_ring->ext_en)
117 hw_idx = 2 * i;
118 #endif
119 return &(((union enetc_rx_bd *)rx_ring->bd_base)[hw_idx]);
120 }
121
enetc_rxbd_next(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,int i)122 static inline union enetc_rx_bd *enetc_rxbd_next(struct enetc_bdr *rx_ring,
123 union enetc_rx_bd *rxbd,
124 int i)
125 {
126 rxbd++;
127 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
128 if (rx_ring->ext_en)
129 rxbd++;
130 #endif
131 if (unlikely(++i == rx_ring->bd_count))
132 rxbd = rx_ring->bd_base;
133
134 return rxbd;
135 }
136
enetc_rxbd_ext(union enetc_rx_bd * rxbd)137 static inline union enetc_rx_bd *enetc_rxbd_ext(union enetc_rx_bd *rxbd)
138 {
139 return ++rxbd;
140 }
141
142 struct enetc_msg_swbd {
143 void *vaddr;
144 dma_addr_t dma;
145 int size;
146 };
147
148 #define ENETC_REV1 0x1
149 enum enetc_errata {
150 ENETC_ERR_TXCSUM = BIT(0),
151 ENETC_ERR_VLAN_ISOL = BIT(1),
152 ENETC_ERR_UCMCSWP = BIT(2),
153 };
154
155 #define ENETC_SI_F_QBV BIT(0)
156 #define ENETC_SI_F_PSFP BIT(1)
157
158 /* PCI IEP device data */
159 struct enetc_si {
160 struct pci_dev *pdev;
161 struct enetc_hw hw;
162 enum enetc_errata errata;
163
164 struct net_device *ndev; /* back ref. */
165
166 struct enetc_cbdr cbd_ring;
167
168 int num_rx_rings; /* how many rings are available in the SI */
169 int num_tx_rings;
170 int num_fs_entries;
171 int num_rss; /* number of RSS buckets */
172 unsigned short pad;
173 int hw_features;
174 };
175
176 #define ENETC_SI_ALIGN 32
177
enetc_si_priv(const struct enetc_si * si)178 static inline void *enetc_si_priv(const struct enetc_si *si)
179 {
180 return (char *)si + ALIGN(sizeof(struct enetc_si), ENETC_SI_ALIGN);
181 }
182
enetc_si_is_pf(struct enetc_si * si)183 static inline bool enetc_si_is_pf(struct enetc_si *si)
184 {
185 return !!(si->hw.port);
186 }
187
188 #define ENETC_MAX_NUM_TXQS 8
189 #define ENETC_INT_NAME_MAX (IFNAMSIZ + 8)
190
191 struct enetc_int_vector {
192 void __iomem *rbier;
193 void __iomem *tbier_base;
194 void __iomem *ricr1;
195 unsigned long tx_rings_map;
196 int count_tx_rings;
197 u32 rx_ictt;
198 u16 comp_cnt;
199 bool rx_dim_en, rx_napi_work;
200 struct napi_struct napi ____cacheline_aligned_in_smp;
201 struct dim rx_dim ____cacheline_aligned_in_smp;
202 char name[ENETC_INT_NAME_MAX];
203
204 struct enetc_bdr rx_ring;
205 struct enetc_bdr tx_ring[];
206 } ____cacheline_aligned_in_smp;
207
208 struct enetc_cls_rule {
209 struct ethtool_rx_flow_spec fs;
210 int used;
211 };
212
213 #define ENETC_MAX_BDR_INT 2 /* fixed to max # of available cpus */
214 struct psfp_cap {
215 u32 max_streamid;
216 u32 max_psfp_filter;
217 u32 max_psfp_gate;
218 u32 max_psfp_gatelist;
219 u32 max_psfp_meter;
220 };
221
222 /* TODO: more hardware offloads */
223 enum enetc_active_offloads {
224 ENETC_F_RX_TSTAMP = BIT(0),
225 ENETC_F_TX_TSTAMP = BIT(1),
226 ENETC_F_QBV = BIT(2),
227 ENETC_F_QCI = BIT(3),
228 };
229
230 /* interrupt coalescing modes */
231 enum enetc_ic_mode {
232 /* one interrupt per frame */
233 ENETC_IC_NONE = 0,
234 /* activated when int coalescing time is set to a non-0 value */
235 ENETC_IC_RX_MANUAL = BIT(0),
236 ENETC_IC_TX_MANUAL = BIT(1),
237 /* use dynamic interrupt moderation */
238 ENETC_IC_RX_ADAPTIVE = BIT(2),
239 };
240
241 #define ENETC_RXIC_PKTTHR min_t(u32, 256, ENETC_RX_RING_DEFAULT_SIZE / 2)
242 #define ENETC_TXIC_PKTTHR min_t(u32, 128, ENETC_TX_RING_DEFAULT_SIZE / 2)
243 #define ENETC_TXIC_TIMETHR enetc_usecs_to_cycles(600)
244
245 struct enetc_ndev_priv {
246 struct net_device *ndev;
247 struct device *dev; /* dma-mapping device */
248 struct enetc_si *si;
249
250 int bdr_int_num; /* number of Rx/Tx ring interrupts */
251 struct enetc_int_vector *int_vector[ENETC_MAX_BDR_INT];
252 u16 num_rx_rings, num_tx_rings;
253 u16 rx_bd_count, tx_bd_count;
254
255 u16 msg_enable;
256 int active_offloads;
257
258 u32 speed; /* store speed for compare update pspeed */
259
260 struct enetc_bdr *tx_ring[16];
261 struct enetc_bdr *rx_ring[16];
262
263 struct enetc_cls_rule *cls_rules;
264
265 struct psfp_cap psfp_cap;
266
267 struct phylink *phylink;
268 int ic_mode;
269 u32 tx_ictt;
270 };
271
272 /* Messaging */
273
274 /* VF-PF set primary MAC address message format */
275 struct enetc_msg_cmd_set_primary_mac {
276 struct enetc_msg_cmd_header header;
277 struct sockaddr mac;
278 };
279
280 #define ENETC_CBD(R, i) (&(((struct enetc_cbd *)((R).bd_base))[i]))
281
282 #define ENETC_CBDR_TIMEOUT 1000 /* usecs */
283
284 /* PTP driver exports */
285 extern int enetc_phc_index;
286
287 /* SI common */
288 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv);
289 void enetc_pci_remove(struct pci_dev *pdev);
290 int enetc_alloc_msix(struct enetc_ndev_priv *priv);
291 void enetc_free_msix(struct enetc_ndev_priv *priv);
292 void enetc_get_si_caps(struct enetc_si *si);
293 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv);
294 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv);
295 void enetc_free_si_resources(struct enetc_ndev_priv *priv);
296
297 int enetc_open(struct net_device *ndev);
298 int enetc_close(struct net_device *ndev);
299 void enetc_start(struct net_device *ndev);
300 void enetc_stop(struct net_device *ndev);
301 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev);
302 struct net_device_stats *enetc_get_stats(struct net_device *ndev);
303 int enetc_set_features(struct net_device *ndev,
304 netdev_features_t features);
305 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd);
306 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
307 void *type_data);
308
309 /* ethtool */
310 void enetc_set_ethtool_ops(struct net_device *ndev);
311
312 /* control buffer descriptor ring (CBDR) */
313 int enetc_set_mac_flt_entry(struct enetc_si *si, int index,
314 char *mac_addr, int si_map);
315 int enetc_clear_mac_flt_entry(struct enetc_si *si, int index);
316 int enetc_set_fs_entry(struct enetc_si *si, struct enetc_cmd_rfse *rfse,
317 int index);
318 void enetc_set_rss_key(struct enetc_hw *hw, const u8 *bytes);
319 int enetc_get_rss_table(struct enetc_si *si, u32 *table, int count);
320 int enetc_set_rss_table(struct enetc_si *si, const u32 *table, int count);
321 int enetc_send_cmd(struct enetc_si *si, struct enetc_cbd *cbd);
322
323 #ifdef CONFIG_FSL_ENETC_QOS
324 int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data);
325 void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed);
326 int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data);
327 int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data);
328 int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
329 void *cb_priv);
330 int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data);
331 int enetc_psfp_init(struct enetc_ndev_priv *priv);
332 int enetc_psfp_clean(struct enetc_ndev_priv *priv);
333
enetc_get_max_cap(struct enetc_ndev_priv * priv)334 static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv)
335 {
336 u32 reg;
337
338 reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR);
339 priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK;
340 /* Port stream filter capability */
341 reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR);
342 priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK;
343 /* Port stream gate capability */
344 reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR);
345 priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK);
346 priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16;
347 /* Port flow meter capability */
348 reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR);
349 priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK;
350 }
351
enetc_psfp_enable(struct enetc_ndev_priv * priv)352 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
353 {
354 struct enetc_hw *hw = &priv->si->hw;
355 int err;
356
357 enetc_get_max_cap(priv);
358
359 err = enetc_psfp_init(priv);
360 if (err)
361 return err;
362
363 enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) |
364 ENETC_PPSFPMR_PSFPEN | ENETC_PPSFPMR_VS |
365 ENETC_PPSFPMR_PVC | ENETC_PPSFPMR_PVZC);
366
367 return 0;
368 }
369
enetc_psfp_disable(struct enetc_ndev_priv * priv)370 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
371 {
372 struct enetc_hw *hw = &priv->si->hw;
373 int err;
374
375 err = enetc_psfp_clean(priv);
376 if (err)
377 return err;
378
379 enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) &
380 ~ENETC_PPSFPMR_PSFPEN & ~ENETC_PPSFPMR_VS &
381 ~ENETC_PPSFPMR_PVC & ~ENETC_PPSFPMR_PVZC);
382
383 memset(&priv->psfp_cap, 0, sizeof(struct psfp_cap));
384
385 return 0;
386 }
387
388 #else
389 #define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP
390 #define enetc_sched_speed_set(priv, speed) (void)0
391 #define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP
392 #define enetc_setup_tc_txtime(ndev, type_data) -EOPNOTSUPP
393 #define enetc_setup_tc_psfp(ndev, type_data) -EOPNOTSUPP
394 #define enetc_setup_tc_block_cb NULL
395
396 #define enetc_get_max_cap(p) \
397 memset(&((p)->psfp_cap), 0, sizeof(struct psfp_cap))
398
enetc_psfp_enable(struct enetc_ndev_priv * priv)399 static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
400 {
401 return 0;
402 }
403
enetc_psfp_disable(struct enetc_ndev_priv * priv)404 static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
405 {
406 return 0;
407 }
408 #endif
409
