1  /* SPDX-License-Identifier: GPL-2.0-only */
2  /*
3   * Copyright (c) 2018 Chelsio Communications, Inc.
4   */
5  
6  #ifndef __CHTLS_H__
7  #define __CHTLS_H__
8  
9  #include <crypto/aes.h>
10  #include <crypto/algapi.h>
11  #include <crypto/hash.h>
12  #include <crypto/sha1.h>
13  #include <crypto/sha2.h>
14  #include <crypto/authenc.h>
15  #include <crypto/ctr.h>
16  #include <crypto/gf128mul.h>
17  #include <crypto/internal/aead.h>
18  #include <crypto/null.h>
19  #include <crypto/internal/skcipher.h>
20  #include <crypto/aead.h>
21  #include <crypto/scatterwalk.h>
22  #include <crypto/internal/hash.h>
23  #include <linux/tls.h>
24  #include <net/tls.h>
25  #include <net/tls_toe.h>
26  
27  #include "t4fw_api.h"
28  #include "t4_msg.h"
29  #include "cxgb4.h"
30  #include "cxgb4_uld.h"
31  #include "l2t.h"
32  #include "chcr_algo.h"
33  #include "chcr_core.h"
34  #include "chcr_crypto.h"
35  
36  #define CHTLS_DRV_VERSION "1.0.0.0-ko"
37  
38  #define TLS_KEYCTX_RXFLIT_CNT_S 24
39  #define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
40  
41  #define TLS_KEYCTX_RXPROT_VER_S 20
42  #define TLS_KEYCTX_RXPROT_VER_M 0xf
43  #define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
44  
45  #define TLS_KEYCTX_RXCIPH_MODE_S 16
46  #define TLS_KEYCTX_RXCIPH_MODE_M 0xf
47  #define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
48  
49  #define TLS_KEYCTX_RXAUTH_MODE_S 12
50  #define TLS_KEYCTX_RXAUTH_MODE_M 0xf
51  #define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
52  
53  #define TLS_KEYCTX_RXCIAU_CTRL_S 11
54  #define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
55  
56  #define TLS_KEYCTX_RX_SEQCTR_S 9
57  #define TLS_KEYCTX_RX_SEQCTR_M 0x3
58  #define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
59  
60  #define TLS_KEYCTX_RX_VALID_S 8
61  #define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
62  
63  #define TLS_KEYCTX_RXCK_SIZE_S 3
64  #define TLS_KEYCTX_RXCK_SIZE_M 0x7
65  #define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
66  
67  #define TLS_KEYCTX_RXMK_SIZE_S 0
68  #define TLS_KEYCTX_RXMK_SIZE_M 0x7
69  #define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
70  
71  #define KEYCTX_TX_WR_IV_S  55
72  #define KEYCTX_TX_WR_IV_M  0x1ffULL
73  #define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
74  #define KEYCTX_TX_WR_IV_G(x) \
75  	(((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
76  
77  #define KEYCTX_TX_WR_AAD_S 47
78  #define KEYCTX_TX_WR_AAD_M 0xffULL
79  #define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
80  #define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
81  				KEYCTX_TX_WR_AAD_M)
82  
83  #define KEYCTX_TX_WR_AADST_S 39
84  #define KEYCTX_TX_WR_AADST_M 0xffULL
85  #define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
86  #define KEYCTX_TX_WR_AADST_G(x) \
87  	(((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
88  
89  #define KEYCTX_TX_WR_CIPHER_S 30
90  #define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
91  #define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
92  #define KEYCTX_TX_WR_CIPHER_G(x) \
93  	(((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
94  
95  #define KEYCTX_TX_WR_CIPHERST_S 23
96  #define KEYCTX_TX_WR_CIPHERST_M 0x7f
97  #define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
98  #define KEYCTX_TX_WR_CIPHERST_G(x) \
99  	(((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
100  
101  #define KEYCTX_TX_WR_AUTH_S 14
102  #define KEYCTX_TX_WR_AUTH_M 0x1ff
103  #define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
104  #define KEYCTX_TX_WR_AUTH_G(x) \
105  	(((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
106  
107  #define KEYCTX_TX_WR_AUTHST_S 7
108  #define KEYCTX_TX_WR_AUTHST_M 0x7f
109  #define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
110  #define KEYCTX_TX_WR_AUTHST_G(x) \
111  	(((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
112  
113  #define KEYCTX_TX_WR_AUTHIN_S 0
114  #define KEYCTX_TX_WR_AUTHIN_M 0x7f
115  #define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
116  #define KEYCTX_TX_WR_AUTHIN_G(x) \
117  	(((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
118  
119  struct sge_opaque_hdr {
120  	void *dev;
121  	dma_addr_t addr[MAX_SKB_FRAGS + 1];
122  };
123  
124  #define MAX_IVS_PAGE			256
125  #define TLS_KEY_CONTEXT_SZ		64
126  #define CIPHER_BLOCK_SIZE		16
127  #define GCM_TAG_SIZE			16
128  #define KEY_ON_MEM_SZ			16
129  #define AEAD_EXPLICIT_DATA_SIZE		8
130  #define TLS_HEADER_LENGTH		5
131  #define SCMD_CIPH_MODE_AES_GCM		2
132  /* Any MFS size should work and come from openssl */
133  #define TLS_MFS				16384
134  
135  #define RSS_HDR sizeof(struct rss_header)
136  #define TLS_WR_CPL_LEN \
137  	(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
138  
139  enum {
140  	CHTLS_KEY_CONTEXT_DSGL,
141  	CHTLS_KEY_CONTEXT_IMM,
142  	CHTLS_KEY_CONTEXT_DDR,
143  };
144  
145  enum {
146  	CHTLS_LISTEN_START,
147  	CHTLS_LISTEN_STOP,
148  };
149  
150  /* Flags for return value of CPL message handlers */
151  enum {
152  	CPL_RET_BUF_DONE =    1,   /* buffer processing done */
153  	CPL_RET_BAD_MSG =     2,   /* bad CPL message */
154  	CPL_RET_UNKNOWN_TID = 4    /* unexpected unknown TID */
155  };
156  
157  #define LISTEN_INFO_HASH_SIZE 32
158  #define RSPQ_HASH_BITS 5
159  struct listen_info {
160  	struct listen_info *next;  /* Link to next entry */
161  	struct sock *sk;           /* The listening socket */
162  	unsigned int stid;         /* The server TID */
163  };
164  
165  enum {
166  	T4_LISTEN_START_PENDING,
167  	T4_LISTEN_STARTED
168  };
169  
170  enum csk_flags {
171  	CSK_CALLBACKS_CHKD,	/* socket callbacks have been sanitized */
172  	CSK_ABORT_REQ_RCVD,	/* received one ABORT_REQ_RSS message */
173  	CSK_TX_MORE_DATA,	/* sending ULP data; don't set SHOVE bit */
174  	CSK_TX_WAIT_IDLE,	/* suspend Tx until in-flight data is ACKed */
175  	CSK_ABORT_SHUTDOWN,	/* shouldn't send more abort requests */
176  	CSK_ABORT_RPL_PENDING,	/* expecting an abort reply */
177  	CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
178  	CSK_TX_DATA_SENT,	/* sent a TX_DATA WR on this connection */
179  	CSK_TX_FAILOVER,	/* Tx traffic failing over */
180  	CSK_UPDATE_RCV_WND,	/* Need to update rcv window */
181  	CSK_RST_ABORTED,	/* outgoing RST was aborted */
182  	CSK_TLS_HANDSHK,	/* TLS Handshake */
183  	CSK_CONN_INLINE,	/* Connection on HW */
184  };
185  
186  enum chtls_cdev_state {
187  	CHTLS_CDEV_STATE_UP = 1
188  };
189  
190  struct listen_ctx {
191  	struct sock *lsk;
192  	struct chtls_dev *cdev;
193  	struct sk_buff_head synq;
194  	u32 state;
195  };
196  
197  struct key_map {
198  	unsigned long *addr;
199  	unsigned int start;
200  	unsigned int available;
201  	unsigned int size;
202  	spinlock_t lock; /* lock for key id request from map */
203  } __packed;
204  
205  struct tls_scmd {
206  	u32 seqno_numivs;
207  	u32 ivgen_hdrlen;
208  };
209  
210  struct chtls_dev {
211  	struct tls_toe_device tlsdev;
212  	struct list_head list;
213  	struct cxgb4_lld_info *lldi;
214  	struct pci_dev *pdev;
215  	struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
216  	spinlock_t listen_lock; /* lock for listen list */
217  	struct net_device **ports;
218  	struct tid_info *tids;
219  	unsigned int pfvf;
220  	const unsigned short *mtus;
221  
222  	struct idr hwtid_idr;
223  	struct idr stid_idr;
224  
225  	spinlock_t idr_lock ____cacheline_aligned_in_smp;
226  
227  	struct net_device *egr_dev[NCHAN * 2];
228  	struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
229  	struct sk_buff *askb;
230  
231  	struct sk_buff_head deferq;
232  	struct work_struct deferq_task;
233  
234  	struct list_head list_node;
235  	struct list_head rcu_node;
236  	struct list_head na_node;
237  	unsigned int send_page_order;
238  	int max_host_sndbuf;
239  	u32 round_robin_cnt;
240  	struct key_map kmap;
241  	unsigned int cdev_state;
242  };
243  
244  struct chtls_listen {
245  	struct chtls_dev *cdev;
246  	struct sock *sk;
247  };
248  
249  struct chtls_hws {
250  	struct sk_buff_head sk_recv_queue;
251  	u8 txqid;
252  	u8 ofld;
253  	u16 type;
254  	u16 rstate;
255  	u16 keyrpl;
256  	u16 pldlen;
257  	u16 rcvpld;
258  	u16 compute;
259  	u16 expansion;
260  	u16 keylen;
261  	u16 pdus;
262  	u16 adjustlen;
263  	u16 ivsize;
264  	u16 txleft;
265  	u32 mfs;
266  	s32 txkey;
267  	s32 rxkey;
268  	u32 fcplenmax;
269  	u32 copied_seq;
270  	u64 tx_seq_no;
271  	struct tls_scmd scmd;
272  	union {
273  		struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
274  		struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
275  	} crypto_info;
276  };
277  
278  struct chtls_sock {
279  	struct sock *sk;
280  	struct chtls_dev *cdev;
281  	struct l2t_entry *l2t_entry;    /* pointer to the L2T entry */
282  	struct net_device *egress_dev;  /* TX_CHAN for act open retry */
283  
284  	struct sk_buff_head txq;
285  	struct sk_buff *wr_skb_head;
286  	struct sk_buff *wr_skb_tail;
287  	struct sk_buff *ctrl_skb_cache;
288  	struct sk_buff *txdata_skb_cache; /* abort path messages */
289  	struct kref kref;
290  	unsigned long flags;
291  	u32 opt2;
292  	u32 wr_credits;
293  	u32 wr_unacked;
294  	u32 wr_max_credits;
295  	u32 wr_nondata;
296  	u32 hwtid;               /* TCP Control Block ID */
297  	u32 txq_idx;
298  	u32 rss_qid;
299  	u32 tid;
300  	u32 idr;
301  	u32 mss;
302  	u32 ulp_mode;
303  	u32 tx_chan;
304  	u32 rx_chan;
305  	u32 sndbuf;
306  	u32 txplen_max;
307  	u32 mtu_idx;           /* MTU table index */
308  	u32 smac_idx;
309  	u8 port_id;
310  	u8 tos;
311  	u16 resv2;
312  	u32 delack_mode;
313  	u32 delack_seq;
314  	u32 snd_win;
315  	u32 rcv_win;
316  
317  	void *passive_reap_next;        /* placeholder for passive */
318  	struct chtls_hws tlshws;
319  	struct synq {
320  		struct sk_buff *next;
321  		struct sk_buff *prev;
322  	} synq;
323  	struct listen_ctx *listen_ctx;
324  };
325  
326  struct tls_hdr {
327  	u8  type;
328  	u16 version;
329  	u16 length;
330  } __packed;
331  
332  struct tlsrx_cmp_hdr {
333  	u8  type;
334  	u16 version;
335  	u16 length;
336  
337  	u64 tls_seq;
338  	u16 reserved1;
339  	u8  res_to_mac_error;
340  } __packed;
341  
342  /* res_to_mac_error fields */
343  #define TLSRX_HDR_PKT_INT_ERROR_S   4
344  #define TLSRX_HDR_PKT_INT_ERROR_M   0x1
345  #define TLSRX_HDR_PKT_INT_ERROR_V(x) \
346  	((x) << TLSRX_HDR_PKT_INT_ERROR_S)
347  #define TLSRX_HDR_PKT_INT_ERROR_G(x) \
348  	(((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
349  #define TLSRX_HDR_PKT_INT_ERROR_F   TLSRX_HDR_PKT_INT_ERROR_V(1U)
350  
351  #define TLSRX_HDR_PKT_SPP_ERROR_S        3
352  #define TLSRX_HDR_PKT_SPP_ERROR_M        0x1
353  #define TLSRX_HDR_PKT_SPP_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_SPP_ERROR)
354  #define TLSRX_HDR_PKT_SPP_ERROR_G(x)     \
355  	(((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
356  #define TLSRX_HDR_PKT_SPP_ERROR_F        TLSRX_HDR_PKT_SPP_ERROR_V(1U)
357  
358  #define TLSRX_HDR_PKT_CCDX_ERROR_S       2
359  #define TLSRX_HDR_PKT_CCDX_ERROR_M       0x1
360  #define TLSRX_HDR_PKT_CCDX_ERROR_V(x)    ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
361  #define TLSRX_HDR_PKT_CCDX_ERROR_G(x)    \
362  	(((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
363  #define TLSRX_HDR_PKT_CCDX_ERROR_F       TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
364  
365  #define TLSRX_HDR_PKT_PAD_ERROR_S        1
366  #define TLSRX_HDR_PKT_PAD_ERROR_M        0x1
367  #define TLSRX_HDR_PKT_PAD_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
368  #define TLSRX_HDR_PKT_PAD_ERROR_G(x)     \
369  	(((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
370  #define TLSRX_HDR_PKT_PAD_ERROR_F        TLSRX_HDR_PKT_PAD_ERROR_V(1U)
371  
372  #define TLSRX_HDR_PKT_MAC_ERROR_S        0
373  #define TLSRX_HDR_PKT_MAC_ERROR_M        0x1
374  #define TLSRX_HDR_PKT_MAC_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_MAC_ERROR)
375  #define TLSRX_HDR_PKT_MAC_ERROR_G(x)     \
376  	(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
377  #define TLSRX_HDR_PKT_MAC_ERROR_F        TLSRX_HDR_PKT_MAC_ERROR_V(1U)
378  
379  #define TLSRX_HDR_PKT_ERROR_M           0x1F
380  #define CONTENT_TYPE_ERROR		0x7F
381  
382  struct ulp_mem_rw {
383  	__be32 cmd;
384  	__be32 len16;             /* command length */
385  	__be32 dlen;              /* data length in 32-byte units */
386  	__be32 lock_addr;
387  };
388  
389  struct tls_key_wr {
390  	__be32 op_to_compl;
391  	__be32 flowid_len16;
392  	__be32 ftid;
393  	u8   reneg_to_write_rx;
394  	u8   protocol;
395  	__be16 mfs;
396  };
397  
398  struct tls_key_req {
399  	struct tls_key_wr wr;
400  	struct ulp_mem_rw req;
401  	struct ulptx_idata sc_imm;
402  };
403  
404  /*
405   * This lives in skb->cb and is used to chain WRs in a linked list.
406   */
407  struct wr_skb_cb {
408  	struct l2t_skb_cb l2t;          /* reserve space for l2t CB */
409  	struct sk_buff *next_wr;        /* next write request */
410  };
411  
412  /* Per-skb backlog handler.  Run when a socket's backlog is processed. */
413  struct blog_skb_cb {
414  	void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
415  	struct chtls_dev *cdev;
416  };
417  
418  /*
419   * Similar to tcp_skb_cb but with ULP elements added to support TLS,
420   * etc.
421   */
422  struct ulp_skb_cb {
423  	struct wr_skb_cb wr;		/* reserve space for write request */
424  	u16 flags;			/* TCP-like flags */
425  	u8 psh;
426  	u8 ulp_mode;			/* ULP mode/submode of sk_buff */
427  	u32 seq;			/* TCP sequence number */
428  	union { /* ULP-specific fields */
429  		struct {
430  			u8  type;
431  			u8  ofld;
432  			u8  iv;
433  		} tls;
434  	} ulp;
435  };
436  
437  #define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
438  #define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
439  
440  /*
441   * Flags for ulp_skb_cb.flags.
442   */
443  enum {
444  	ULPCB_FLAG_NEED_HDR  = 1 << 0,	/* packet needs a TX_DATA_WR header */
445  	ULPCB_FLAG_NO_APPEND = 1 << 1,	/* don't grow this skb */
446  	ULPCB_FLAG_BARRIER   = 1 << 2,	/* set TX_WAIT_IDLE after sending */
447  	ULPCB_FLAG_HOLD      = 1 << 3,	/* skb not ready for Tx yet */
448  	ULPCB_FLAG_COMPL     = 1 << 4,	/* request WR completion */
449  	ULPCB_FLAG_URG       = 1 << 5,	/* urgent data */
450  	ULPCB_FLAG_TLS_HDR   = 1 << 6,  /* payload with tls hdr */
451  	ULPCB_FLAG_NO_HDR    = 1 << 7,  /* not a ofld wr */
452  };
453  
454  /* The ULP mode/submode of an skbuff */
455  #define skb_ulp_mode(skb)  (ULP_SKB_CB(skb)->ulp_mode)
456  #define TCP_PAGE(sk)   (sk->sk_frag.page)
457  #define TCP_OFF(sk)    (sk->sk_frag.offset)
458  
to_chtls_dev(struct tls_toe_device * tlsdev)459  static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
460  {
461  	return container_of(tlsdev, struct chtls_dev, tlsdev);
462  }
463  
csk_set_flag(struct chtls_sock * csk,enum csk_flags flag)464  static inline void csk_set_flag(struct chtls_sock *csk,
465  				enum csk_flags flag)
466  {
467  	__set_bit(flag, &csk->flags);
468  }
469  
csk_reset_flag(struct chtls_sock * csk,enum csk_flags flag)470  static inline void csk_reset_flag(struct chtls_sock *csk,
471  				  enum csk_flags flag)
472  {
473  	__clear_bit(flag, &csk->flags);
474  }
475  
csk_conn_inline(const struct chtls_sock * csk)476  static inline bool csk_conn_inline(const struct chtls_sock *csk)
477  {
478  	return test_bit(CSK_CONN_INLINE, &csk->flags);
479  }
480  
csk_flag(const struct sock * sk,enum csk_flags flag)481  static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
482  {
483  	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
484  
485  	if (!csk_conn_inline(csk))
486  		return 0;
487  	return test_bit(flag, &csk->flags);
488  }
489  
csk_flag_nochk(const struct chtls_sock * csk,enum csk_flags flag)490  static inline int csk_flag_nochk(const struct chtls_sock *csk,
491  				 enum csk_flags flag)
492  {
493  	return test_bit(flag, &csk->flags);
494  }
495  
cplhdr(struct sk_buff * skb)496  static inline void *cplhdr(struct sk_buff *skb)
497  {
498  	return skb->data;
499  }
500  
is_neg_adv(unsigned int status)501  static inline int is_neg_adv(unsigned int status)
502  {
503  	return status == CPL_ERR_RTX_NEG_ADVICE ||
504  	       status == CPL_ERR_KEEPALV_NEG_ADVICE ||
505  	       status == CPL_ERR_PERSIST_NEG_ADVICE;
506  }
507  
process_cpl_msg(void (* fn)(struct sock *,struct sk_buff *),struct sock * sk,struct sk_buff * skb)508  static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
509  				   struct sock *sk,
510  				   struct sk_buff *skb)
511  {
512  	skb_reset_mac_header(skb);
513  	skb_reset_network_header(skb);
514  	skb_reset_transport_header(skb);
515  
516  	bh_lock_sock(sk);
517  	if (unlikely(sock_owned_by_user(sk))) {
518  		BLOG_SKB_CB(skb)->backlog_rcv = fn;
519  		__sk_add_backlog(sk, skb);
520  	} else {
521  		fn(sk, skb);
522  	}
523  	bh_unlock_sock(sk);
524  }
525  
chtls_sock_free(struct kref * ref)526  static inline void chtls_sock_free(struct kref *ref)
527  {
528  	struct chtls_sock *csk = container_of(ref, struct chtls_sock,
529  					      kref);
530  	kfree(csk);
531  }
532  
__chtls_sock_put(const char * fn,struct chtls_sock * csk)533  static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
534  {
535  	kref_put(&csk->kref, chtls_sock_free);
536  }
537  
__chtls_sock_get(const char * fn,struct chtls_sock * csk)538  static inline void __chtls_sock_get(const char *fn,
539  				    struct chtls_sock *csk)
540  {
541  	kref_get(&csk->kref);
542  }
543  
send_or_defer(struct sock * sk,struct tcp_sock * tp,struct sk_buff * skb,int through_l2t)544  static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
545  				 struct sk_buff *skb, int through_l2t)
546  {
547  	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
548  
549  	if (through_l2t) {
550  		/* send through L2T */
551  		cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
552  	} else {
553  		/* send directly */
554  		cxgb4_ofld_send(csk->egress_dev, skb);
555  	}
556  }
557  
558  typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
559  extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
560  void chtls_install_cpl_ops(struct sock *sk);
561  int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
562  void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
563  int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
564  void chtls_close(struct sock *sk, long timeout);
565  int chtls_disconnect(struct sock *sk, int flags);
566  void chtls_shutdown(struct sock *sk, int how);
567  void chtls_destroy_sock(struct sock *sk);
568  int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
569  int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
570  		  size_t len, int flags, int *addr_len);
571  int chtls_sendpage(struct sock *sk, struct page *page,
572  		   int offset, size_t size, int flags);
573  int send_tx_flowc_wr(struct sock *sk, int compl,
574  		     u32 snd_nxt, u32 rcv_nxt);
575  void chtls_tcp_push(struct sock *sk, int flags);
576  int chtls_push_frames(struct chtls_sock *csk, int comp);
577  int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
578  void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
579  				 u64 mask, u64 val, u8 cookie,
580  				 int through_l2t);
581  int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
582  void chtls_set_quiesce_ctrl(struct sock *sk, int val);
583  void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
584  unsigned int keyid_to_addr(int start_addr, int keyid);
585  void free_tls_keyid(struct sock *sk);
586  #endif
587