1 /*
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <crypto/aead.h>
35 #include <net/xfrm.h>
36 #include <net/esp.h>
37
38 #include "en_accel/ipsec_rxtx.h"
39 #include "en_accel/ipsec.h"
40 #include "accel/accel.h"
41 #include "en.h"
42
43 enum {
44 MLX5E_IPSEC_RX_SYNDROME_DECRYPTED = 0x11,
45 MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED = 0x12,
46 MLX5E_IPSEC_RX_SYNDROME_BAD_PROTO = 0x17,
47 };
48
49 struct mlx5e_ipsec_rx_metadata {
50 unsigned char nexthdr;
51 __be32 sa_handle;
52 } __packed;
53
54 enum {
55 MLX5E_IPSEC_TX_SYNDROME_OFFLOAD = 0x8,
56 MLX5E_IPSEC_TX_SYNDROME_OFFLOAD_WITH_LSO_TCP = 0x9,
57 };
58
59 struct mlx5e_ipsec_tx_metadata {
60 __be16 mss_inv; /* 1/MSS in 16bit fixed point, only for LSO */
61 __be16 seq; /* LSBs of the first TCP seq, only for LSO */
62 u8 esp_next_proto; /* Next protocol of ESP */
63 } __packed;
64
65 struct mlx5e_ipsec_metadata {
66 unsigned char syndrome;
67 union {
68 unsigned char raw[5];
69 /* from FPGA to host, on successful decrypt */
70 struct mlx5e_ipsec_rx_metadata rx;
71 /* from host to FPGA */
72 struct mlx5e_ipsec_tx_metadata tx;
73 } __packed content;
74 /* packet type ID field */
75 __be16 ethertype;
76 } __packed;
77
78 #define MAX_LSO_MSS 2048
79
80 /* Pre-calculated (Q0.16) fixed-point inverse 1/x function */
81 static __be16 mlx5e_ipsec_inverse_table[MAX_LSO_MSS];
82
mlx5e_ipsec_mss_inv(struct sk_buff * skb)83 static inline __be16 mlx5e_ipsec_mss_inv(struct sk_buff *skb)
84 {
85 return mlx5e_ipsec_inverse_table[skb_shinfo(skb)->gso_size];
86 }
87
mlx5e_ipsec_add_metadata(struct sk_buff * skb)88 static struct mlx5e_ipsec_metadata *mlx5e_ipsec_add_metadata(struct sk_buff *skb)
89 {
90 struct mlx5e_ipsec_metadata *mdata;
91 struct ethhdr *eth;
92
93 if (unlikely(skb_cow_head(skb, sizeof(*mdata))))
94 return ERR_PTR(-ENOMEM);
95
96 eth = (struct ethhdr *)skb_push(skb, sizeof(*mdata));
97 skb->mac_header -= sizeof(*mdata);
98 mdata = (struct mlx5e_ipsec_metadata *)(eth + 1);
99
100 memmove(skb->data, skb->data + sizeof(*mdata),
101 2 * ETH_ALEN);
102
103 eth->h_proto = cpu_to_be16(MLX5E_METADATA_ETHER_TYPE);
104
105 memset(mdata->content.raw, 0, sizeof(mdata->content.raw));
106 return mdata;
107 }
108
mlx5e_ipsec_remove_trailer(struct sk_buff * skb,struct xfrm_state * x)109 static int mlx5e_ipsec_remove_trailer(struct sk_buff *skb, struct xfrm_state *x)
110 {
111 unsigned int alen = crypto_aead_authsize(x->data);
112 struct ipv6hdr *ipv6hdr = ipv6_hdr(skb);
113 struct iphdr *ipv4hdr = ip_hdr(skb);
114 unsigned int trailer_len;
115 u8 plen;
116 int ret;
117
118 ret = skb_copy_bits(skb, skb->len - alen - 2, &plen, 1);
119 if (unlikely(ret))
120 return ret;
121
122 trailer_len = alen + plen + 2;
123
124 pskb_trim(skb, skb->len - trailer_len);
125 if (skb->protocol == htons(ETH_P_IP)) {
126 ipv4hdr->tot_len = htons(ntohs(ipv4hdr->tot_len) - trailer_len);
127 ip_send_check(ipv4hdr);
128 } else {
129 ipv6hdr->payload_len = htons(ntohs(ipv6hdr->payload_len) -
130 trailer_len);
131 }
132 return 0;
133 }
134
mlx5e_ipsec_set_swp(struct sk_buff * skb,struct mlx5_wqe_eth_seg * eseg,u8 mode,struct xfrm_offload * xo)135 static void mlx5e_ipsec_set_swp(struct sk_buff *skb,
136 struct mlx5_wqe_eth_seg *eseg, u8 mode,
137 struct xfrm_offload *xo)
138 {
139 u8 proto;
140
141 /* Tunnel Mode:
142 * SWP: OutL3 InL3 InL4
143 * Pkt: MAC IP ESP IP L4
144 *
145 * Transport Mode:
146 * SWP: OutL3 InL4
147 * InL3
148 * Pkt: MAC IP ESP L4
149 *
150 * Offsets are in 2-byte words, counting from start of frame
151 */
152 eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
153 if (skb->protocol == htons(ETH_P_IPV6))
154 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
155
156 if (mode == XFRM_MODE_TUNNEL) {
157 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
158 if (xo->proto == IPPROTO_IPV6) {
159 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
160 proto = inner_ipv6_hdr(skb)->nexthdr;
161 } else {
162 proto = inner_ip_hdr(skb)->protocol;
163 }
164 } else {
165 eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
166 if (skb->protocol == htons(ETH_P_IPV6))
167 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
168 proto = xo->proto;
169 }
170 switch (proto) {
171 case IPPROTO_UDP:
172 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
173 /* Fall through */
174 case IPPROTO_TCP:
175 eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
176 break;
177 }
178 }
179
mlx5e_ipsec_set_iv_esn(struct sk_buff * skb,struct xfrm_state * x,struct xfrm_offload * xo)180 void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x,
181 struct xfrm_offload *xo)
182 {
183 struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
184 __u32 oseq = replay_esn->oseq;
185 int iv_offset;
186 __be64 seqno;
187 u32 seq_hi;
188
189 if (unlikely(skb_is_gso(skb) && oseq < MLX5E_IPSEC_ESN_SCOPE_MID &&
190 MLX5E_IPSEC_ESN_SCOPE_MID < (oseq - skb_shinfo(skb)->gso_segs))) {
191 seq_hi = xo->seq.hi - 1;
192 } else {
193 seq_hi = xo->seq.hi;
194 }
195
196 /* Place the SN in the IV field */
197 seqno = cpu_to_be64(xo->seq.low + ((u64)seq_hi << 32));
198 iv_offset = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr);
199 skb_store_bits(skb, iv_offset, &seqno, 8);
200 }
201
mlx5e_ipsec_set_iv(struct sk_buff * skb,struct xfrm_state * x,struct xfrm_offload * xo)202 void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x,
203 struct xfrm_offload *xo)
204 {
205 int iv_offset;
206 __be64 seqno;
207
208 /* Place the SN in the IV field */
209 seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
210 iv_offset = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr);
211 skb_store_bits(skb, iv_offset, &seqno, 8);
212 }
213
mlx5e_ipsec_set_metadata(struct sk_buff * skb,struct mlx5e_ipsec_metadata * mdata,struct xfrm_offload * xo)214 static void mlx5e_ipsec_set_metadata(struct sk_buff *skb,
215 struct mlx5e_ipsec_metadata *mdata,
216 struct xfrm_offload *xo)
217 {
218 struct ip_esp_hdr *esph;
219 struct tcphdr *tcph;
220
221 if (skb_is_gso(skb)) {
222 /* Add LSO metadata indication */
223 esph = ip_esp_hdr(skb);
224 tcph = inner_tcp_hdr(skb);
225 netdev_dbg(skb->dev, " Offloading GSO packet outer L3 %u; L4 %u; Inner L3 %u; L4 %u\n",
226 skb->network_header,
227 skb->transport_header,
228 skb->inner_network_header,
229 skb->inner_transport_header);
230 netdev_dbg(skb->dev, " Offloading GSO packet of len %u; mss %u; TCP sp %u dp %u seq 0x%x ESP seq 0x%x\n",
231 skb->len, skb_shinfo(skb)->gso_size,
232 ntohs(tcph->source), ntohs(tcph->dest),
233 ntohl(tcph->seq), ntohl(esph->seq_no));
234 mdata->syndrome = MLX5E_IPSEC_TX_SYNDROME_OFFLOAD_WITH_LSO_TCP;
235 mdata->content.tx.mss_inv = mlx5e_ipsec_mss_inv(skb);
236 mdata->content.tx.seq = htons(ntohl(tcph->seq) & 0xFFFF);
237 } else {
238 mdata->syndrome = MLX5E_IPSEC_TX_SYNDROME_OFFLOAD;
239 }
240 mdata->content.tx.esp_next_proto = xo->proto;
241
242 netdev_dbg(skb->dev, " TX metadata syndrome %u proto %u mss_inv %04x seq %04x\n",
243 mdata->syndrome, mdata->content.tx.esp_next_proto,
244 ntohs(mdata->content.tx.mss_inv),
245 ntohs(mdata->content.tx.seq));
246 }
247
mlx5e_ipsec_handle_tx_skb(struct net_device * netdev,struct mlx5e_tx_wqe * wqe,struct sk_buff * skb)248 struct sk_buff *mlx5e_ipsec_handle_tx_skb(struct net_device *netdev,
249 struct mlx5e_tx_wqe *wqe,
250 struct sk_buff *skb)
251 {
252 struct mlx5e_priv *priv = netdev_priv(netdev);
253 struct xfrm_offload *xo = xfrm_offload(skb);
254 struct mlx5e_ipsec_metadata *mdata;
255 struct mlx5e_ipsec_sa_entry *sa_entry;
256 struct xfrm_state *x;
257
258 if (!xo)
259 return skb;
260
261 if (unlikely(skb->sp->len != 1)) {
262 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_bundle);
263 goto drop;
264 }
265
266 x = xfrm_input_state(skb);
267 if (unlikely(!x)) {
268 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_no_state);
269 goto drop;
270 }
271
272 if (unlikely(!x->xso.offload_handle ||
273 (skb->protocol != htons(ETH_P_IP) &&
274 skb->protocol != htons(ETH_P_IPV6)))) {
275 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_not_ip);
276 goto drop;
277 }
278
279 if (!skb_is_gso(skb))
280 if (unlikely(mlx5e_ipsec_remove_trailer(skb, x))) {
281 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_trailer);
282 goto drop;
283 }
284 mdata = mlx5e_ipsec_add_metadata(skb);
285 if (IS_ERR(mdata)) {
286 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_metadata);
287 goto drop;
288 }
289 mlx5e_ipsec_set_swp(skb, &wqe->eth, x->props.mode, xo);
290 sa_entry = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
291 sa_entry->set_iv_op(skb, x, xo);
292 mlx5e_ipsec_set_metadata(skb, mdata, xo);
293
294 return skb;
295
296 drop:
297 kfree_skb(skb);
298 return NULL;
299 }
300
301 static inline struct xfrm_state *
mlx5e_ipsec_build_sp(struct net_device * netdev,struct sk_buff * skb,struct mlx5e_ipsec_metadata * mdata)302 mlx5e_ipsec_build_sp(struct net_device *netdev, struct sk_buff *skb,
303 struct mlx5e_ipsec_metadata *mdata)
304 {
305 struct mlx5e_priv *priv = netdev_priv(netdev);
306 struct xfrm_offload *xo;
307 struct xfrm_state *xs;
308 u32 sa_handle;
309
310 skb->sp = secpath_dup(skb->sp);
311 if (unlikely(!skb->sp)) {
312 atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc);
313 return NULL;
314 }
315
316 sa_handle = be32_to_cpu(mdata->content.rx.sa_handle);
317 xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle);
318 if (unlikely(!xs)) {
319 atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
320 return NULL;
321 }
322
323 skb->sp->xvec[skb->sp->len++] = xs;
324 skb->sp->olen++;
325
326 xo = xfrm_offload(skb);
327 xo->flags = CRYPTO_DONE;
328 switch (mdata->syndrome) {
329 case MLX5E_IPSEC_RX_SYNDROME_DECRYPTED:
330 xo->status = CRYPTO_SUCCESS;
331 if (likely(priv->ipsec->no_trailer)) {
332 xo->flags |= XFRM_ESP_NO_TRAILER;
333 xo->proto = mdata->content.rx.nexthdr;
334 }
335 break;
336 case MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED:
337 xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
338 break;
339 case MLX5E_IPSEC_RX_SYNDROME_BAD_PROTO:
340 xo->status = CRYPTO_INVALID_PROTOCOL;
341 break;
342 default:
343 atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome);
344 return NULL;
345 }
346 return xs;
347 }
348
mlx5e_ipsec_handle_rx_skb(struct net_device * netdev,struct sk_buff * skb,u32 * cqe_bcnt)349 struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
350 struct sk_buff *skb, u32 *cqe_bcnt)
351 {
352 struct mlx5e_ipsec_metadata *mdata;
353 struct xfrm_state *xs;
354
355 if (!is_metadata_hdr_valid(skb))
356 return skb;
357
358 /* Use the metadata */
359 mdata = (struct mlx5e_ipsec_metadata *)(skb->data + ETH_HLEN);
360 xs = mlx5e_ipsec_build_sp(netdev, skb, mdata);
361 if (unlikely(!xs)) {
362 kfree_skb(skb);
363 return NULL;
364 }
365
366 remove_metadata_hdr(skb);
367 *cqe_bcnt -= MLX5E_METADATA_ETHER_LEN;
368
369 return skb;
370 }
371
mlx5e_ipsec_feature_check(struct sk_buff * skb,struct net_device * netdev,netdev_features_t features)372 bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev,
373 netdev_features_t features)
374 {
375 struct xfrm_state *x;
376
377 if (skb->sp && skb->sp->len) {
378 x = skb->sp->xvec[0];
379 if (x && x->xso.offload_handle)
380 return true;
381 }
382 return false;
383 }
384
mlx5e_ipsec_build_inverse_table(void)385 void mlx5e_ipsec_build_inverse_table(void)
386 {
387 u16 mss_inv;
388 u32 mss;
389
390 /* Calculate 1/x inverse table for use in GSO data path.
391 * Using this table, we provide the IPSec accelerator with the value of
392 * 1/gso_size so that it can infer the position of each segment inside
393 * the GSO, and increment the ESP sequence number, and generate the IV.
394 * The HW needs this value in Q0.16 fixed-point number format
395 */
396 mlx5e_ipsec_inverse_table[1] = htons(0xFFFF);
397 for (mss = 2; mss < MAX_LSO_MSS; mss++) {
398 mss_inv = div_u64(1ULL << 32, mss) >> 16;
399 mlx5e_ipsec_inverse_table[mss] = htons(mss_inv);
400 }
401 }
402
