1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * xfrm_device.c - IPsec device offloading code.
25 	struct xfrm_offload *xo = xfrm_offload(skb);  in __xfrm_transport_prep()  local
28 	if (xo->flags & XFRM_GSO_SEGMENT)  in __xfrm_transport_prep()
29 		skb->transport_header -= x->props.header_len;  in __xfrm_transport_prep()
31 	pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);  in __xfrm_transport_prep()
38 	struct xfrm_offload *xo = xfrm_offload(skb);  in __xfrm_mode_tunnel_prep()  local
40 	if (xo->flags & XFRM_GSO_SEGMENT)  in __xfrm_mode_tunnel_prep()
41 		skb->transport_header = skb->network_header + hsize;  in __xfrm_mode_tunnel_prep()
44 	pskb_pull(skb, skb->mac_len + x->props.header_len);  in __xfrm_mode_tunnel_prep()
50 	struct xfrm_offload *xo = xfrm_offload(skb);  in __xfrm_mode_beet_prep()  local
53 	if (xo->flags & XFRM_GSO_SEGMENT)  in __xfrm_mode_beet_prep()
54 		skb->transport_header = skb->network_header + hsize;  in __xfrm_mode_beet_prep()
57 	if (x->sel.family != AF_INET6) {  in __xfrm_mode_beet_prep()
59 		if (x->outer_mode.family == AF_INET6)  in __xfrm_mode_beet_prep()
60 			phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);  in __xfrm_mode_beet_prep()
63 	pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));  in __xfrm_mode_beet_prep()
69 	switch (x->outer_mode.encap) {  in xfrm_outer_mode_prep()
71 		if (x->outer_mode.family == AF_INET)  in xfrm_outer_mode_prep()
74 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_prep()
79 		if (x->outer_mode.family == AF_INET)  in xfrm_outer_mode_prep()
82 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_prep()
87 		if (x->outer_mode.family == AF_INET)  in xfrm_outer_mode_prep()
90 		if (x->outer_mode.family == AF_INET6)  in xfrm_outer_mode_prep()
102 	struct xfrm_offload *xo = xfrm_offload(skb);  in xmit_xfrm_check_overflow()  local
103 	__u32 seq = xo->seq.low;  in xmit_xfrm_check_overflow()
105 	seq += skb_shinfo(skb)->gso_segs;  in xmit_xfrm_check_overflow()
106 	if (unlikely(seq < xo->seq.low))  in xmit_xfrm_check_overflow()
120 	struct xfrm_offload *xo = xfrm_offload(skb);  in validate_xmit_xfrm()  local
121 	struct net_device *dev = skb->dev;  in validate_xmit_xfrm()
124 	if (!xo || (xo->flags & XFRM_XMIT))  in validate_xmit_xfrm()
131 	x = sp->xvec[sp->len - 1];  in validate_xmit_xfrm()
132 	if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)  in validate_xmit_xfrm()
136 	if ((x->xso.dev != dev) && (x->xso.real_dev == dev))  in validate_xmit_xfrm()
141 	err = !skb_queue_empty(&sd->xfrm_backlog);  in validate_xmit_xfrm()
149 	if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) ||  in validate_xmit_xfrm()
167 	if (!skb->next) {  in validate_xmit_xfrm()
168 		esp_features |= skb->dev->gso_partial_features;  in validate_xmit_xfrm()
171 		xo->flags |= XFRM_DEV_RESUME;  in validate_xmit_xfrm()
173 		err = x->type_offload->xmit(x, skb, esp_features);  in validate_xmit_xfrm()
175 			if (err == -EINPROGRESS)  in validate_xmit_xfrm()
183 		skb_push(skb, skb->data - skb_mac_header(skb));  in validate_xmit_xfrm()
189 		esp_features |= skb->dev->gso_partial_features;  in validate_xmit_xfrm()
192 		xo = xfrm_offload(skb2);  in validate_xmit_xfrm()
193 		xo->flags |= XFRM_DEV_RESUME;  in validate_xmit_xfrm()
197 		err = x->type_offload->xmit(x, skb2, esp_features);  in validate_xmit_xfrm()
199 			skb2->next = nskb;  in validate_xmit_xfrm()
200 		} else if (err != -EINPROGRESS) {  in validate_xmit_xfrm()
202 			skb2->next = nskb;  in validate_xmit_xfrm()
209 				pskb->next = nskb;  in validate_xmit_xfrm()
214 		skb_push(skb2, skb2->data - skb_mac_header(skb2));  in validate_xmit_xfrm()
229 	struct xfrm_dev_offload *xso = &x->xso;  in xfrm_dev_state_add()
233 	if (!x->type_offload) {  in xfrm_dev_state_add()
235 		return -EINVAL;  in xfrm_dev_state_add()
239 	if (x->encap || x->tfcpad) {  in xfrm_dev_state_add()
241 		return -EINVAL;  in xfrm_dev_state_add()
244 	if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) {  in xfrm_dev_state_add()
246 		return -EINVAL;  in xfrm_dev_state_add()
249 	dev = dev_get_by_index(net, xuo->ifindex);  in xfrm_dev_state_add()
251 		if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {  in xfrm_dev_state_add()
252 			saddr = &x->props.saddr;  in xfrm_dev_state_add()
253 			daddr = &x->id.daddr;  in xfrm_dev_state_add()
255 			saddr = &x->id.daddr;  in xfrm_dev_state_add()
256 			daddr = &x->props.saddr;  in xfrm_dev_state_add()
260 					x->props.family,  in xfrm_dev_state_add()
265 		dev = dst->dev;  in xfrm_dev_state_add()
271 	if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {  in xfrm_dev_state_add()
272 		xso->dev = NULL;  in xfrm_dev_state_add()
277 	if (x->props.flags & XFRM_STATE_ESN &&  in xfrm_dev_state_add()
278 	    !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {  in xfrm_dev_state_add()
280 		xso->dev = NULL;  in xfrm_dev_state_add()
282 		return -EINVAL;  in xfrm_dev_state_add()
285 	xso->dev = dev;  in xfrm_dev_state_add()
286 	netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);  in xfrm_dev_state_add()
287 	xso->real_dev = dev;  in xfrm_dev_state_add()
289 	if (xuo->flags & XFRM_OFFLOAD_INBOUND)  in xfrm_dev_state_add()
290 		xso->dir = XFRM_DEV_OFFLOAD_IN;  in xfrm_dev_state_add()
292 		xso->dir = XFRM_DEV_OFFLOAD_OUT;  in xfrm_dev_state_add()
294 	err = dev->xfrmdev_ops->xdo_dev_state_add(x);  in xfrm_dev_state_add()
296 		xso->dev = NULL;  in xfrm_dev_state_add()
297 		xso->dir = 0;  in xfrm_dev_state_add()
298 		xso->real_dev = NULL;  in xfrm_dev_state_add()
299 		netdev_put(dev, &xso->dev_tracker);  in xfrm_dev_state_add()
301 		if (err != -EOPNOTSUPP) {  in xfrm_dev_state_add()
316 	struct net_device *dev = x->xso.dev;  in xfrm_dev_offload_ok()
318 	if (!x->type_offload || x->encap)  in xfrm_dev_offload_ok()
321 	if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&  in xfrm_dev_offload_ok()
322 	    (!xdst->child->xfrm)) {  in xfrm_dev_offload_ok()
323 		mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);  in xfrm_dev_offload_ok()
324 		if (skb->len <= mtu)  in xfrm_dev_offload_ok()
334 	if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)  in xfrm_dev_offload_ok()
335 		return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);  in xfrm_dev_offload_ok()
343 	struct net_device *dev = skb->dev;  in xfrm_dev_resume()
360 		skb_queue_tail(&sd->xfrm_backlog, skb);  in xfrm_dev_resume()
370 	struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;  in xfrm_dev_backlog()
379 	spin_lock(&xfrm_backlog->lock);  in xfrm_dev_backlog()
381 	spin_unlock(&xfrm_backlog->lock);  in xfrm_dev_backlog()
394 	if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&  in xfrm_api_check()
395 	    !(dev->features & NETIF_F_HW_ESP))  in xfrm_api_check()
398 	if ((dev->features & NETIF_F_HW_ESP) &&  in xfrm_api_check()
399 	    (!(dev->xfrmdev_ops &&  in xfrm_api_check()
400 	       dev->xfrmdev_ops->xdo_dev_state_add &&  in xfrm_api_check()
401 	       dev->xfrmdev_ops->xdo_dev_state_delete)))  in xfrm_api_check()
404 	if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))  in xfrm_api_check()
413 	if (dev->features & NETIF_F_HW_ESP)  in xfrm_dev_down()