1415 static inline void __flow_hash_consistentify(struct flow_keys *keys)  in __flow_hash_consistentify()  argument
1419 	switch (keys->control.addr_type) {  in __flow_hash_consistentify()
1421 		addr_diff = (__force u32)keys->addrs.v4addrs.dst -  in __flow_hash_consistentify()
1422 			    (__force u32)keys->addrs.v4addrs.src;  in __flow_hash_consistentify()
1425 		     ((__force u16)keys->ports.dst <  in __flow_hash_consistentify()
1426 		      (__force u16)keys->ports.src))) {  in __flow_hash_consistentify()
1427 			swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);  in __flow_hash_consistentify()
1428 			swap(keys->ports.src, keys->ports.dst);  in __flow_hash_consistentify()
1432 		addr_diff = memcmp(&keys->addrs.v6addrs.dst,  in __flow_hash_consistentify()
1433 				   &keys->addrs.v6addrs.src,  in __flow_hash_consistentify()
1434 				   sizeof(keys->addrs.v6addrs.dst));  in __flow_hash_consistentify()
1437 		     ((__force u16)keys->ports.dst <  in __flow_hash_consistentify()
1438 		      (__force u16)keys->ports.src))) {  in __flow_hash_consistentify()
1440 				swap(keys->addrs.v6addrs.src.s6_addr32[i],  in __flow_hash_consistentify()
1441 				     keys->addrs.v6addrs.dst.s6_addr32[i]);  in __flow_hash_consistentify()
1442 			swap(keys->ports.src, keys->ports.dst);  in __flow_hash_consistentify()
1448 static inline u32 __flow_hash_from_keys(struct flow_keys *keys,  in __flow_hash_from_keys()  argument
1453 	__flow_hash_consistentify(keys);  in __flow_hash_from_keys()
1455 	hash = siphash(flow_keys_hash_start(keys),  in __flow_hash_from_keys()
1456 		       flow_keys_hash_length(keys), keyval);  in __flow_hash_from_keys()
1463 u32 flow_hash_from_keys(struct flow_keys *keys)  in flow_hash_from_keys()  argument
1466 	return __flow_hash_from_keys(keys, &hashrnd);  in flow_hash_from_keys()
1471 				  struct flow_keys *keys,  in ___skb_get_hash()  argument
1474 	skb_flow_dissect_flow_keys(skb, keys,  in ___skb_get_hash()
1477 	return __flow_hash_from_keys(keys, keyval);  in ___skb_get_hash()
1511 	struct flow_keys keys;  in __skb_get_hash_symmetric()  local
1515 	memset(&keys, 0, sizeof(keys));  in __skb_get_hash_symmetric()
1517 			   &keys, NULL, 0, 0, 0,  in __skb_get_hash_symmetric()
1520 	return __flow_hash_from_keys(&keys, &hashrnd);  in __skb_get_hash_symmetric()
1535 	struct flow_keys keys;  in __skb_get_hash()  local
1540 	hash = ___skb_get_hash(skb, &keys, &hashrnd);  in __skb_get_hash()
1542 	__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));  in __skb_get_hash()
1549 	struct flow_keys keys;  in skb_get_hash_perturb()  local
1551 	return ___skb_get_hash(skb, &keys, perturb);  in skb_get_hash_perturb()
1556 		   const struct flow_keys_basic *keys, int hlen)  in __skb_get_poff()  argument
1558 	u32 poff = keys->control.thoff;  in __skb_get_poff()
1561 	if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&  in __skb_get_poff()
1562 	    !(keys->control.flags & FLOW_DIS_FIRST_FRAG))  in __skb_get_poff()
1565 	switch (keys->basic.ip_proto) {  in __skb_get_poff()
1617 	struct flow_keys_basic keys;  in skb_get_poff()  local
1619 	if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,  in skb_get_poff()
1623 	return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));  in skb_get_poff()
1626 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)  in __get_hash_from_flowi6()  argument
1628 	memset(keys, 0, sizeof(*keys));  in __get_hash_from_flowi6()
1630 	memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,  in __get_hash_from_flowi6()
1631 	    sizeof(keys->addrs.v6addrs.src));  in __get_hash_from_flowi6()
1632 	memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,  in __get_hash_from_flowi6()
1633 	    sizeof(keys->addrs.v6addrs.dst));  in __get_hash_from_flowi6()
1634 	keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;  in __get_hash_from_flowi6()
1635 	keys->ports.src = fl6->fl6_sport;  in __get_hash_from_flowi6()
1636 	keys->ports.dst = fl6->fl6_dport;  in __get_hash_from_flowi6()
1637 	keys->keyid.keyid = fl6->fl6_gre_key;  in __get_hash_from_flowi6()
1638 	keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);  in __get_hash_from_flowi6()
1639 	keys->basic.ip_proto = fl6->flowi6_proto;  in __get_hash_from_flowi6()
1641 	return flow_hash_from_keys(keys);  in __get_hash_from_flowi6()