| #include <linux/skbuff.h> |
| #include <linux/export.h> |
| #include <linux/ip.h> |
| #include <linux/ipv6.h> |
| #include <linux/if_vlan.h> |
| #include <net/ip.h> |
| #include <net/ipv6.h> |
| #include <linux/igmp.h> |
| #include <linux/icmp.h> |
| #include <linux/sctp.h> |
| #include <linux/dccp.h> |
| #include <linux/if_tunnel.h> |
| #include <linux/if_pppox.h> |
| #include <linux/ppp_defs.h> |
| #include <net/flow_dissector.h> |
| #include <scsi/fc/fc_fcoe.h> |
| |
| /* copy saddr & daddr, possibly using 64bit load/store |
| * Equivalent to : flow->src = iph->saddr; |
| * flow->dst = iph->daddr; |
| */ |
| static void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph) |
| { |
| BUILD_BUG_ON(offsetof(typeof(*flow), dst) != |
| offsetof(typeof(*flow), src) + sizeof(flow->src)); |
| memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst)); |
| } |
| |
| /** |
| * __skb_flow_get_ports - extract the upper layer ports and return them |
| * @skb: sk_buff to extract the ports from |
| * @thoff: transport header offset |
| * @ip_proto: protocol for which to get port offset |
| * @data: raw buffer pointer to the packet, if NULL use skb->data |
| * @hlen: packet header length, if @data is NULL use skb_headlen(skb) |
| * |
| * The function will try to retrieve the ports at offset thoff + poff where poff |
| * is the protocol port offset returned from proto_ports_offset |
| */ |
| __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, |
| void *data, int hlen) |
| { |
| int poff = proto_ports_offset(ip_proto); |
| |
| if (!data) { |
| data = skb->data; |
| hlen = skb_headlen(skb); |
| } |
| |
| if (poff >= 0) { |
| __be32 *ports, _ports; |
| |
| ports = __skb_header_pointer(skb, thoff + poff, |
| sizeof(_ports), data, hlen, &_ports); |
| if (ports) |
| return *ports; |
| } |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(__skb_flow_get_ports); |
| |
| /** |
| * __skb_flow_dissect - extract the flow_keys struct and return it |
| * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified |
| * @data: raw buffer pointer to the packet, if NULL use skb->data |
| * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol |
| * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) |
| * @hlen: packet header length, if @data is NULL use skb_headlen(skb) |
| * |
| * The function will try to retrieve the struct flow_keys from either the skbuff |
| * or a raw buffer specified by the rest parameters |
| */ |
| bool __skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow, |
| void *data, __be16 proto, int nhoff, int hlen) |
| { |
| u8 ip_proto; |
| |
| if (!data) { |
| data = skb->data; |
| proto = skb->protocol; |
| nhoff = skb_network_offset(skb); |
| hlen = skb_headlen(skb); |
| } |
| |
| memset(flow, 0, sizeof(*flow)); |
| |
| again: |
| switch (proto) { |
| case htons(ETH_P_IP): { |
| const struct iphdr *iph; |
| struct iphdr _iph; |
| ip: |
| iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); |
| if (!iph || iph->ihl < 5) |
| return false; |
| nhoff += iph->ihl * 4; |
| |
| ip_proto = iph->protocol; |
| if (ip_is_fragment(iph)) |
| ip_proto = 0; |
| |
| /* skip the address processing if skb is NULL. The assumption |
| * here is that if there is no skb we are not looking for flow |
| * info but lengths and protocols. |
| */ |
| if (!skb) |
| break; |
| |
| iph_to_flow_copy_addrs(flow, iph); |
| break; |
| } |
| case htons(ETH_P_IPV6): { |
| const struct ipv6hdr *iph; |
| struct ipv6hdr _iph; |
| __be32 flow_label; |
| |
| ipv6: |
| iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); |
| if (!iph) |
| return false; |
| |
| ip_proto = iph->nexthdr; |
| nhoff += sizeof(struct ipv6hdr); |
| |
| /* see comment above in IPv4 section */ |
| if (!skb) |
| break; |
| |
| flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr); |
| flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr); |
| |
| flow_label = ip6_flowlabel(iph); |
| if (flow_label) { |
| /* Awesome, IPv6 packet has a flow label so we can |
| * use that to represent the ports without any |
| * further dissection. |
| */ |
| flow->n_proto = proto; |
| flow->ip_proto = ip_proto; |
| flow->ports = flow_label; |
| flow->thoff = (u16)nhoff; |
| |
| return true; |
| } |
| |
| break; |
| } |
| case htons(ETH_P_8021AD): |
| case htons(ETH_P_8021Q): { |
| const struct vlan_hdr *vlan; |
| struct vlan_hdr _vlan; |
| |
| vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan); |
| if (!vlan) |
| return false; |
| |
| proto = vlan->h_vlan_encapsulated_proto; |
| nhoff += sizeof(*vlan); |
| goto again; |
| } |
| case htons(ETH_P_PPP_SES): { |
| struct { |
| struct pppoe_hdr hdr; |
| __be16 proto; |
| } *hdr, _hdr; |
| hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); |
| if (!hdr) |
| return false; |
| proto = hdr->proto; |
| nhoff += PPPOE_SES_HLEN; |
| switch (proto) { |
| case htons(PPP_IP): |
| goto ip; |
| case htons(PPP_IPV6): |
| goto ipv6; |
| default: |
| return false; |
| } |
| } |
| case htons(ETH_P_TIPC): { |
| struct { |
| __be32 pre[3]; |
| __be32 srcnode; |
| } *hdr, _hdr; |
| hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); |
| if (!hdr) |
| return false; |
| flow->src = hdr->srcnode; |
| flow->dst = 0; |
| flow->n_proto = proto; |
| flow->thoff = (u16)nhoff; |
| return true; |
| } |
| case htons(ETH_P_FCOE): |
| flow->thoff = (u16)(nhoff + FCOE_HEADER_LEN); |
| /* fall through */ |
| default: |
| return false; |
| } |
| |
| switch (ip_proto) { |
| case IPPROTO_GRE: { |
| struct gre_hdr { |
| __be16 flags; |
| __be16 proto; |
| } *hdr, _hdr; |
| |
| hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); |
| if (!hdr) |
| return false; |
| /* |
| * Only look inside GRE if version zero and no |
| * routing |
| */ |
| if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) { |
| proto = hdr->proto; |
| nhoff += 4; |
| if (hdr->flags & GRE_CSUM) |
| nhoff += 4; |
| if (hdr->flags & GRE_KEY) |
| nhoff += 4; |
| if (hdr->flags & GRE_SEQ) |
| nhoff += 4; |
| if (proto == htons(ETH_P_TEB)) { |
| const struct ethhdr *eth; |
| struct ethhdr _eth; |
| |
| eth = __skb_header_pointer(skb, nhoff, |
| sizeof(_eth), |
| data, hlen, &_eth); |
| if (!eth) |
| return false; |
| proto = eth->h_proto; |
| nhoff += sizeof(*eth); |
| } |
| goto again; |
| } |
| break; |
| } |
| case IPPROTO_IPIP: |
| proto = htons(ETH_P_IP); |
| goto ip; |
| case IPPROTO_IPV6: |
| proto = htons(ETH_P_IPV6); |
| goto ipv6; |
| default: |
| break; |
| } |
| |
| flow->n_proto = proto; |
| flow->ip_proto = ip_proto; |
| flow->thoff = (u16) nhoff; |
| |
| /* unless skb is set we don't need to record port info */ |
| if (skb) |
| flow->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, |
| data, hlen); |
| |
| return true; |
| } |
| EXPORT_SYMBOL(__skb_flow_dissect); |
| |
| static u32 hashrnd __read_mostly; |
| static __always_inline void __flow_hash_secret_init(void) |
| { |
| net_get_random_once(&hashrnd, sizeof(hashrnd)); |
| } |
| |
| static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval) |
| { |
| return jhash_3words(a, b, c, keyval); |
| } |
| |
| static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval) |
| { |
| u32 hash; |
| |
| /* get a consistent hash (same value on both flow directions) */ |
| if (((__force u32)keys->dst < (__force u32)keys->src) || |
| (((__force u32)keys->dst == (__force u32)keys->src) && |
| ((__force u16)keys->port16[1] < (__force u16)keys->port16[0]))) { |
| swap(keys->dst, keys->src); |
| swap(keys->port16[0], keys->port16[1]); |
| } |
| |
| hash = __flow_hash_3words((__force u32)keys->dst, |
| (__force u32)keys->src, |
| (__force u32)keys->ports, |
| keyval); |
| if (!hash) |
| hash = 1; |
| |
| return hash; |
| } |
| |
| u32 flow_hash_from_keys(struct flow_keys *keys) |
| { |
| __flow_hash_secret_init(); |
| return __flow_hash_from_keys(keys, hashrnd); |
| } |
| EXPORT_SYMBOL(flow_hash_from_keys); |
| |
| static inline u32 ___skb_get_hash(const struct sk_buff *skb, |
| struct flow_keys *keys, u32 keyval) |
| { |
| if (!skb_flow_dissect(skb, keys)) |
| return 0; |
| |
| return __flow_hash_from_keys(keys, keyval); |
| } |
| |
| struct _flow_keys_digest_data { |
| __be16 n_proto; |
| u8 ip_proto; |
| u8 padding; |
| __be32 ports; |
| __be32 src; |
| __be32 dst; |
| }; |
| |
| void make_flow_keys_digest(struct flow_keys_digest *digest, |
| const struct flow_keys *flow) |
| { |
| struct _flow_keys_digest_data *data = |
| (struct _flow_keys_digest_data *)digest; |
| |
| BUILD_BUG_ON(sizeof(*data) > sizeof(*digest)); |
| |
| memset(digest, 0, sizeof(*digest)); |
| |
| data->n_proto = flow->n_proto; |
| data->ip_proto = flow->ip_proto; |
| data->ports = flow->ports; |
| data->src = flow->src; |
| data->dst = flow->dst; |
| } |
| EXPORT_SYMBOL(make_flow_keys_digest); |
| |
| /** |
| * __skb_get_hash: calculate a flow hash |
| * @skb: sk_buff to calculate flow hash from |
| * |
| * This function calculates a flow hash based on src/dst addresses |
| * and src/dst port numbers. Sets hash in skb to non-zero hash value |
| * on success, zero indicates no valid hash. Also, sets l4_hash in skb |
| * if hash is a canonical 4-tuple hash over transport ports. |
| */ |
| void __skb_get_hash(struct sk_buff *skb) |
| { |
| struct flow_keys keys; |
| u32 hash; |
| |
| __flow_hash_secret_init(); |
| |
| hash = ___skb_get_hash(skb, &keys, hashrnd); |
| if (!hash) |
| return; |
| if (keys.ports) |
| skb->l4_hash = 1; |
| skb->sw_hash = 1; |
| skb->hash = hash; |
| } |
| EXPORT_SYMBOL(__skb_get_hash); |
| |
| __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb) |
| { |
| struct flow_keys keys; |
| |
| return ___skb_get_hash(skb, &keys, perturb); |
| } |
| EXPORT_SYMBOL(skb_get_hash_perturb); |
| |
| u32 __skb_get_poff(const struct sk_buff *skb, void *data, |
| const struct flow_keys *keys, int hlen) |
| { |
| u32 poff = keys->thoff; |
| |
| switch (keys->ip_proto) { |
| case IPPROTO_TCP: { |
| /* access doff as u8 to avoid unaligned access */ |
| const u8 *doff; |
| u8 _doff; |
| |
| doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), |
| data, hlen, &_doff); |
| if (!doff) |
| return poff; |
| |
| poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); |
| break; |
| } |
| case IPPROTO_UDP: |
| case IPPROTO_UDPLITE: |
| poff += sizeof(struct udphdr); |
| break; |
| /* For the rest, we do not really care about header |
| * extensions at this point for now. |
| */ |
| case IPPROTO_ICMP: |
| poff += sizeof(struct icmphdr); |
| break; |
| case IPPROTO_ICMPV6: |
| poff += sizeof(struct icmp6hdr); |
| break; |
| case IPPROTO_IGMP: |
| poff += sizeof(struct igmphdr); |
| break; |
| case IPPROTO_DCCP: |
| poff += sizeof(struct dccp_hdr); |
| break; |
| case IPPROTO_SCTP: |
| poff += sizeof(struct sctphdr); |
| break; |
| } |
| |
| return poff; |
| } |
| |
| /* skb_get_poff() returns the offset to the payload as far as it could |
| * be dissected. The main user is currently BPF, so that we can dynamically |
| * truncate packets without needing to push actual payload to the user |
| * space and can analyze headers only, instead. |
| */ |
| u32 skb_get_poff(const struct sk_buff *skb) |
| { |
| struct flow_keys keys; |
| |
| if (!skb_flow_dissect(skb, &keys)) |
| return 0; |
| |
| return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); |
| } |