Possible problem with e6afc8ac ("udp: remove headers from UDP packets before queueing")
[cascardo/linux.git] / net / ipv6 / udp.c
index 6bc5c66..f421c9f 100644 (file)
@@ -213,37 +213,28 @@ static struct sock *udp6_lib_lookup2(struct net *net,
                struct sk_buff *skb)
 {
        struct sock *sk, *result;
-       struct hlist_nulls_node *node;
        int score, badness, matches = 0, reuseport = 0;
-       bool select_ok = true;
        u32 hash = 0;
 
-begin:
        result = NULL;
        badness = -1;
-       udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {
+       udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
                score = compute_score2(sk, net, saddr, sport,
                                      daddr, hnum, dif);
                if (score > badness) {
-                       result = sk;
-                       badness = score;
                        reuseport = sk->sk_reuseport;
                        if (reuseport) {
                                hash = udp6_ehashfn(net, daddr, hnum,
                                                    saddr, sport);
-                               if (select_ok) {
-                                       struct sock *sk2;
 
-                                       sk2 = reuseport_select_sock(sk, hash, skb,
+                               result = reuseport_select_sock(sk, hash, skb,
                                                        sizeof(struct udphdr));
-                                       if (sk2) {
-                                               result = sk2;
-                                               select_ok = false;
-                                               goto found;
-                                       }
-                               }
+                               if (result)
+                                       return result;
                                matches = 1;
                        }
+                       result = sk;
+                       badness = score;
                } else if (score == badness && reuseport) {
                        matches++;
                        if (reciprocal_scale(hash, matches) == 0)
@@ -251,27 +242,10 @@ begin:
                        hash = next_pseudo_random32(hash);
                }
        }
-       /*
-        * if the nulls value we got at the end of this lookup is
-        * not the expected one, we must restart lookup.
-        * We probably met an item that was moved to another chain.
-        */
-       if (get_nulls_value(node) != slot2)
-               goto begin;
-
-       if (result) {
-found:
-               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
-                       result = NULL;
-               else if (unlikely(compute_score2(result, net, saddr, sport,
-                                 daddr, hnum, dif) < badness)) {
-                       sock_put(result);
-                       goto begin;
-               }
-       }
        return result;
 }
 
+/* rcu_read_lock() must be held */
 struct sock *__udp6_lib_lookup(struct net *net,
                                      const struct in6_addr *saddr, __be16 sport,
                                      const struct in6_addr *daddr, __be16 dport,
@@ -279,15 +253,12 @@ struct sock *__udp6_lib_lookup(struct net *net,
                                      struct sk_buff *skb)
 {
        struct sock *sk, *result;
-       struct hlist_nulls_node *node;
        unsigned short hnum = ntohs(dport);
        unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
        struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
        int score, badness, matches = 0, reuseport = 0;
-       bool select_ok = true;
        u32 hash = 0;
 
-       rcu_read_lock();
        if (hslot->count > 10) {
                hash2 = udp6_portaddr_hash(net, daddr, hnum);
                slot2 = hash2 & udptable->mask;
@@ -309,34 +280,26 @@ struct sock *__udp6_lib_lookup(struct net *net,
                                                  &in6addr_any, hnum, dif,
                                                  hslot2, slot2, skb);
                }
-               rcu_read_unlock();
                return result;
        }
 begin:
        result = NULL;
        badness = -1;
-       sk_nulls_for_each_rcu(sk, node, &hslot->head) {
+       sk_for_each_rcu(sk, &hslot->head) {
                score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);
                if (score > badness) {
-                       result = sk;
-                       badness = score;
                        reuseport = sk->sk_reuseport;
                        if (reuseport) {
                                hash = udp6_ehashfn(net, daddr, hnum,
                                                    saddr, sport);
-                               if (select_ok) {
-                                       struct sock *sk2;
-
-                                       sk2 = reuseport_select_sock(sk, hash, skb,
+                               result = reuseport_select_sock(sk, hash, skb,
                                                        sizeof(struct udphdr));
-                                       if (sk2) {
-                                               result = sk2;
-                                               select_ok = false;
-                                               goto found;
-                                       }
-                               }
+                               if (result)
+                                       return result;
                                matches = 1;
                        }
+                       result = sk;
+                       badness = score;
                } else if (score == badness && reuseport) {
                        matches++;
                        if (reciprocal_scale(hash, matches) == 0)
@@ -344,25 +307,6 @@ begin:
                        hash = next_pseudo_random32(hash);
                }
        }
-       /*
-        * if the nulls value we got at the end of this lookup is
-        * not the expected one, we must restart lookup.
-        * We probably met an item that was moved to another chain.
-        */
-       if (get_nulls_value(node) != slot)
-               goto begin;
-
-       if (result) {
-found:
-               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
-                       result = NULL;
-               else if (unlikely(compute_score(result, net, hnum, saddr, sport,
-                                       daddr, dport, dif) < badness)) {
-                       sock_put(result);
-                       goto begin;
-               }
-       }
-       rcu_read_unlock();
        return result;
 }
 EXPORT_SYMBOL_GPL(__udp6_lib_lookup);
@@ -371,23 +315,46 @@ static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb,
                                          __be16 sport, __be16 dport,
                                          struct udp_table *udptable)
 {
-       struct sock *sk;
        const struct ipv6hdr *iph = ipv6_hdr(skb);
+       struct sock *sk;
 
        sk = skb_steal_sock(skb);
        if (unlikely(sk))
                return sk;
-       return __udp6_lib_lookup(dev_net(skb_dst(skb)->dev), &iph->saddr, sport,
+       return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
                                 &iph->daddr, dport, inet6_iif(skb),
                                 udptable, skb);
 }
 
+struct sock *udp6_lib_lookup_skb(struct sk_buff *skb,
+                                __be16 sport, __be16 dport)
+{
+       const struct ipv6hdr *iph = ipv6_hdr(skb);
+
+       return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
+                                &iph->daddr, dport, inet6_iif(skb),
+                                &udp_table, skb);
+}
+EXPORT_SYMBOL_GPL(udp6_lib_lookup_skb);
+
+/* Must be called under rcu_read_lock().
+ * Does increment socket refcount.
+ */
+#if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \
+    IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY)
 struct sock *udp6_lib_lookup(struct net *net, const struct in6_addr *saddr, __be16 sport,
                             const struct in6_addr *daddr, __be16 dport, int dif)
 {
-       return __udp6_lib_lookup(net, saddr, sport, daddr, dport, dif, &udp_table, NULL);
+       struct sock *sk;
+
+       sk =  __udp6_lib_lookup(net, saddr, sport, daddr, dport,
+                               dif, &udp_table, NULL);
+       if (sk && !atomic_inc_not_zero(&sk->sk_refcnt))
+               sk = NULL;
+       return sk;
 }
 EXPORT_SYMBOL_GPL(udp6_lib_lookup);
+#endif
 
 /*
  *     This should be easy, if there is something there we
@@ -401,7 +368,7 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
        struct inet_sock *inet = inet_sk(sk);
        struct sk_buff *skb;
        unsigned int ulen, copied;
-       int peeked, off = 0;
+       int peeked, peeking, off;
        int err;
        int is_udplite = IS_UDPLITE(sk);
        bool checksum_valid = false;
@@ -415,15 +382,16 @@ int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                return ipv6_recv_rxpmtu(sk, msg, len, addr_len);
 
 try_again:
+       peeking = off = sk_peek_offset(sk, flags);
        skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),
                                  &peeked, &off, &err);
        if (!skb)
-               goto out;
+               return err;
 
-       ulen = skb->len - sizeof(struct udphdr);
+       ulen = skb->len;
        copied = len;
-       if (copied > ulen)
-               copied = ulen;
+       if (copied > ulen - off)
+               copied = ulen - off;
        else if (copied < ulen)
                msg->msg_flags |= MSG_TRUNC;
 
@@ -435,17 +403,16 @@ try_again:
         * coverage checksum (UDP-Lite), do it before the copy.
         */
 
-       if (copied < ulen || UDP_SKB_CB(skb)->partial_cov) {
+       if (copied < ulen || UDP_SKB_CB(skb)->partial_cov || peeking) {
                checksum_valid = !udp_lib_checksum_complete(skb);
                if (!checksum_valid)
                        goto csum_copy_err;
        }
 
        if (checksum_valid || skb_csum_unnecessary(skb))
-               err = skb_copy_datagram_msg(skb, sizeof(struct udphdr),
-                                           msg, copied);
+               err = skb_copy_datagram_msg(skb, off, msg, copied);
        else {
-               err = skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr), msg);
+               err = skb_copy_and_csum_datagram_msg(skb, off, msg);
                if (err == -EINVAL)
                        goto csum_copy_err;
        }
@@ -454,23 +421,22 @@ try_again:
                if (!peeked) {
                        atomic_inc(&sk->sk_drops);
                        if (is_udp4)
-                               UDP_INC_STATS_USER(sock_net(sk),
-                                                  UDP_MIB_INERRORS,
-                                                  is_udplite);
+                               UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
+                                             is_udplite);
                        else
-                               UDP6_INC_STATS_USER(sock_net(sk),
-                                                   UDP_MIB_INERRORS,
-                                                   is_udplite);
+                               UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
+                                              is_udplite);
                }
-               goto out_free;
+               skb_free_datagram_locked(sk, skb);
+               return err;
        }
        if (!peeked) {
                if (is_udp4)
-                       UDP_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_INDATAGRAMS, is_udplite);
+                       UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS,
+                                     is_udplite);
                else
-                       UDP6_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_INDATAGRAMS, is_udplite);
+                       UDP6_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS,
+                                      is_udplite);
        }
 
        sock_recv_ts_and_drops(msg, sk, skb);
@@ -510,24 +476,22 @@ try_again:
        if (flags & MSG_TRUNC)
                err = ulen;
 
-out_free:
-       skb_free_datagram_locked(sk, skb);
-out:
+       __skb_free_datagram_locked(sk, skb, peeking ? -err : err);
        return err;
 
 csum_copy_err:
        slow = lock_sock_fast(sk);
        if (!skb_kill_datagram(sk, skb, flags)) {
                if (is_udp4) {
-                       UDP_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_CSUMERRORS, is_udplite);
-                       UDP_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_INERRORS, is_udplite);
+                       UDP_INC_STATS(sock_net(sk),
+                                     UDP_MIB_CSUMERRORS, is_udplite);
+                       UDP_INC_STATS(sock_net(sk),
+                                     UDP_MIB_INERRORS, is_udplite);
                } else {
-                       UDP6_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_CSUMERRORS, is_udplite);
-                       UDP6_INC_STATS_USER(sock_net(sk),
-                                       UDP_MIB_INERRORS, is_udplite);
+                       UDP6_INC_STATS(sock_net(sk),
+                                      UDP_MIB_CSUMERRORS, is_udplite);
+                       UDP6_INC_STATS(sock_net(sk),
+                                      UDP_MIB_INERRORS, is_udplite);
                }
        }
        unlock_sock_fast(sk, slow);
@@ -555,8 +519,8 @@ void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
        sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source,
                               inet6_iif(skb), udptable, skb);
        if (!sk) {
-               ICMP6_INC_STATS_BH(net, __in6_dev_get(skb->dev),
-                                  ICMP6_MIB_INERRORS);
+               __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
+                                 ICMP6_MIB_INERRORS);
                return;
        }
 
@@ -585,7 +549,7 @@ void __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
        sk->sk_err = err;
        sk->sk_error_report(sk);
 out:
-       sock_put(sk);
+       return;
 }
 
 static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
@@ -598,15 +562,15 @@ static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
                sk_incoming_cpu_update(sk);
        }
 
-       rc = sock_queue_rcv_skb(sk, skb);
+       rc = __sock_queue_rcv_skb(sk, skb);
        if (rc < 0) {
                int is_udplite = IS_UDPLITE(sk);
 
                /* Note that an ENOMEM error is charged twice */
                if (rc == -ENOMEM)
-                       UDP6_INC_STATS_BH(sock_net(sk),
-                                       UDP_MIB_RCVBUFERRORS, is_udplite);
-               UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
+                       UDP6_INC_STATS(sock_net(sk),
+                                        UDP_MIB_RCVBUFERRORS, is_udplite);
+               UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
                kfree_skb(skb);
                return -1;
        }
@@ -653,7 +617,7 @@ int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 
                /* if we're overly short, let UDP handle it */
                encap_rcv = ACCESS_ONCE(up->encap_rcv);
-               if (skb->len > sizeof(struct udphdr) && encap_rcv) {
+               if (encap_rcv) {
                        int ret;
 
                        /* Verify checksum before giving to encap */
@@ -662,9 +626,9 @@ int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 
                        ret = encap_rcv(sk, skb);
                        if (ret <= 0) {
-                               UDP_INC_STATS_BH(sock_net(sk),
-                                                UDP_MIB_INDATAGRAMS,
-                                                is_udplite);
+                               __UDP_INC_STATS(sock_net(sk),
+                                               UDP_MIB_INDATAGRAMS,
+                                               is_udplite);
                                return -ret;
                        }
                }
@@ -689,14 +653,17 @@ int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
                }
        }
 
-       if (rcu_access_pointer(sk->sk_filter)) {
-               if (udp_lib_checksum_complete(skb))
-                       goto csum_error;
-       }
+       if (rcu_access_pointer(sk->sk_filter) &&
+           udp_lib_checksum_complete(skb))
+               goto csum_error;
 
+       if (sk_filter(sk, skb))
+               goto drop;
+
+       udp_csum_pull_header(skb);
        if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
-               UDP6_INC_STATS_BH(sock_net(sk),
-                                 UDP_MIB_RCVBUFERRORS, is_udplite);
+               __UDP6_INC_STATS(sock_net(sk),
+                                UDP_MIB_RCVBUFERRORS, is_udplite);
                goto drop;
        }
 
@@ -715,9 +682,9 @@ int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
        return rc;
 
 csum_error:
-       UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
+       __UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
 drop:
-       UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
+       __UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
        atomic_inc(&sk->sk_drops);
        kfree_skb(skb);
        return -1;
@@ -747,33 +714,6 @@ static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk,
        return true;
 }
 
-static void flush_stack(struct sock **stack, unsigned int count,
-                       struct sk_buff *skb, unsigned int final)
-{
-       struct sk_buff *skb1 = NULL;
-       struct sock *sk;
-       unsigned int i;
-
-       for (i = 0; i < count; i++) {
-               sk = stack[i];
-               if (likely(!skb1))
-                       skb1 = (i == final) ? skb : skb_clone(skb, GFP_ATOMIC);
-               if (!skb1) {
-                       atomic_inc(&sk->sk_drops);
-                       UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
-                                         IS_UDPLITE(sk));
-                       UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,
-                                         IS_UDPLITE(sk));
-               }
-
-               if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)
-                       skb1 = NULL;
-               sock_put(sk);
-       }
-       if (unlikely(skb1))
-               kfree_skb(skb1);
-}
-
 static void udp6_csum_zero_error(struct sk_buff *skb)
 {
        /* RFC 2460 section 8.1 says that we SHOULD log
@@ -792,15 +732,15 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
                const struct in6_addr *saddr, const struct in6_addr *daddr,
                struct udp_table *udptable, int proto)
 {
-       struct sock *sk, *stack[256 / sizeof(struct sock *)];
+       struct sock *sk, *first = NULL;
        const struct udphdr *uh = udp_hdr(skb);
-       struct hlist_nulls_node *node;
        unsigned short hnum = ntohs(uh->dest);
        struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
-       int dif = inet6_iif(skb);
-       unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
+       unsigned int offset = offsetof(typeof(*sk), sk_node);
        unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
-       bool inner_flushed = false;
+       int dif = inet6_iif(skb);
+       struct hlist_node *node;
+       struct sk_buff *nskb;
 
        if (use_hash2) {
                hash2_any = udp6_portaddr_hash(net, &in6addr_any, hnum) &
@@ -811,27 +751,32 @@ start_lookup:
                offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
        }
 
-       spin_lock(&hslot->lock);
-       sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
-               if (__udp_v6_is_mcast_sock(net, sk,
-                                          uh->dest, daddr,
-                                          uh->source, saddr,
-                                          dif, hnum) &&
-                   /* If zero checksum and no_check is not on for
-                    * the socket then skip it.
-                    */
-                   (uh->check || udp_sk(sk)->no_check6_rx)) {
-                       if (unlikely(count == ARRAY_SIZE(stack))) {
-                               flush_stack(stack, count, skb, ~0);
-                               inner_flushed = true;
-                               count = 0;
-                       }
-                       stack[count++] = sk;
-                       sock_hold(sk);
+       sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
+               if (!__udp_v6_is_mcast_sock(net, sk, uh->dest, daddr,
+                                           uh->source, saddr, dif, hnum))
+                       continue;
+               /* If zero checksum and no_check is not on for
+                * the socket then skip it.
+                */
+               if (!uh->check && !udp_sk(sk)->no_check6_rx)
+                       continue;
+               if (!first) {
+                       first = sk;
+                       continue;
+               }
+               nskb = skb_clone(skb, GFP_ATOMIC);
+               if (unlikely(!nskb)) {
+                       atomic_inc(&sk->sk_drops);
+                       __UDP6_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
+                                        IS_UDPLITE(sk));
+                       __UDP6_INC_STATS(net, UDP_MIB_INERRORS,
+                                        IS_UDPLITE(sk));
+                       continue;
                }
-       }
 
-       spin_unlock(&hslot->lock);
+               if (udpv6_queue_rcv_skb(sk, nskb) > 0)
+                       consume_skb(nskb);
+       }
 
        /* Also lookup *:port if we are using hash2 and haven't done so yet. */
        if (use_hash2 && hash2 != hash2_any) {
@@ -839,13 +784,13 @@ start_lookup:
                goto start_lookup;
        }
 
-       if (count) {
-               flush_stack(stack, count, skb, count - 1);
+       if (first) {
+               if (udpv6_queue_rcv_skb(first, skb) > 0)
+                       consume_skb(skb);
        } else {
-               if (!inner_flushed)
-                       UDP6_INC_STATS_BH(net, UDP_MIB_IGNOREDMULTI,
-                                         proto == IPPROTO_UDPLITE);
-               consume_skb(skb);
+               kfree_skb(skb);
+               __UDP6_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
+                                proto == IPPROTO_UDPLITE);
        }
        return 0;
 }
@@ -853,10 +798,10 @@ start_lookup:
 int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
                   int proto)
 {
+       const struct in6_addr *saddr, *daddr;
        struct net *net = dev_net(skb->dev);
-       struct sock *sk;
        struct udphdr *uh;
-       const struct in6_addr *saddr, *daddr;
+       struct sock *sk;
        u32 ulen = 0;
 
        if (!pskb_may_pull(skb, sizeof(struct udphdr)))
@@ -910,7 +855,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
                int ret;
 
                if (!uh->check && !udp_sk(sk)->no_check6_rx) {
-                       sock_put(sk);
                        udp6_csum_zero_error(skb);
                        goto csum_error;
                }
@@ -920,7 +864,6 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
                                                 ip6_compute_pseudo);
 
                ret = udpv6_queue_rcv_skb(sk, skb);
-               sock_put(sk);
 
                /* a return value > 0 means to resubmit the input */
                if (ret > 0)
@@ -940,7 +883,7 @@ int __udp6_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
        if (udp_lib_checksum_complete(skb))
                goto csum_error;
 
-       UDP6_INC_STATS_BH(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
+       __UDP6_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
 
        kfree_skb(skb);
@@ -954,9 +897,9 @@ short_packet:
                            daddr, ntohs(uh->dest));
        goto discard;
 csum_error:
-       UDP6_INC_STATS_BH(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
+       __UDP6_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
 discard:
-       UDP6_INC_STATS_BH(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
+       __UDP6_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
        kfree_skb(skb);
        return 0;
 }
@@ -1068,13 +1011,14 @@ send:
        err = ip6_send_skb(skb);
        if (err) {
                if (err == -ENOBUFS && !inet6_sk(sk)->recverr) {
-                       UDP6_INC_STATS_USER(sock_net(sk),
-                                           UDP_MIB_SNDBUFERRORS, is_udplite);
+                       UDP6_INC_STATS(sock_net(sk),
+                                      UDP_MIB_SNDBUFERRORS, is_udplite);
                        err = 0;
                }
-       } else
-               UDP6_INC_STATS_USER(sock_net(sk),
-                                   UDP_MIB_OUTDATAGRAMS, is_udplite);
+       } else {
+               UDP6_INC_STATS(sock_net(sk),
+                              UDP_MIB_OUTDATAGRAMS, is_udplite);
+       }
        return err;
 }
 
@@ -1118,16 +1062,19 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
        struct ip6_flowlabel *flowlabel = NULL;
        struct flowi6 fl6;
        struct dst_entry *dst;
+       struct ipcm6_cookie ipc6;
        int addr_len = msg->msg_namelen;
        int ulen = len;
-       int hlimit = -1;
-       int tclass = -1;
-       int dontfrag = -1;
        int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
        int err;
        int connected = 0;
        int is_udplite = IS_UDPLITE(sk);
        int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
+       struct sockcm_cookie sockc;
+
+       ipc6.hlimit = -1;
+       ipc6.tclass = -1;
+       ipc6.dontfrag = -1;
 
        /* destination address check */
        if (sin6) {
@@ -1247,14 +1194,15 @@ do_udp_sendmsg:
                fl6.flowi6_oif = np->sticky_pktinfo.ipi6_ifindex;
 
        fl6.flowi6_mark = sk->sk_mark;
+       sockc.tsflags = sk->sk_tsflags;
 
        if (msg->msg_controllen) {
                opt = &opt_space;
                memset(opt, 0, sizeof(struct ipv6_txoptions));
                opt->tot_len = sizeof(*opt);
+               ipc6.opt = opt;
 
-               err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6, opt,
-                                           &hlimit, &tclass, &dontfrag);
+               err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6, &ipc6, &sockc);
                if (err < 0) {
                        fl6_sock_release(flowlabel);
                        return err;
@@ -1275,6 +1223,7 @@ do_udp_sendmsg:
        if (flowlabel)
                opt = fl6_merge_options(&opt_space, flowlabel, opt);
        opt = ipv6_fixup_options(&opt_space, opt);
+       ipc6.opt = opt;
 
        fl6.flowi6_proto = sk->sk_protocol;
        if (!ipv6_addr_any(daddr))
@@ -1304,11 +1253,11 @@ do_udp_sendmsg:
                goto out;
        }
 
-       if (hlimit < 0)
-               hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
+       if (ipc6.hlimit < 0)
+               ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);
 
-       if (tclass < 0)
-               tclass = np->tclass;
+       if (ipc6.tclass < 0)
+               ipc6.tclass = np->tclass;
 
        if (msg->msg_flags&MSG_CONFIRM)
                goto do_confirm;
@@ -1319,9 +1268,9 @@ back_from_confirm:
                struct sk_buff *skb;
 
                skb = ip6_make_skb(sk, getfrag, msg, ulen,
-                                  sizeof(struct udphdr), hlimit, tclass, opt,
+                                  sizeof(struct udphdr), &ipc6,
                                   &fl6, (struct rt6_info *)dst,
-                                  msg->msg_flags, dontfrag);
+                                  msg->msg_flags, &sockc);
                err = PTR_ERR(skb);
                if (!IS_ERR_OR_NULL(skb))
                        err = udp_v6_send_skb(skb, &fl6);
@@ -1342,13 +1291,12 @@ back_from_confirm:
        up->pending = AF_INET6;
 
 do_append_data:
-       if (dontfrag < 0)
-               dontfrag = np->dontfrag;
+       if (ipc6.dontfrag < 0)
+               ipc6.dontfrag = np->dontfrag;
        up->len += ulen;
-       err = ip6_append_data(sk, getfrag, msg, ulen,
-               sizeof(struct udphdr), hlimit, tclass, opt, &fl6,
-               (struct rt6_info *)dst,
-               corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags, dontfrag);
+       err = ip6_append_data(sk, getfrag, msg, ulen, sizeof(struct udphdr),
+                             &ipc6, &fl6, (struct rt6_info *)dst,
+                             corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags, &sockc);
        if (err)
                udp_v6_flush_pending_frames(sk);
        else if (!corkreq)
@@ -1391,8 +1339,8 @@ out:
         * seems like overkill.
         */
        if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
-               UDP6_INC_STATS_USER(sock_net(sk),
-                               UDP_MIB_SNDBUFERRORS, is_udplite);
+               UDP6_INC_STATS(sock_net(sk),
+                              UDP_MIB_SNDBUFERRORS, is_udplite);
        }
        return err;