soreuseport: Prep for fast reuseport TCP socket selection

[cascardo/linux.git] / net / core / sock.c

diff --git a/net/core/sock.c b/net/core/sock.c
index 5127023..46dc8ad 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -195,44 +195,6 @@ bool sk_net_capable(const struct sock *sk, int cap)
 }
 EXPORT_SYMBOL(sk_net_capable);
 
-
-#ifdef CONFIG_MEMCG_KMEM
-int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
-{
-       struct proto *proto;
-       int ret = 0;
-
-       mutex_lock(&proto_list_mutex);
-       list_for_each_entry(proto, &proto_list, node) {
-               if (proto->init_cgroup) {
-                       ret = proto->init_cgroup(memcg, ss);
-                       if (ret)
-                               goto out;
-               }
-       }
-
-       mutex_unlock(&proto_list_mutex);
-       return ret;
-out:
-       list_for_each_entry_continue_reverse(proto, &proto_list, node)
-               if (proto->destroy_cgroup)
-                       proto->destroy_cgroup(memcg);
-       mutex_unlock(&proto_list_mutex);
-       return ret;
-}
-
-void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
-{
-       struct proto *proto;
-
-       mutex_lock(&proto_list_mutex);
-       list_for_each_entry_reverse(proto, &proto_list, node)
-               if (proto->destroy_cgroup)
-                       proto->destroy_cgroup(memcg);
-       mutex_unlock(&proto_list_mutex);
-}
-#endif
-
 /*
  * Each address family might have different locking rules, so we have
  * one slock key per address family:
@@ -240,11 +202,6 @@ void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
 static struct lock_class_key af_family_keys[AF_MAX];
 static struct lock_class_key af_family_slock_keys[AF_MAX];
 
-#if defined(CONFIG_MEMCG_KMEM)
-struct static_key memcg_socket_limit_enabled;
-EXPORT_SYMBOL(memcg_socket_limit_enabled);
-#endif
-
 /*
  * Make lock validator output more readable. (we pre-construct these
  * strings build-time, so that runtime initialization of socket
@@ -1507,12 +1464,6 @@ void sk_free(struct sock *sk)
 }
 EXPORT_SYMBOL(sk_free);
 
-static void sk_update_clone(const struct sock *sk, struct sock *newsk)
-{
-       if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
-               sock_update_memcg(newsk);
-}
-
 /**
  *     sk_clone_lock - clone a socket, and lock its clone
  *     @sk: the socket to clone
@@ -1580,6 +1531,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
                        newsk = NULL;
                        goto out;
                }
+               RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
 
                newsk->sk_err      = 0;
                newsk->sk_priority = 0;
@@ -1607,7 +1559,8 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
                sk_set_socket(newsk, NULL);
                newsk->sk_wq = NULL;
 
-               sk_update_clone(sk, newsk);
+               if (mem_cgroup_sockets_enabled && sk->sk_memcg)
+                       sock_update_memcg(newsk);
 
                if (newsk->sk_prot->sockets_allocated)
                        sk_sockets_allocated_inc(newsk);
@@ -2089,27 +2042,27 @@ int __sk_mem_schedule(struct sock *sk, int size, int kind)
        struct proto *prot = sk->sk_prot;
        int amt = sk_mem_pages(size);
        long allocated;
-       int parent_status = UNDER_LIMIT;
 
        sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
 
-       allocated = sk_memory_allocated_add(sk, amt, &parent_status);
+       allocated = sk_memory_allocated_add(sk, amt);
+
+       if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
+           !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
+               goto suppress_allocation;
 
        /* Under limit. */
-       if (parent_status == UNDER_LIMIT &&
-                       allocated <= sk_prot_mem_limits(sk, 0)) {
+       if (allocated <= sk_prot_mem_limits(sk, 0)) {
                sk_leave_memory_pressure(sk);
                return 1;
        }
 
-       /* Under pressure. (we or our parents) */
-       if ((parent_status > SOFT_LIMIT) ||
-                       allocated > sk_prot_mem_limits(sk, 1))
+       /* Under pressure. */
+       if (allocated > sk_prot_mem_limits(sk, 1))
                sk_enter_memory_pressure(sk);
 
-       /* Over hard limit (we or our parents) */
-       if ((parent_status == OVER_LIMIT) ||
-                       (allocated > sk_prot_mem_limits(sk, 2)))
+       /* Over hard limit. */
+       if (allocated > sk_prot_mem_limits(sk, 2))
                goto suppress_allocation;
 
        /* guarantee minimum buffer size under pressure */
@@ -2158,6 +2111,9 @@ suppress_allocation:
 
        sk_memory_allocated_sub(sk, amt);
 
+       if (mem_cgroup_sockets_enabled && sk->sk_memcg)
+               mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
+
        return 0;
 }
 EXPORT_SYMBOL(__sk_mem_schedule);
@@ -2173,6 +2129,9 @@ void __sk_mem_reclaim(struct sock *sk, int amount)
        sk_memory_allocated_sub(sk, amount);
        sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
 
+       if (mem_cgroup_sockets_enabled && sk->sk_memcg)
+               mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
+
        if (sk_under_memory_pressure(sk) &&
            (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
                sk_leave_memory_pressure(sk);