soreuseport: fast reuseport TCP socket selection

[cascardo/linux.git] / net / ipv4 / inet_connection_sock.c

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 12c8d38..c16a2e6 100644 (file)
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -24,6 +24,7 @@
 #include <net/tcp_states.h>
 #include <net/xfrm.h>
 #include <net/tcp.h>
+#include <net/sock_reuseport.h>
 
 #ifdef INET_CSK_DEBUG
 const char inet_csk_timer_bug_msg[] = "inet_csk BUG: unknown timer value\n";
@@ -67,7 +68,8 @@ int inet_csk_bind_conflict(const struct sock *sk,
                        if ((!reuse || !sk2->sk_reuse ||
                            sk2->sk_state == TCP_LISTEN) &&
                            (!reuseport || !sk2->sk_reuseport ||
-                           (sk2->sk_state != TCP_TIME_WAIT &&
+                            rcu_access_pointer(sk->sk_reuseport_cb) ||
+                            (sk2->sk_state != TCP_TIME_WAIT &&
                             !uid_eq(uid, sock_i_uid(sk2))))) {
 
                                if (!sk2->sk_rcv_saddr || !sk->sk_rcv_saddr ||
@@ -132,6 +134,7 @@ again:
                                              sk->sk_state != TCP_LISTEN) ||
                                             (tb->fastreuseport > 0 &&
                                              sk->sk_reuseport &&
+                                             !rcu_access_pointer(sk->sk_reuseport_cb) &&
                                              uid_eq(tb->fastuid, uid))) &&
                                            (tb->num_owners < smallest_size || smallest_size == -1)) {
                                                smallest_size = tb->num_owners;
@@ -193,15 +196,18 @@ tb_found:
                if (((tb->fastreuse > 0 &&
                      sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
                     (tb->fastreuseport > 0 &&
-                     sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
-                   smallest_size == -1) {
+                     sk->sk_reuseport &&
+                     !rcu_access_pointer(sk->sk_reuseport_cb) &&
+                     uid_eq(tb->fastuid, uid))) && smallest_size == -1) {
                        goto success;
                } else {
                        ret = 1;
                        if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) {
                                if (((sk->sk_reuse && sk->sk_state != TCP_LISTEN) ||
                                     (tb->fastreuseport > 0 &&
-                                     sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
+                                     sk->sk_reuseport &&
+                                     !rcu_access_pointer(sk->sk_reuseport_cb) &&
+                                     uid_eq(tb->fastuid, uid))) &&
                                    smallest_size != -1 && --attempts >= 0) {
                                        spin_unlock(&head->lock);
                                        goto again;