In order to support fast reuseport lookups in TCP, the hash function
defined in struct proto must be capable of returning an error code.
This patch changes the function signature of all related hash functions
to return an integer and handles or propagates this return value at
all call sites.
Signed-off-by: Craig Gallek <kraig@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
15 files changed:
bool inet_ehash_insert(struct sock *sk, struct sock *osk);
bool inet_ehash_nolisten(struct sock *sk, struct sock *osk);
void __inet_hash(struct sock *sk, struct sock *osk);
bool inet_ehash_insert(struct sock *sk, struct sock *osk);
bool inet_ehash_nolisten(struct sock *sk, struct sock *osk);
void __inet_hash(struct sock *sk, struct sock *osk);
-void inet_hash(struct sock *sk);
+int inet_hash(struct sock *sk);
void inet_unhash(struct sock *sk);
struct sock *__inet_lookup_listener(struct net *net,
void inet_unhash(struct sock *sk);
struct sock *__inet_lookup_listener(struct net *net,
struct sock *pn_find_sock_by_sa(struct net *net, const struct sockaddr_pn *sa);
void pn_deliver_sock_broadcast(struct net *net, struct sk_buff *skb);
void phonet_get_local_port_range(int *min, int *max);
struct sock *pn_find_sock_by_sa(struct net *net, const struct sockaddr_pn *sa);
void pn_deliver_sock_broadcast(struct net *net, struct sk_buff *skb);
void phonet_get_local_port_range(int *min, int *max);
-void pn_sock_hash(struct sock *sk);
+int pn_sock_hash(struct sock *sk);
void pn_sock_unhash(struct sock *sk);
int pn_sock_get_port(struct sock *sk, unsigned short sport);
void pn_sock_unhash(struct sock *sk);
int pn_sock_get_port(struct sock *sk, unsigned short sport);
};
int ping_get_port(struct sock *sk, unsigned short ident);
};
int ping_get_port(struct sock *sk, unsigned short ident);
-void ping_hash(struct sock *sk);
+int ping_hash(struct sock *sk);
void ping_unhash(struct sock *sk);
int ping_init_sock(struct sock *sk);
void ping_unhash(struct sock *sk);
int ping_init_sock(struct sock *sk);
-void raw_hash_sk(struct sock *sk);
+int raw_hash_sk(struct sock *sk);
void raw_unhash_sk(struct sock *sk);
struct raw_sock {
void raw_unhash_sk(struct sock *sk);
struct raw_sock {
void (*release_cb)(struct sock *sk);
/* Keeping track of sk's, looking them up, and port selection methods. */
void (*release_cb)(struct sock *sk);
/* Keeping track of sk's, looking them up, and port selection methods. */
- void (*hash)(struct sock *sk);
+ int (*hash)(struct sock *sk);
void (*unhash)(struct sock *sk);
void (*rehash)(struct sock *sk);
int (*get_port)(struct sock *sk, unsigned short snum);
void (*unhash)(struct sock *sk);
void (*rehash)(struct sock *sk);
int (*get_port)(struct sock *sk, unsigned short snum);
/* With per-bucket locks this operation is not-atomic, so that
* this version is not worse.
*/
/* With per-bucket locks this operation is not-atomic, so that
* this version is not worse.
*/
-static inline void __sk_prot_rehash(struct sock *sk)
+static inline int __sk_prot_rehash(struct sock *sk)
{
sk->sk_prot->unhash(sk);
{
sk->sk_prot->unhash(sk);
+ return sk->sk_prot->hash(sk);
}
void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
}
void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
}
/* hash routines shared between UDPv4/6 and UDP-Litev4/6 */
}
/* hash routines shared between UDPv4/6 and UDP-Litev4/6 */
-static inline void udp_lib_hash(struct sock *sk)
+static inline int udp_lib_hash(struct sock *sk)
}
void udp_lib_unhash(struct sock *sk);
}
void udp_lib_unhash(struct sock *sk);
static HLIST_HEAD(raw_head);
static DEFINE_RWLOCK(raw_lock);
static HLIST_HEAD(raw_head);
static DEFINE_RWLOCK(raw_lock);
-static void raw_hash(struct sock *sk)
+static int raw_hash(struct sock *sk)
{
write_lock_bh(&raw_lock);
sk_add_node(sk, &raw_head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&raw_lock);
{
write_lock_bh(&raw_lock);
sk_add_node(sk, &raw_head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&raw_lock);
}
static void raw_unhash(struct sock *sk)
}
static void raw_unhash(struct sock *sk)
return container_of(sk, struct dgram_sock, sk);
}
return container_of(sk, struct dgram_sock, sk);
}
-static void dgram_hash(struct sock *sk)
+static int dgram_hash(struct sock *sk)
{
write_lock_bh(&dgram_lock);
sk_add_node(sk, &dgram_head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&dgram_lock);
{
write_lock_bh(&dgram_lock);
sk_add_node(sk, &dgram_head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&dgram_lock);
}
static void dgram_unhash(struct sock *sk)
}
static void dgram_unhash(struct sock *sk)
/* Checksums on by default */
sock_set_flag(sk, SOCK_ZAPPED);
/* Checksums on by default */
sock_set_flag(sk, SOCK_ZAPPED);
- if (sk->sk_prot->hash)
- sk->sk_prot->hash(sk);
+ if (sk->sk_prot->hash) {
+ rc = sk->sk_prot->hash(sk);
+ if (rc) {
+ sk_common_release(sk);
+ goto out;
+ }
+ }
if (sk->sk_prot->init) {
rc = sk->sk_prot->init(sk);
if (sk->sk_prot->init) {
rc = sk->sk_prot->init(sk);
*/
inet->inet_sport = htons(inet->inet_num);
/* Add to protocol hash chains. */
*/
inet->inet_sport = htons(inet->inet_num);
/* Add to protocol hash chains. */
+ err = sk->sk_prot->hash(sk);
+ if (err) {
+ sk_common_release(sk);
+ goto out;
+ }
}
if (sk->sk_prot->init) {
}
if (sk->sk_prot->init) {
* Besides that, it does not check for connection
* uniqueness. Wait for troubles.
*/
* Besides that, it does not check for connection
* uniqueness. Wait for troubles.
*/
- __sk_prot_rehash(sk);
- return 0;
+ return __sk_prot_rehash(sk);
}
int inet_sk_rebuild_header(struct sock *sk)
}
int inet_sk_rebuild_header(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
struct inet_sock *inet = inet_sk(sk);
{
struct inet_connection_sock *icsk = inet_csk(sk);
struct inet_sock *inet = inet_sk(sk);
reqsk_queue_alloc(&icsk->icsk_accept_queue);
reqsk_queue_alloc(&icsk->icsk_accept_queue);
inet->inet_sport = htons(inet->inet_num);
sk_dst_reset(sk);
inet->inet_sport = htons(inet->inet_num);
sk_dst_reset(sk);
+ err = sk->sk_prot->hash(sk);
+ if (likely(!err))
+ return 0;
}
sk->sk_state = TCP_CLOSE;
}
sk->sk_state = TCP_CLOSE;
}
EXPORT_SYMBOL_GPL(inet_csk_listen_start);
}
EXPORT_SYMBOL_GPL(inet_csk_listen_start);
}
EXPORT_SYMBOL(__inet_hash);
}
EXPORT_SYMBOL(__inet_hash);
-void inet_hash(struct sock *sk)
+int inet_hash(struct sock *sk)
{
if (sk->sk_state != TCP_CLOSE) {
local_bh_disable();
__inet_hash(sk, NULL);
local_bh_enable();
}
{
if (sk->sk_state != TCP_CLOSE) {
local_bh_disable();
__inet_hash(sk, NULL);
local_bh_enable();
}
}
EXPORT_SYMBOL_GPL(inet_hash);
}
EXPORT_SYMBOL_GPL(inet_hash);
}
EXPORT_SYMBOL_GPL(ping_get_port);
}
EXPORT_SYMBOL_GPL(ping_get_port);
-void ping_hash(struct sock *sk)
+int ping_hash(struct sock *sk)
{
pr_debug("ping_hash(sk->port=%u)\n", inet_sk(sk)->inet_num);
BUG(); /* "Please do not press this button again." */
{
pr_debug("ping_hash(sk->port=%u)\n", inet_sk(sk)->inet_num);
BUG(); /* "Please do not press this button again." */
}
void ping_unhash(struct sock *sk)
}
void ping_unhash(struct sock *sk)
.lock = __RW_LOCK_UNLOCKED(raw_v4_hashinfo.lock),
};
.lock = __RW_LOCK_UNLOCKED(raw_v4_hashinfo.lock),
};
-void raw_hash_sk(struct sock *sk)
+int raw_hash_sk(struct sock *sk)
{
struct raw_hashinfo *h = sk->sk_prot->h.raw_hash;
struct hlist_head *head;
{
struct raw_hashinfo *h = sk->sk_prot->h.raw_hash;
struct hlist_head *head;
sk_add_node(sk, head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&h->lock);
sk_add_node(sk, head);
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
write_unlock_bh(&h->lock);
}
EXPORT_SYMBOL_GPL(raw_hash_sk);
}
EXPORT_SYMBOL_GPL(raw_hash_sk);
* creation time automatically shares.
*/
inet->inet_sport = htons(inet->inet_num);
* creation time automatically shares.
*/
inet->inet_sport = htons(inet->inet_num);
+ err = sk->sk_prot->hash(sk);
+ if (err) {
+ sk_common_release(sk);
+ goto out;
+ }
}
if (sk->sk_prot->init) {
err = sk->sk_prot->init(sk);
}
if (sk->sk_prot->init) {
err = sk->sk_prot->init(sk);
-void pn_sock_hash(struct sock *sk)
+int pn_sock_hash(struct sock *sk)
{
struct hlist_head *hlist = pn_hash_list(pn_sk(sk)->sobject);
mutex_lock(&pnsocks.lock);
sk_add_node_rcu(sk, hlist);
mutex_unlock(&pnsocks.lock);
{
struct hlist_head *hlist = pn_hash_list(pn_sk(sk)->sobject);
mutex_lock(&pnsocks.lock);
sk_add_node_rcu(sk, hlist);
mutex_unlock(&pnsocks.lock);
}
EXPORT_SYMBOL(pn_sock_hash);
}
EXPORT_SYMBOL(pn_sock_hash);
pn->resource = spn->spn_resource;
/* Enable RX on the socket */
pn->resource = spn->spn_resource;
/* Enable RX on the socket */
+ err = sk->sk_prot->hash(sk);
out_port:
mutex_unlock(&port_mutex);
out:
out_port:
mutex_unlock(&port_mutex);
out:
-static void sctp_hash(struct sock *sk)
+static int sctp_hash(struct sock *sk)
}
static void sctp_unhash(struct sock *sk)
}
static void sctp_unhash(struct sock *sk)