net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #include <linux/kmemleak.h>
 112 #endif
 113 #include <net/secure_seq.h>
 114 #include <net/ip_tunnels.h>
 115
 116 #define RT_FL_TOS(oldflp4) \
 117         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 118
 119 #define RT_GC_TIMEOUT (300*HZ)
 120
 121 static int ip_rt_max_size;
 122 static int ip_rt_redirect_number __read_mostly  = 9;
 123 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 124 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 125 static int ip_rt_error_cost __read_mostly       = HZ;
 126 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 127 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 128 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 129 static int ip_rt_min_advmss __read_mostly       = 256;
 130
 131 /*
 132  *      Interface to generic destination cache.
 133  */
 134
 135 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 136 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 137 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 138 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 139 static void              ipv4_link_failure(struct sk_buff *skb);
 140 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 141                                            struct sk_buff *skb, u32 mtu);
 142 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 143                                         struct sk_buff *skb);
 144 static void             ipv4_dst_destroy(struct dst_entry *dst);
 145
 146 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 147 {
 148         WARN_ON(1);
 149         return NULL;
 150 }
 151
 152 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 153                                            struct sk_buff *skb,
 154                                            const void *daddr);
 155
 156 static struct dst_ops ipv4_dst_ops = {
 157         .family =               AF_INET,
 158         .check =                ipv4_dst_check,
 159         .default_advmss =       ipv4_default_advmss,
 160         .mtu =                  ipv4_mtu,
 161         .cow_metrics =          ipv4_cow_metrics,
 162         .destroy =              ipv4_dst_destroy,
 163         .negative_advice =      ipv4_negative_advice,
 164         .link_failure =         ipv4_link_failure,
 165         .update_pmtu =          ip_rt_update_pmtu,
 166         .redirect =             ip_do_redirect,
 167         .local_out =            __ip_local_out,
 168         .neigh_lookup =         ipv4_neigh_lookup,
 169 };
 170
 171 #define ECN_OR_COST(class)      TC_PRIO_##class
 172
 173 const __u8 ip_tos2prio[16] = {
 174         TC_PRIO_BESTEFFORT,
 175         ECN_OR_COST(BESTEFFORT),
 176         TC_PRIO_BESTEFFORT,
 177         ECN_OR_COST(BESTEFFORT),
 178         TC_PRIO_BULK,
 179         ECN_OR_COST(BULK),
 180         TC_PRIO_BULK,
 181         ECN_OR_COST(BULK),
 182         TC_PRIO_INTERACTIVE,
 183         ECN_OR_COST(INTERACTIVE),
 184         TC_PRIO_INTERACTIVE,
 185         ECN_OR_COST(INTERACTIVE),
 186         TC_PRIO_INTERACTIVE_BULK,
 187         ECN_OR_COST(INTERACTIVE_BULK),
 188         TC_PRIO_INTERACTIVE_BULK,
 189         ECN_OR_COST(INTERACTIVE_BULK)
 190 };
 191 EXPORT_SYMBOL(ip_tos2prio);
 192
 193 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 194 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 195
 196 #ifdef CONFIG_PROC_FS
 197 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 198 {
 199         if (*pos)
 200                 return NULL;
 201         return SEQ_START_TOKEN;
 202 }
 203
 204 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 205 {
 206         ++*pos;
 207         return NULL;
 208 }
 209
 210 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 211 {
 212 }
 213
 214 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 215 {
 216         if (v == SEQ_START_TOKEN)
 217                 seq_printf(seq, "%-127s\n",
 218                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 219                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 220                            "HHUptod\tSpecDst");
 221         return 0;
 222 }
 223
 224 static const struct seq_operations rt_cache_seq_ops = {
 225         .start  = rt_cache_seq_start,
 226         .next   = rt_cache_seq_next,
 227         .stop   = rt_cache_seq_stop,
 228         .show   = rt_cache_seq_show,
 229 };
 230
 231 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 232 {
 233         return seq_open(file, &rt_cache_seq_ops);
 234 }
 235
 236 static const struct file_operations rt_cache_seq_fops = {
 237         .owner   = THIS_MODULE,
 238         .open    = rt_cache_seq_open,
 239         .read    = seq_read,
 240         .llseek  = seq_lseek,
 241         .release = seq_release,
 242 };
 243
 244
 245 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 246 {
 247         int cpu;
 248
 249         if (*pos == 0)
 250                 return SEQ_START_TOKEN;
 251
 252         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 253                 if (!cpu_possible(cpu))
 254                         continue;
 255                 *pos = cpu+1;
 256                 return &per_cpu(rt_cache_stat, cpu);
 257         }
 258         return NULL;
 259 }
 260
 261 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 262 {
 263         int cpu;
 264
 265         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 266                 if (!cpu_possible(cpu))
 267                         continue;
 268                 *pos = cpu+1;
 269                 return &per_cpu(rt_cache_stat, cpu);
 270         }
 271         return NULL;
 272
 273 }
 274
 275 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 276 {
 277
 278 }
 279
 280 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 281 {
 282         struct rt_cache_stat *st = v;
 283
 284         if (v == SEQ_START_TOKEN) {
 285                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 286                 return 0;
 287         }
 288
 289         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 290                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 291                    dst_entries_get_slow(&ipv4_dst_ops),
 292                    0, /* st->in_hit */
 293                    st->in_slow_tot,
 294                    st->in_slow_mc,
 295                    st->in_no_route,
 296                    st->in_brd,
 297                    st->in_martian_dst,
 298                    st->in_martian_src,
 299
 300                    0, /* st->out_hit */
 301                    st->out_slow_tot,
 302                    st->out_slow_mc,
 303
 304                    0, /* st->gc_total */
 305                    0, /* st->gc_ignored */
 306                    0, /* st->gc_goal_miss */
 307                    0, /* st->gc_dst_overflow */
 308                    0, /* st->in_hlist_search */
 309                    0  /* st->out_hlist_search */
 310                 );
 311         return 0;
 312 }
 313
 314 static const struct seq_operations rt_cpu_seq_ops = {
 315         .start  = rt_cpu_seq_start,
 316         .next   = rt_cpu_seq_next,
 317         .stop   = rt_cpu_seq_stop,
 318         .show   = rt_cpu_seq_show,
 319 };
 320
 321
 322 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 323 {
 324         return seq_open(file, &rt_cpu_seq_ops);
 325 }
 326
 327 static const struct file_operations rt_cpu_seq_fops = {
 328         .owner   = THIS_MODULE,
 329         .open    = rt_cpu_seq_open,
 330         .read    = seq_read,
 331         .llseek  = seq_lseek,
 332         .release = seq_release,
 333 };
 334
 335 #ifdef CONFIG_IP_ROUTE_CLASSID
 336 static int rt_acct_proc_show(struct seq_file *m, void *v)
 337 {
 338         struct ip_rt_acct *dst, *src;
 339         unsigned int i, j;
 340
 341         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 342         if (!dst)
 343                 return -ENOMEM;
 344
 345         for_each_possible_cpu(i) {
 346                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 347                 for (j = 0; j < 256; j++) {
 348                         dst[j].o_bytes   += src[j].o_bytes;
 349                         dst[j].o_packets += src[j].o_packets;
 350                         dst[j].i_bytes   += src[j].i_bytes;
 351                         dst[j].i_packets += src[j].i_packets;
 352                 }
 353         }
 354
 355         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 356         kfree(dst);
 357         return 0;
 358 }
 359
 360 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 361 {
 362         return single_open(file, rt_acct_proc_show, NULL);
 363 }
 364
 365 static const struct file_operations rt_acct_proc_fops = {
 366         .owner          = THIS_MODULE,
 367         .open           = rt_acct_proc_open,
 368         .read           = seq_read,
 369         .llseek         = seq_lseek,
 370         .release        = single_release,
 371 };
 372 #endif
 373
 374 static int __net_init ip_rt_do_proc_init(struct net *net)
 375 {
 376         struct proc_dir_entry *pde;
 377
 378         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 379                           &rt_cache_seq_fops);
 380         if (!pde)
 381                 goto err1;
 382
 383         pde = proc_create("rt_cache", S_IRUGO,
 384                           net->proc_net_stat, &rt_cpu_seq_fops);
 385         if (!pde)
 386                 goto err2;
 387
 388 #ifdef CONFIG_IP_ROUTE_CLASSID
 389         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 390         if (!pde)
 391                 goto err3;
 392 #endif
 393         return 0;
 394
 395 #ifdef CONFIG_IP_ROUTE_CLASSID
 396 err3:
 397         remove_proc_entry("rt_cache", net->proc_net_stat);
 398 #endif
 399 err2:
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 err1:
 402         return -ENOMEM;
 403 }
 404
 405 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 406 {
 407         remove_proc_entry("rt_cache", net->proc_net_stat);
 408         remove_proc_entry("rt_cache", net->proc_net);
 409 #ifdef CONFIG_IP_ROUTE_CLASSID
 410         remove_proc_entry("rt_acct", net->proc_net);
 411 #endif
 412 }
 413
 414 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 415         .init = ip_rt_do_proc_init,
 416         .exit = ip_rt_do_proc_exit,
 417 };
 418
 419 static int __init ip_rt_proc_init(void)
 420 {
 421         return register_pernet_subsys(&ip_rt_proc_ops);
 422 }
 423
 424 #else
 425 static inline int ip_rt_proc_init(void)
 426 {
 427         return 0;
 428 }
 429 #endif /* CONFIG_PROC_FS */
 430
 431 static inline bool rt_is_expired(const struct rtable *rth)
 432 {
 433         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 434 }
 435
 436 void rt_cache_flush(struct net *net)
 437 {
 438         rt_genid_bump_ipv4(net);
 439 }
 440
 441 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 442                                            struct sk_buff *skb,
 443                                            const void *daddr)
 444 {
 445         struct net_device *dev = dst->dev;
 446         const __be32 *pkey = daddr;
 447         const struct rtable *rt;
 448         struct neighbour *n;
 449
 450         rt = (const struct rtable *) dst;
 451         if (rt->rt_gateway)
 452                 pkey = (const __be32 *) &rt->rt_gateway;
 453         else if (skb)
 454                 pkey = &ip_hdr(skb)->daddr;
 455
 456         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 457         if (n)
 458                 return n;
 459         return neigh_create(&arp_tbl, pkey, dev);
 460 }
 461
 462 #define IP_IDENTS_SZ 2048u
 463
 464 static atomic_t *ip_idents __read_mostly;
 465 static u32 *ip_tstamps __read_mostly;
 466
 467 /* In order to protect privacy, we add a perturbation to identifiers
 468  * if one generator is seldom used. This makes hard for an attacker
 469  * to infer how many packets were sent between two points in time.
 470  */
 471 u32 ip_idents_reserve(u32 hash, int segs)
 472 {
 473         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 474         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 475         u32 old = ACCESS_ONCE(*p_tstamp);
 476         u32 now = (u32)jiffies;
 477         u32 delta = 0;
 478
 479         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 480                 delta = prandom_u32_max(now - old);
 481
 482         return atomic_add_return(segs + delta, p_id) - segs;
 483 }
 484 EXPORT_SYMBOL(ip_idents_reserve);
 485
 486 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 487 {
 488         static u32 ip_idents_hashrnd __read_mostly;
 489         u32 hash, id;
 490
 491         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 492
 493         hash = jhash_3words((__force u32)iph->daddr,
 494                             (__force u32)iph->saddr,
 495                             iph->protocol ^ net_hash_mix(net),
 496                             ip_idents_hashrnd);
 497         id = ip_idents_reserve(hash, segs);
 498         iph->id = htons(id);
 499 }
 500 EXPORT_SYMBOL(__ip_select_ident);
 501
 502 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 503                              const struct iphdr *iph,
 504                              int oif, u8 tos,
 505                              u8 prot, u32 mark, int flow_flags)
 506 {
 507         if (sk) {
 508                 const struct inet_sock *inet = inet_sk(sk);
 509
 510                 oif = sk->sk_bound_dev_if;
 511                 mark = sk->sk_mark;
 512                 tos = RT_CONN_FLAGS(sk);
 513                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 514         }
 515         flowi4_init_output(fl4, oif, mark, tos,
 516                            RT_SCOPE_UNIVERSE, prot,
 517                            flow_flags,
 518                            iph->daddr, iph->saddr, 0, 0);
 519 }
 520
 521 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 522                                const struct sock *sk)
 523 {
 524         const struct iphdr *iph = ip_hdr(skb);
 525         int oif = skb->dev->ifindex;
 526         u8 tos = RT_TOS(iph->tos);
 527         u8 prot = iph->protocol;
 528         u32 mark = skb->mark;
 529
 530         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 531 }
 532
 533 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 534 {
 535         const struct inet_sock *inet = inet_sk(sk);
 536         const struct ip_options_rcu *inet_opt;
 537         __be32 daddr = inet->inet_daddr;
 538
 539         rcu_read_lock();
 540         inet_opt = rcu_dereference(inet->inet_opt);
 541         if (inet_opt && inet_opt->opt.srr)
 542                 daddr = inet_opt->opt.faddr;
 543         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 544                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 545                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 546                            inet_sk_flowi_flags(sk),
 547                            daddr, inet->inet_saddr, 0, 0);
 548         rcu_read_unlock();
 549 }
 550
 551 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 552                                  const struct sk_buff *skb)
 553 {
 554         if (skb)
 555                 build_skb_flow_key(fl4, skb, sk);
 556         else
 557                 build_sk_flow_key(fl4, sk);
 558 }
 559
 560 static inline void rt_free(struct rtable *rt)
 561 {
 562         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 563 }
 564
 565 static DEFINE_SPINLOCK(fnhe_lock);
 566
 567 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 568 {
 569         struct rtable *rt;
 570
 571         rt = rcu_dereference(fnhe->fnhe_rth_input);
 572         if (rt) {
 573                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 574                 rt_free(rt);
 575         }
 576         rt = rcu_dereference(fnhe->fnhe_rth_output);
 577         if (rt) {
 578                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 579                 rt_free(rt);
 580         }
 581 }
 582
 583 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 584 {
 585         struct fib_nh_exception *fnhe, *oldest;
 586
 587         oldest = rcu_dereference(hash->chain);
 588         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 589              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 590                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 591                         oldest = fnhe;
 592         }
 593         fnhe_flush_routes(oldest);
 594         return oldest;
 595 }
 596
 597 static inline u32 fnhe_hashfun(__be32 daddr)
 598 {
 599         static u32 fnhe_hashrnd __read_mostly;
 600         u32 hval;
 601
 602         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 603         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 604         return hash_32(hval, FNHE_HASH_SHIFT);
 605 }
 606
 607 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 608 {
 609         rt->rt_pmtu = fnhe->fnhe_pmtu;
 610         rt->dst.expires = fnhe->fnhe_expires;
 611
 612         if (fnhe->fnhe_gw) {
 613                 rt->rt_flags |= RTCF_REDIRECTED;
 614                 rt->rt_gateway = fnhe->fnhe_gw;
 615                 rt->rt_uses_gateway = 1;
 616         }
 617 }
 618
 619 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 620                                   u32 pmtu, unsigned long expires)
 621 {
 622         struct fnhe_hash_bucket *hash;
 623         struct fib_nh_exception *fnhe;
 624         struct rtable *rt;
 625         unsigned int i;
 626         int depth;
 627         u32 hval = fnhe_hashfun(daddr);
 628
 629         spin_lock_bh(&fnhe_lock);
 630
 631         hash = rcu_dereference(nh->nh_exceptions);
 632         if (!hash) {
 633                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 634                 if (!hash)
 635                         goto out_unlock;
 636                 rcu_assign_pointer(nh->nh_exceptions, hash);
 637         }
 638
 639         hash += hval;
 640
 641         depth = 0;
 642         for (fnhe = rcu_dereference(hash->chain); fnhe;
 643              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 644                 if (fnhe->fnhe_daddr == daddr)
 645                         break;
 646                 depth++;
 647         }
 648
 649         if (fnhe) {
 650                 if (gw)
 651                         fnhe->fnhe_gw = gw;
 652                 if (pmtu) {
 653                         fnhe->fnhe_pmtu = pmtu;
 654                         fnhe->fnhe_expires = max(1UL, expires);
 655                 }
 656                 /* Update all cached dsts too */
 657                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 658                 if (rt)
 659                         fill_route_from_fnhe(rt, fnhe);
 660                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 661                 if (rt)
 662                         fill_route_from_fnhe(rt, fnhe);
 663         } else {
 664                 if (depth > FNHE_RECLAIM_DEPTH)
 665                         fnhe = fnhe_oldest(hash);
 666                 else {
 667                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 668                         if (!fnhe)
 669                                 goto out_unlock;
 670
 671                         fnhe->fnhe_next = hash->chain;
 672                         rcu_assign_pointer(hash->chain, fnhe);
 673                 }
 674                 fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
 675                 fnhe->fnhe_daddr = daddr;
 676                 fnhe->fnhe_gw = gw;
 677                 fnhe->fnhe_pmtu = pmtu;
 678                 fnhe->fnhe_expires = expires;
 679
 680                 /* Exception created; mark the cached routes for the nexthop
 681                  * stale, so anyone caching it rechecks if this exception
 682                  * applies to them.
 683                  */
 684                 rt = rcu_dereference(nh->nh_rth_input);
 685                 if (rt)
 686                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 687
 688                 for_each_possible_cpu(i) {
 689                         struct rtable __rcu **prt;
 690                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 691                         rt = rcu_dereference(*prt);
 692                         if (rt)
 693                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 694                 }
 695         }
 696
 697         fnhe->fnhe_stamp = jiffies;
 698
 699 out_unlock:
 700         spin_unlock_bh(&fnhe_lock);
 701 }
 702
 703 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 704                              bool kill_route)
 705 {
 706         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 707         __be32 old_gw = ip_hdr(skb)->saddr;
 708         struct net_device *dev = skb->dev;
 709         struct in_device *in_dev;
 710         struct fib_result res;
 711         struct neighbour *n;
 712         struct net *net;
 713
 714         switch (icmp_hdr(skb)->code & 7) {
 715         case ICMP_REDIR_NET:
 716         case ICMP_REDIR_NETTOS:
 717         case ICMP_REDIR_HOST:
 718         case ICMP_REDIR_HOSTTOS:
 719                 break;
 720
 721         default:
 722                 return;
 723         }
 724
 725         if (rt->rt_gateway != old_gw)
 726                 return;
 727
 728         in_dev = __in_dev_get_rcu(dev);
 729         if (!in_dev)
 730                 return;
 731
 732         net = dev_net(dev);
 733         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 734             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 735             ipv4_is_zeronet(new_gw))
 736                 goto reject_redirect;
 737
 738         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 739                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 740                         goto reject_redirect;
 741                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 742                         goto reject_redirect;
 743         } else {
 744                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 745                         goto reject_redirect;
 746         }
 747
 748         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 749         if (!IS_ERR(n)) {
 750                 if (!(n->nud_state & NUD_VALID)) {
 751                         neigh_event_send(n, NULL);
 752                 } else {
 753                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 754                                 struct fib_nh *nh = &FIB_RES_NH(res);
 755
 756                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 757                                                       0, 0);
 758                         }
 759                         if (kill_route)
 760                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 761                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 762                 }
 763                 neigh_release(n);
 764         }
 765         return;
 766
 767 reject_redirect:
 768 #ifdef CONFIG_IP_ROUTE_VERBOSE
 769         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 770                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 771                 __be32 daddr = iph->daddr;
 772                 __be32 saddr = iph->saddr;
 773
 774                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 775                                      "  Advised path = %pI4 -> %pI4\n",
 776                                      &old_gw, dev->name, &new_gw,
 777                                      &saddr, &daddr);
 778         }
 779 #endif
 780         ;
 781 }
 782
 783 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 784 {
 785         struct rtable *rt;
 786         struct flowi4 fl4;
 787         const struct iphdr *iph = (const struct iphdr *) skb->data;
 788         int oif = skb->dev->ifindex;
 789         u8 tos = RT_TOS(iph->tos);
 790         u8 prot = iph->protocol;
 791         u32 mark = skb->mark;
 792
 793         rt = (struct rtable *) dst;
 794
 795         __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
 796         __ip_do_redirect(rt, skb, &fl4, true);
 797 }
 798
 799 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 800 {
 801         struct rtable *rt = (struct rtable *)dst;
 802         struct dst_entry *ret = dst;
 803
 804         if (rt) {
 805                 if (dst->obsolete > 0) {
 806                         ip_rt_put(rt);
 807                         ret = NULL;
 808                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 809                            rt->dst.expires) {
 810                         ip_rt_put(rt);
 811                         ret = NULL;
 812                 }
 813         }
 814         return ret;
 815 }
 816
 817 /*
 818  * Algorithm:
 819  *      1. The first ip_rt_redirect_number redirects are sent
 820  *         with exponential backoff, then we stop sending them at all,
 821  *         assuming that the host ignores our redirects.
 822  *      2. If we did not see packets requiring redirects
 823  *         during ip_rt_redirect_silence, we assume that the host
 824  *         forgot redirected route and start to send redirects again.
 825  *
 826  * This algorithm is much cheaper and more intelligent than dumb load limiting
 827  * in icmp.c.
 828  *
 829  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 830  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 831  */
 832
 833 void ip_rt_send_redirect(struct sk_buff *skb)
 834 {
 835         struct rtable *rt = skb_rtable(skb);
 836         struct in_device *in_dev;
 837         struct inet_peer *peer;
 838         struct net *net;
 839         int log_martians;
 840
 841         rcu_read_lock();
 842         in_dev = __in_dev_get_rcu(rt->dst.dev);
 843         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 844                 rcu_read_unlock();
 845                 return;
 846         }
 847         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 848         rcu_read_unlock();
 849
 850         net = dev_net(rt->dst.dev);
 851         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 852         if (!peer) {
 853                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 854                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 855                 return;
 856         }
 857
 858         /* No redirected packets during ip_rt_redirect_silence;
 859          * reset the algorithm.
 860          */
 861         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 862                 peer->rate_tokens = 0;
 863
 864         /* Too many ignored redirects; do not send anything
 865          * set dst.rate_last to the last seen redirected packet.
 866          */
 867         if (peer->rate_tokens >= ip_rt_redirect_number) {
 868                 peer->rate_last = jiffies;
 869                 goto out_put_peer;
 870         }
 871
 872         /* Check for load limit; set rate_last to the latest sent
 873          * redirect.
 874          */
 875         if (peer->rate_tokens == 0 ||
 876             time_after(jiffies,
 877                        (peer->rate_last +
 878                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 879                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 880
 881                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 882                 peer->rate_last = jiffies;
 883                 ++peer->rate_tokens;
 884 #ifdef CONFIG_IP_ROUTE_VERBOSE
 885                 if (log_martians &&
 886                     peer->rate_tokens == ip_rt_redirect_number)
 887                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 888                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 889                                              &ip_hdr(skb)->daddr, &gw);
 890 #endif
 891         }
 892 out_put_peer:
 893         inet_putpeer(peer);
 894 }
 895
 896 static int ip_error(struct sk_buff *skb)
 897 {
 898         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 899         struct rtable *rt = skb_rtable(skb);
 900         struct inet_peer *peer;
 901         unsigned long now;
 902         struct net *net;
 903         bool send;
 904         int code;
 905
 906         /* IP on this device is disabled. */
 907         if (!in_dev)
 908                 goto out;
 909
 910         net = dev_net(rt->dst.dev);
 911         if (!IN_DEV_FORWARD(in_dev)) {
 912                 switch (rt->dst.error) {
 913                 case EHOSTUNREACH:
 914                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 915                         break;
 916
 917                 case ENETUNREACH:
 918                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 919                         break;
 920                 }
 921                 goto out;
 922         }
 923
 924         switch (rt->dst.error) {
 925         case EINVAL:
 926         default:
 927                 goto out;
 928         case EHOSTUNREACH:
 929                 code = ICMP_HOST_UNREACH;
 930                 break;
 931         case ENETUNREACH:
 932                 code = ICMP_NET_UNREACH;
 933                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 934                 break;
 935         case EACCES:
 936                 code = ICMP_PKT_FILTERED;
 937                 break;
 938         }
 939
 940         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 941
 942         send = true;
 943         if (peer) {
 944                 now = jiffies;
 945                 peer->rate_tokens += now - peer->rate_last;
 946                 if (peer->rate_tokens > ip_rt_error_burst)
 947                         peer->rate_tokens = ip_rt_error_burst;
 948                 peer->rate_last = now;
 949                 if (peer->rate_tokens >= ip_rt_error_cost)
 950                         peer->rate_tokens -= ip_rt_error_cost;
 951                 else
 952                         send = false;
 953                 inet_putpeer(peer);
 954         }
 955         if (send)
 956                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 957
 958 out:    kfree_skb(skb);
 959         return 0;
 960 }
 961
 962 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 963 {
 964         struct dst_entry *dst = &rt->dst;
 965         struct fib_result res;
 966
 967         if (dst_metric_locked(dst, RTAX_MTU))
 968                 return;
 969
 970         if (ipv4_mtu(dst) < mtu)
 971                 return;
 972
 973         if (mtu < ip_rt_min_pmtu)
 974                 mtu = ip_rt_min_pmtu;
 975
 976         if (rt->rt_pmtu == mtu &&
 977             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
 978                 return;
 979
 980         rcu_read_lock();
 981         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
 982                 struct fib_nh *nh = &FIB_RES_NH(res);
 983
 984                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 985                                       jiffies + ip_rt_mtu_expires);
 986         }
 987         rcu_read_unlock();
 988 }
 989
 990 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 991                               struct sk_buff *skb, u32 mtu)
 992 {
 993         struct rtable *rt = (struct rtable *) dst;
 994         struct flowi4 fl4;
 995
 996         ip_rt_build_flow_key(&fl4, sk, skb);
 997         __ip_rt_update_pmtu(rt, &fl4, mtu);
 998 }
 999
1000 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1001                       int oif, u32 mark, u8 protocol, int flow_flags)
1002 {
1003         const struct iphdr *iph = (const struct iphdr *) skb->data;
1004         struct flowi4 fl4;
1005         struct rtable *rt;
1006
1007         if (!mark)
1008                 mark = IP4_REPLY_MARK(net, skb->mark);
1009
1010         __build_flow_key(&fl4, NULL, iph, oif,
1011                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1012         rt = __ip_route_output_key(net, &fl4);
1013         if (!IS_ERR(rt)) {
1014                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1015                 ip_rt_put(rt);
1016         }
1017 }
1018 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1019
1020 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1021 {
1022         const struct iphdr *iph = (const struct iphdr *) skb->data;
1023         struct flowi4 fl4;
1024         struct rtable *rt;
1025
1026         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1027
1028         if (!fl4.flowi4_mark)
1029                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1030
1031         rt = __ip_route_output_key(sock_net(sk), &fl4);
1032         if (!IS_ERR(rt)) {
1033                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1034                 ip_rt_put(rt);
1035         }
1036 }
1037
1038 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1039 {
1040         const struct iphdr *iph = (const struct iphdr *) skb->data;
1041         struct flowi4 fl4;
1042         struct rtable *rt;
1043         struct dst_entry *odst = NULL;
1044         bool new = false;
1045
1046         bh_lock_sock(sk);
1047
1048         if (!ip_sk_accept_pmtu(sk))
1049                 goto out;
1050
1051         odst = sk_dst_get(sk);
1052
1053         if (sock_owned_by_user(sk) || !odst) {
1054                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1055                 goto out;
1056         }
1057
1058         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1059
1060         rt = (struct rtable *)odst;
1061         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1062                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1063                 if (IS_ERR(rt))
1064                         goto out;
1065
1066                 new = true;
1067         }
1068
1069         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1070
1071         if (!dst_check(&rt->dst, 0)) {
1072                 if (new)
1073                         dst_release(&rt->dst);
1074
1075                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1076                 if (IS_ERR(rt))
1077                         goto out;
1078
1079                 new = true;
1080         }
1081
1082         if (new)
1083                 sk_dst_set(sk, &rt->dst);
1084
1085 out:
1086         bh_unlock_sock(sk);
1087         dst_release(odst);
1088 }
1089 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1090
1091 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1092                    int oif, u32 mark, u8 protocol, int flow_flags)
1093 {
1094         const struct iphdr *iph = (const struct iphdr *) skb->data;
1095         struct flowi4 fl4;
1096         struct rtable *rt;
1097
1098         __build_flow_key(&fl4, NULL, iph, oif,
1099                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1100         rt = __ip_route_output_key(net, &fl4);
1101         if (!IS_ERR(rt)) {
1102                 __ip_do_redirect(rt, skb, &fl4, false);
1103                 ip_rt_put(rt);
1104         }
1105 }
1106 EXPORT_SYMBOL_GPL(ipv4_redirect);
1107
1108 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1109 {
1110         const struct iphdr *iph = (const struct iphdr *) skb->data;
1111         struct flowi4 fl4;
1112         struct rtable *rt;
1113
1114         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1115         rt = __ip_route_output_key(sock_net(sk), &fl4);
1116         if (!IS_ERR(rt)) {
1117                 __ip_do_redirect(rt, skb, &fl4, false);
1118                 ip_rt_put(rt);
1119         }
1120 }
1121 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1122
1123 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1124 {
1125         struct rtable *rt = (struct rtable *) dst;
1126
1127         /* All IPV4 dsts are created with ->obsolete set to the value
1128          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1129          * into this function always.
1130          *
1131          * When a PMTU/redirect information update invalidates a route,
1132          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1133          * DST_OBSOLETE_DEAD by dst_free().
1134          */
1135         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1136                 return NULL;
1137         return dst;
1138 }
1139
1140 static void ipv4_link_failure(struct sk_buff *skb)
1141 {
1142         struct rtable *rt;
1143
1144         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1145
1146         rt = skb_rtable(skb);
1147         if (rt)
1148                 dst_set_expires(&rt->dst, 0);
1149 }
1150
1151 static int ip_rt_bug(struct sock *sk, struct sk_buff *skb)
1152 {
1153         pr_debug("%s: %pI4 -> %pI4, %s\n",
1154                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1155                  skb->dev ? skb->dev->name : "?");
1156         kfree_skb(skb);
1157         WARN_ON(1);
1158         return 0;
1159 }
1160
1161 /*
1162    We do not cache source address of outgoing interface,
1163    because it is used only by IP RR, TS and SRR options,
1164    so that it out of fast path.
1165
1166    BTW remember: "addr" is allowed to be not aligned
1167    in IP options!
1168  */
1169
1170 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1171 {
1172         __be32 src;
1173
1174         if (rt_is_output_route(rt))
1175                 src = ip_hdr(skb)->saddr;
1176         else {
1177                 struct fib_result res;
1178                 struct flowi4 fl4;
1179                 struct iphdr *iph;
1180
1181                 iph = ip_hdr(skb);
1182
1183                 memset(&fl4, 0, sizeof(fl4));
1184                 fl4.daddr = iph->daddr;
1185                 fl4.saddr = iph->saddr;
1186                 fl4.flowi4_tos = RT_TOS(iph->tos);
1187                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1188                 fl4.flowi4_iif = skb->dev->ifindex;
1189                 fl4.flowi4_mark = skb->mark;
1190
1191                 rcu_read_lock();
1192                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1193                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1194                 else
1195                         src = inet_select_addr(rt->dst.dev,
1196                                                rt_nexthop(rt, iph->daddr),
1197                                                RT_SCOPE_UNIVERSE);
1198                 rcu_read_unlock();
1199         }
1200         memcpy(addr, &src, 4);
1201 }
1202
1203 #ifdef CONFIG_IP_ROUTE_CLASSID
1204 static void set_class_tag(struct rtable *rt, u32 tag)
1205 {
1206         if (!(rt->dst.tclassid & 0xFFFF))
1207                 rt->dst.tclassid |= tag & 0xFFFF;
1208         if (!(rt->dst.tclassid & 0xFFFF0000))
1209                 rt->dst.tclassid |= tag & 0xFFFF0000;
1210 }
1211 #endif
1212
1213 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1214 {
1215         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1216
1217         if (advmss == 0) {
1218                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1219                                ip_rt_min_advmss);
1220                 if (advmss > 65535 - 40)
1221                         advmss = 65535 - 40;
1222         }
1223         return advmss;
1224 }
1225
1226 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1227 {
1228         const struct rtable *rt = (const struct rtable *) dst;
1229         unsigned int mtu = rt->rt_pmtu;
1230
1231         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1232                 mtu = dst_metric_raw(dst, RTAX_MTU);
1233
1234         if (mtu)
1235                 return mtu;
1236
1237         mtu = dst->dev->mtu;
1238
1239         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1240                 if (rt->rt_uses_gateway && mtu > 576)
1241                         mtu = 576;
1242         }
1243
1244         return min_t(unsigned int, mtu, IP_MAX_MTU);
1245 }
1246
1247 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1248 {
1249         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1250         struct fib_nh_exception *fnhe;
1251         u32 hval;
1252
1253         if (!hash)
1254                 return NULL;
1255
1256         hval = fnhe_hashfun(daddr);
1257
1258         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1259              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1260                 if (fnhe->fnhe_daddr == daddr)
1261                         return fnhe;
1262         }
1263         return NULL;
1264 }
1265
1266 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1267                               __be32 daddr)
1268 {
1269         bool ret = false;
1270
1271         spin_lock_bh(&fnhe_lock);
1272
1273         if (daddr == fnhe->fnhe_daddr) {
1274                 struct rtable __rcu **porig;
1275                 struct rtable *orig;
1276                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1277
1278                 if (rt_is_input_route(rt))
1279                         porig = &fnhe->fnhe_rth_input;
1280                 else
1281                         porig = &fnhe->fnhe_rth_output;
1282                 orig = rcu_dereference(*porig);
1283
1284                 if (fnhe->fnhe_genid != genid) {
1285                         fnhe->fnhe_genid = genid;
1286                         fnhe->fnhe_gw = 0;
1287                         fnhe->fnhe_pmtu = 0;
1288                         fnhe->fnhe_expires = 0;
1289                         fnhe_flush_routes(fnhe);
1290                         orig = NULL;
1291                 }
1292                 fill_route_from_fnhe(rt, fnhe);
1293                 if (!rt->rt_gateway)
1294                         rt->rt_gateway = daddr;
1295
1296                 if (!(rt->dst.flags & DST_NOCACHE)) {
1297                         rcu_assign_pointer(*porig, rt);
1298                         if (orig)
1299                                 rt_free(orig);
1300                         ret = true;
1301                 }
1302
1303                 fnhe->fnhe_stamp = jiffies;
1304         }
1305         spin_unlock_bh(&fnhe_lock);
1306
1307         return ret;
1308 }
1309
1310 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1311 {
1312         struct rtable *orig, *prev, **p;
1313         bool ret = true;
1314
1315         if (rt_is_input_route(rt)) {
1316                 p = (struct rtable **)&nh->nh_rth_input;
1317         } else {
1318                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1319         }
1320         orig = *p;
1321
1322         prev = cmpxchg(p, orig, rt);
1323         if (prev == orig) {
1324                 if (orig)
1325                         rt_free(orig);
1326         } else
1327                 ret = false;
1328
1329         return ret;
1330 }
1331
1332 struct uncached_list {
1333         spinlock_t              lock;
1334         struct list_head        head;
1335 };
1336
1337 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1338
1339 static void rt_add_uncached_list(struct rtable *rt)
1340 {
1341         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1342
1343         rt->rt_uncached_list = ul;
1344
1345         spin_lock_bh(&ul->lock);
1346         list_add_tail(&rt->rt_uncached, &ul->head);
1347         spin_unlock_bh(&ul->lock);
1348 }
1349
1350 static void ipv4_dst_destroy(struct dst_entry *dst)
1351 {
1352         struct rtable *rt = (struct rtable *) dst;
1353
1354         if (!list_empty(&rt->rt_uncached)) {
1355                 struct uncached_list *ul = rt->rt_uncached_list;
1356
1357                 spin_lock_bh(&ul->lock);
1358                 list_del(&rt->rt_uncached);
1359                 spin_unlock_bh(&ul->lock);
1360         }
1361         lwtstate_put(rt->rt_lwtstate);
1362 }
1363
1364 void rt_flush_dev(struct net_device *dev)
1365 {
1366         struct net *net = dev_net(dev);
1367         struct rtable *rt;
1368         int cpu;
1369
1370         for_each_possible_cpu(cpu) {
1371                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1372
1373                 spin_lock_bh(&ul->lock);
1374                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1375                         if (rt->dst.dev != dev)
1376                                 continue;
1377                         rt->dst.dev = net->loopback_dev;
1378                         dev_hold(rt->dst.dev);
1379                         dev_put(dev);
1380                 }
1381                 spin_unlock_bh(&ul->lock);
1382         }
1383 }
1384
1385 static bool rt_cache_valid(const struct rtable *rt)
1386 {
1387         return  rt &&
1388                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1389                 !rt_is_expired(rt);
1390 }
1391
1392 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1393                            const struct fib_result *res,
1394                            struct fib_nh_exception *fnhe,
1395                            struct fib_info *fi, u16 type, u32 itag)
1396 {
1397         bool cached = false;
1398
1399         if (fi) {
1400                 struct fib_nh *nh = &FIB_RES_NH(*res);
1401
1402                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1403                         rt->rt_gateway = nh->nh_gw;
1404                         rt->rt_uses_gateway = 1;
1405                 }
1406                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1407 #ifdef CONFIG_IP_ROUTE_CLASSID
1408                 rt->dst.tclassid = nh->nh_tclassid;
1409 #endif
1410                 rt->rt_lwtstate = lwtstate_get(nh->nh_lwtstate);
1411                 if (unlikely(fnhe))
1412                         cached = rt_bind_exception(rt, fnhe, daddr);
1413                 else if (!(rt->dst.flags & DST_NOCACHE))
1414                         cached = rt_cache_route(nh, rt);
1415                 if (unlikely(!cached)) {
1416                         /* Routes we intend to cache in nexthop exception or
1417                          * FIB nexthop have the DST_NOCACHE bit clear.
1418                          * However, if we are unsuccessful at storing this
1419                          * route into the cache we really need to set it.
1420                          */
1421                         rt->dst.flags |= DST_NOCACHE;
1422                         if (!rt->rt_gateway)
1423                                 rt->rt_gateway = daddr;
1424                         rt_add_uncached_list(rt);
1425                 }
1426         } else
1427                 rt_add_uncached_list(rt);
1428
1429 #ifdef CONFIG_IP_ROUTE_CLASSID
1430 #ifdef CONFIG_IP_MULTIPLE_TABLES
1431         set_class_tag(rt, res->tclassid);
1432 #endif
1433         set_class_tag(rt, itag);
1434 #endif
1435 }
1436
1437 static struct rtable *rt_dst_alloc(struct net_device *dev,
1438                                    bool nopolicy, bool noxfrm, bool will_cache)
1439 {
1440         return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1441                          (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1442                          (nopolicy ? DST_NOPOLICY : 0) |
1443                          (noxfrm ? DST_NOXFRM : 0));
1444 }
1445
1446 /* called in rcu_read_lock() section */
1447 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1448                                 u8 tos, struct net_device *dev, int our)
1449 {
1450         struct rtable *rth;
1451         struct in_device *in_dev = __in_dev_get_rcu(dev);
1452         u32 itag = 0;
1453         int err;
1454
1455         /* Primary sanity checks. */
1456
1457         if (!in_dev)
1458                 return -EINVAL;
1459
1460         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1461             skb->protocol != htons(ETH_P_IP))
1462                 goto e_inval;
1463
1464         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1465                 if (ipv4_is_loopback(saddr))
1466                         goto e_inval;
1467
1468         if (ipv4_is_zeronet(saddr)) {
1469                 if (!ipv4_is_local_multicast(daddr))
1470                         goto e_inval;
1471         } else {
1472                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1473                                           in_dev, &itag);
1474                 if (err < 0)
1475                         goto e_err;
1476         }
1477         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1478                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1479         if (!rth)
1480                 goto e_nobufs;
1481
1482 #ifdef CONFIG_IP_ROUTE_CLASSID
1483         rth->dst.tclassid = itag;
1484 #endif
1485         rth->dst.output = ip_rt_bug;
1486
1487         rth->rt_genid   = rt_genid_ipv4(dev_net(dev));
1488         rth->rt_flags   = RTCF_MULTICAST;
1489         rth->rt_type    = RTN_MULTICAST;
1490         rth->rt_is_input= 1;
1491         rth->rt_iif     = 0;
1492         rth->rt_pmtu    = 0;
1493         rth->rt_gateway = 0;
1494         rth->rt_uses_gateway = 0;
1495         INIT_LIST_HEAD(&rth->rt_uncached);
1496         rth->rt_lwtstate = NULL;
1497         if (our) {
1498                 rth->dst.input= ip_local_deliver;
1499                 rth->rt_flags |= RTCF_LOCAL;
1500         }
1501
1502 #ifdef CONFIG_IP_MROUTE
1503         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1504                 rth->dst.input = ip_mr_input;
1505 #endif
1506         RT_CACHE_STAT_INC(in_slow_mc);
1507
1508         skb_dst_set(skb, &rth->dst);
1509         return 0;
1510
1511 e_nobufs:
1512         return -ENOBUFS;
1513 e_inval:
1514         return -EINVAL;
1515 e_err:
1516         return err;
1517 }
1518
1519
1520 static void ip_handle_martian_source(struct net_device *dev,
1521                                      struct in_device *in_dev,
1522                                      struct sk_buff *skb,
1523                                      __be32 daddr,
1524                                      __be32 saddr)
1525 {
1526         RT_CACHE_STAT_INC(in_martian_src);
1527 #ifdef CONFIG_IP_ROUTE_VERBOSE
1528         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1529                 /*
1530                  *      RFC1812 recommendation, if source is martian,
1531                  *      the only hint is MAC header.
1532                  */
1533                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1534                         &daddr, &saddr, dev->name);
1535                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1536                         print_hex_dump(KERN_WARNING, "ll header: ",
1537                                        DUMP_PREFIX_OFFSET, 16, 1,
1538                                        skb_mac_header(skb),
1539                                        dev->hard_header_len, true);
1540                 }
1541         }
1542 #endif
1543 }
1544
1545 /* called in rcu_read_lock() section */
1546 static int __mkroute_input(struct sk_buff *skb,
1547                            const struct fib_result *res,
1548                            struct in_device *in_dev,
1549                            __be32 daddr, __be32 saddr, u32 tos)
1550 {
1551         struct fib_nh_exception *fnhe;
1552         struct rtable *rth;
1553         int err;
1554         struct in_device *out_dev;
1555         bool do_cache;
1556         u32 itag = 0;
1557
1558         /* get a working reference to the output device */
1559         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1560         if (!out_dev) {
1561                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1562                 return -EINVAL;
1563         }
1564
1565         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1566                                   in_dev->dev, in_dev, &itag);
1567         if (err < 0) {
1568                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1569                                          saddr);
1570
1571                 goto cleanup;
1572         }
1573
1574         do_cache = res->fi && !itag;
1575         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1576             skb->protocol == htons(ETH_P_IP) &&
1577             (IN_DEV_SHARED_MEDIA(out_dev) ||
1578              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1579                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1580
1581         if (skb->protocol != htons(ETH_P_IP)) {
1582                 /* Not IP (i.e. ARP). Do not create route, if it is
1583                  * invalid for proxy arp. DNAT routes are always valid.
1584                  *
1585                  * Proxy arp feature have been extended to allow, ARP
1586                  * replies back to the same interface, to support
1587                  * Private VLAN switch technologies. See arp.c.
1588                  */
1589                 if (out_dev == in_dev &&
1590                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1591                         err = -EINVAL;
1592                         goto cleanup;
1593                 }
1594         }
1595
1596         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1597         if (do_cache) {
1598                 if (fnhe)
1599                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1600                 else
1601                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1602
1603                 if (rt_cache_valid(rth)) {
1604                         skb_dst_set_noref(skb, &rth->dst);
1605                         goto out;
1606                 }
1607         }
1608
1609         rth = rt_dst_alloc(out_dev->dev,
1610                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1611                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1612         if (!rth) {
1613                 err = -ENOBUFS;
1614                 goto cleanup;
1615         }
1616
1617         rth->rt_genid = rt_genid_ipv4(dev_net(rth->dst.dev));
1618         rth->rt_flags = 0;
1619         rth->rt_type = res->type;
1620         rth->rt_is_input = 1;
1621         rth->rt_iif     = 0;
1622         rth->rt_pmtu    = 0;
1623         rth->rt_gateway = 0;
1624         rth->rt_uses_gateway = 0;
1625         INIT_LIST_HEAD(&rth->rt_uncached);
1626         rth->rt_lwtstate = NULL;
1627         RT_CACHE_STAT_INC(in_slow_tot);
1628
1629         rth->dst.input = ip_forward;
1630         rth->dst.output = ip_output;
1631
1632         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1633         if (lwtunnel_output_redirect(rth->rt_lwtstate))
1634                 rth->dst.output = lwtunnel_output;
1635         skb_dst_set(skb, &rth->dst);
1636 out:
1637         err = 0;
1638  cleanup:
1639         return err;
1640 }
1641
1642 static int ip_mkroute_input(struct sk_buff *skb,
1643                             struct fib_result *res,
1644                             const struct flowi4 *fl4,
1645                             struct in_device *in_dev,
1646                             __be32 daddr, __be32 saddr, u32 tos)
1647 {
1648 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1649         if (res->fi && res->fi->fib_nhs > 1)
1650                 fib_select_multipath(res);
1651 #endif
1652
1653         /* create a routing cache entry */
1654         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1655 }
1656
1657 /*
1658  *      NOTE. We drop all the packets that has local source
1659  *      addresses, because every properly looped back packet
1660  *      must have correct destination already attached by output routine.
1661  *
1662  *      Such approach solves two big problems:
1663  *      1. Not simplex devices are handled properly.
1664  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1665  *      called with rcu_read_lock()
1666  */
1667
1668 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1669                                u8 tos, struct net_device *dev)
1670 {
1671         struct fib_result res;
1672         struct in_device *in_dev = __in_dev_get_rcu(dev);
1673         struct ip_tunnel_info *tun_info;
1674         struct flowi4   fl4;
1675         unsigned int    flags = 0;
1676         u32             itag = 0;
1677         struct rtable   *rth;
1678         int             err = -EINVAL;
1679         struct net    *net = dev_net(dev);
1680         bool do_cache;
1681
1682         /* IP on this device is disabled. */
1683
1684         if (!in_dev)
1685                 goto out;
1686
1687         /* Check for the most weird martians, which can be not detected
1688            by fib_lookup.
1689          */
1690
1691         tun_info = skb_tunnel_info(skb, AF_INET);
1692         if (tun_info && tun_info->mode == IP_TUNNEL_INFO_RX)
1693                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1694         else
1695                 fl4.flowi4_tun_key.tun_id = 0;
1696         skb_dst_drop(skb);
1697
1698         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1699                 goto martian_source;
1700
1701         res.fi = NULL;
1702         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1703                 goto brd_input;
1704
1705         /* Accept zero addresses only to limited broadcast;
1706          * I even do not know to fix it or not. Waiting for complains :-)
1707          */
1708         if (ipv4_is_zeronet(saddr))
1709                 goto martian_source;
1710
1711         if (ipv4_is_zeronet(daddr))
1712                 goto martian_destination;
1713
1714         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1715          * and call it once if daddr or/and saddr are loopback addresses
1716          */
1717         if (ipv4_is_loopback(daddr)) {
1718                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1719                         goto martian_destination;
1720         } else if (ipv4_is_loopback(saddr)) {
1721                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1722                         goto martian_source;
1723         }
1724
1725         /*
1726          *      Now we are ready to route packet.
1727          */
1728         fl4.flowi4_oif = 0;
1729         fl4.flowi4_iif = dev->ifindex;
1730         fl4.flowi4_mark = skb->mark;
1731         fl4.flowi4_tos = tos;
1732         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1733         fl4.daddr = daddr;
1734         fl4.saddr = saddr;
1735         err = fib_lookup(net, &fl4, &res, 0);
1736         if (err != 0) {
1737                 if (!IN_DEV_FORWARD(in_dev))
1738                         err = -EHOSTUNREACH;
1739                 goto no_route;
1740         }
1741
1742         if (res.type == RTN_BROADCAST)
1743                 goto brd_input;
1744
1745         if (res.type == RTN_LOCAL) {
1746                 err = fib_validate_source(skb, saddr, daddr, tos,
1747                                           0, dev, in_dev, &itag);
1748                 if (err < 0)
1749                         goto martian_source_keep_err;
1750                 goto local_input;
1751         }
1752
1753         if (!IN_DEV_FORWARD(in_dev)) {
1754                 err = -EHOSTUNREACH;
1755                 goto no_route;
1756         }
1757         if (res.type != RTN_UNICAST)
1758                 goto martian_destination;
1759
1760         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1761 out:    return err;
1762
1763 brd_input:
1764         if (skb->protocol != htons(ETH_P_IP))
1765                 goto e_inval;
1766
1767         if (!ipv4_is_zeronet(saddr)) {
1768                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1769                                           in_dev, &itag);
1770                 if (err < 0)
1771                         goto martian_source_keep_err;
1772         }
1773         flags |= RTCF_BROADCAST;
1774         res.type = RTN_BROADCAST;
1775         RT_CACHE_STAT_INC(in_brd);
1776
1777 local_input:
1778         do_cache = false;
1779         if (res.fi) {
1780                 if (!itag) {
1781                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1782                         if (rt_cache_valid(rth)) {
1783                                 skb_dst_set_noref(skb, &rth->dst);
1784                                 err = 0;
1785                                 goto out;
1786                         }
1787                         do_cache = true;
1788                 }
1789         }
1790
1791         rth = rt_dst_alloc(net->loopback_dev,
1792                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1793         if (!rth)
1794                 goto e_nobufs;
1795
1796         rth->dst.input= ip_local_deliver;
1797         rth->dst.output= ip_rt_bug;
1798 #ifdef CONFIG_IP_ROUTE_CLASSID
1799         rth->dst.tclassid = itag;
1800 #endif
1801
1802         rth->rt_genid = rt_genid_ipv4(net);
1803         rth->rt_flags   = flags|RTCF_LOCAL;
1804         rth->rt_type    = res.type;
1805         rth->rt_is_input = 1;
1806         rth->rt_iif     = 0;
1807         rth->rt_pmtu    = 0;
1808         rth->rt_gateway = 0;
1809         rth->rt_uses_gateway = 0;
1810         INIT_LIST_HEAD(&rth->rt_uncached);
1811         rth->rt_lwtstate = NULL;
1812
1813         RT_CACHE_STAT_INC(in_slow_tot);
1814         if (res.type == RTN_UNREACHABLE) {
1815                 rth->dst.input= ip_error;
1816                 rth->dst.error= -err;
1817                 rth->rt_flags   &= ~RTCF_LOCAL;
1818         }
1819         if (do_cache) {
1820                 if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
1821                         rth->dst.flags |= DST_NOCACHE;
1822                         rt_add_uncached_list(rth);
1823                 }
1824         }
1825         skb_dst_set(skb, &rth->dst);
1826         err = 0;
1827         goto out;
1828
1829 no_route:
1830         RT_CACHE_STAT_INC(in_no_route);
1831         res.type = RTN_UNREACHABLE;
1832         res.fi = NULL;
1833         goto local_input;
1834
1835         /*
1836          *      Do not cache martian addresses: they should be logged (RFC1812)
1837          */
1838 martian_destination:
1839         RT_CACHE_STAT_INC(in_martian_dst);
1840 #ifdef CONFIG_IP_ROUTE_VERBOSE
1841         if (IN_DEV_LOG_MARTIANS(in_dev))
1842                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1843                                      &daddr, &saddr, dev->name);
1844 #endif
1845
1846 e_inval:
1847         err = -EINVAL;
1848         goto out;
1849
1850 e_nobufs:
1851         err = -ENOBUFS;
1852         goto out;
1853
1854 martian_source:
1855         err = -EINVAL;
1856 martian_source_keep_err:
1857         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1858         goto out;
1859 }
1860
1861 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1862                          u8 tos, struct net_device *dev)
1863 {
1864         int res;
1865
1866         rcu_read_lock();
1867
1868         /* Multicast recognition logic is moved from route cache to here.
1869            The problem was that too many Ethernet cards have broken/missing
1870            hardware multicast filters :-( As result the host on multicasting
1871            network acquires a lot of useless route cache entries, sort of
1872            SDR messages from all the world. Now we try to get rid of them.
1873            Really, provided software IP multicast filter is organized
1874            reasonably (at least, hashed), it does not result in a slowdown
1875            comparing with route cache reject entries.
1876            Note, that multicast routers are not affected, because
1877            route cache entry is created eventually.
1878          */
1879         if (ipv4_is_multicast(daddr)) {
1880                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1881
1882                 if (in_dev) {
1883                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1884                                                   ip_hdr(skb)->protocol);
1885                         if (our
1886 #ifdef CONFIG_IP_MROUTE
1887                                 ||
1888                             (!ipv4_is_local_multicast(daddr) &&
1889                              IN_DEV_MFORWARD(in_dev))
1890 #endif
1891                            ) {
1892                                 int res = ip_route_input_mc(skb, daddr, saddr,
1893                                                             tos, dev, our);
1894                                 rcu_read_unlock();
1895                                 return res;
1896                         }
1897                 }
1898                 rcu_read_unlock();
1899                 return -EINVAL;
1900         }
1901         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1902         rcu_read_unlock();
1903         return res;
1904 }
1905 EXPORT_SYMBOL(ip_route_input_noref);
1906
1907 /* called with rcu_read_lock() */
1908 static struct rtable *__mkroute_output(const struct fib_result *res,
1909                                        const struct flowi4 *fl4, int orig_oif,
1910                                        struct net_device *dev_out,
1911                                        unsigned int flags)
1912 {
1913         struct fib_info *fi = res->fi;
1914         struct fib_nh_exception *fnhe;
1915         struct in_device *in_dev;
1916         u16 type = res->type;
1917         struct rtable *rth;
1918         bool do_cache;
1919
1920         in_dev = __in_dev_get_rcu(dev_out);
1921         if (!in_dev)
1922                 return ERR_PTR(-EINVAL);
1923
1924         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1925                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
1926                         return ERR_PTR(-EINVAL);
1927
1928         if (ipv4_is_lbcast(fl4->daddr))
1929                 type = RTN_BROADCAST;
1930         else if (ipv4_is_multicast(fl4->daddr))
1931                 type = RTN_MULTICAST;
1932         else if (ipv4_is_zeronet(fl4->daddr))
1933                 return ERR_PTR(-EINVAL);
1934
1935         if (dev_out->flags & IFF_LOOPBACK)
1936                 flags |= RTCF_LOCAL;
1937
1938         do_cache = true;
1939         if (type == RTN_BROADCAST) {
1940                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
1941                 fi = NULL;
1942         } else if (type == RTN_MULTICAST) {
1943                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
1944                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
1945                                      fl4->flowi4_proto))
1946                         flags &= ~RTCF_LOCAL;
1947                 else
1948                         do_cache = false;
1949                 /* If multicast route do not exist use
1950                  * default one, but do not gateway in this case.
1951                  * Yes, it is hack.
1952                  */
1953                 if (fi && res->prefixlen < 4)
1954                         fi = NULL;
1955         }
1956
1957         fnhe = NULL;
1958         do_cache &= fi != NULL;
1959         if (do_cache) {
1960                 struct rtable __rcu **prth;
1961                 struct fib_nh *nh = &FIB_RES_NH(*res);
1962
1963                 fnhe = find_exception(nh, fl4->daddr);
1964                 if (fnhe)
1965                         prth = &fnhe->fnhe_rth_output;
1966                 else {
1967                         if (unlikely(fl4->flowi4_flags &
1968                                      FLOWI_FLAG_KNOWN_NH &&
1969                                      !(nh->nh_gw &&
1970                                        nh->nh_scope == RT_SCOPE_LINK))) {
1971                                 do_cache = false;
1972                                 goto add;
1973                         }
1974                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
1975                 }
1976                 rth = rcu_dereference(*prth);
1977                 if (rt_cache_valid(rth)) {
1978                         dst_hold(&rth->dst);
1979                         return rth;
1980                 }
1981         }
1982
1983 add:
1984         rth = rt_dst_alloc(dev_out,
1985                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1986                            IN_DEV_CONF_GET(in_dev, NOXFRM),
1987                            do_cache);
1988         if (!rth)
1989                 return ERR_PTR(-ENOBUFS);
1990
1991         rth->dst.output = ip_output;
1992
1993         rth->rt_genid = rt_genid_ipv4(dev_net(dev_out));
1994         rth->rt_flags   = flags;
1995         rth->rt_type    = type;
1996         rth->rt_is_input = 0;
1997         rth->rt_iif     = orig_oif ? : 0;
1998         rth->rt_pmtu    = 0;
1999         rth->rt_gateway = 0;
2000         rth->rt_uses_gateway = 0;
2001         INIT_LIST_HEAD(&rth->rt_uncached);
2002         rth->rt_lwtstate = NULL;
2003         RT_CACHE_STAT_INC(out_slow_tot);
2004
2005         if (flags & RTCF_LOCAL)
2006                 rth->dst.input = ip_local_deliver;
2007         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2008                 if (flags & RTCF_LOCAL &&
2009                     !(dev_out->flags & IFF_LOOPBACK)) {
2010                         rth->dst.output = ip_mc_output;
2011                         RT_CACHE_STAT_INC(out_slow_mc);
2012                 }
2013 #ifdef CONFIG_IP_MROUTE
2014                 if (type == RTN_MULTICAST) {
2015                         if (IN_DEV_MFORWARD(in_dev) &&
2016                             !ipv4_is_local_multicast(fl4->daddr)) {
2017                                 rth->dst.input = ip_mr_input;
2018                                 rth->dst.output = ip_mc_output;
2019                         }
2020                 }
2021 #endif
2022         }
2023
2024         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
2025
2026         return rth;
2027 }
2028
2029 /*
2030  * Major route resolver routine.
2031  */
2032
2033 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4)
2034 {
2035         struct net_device *dev_out = NULL;
2036         __u8 tos = RT_FL_TOS(fl4);
2037         unsigned int flags = 0;
2038         struct fib_result res;
2039         struct rtable *rth;
2040         int orig_oif;
2041
2042         res.tclassid    = 0;
2043         res.fi          = NULL;
2044         res.table       = NULL;
2045
2046         orig_oif = fl4->flowi4_oif;
2047
2048         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2049         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2050         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2051                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2052
2053         rcu_read_lock();
2054         if (fl4->saddr) {
2055                 rth = ERR_PTR(-EINVAL);
2056                 if (ipv4_is_multicast(fl4->saddr) ||
2057                     ipv4_is_lbcast(fl4->saddr) ||
2058                     ipv4_is_zeronet(fl4->saddr))
2059                         goto out;
2060
2061                 /* I removed check for oif == dev_out->oif here.
2062                    It was wrong for two reasons:
2063                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2064                       is assigned to multiple interfaces.
2065                    2. Moreover, we are allowed to send packets with saddr
2066                       of another iface. --ANK
2067                  */
2068
2069                 if (fl4->flowi4_oif == 0 &&
2070                     (ipv4_is_multicast(fl4->daddr) ||
2071                      ipv4_is_lbcast(fl4->daddr))) {
2072                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2073                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2074                         if (!dev_out)
2075                                 goto out;
2076
2077                         /* Special hack: user can direct multicasts
2078                            and limited broadcast via necessary interface
2079                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2080                            This hack is not just for fun, it allows
2081                            vic,vat and friends to work.
2082                            They bind socket to loopback, set ttl to zero
2083                            and expect that it will work.
2084                            From the viewpoint of routing cache they are broken,
2085                            because we are not allowed to build multicast path
2086                            with loopback source addr (look, routing cache
2087                            cannot know, that ttl is zero, so that packet
2088                            will not leave this host and route is valid).
2089                            Luckily, this hack is good workaround.
2090                          */
2091
2092                         fl4->flowi4_oif = dev_out->ifindex;
2093                         goto make_route;
2094                 }
2095
2096                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2097                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2098                         if (!__ip_dev_find(net, fl4->saddr, false))
2099                                 goto out;
2100                 }
2101         }
2102
2103
2104         if (fl4->flowi4_oif) {
2105                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2106                 rth = ERR_PTR(-ENODEV);
2107                 if (!dev_out)
2108                         goto out;
2109
2110                 /* RACE: Check return value of inet_select_addr instead. */
2111                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2112                         rth = ERR_PTR(-ENETUNREACH);
2113                         goto out;
2114                 }
2115                 if (ipv4_is_local_multicast(fl4->daddr) ||
2116                     ipv4_is_lbcast(fl4->daddr) ||
2117                     fl4->flowi4_proto == IPPROTO_IGMP) {
2118                         if (!fl4->saddr)
2119                                 fl4->saddr = inet_select_addr(dev_out, 0,
2120                                                               RT_SCOPE_LINK);
2121                         goto make_route;
2122                 }
2123                 if (!fl4->saddr) {
2124                         if (ipv4_is_multicast(fl4->daddr))
2125                                 fl4->saddr = inet_select_addr(dev_out, 0,
2126                                                               fl4->flowi4_scope);
2127                         else if (!fl4->daddr)
2128                                 fl4->saddr = inet_select_addr(dev_out, 0,
2129                                                               RT_SCOPE_HOST);
2130                 }
2131         }
2132
2133         if (!fl4->daddr) {
2134                 fl4->daddr = fl4->saddr;
2135                 if (!fl4->daddr)
2136                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2137                 dev_out = net->loopback_dev;
2138                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2139                 res.type = RTN_LOCAL;
2140                 flags |= RTCF_LOCAL;
2141                 goto make_route;
2142         }
2143
2144         if (fib_lookup(net, fl4, &res, 0)) {
2145                 res.fi = NULL;
2146                 res.table = NULL;
2147                 if (fl4->flowi4_oif) {
2148                         /* Apparently, routing tables are wrong. Assume,
2149                            that the destination is on link.
2150
2151                            WHY? DW.
2152                            Because we are allowed to send to iface
2153                            even if it has NO routes and NO assigned
2154                            addresses. When oif is specified, routing
2155                            tables are looked up with only one purpose:
2156                            to catch if destination is gatewayed, rather than
2157                            direct. Moreover, if MSG_DONTROUTE is set,
2158                            we send packet, ignoring both routing tables
2159                            and ifaddr state. --ANK
2160
2161
2162                            We could make it even if oif is unknown,
2163                            likely IPv6, but we do not.
2164                          */
2165
2166                         if (fl4->saddr == 0)
2167                                 fl4->saddr = inet_select_addr(dev_out, 0,
2168                                                               RT_SCOPE_LINK);
2169                         res.type = RTN_UNICAST;
2170                         goto make_route;
2171                 }
2172                 rth = ERR_PTR(-ENETUNREACH);
2173                 goto out;
2174         }
2175
2176         if (res.type == RTN_LOCAL) {
2177                 if (!fl4->saddr) {
2178                         if (res.fi->fib_prefsrc)
2179                                 fl4->saddr = res.fi->fib_prefsrc;
2180                         else
2181                                 fl4->saddr = fl4->daddr;
2182                 }
2183                 dev_out = net->loopback_dev;
2184                 fl4->flowi4_oif = dev_out->ifindex;
2185                 flags |= RTCF_LOCAL;
2186                 goto make_route;
2187         }
2188
2189 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2190         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2191                 fib_select_multipath(&res);
2192         else
2193 #endif
2194         if (!res.prefixlen &&
2195             res.table->tb_num_default > 1 &&
2196             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2197                 fib_select_default(fl4, &res);
2198
2199         if (!fl4->saddr)
2200                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2201
2202         dev_out = FIB_RES_DEV(res);
2203         fl4->flowi4_oif = dev_out->ifindex;
2204
2205
2206 make_route:
2207         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2208
2209 out:
2210         rcu_read_unlock();
2211         return rth;
2212 }
2213 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2214
2215 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2216 {
2217         return NULL;
2218 }
2219
2220 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2221 {
2222         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2223
2224         return mtu ? : dst->dev->mtu;
2225 }
2226
2227 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2228                                           struct sk_buff *skb, u32 mtu)
2229 {
2230 }
2231
2232 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2233                                        struct sk_buff *skb)
2234 {
2235 }
2236
2237 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2238                                           unsigned long old)
2239 {
2240         return NULL;
2241 }
2242
2243 static struct dst_ops ipv4_dst_blackhole_ops = {
2244         .family                 =       AF_INET,
2245         .check                  =       ipv4_blackhole_dst_check,
2246         .mtu                    =       ipv4_blackhole_mtu,
2247         .default_advmss         =       ipv4_default_advmss,
2248         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2249         .redirect               =       ipv4_rt_blackhole_redirect,
2250         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2251         .neigh_lookup           =       ipv4_neigh_lookup,
2252 };
2253
2254 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2255 {
2256         struct rtable *ort = (struct rtable *) dst_orig;
2257         struct rtable *rt;
2258
2259         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2260         if (rt) {
2261                 struct dst_entry *new = &rt->dst;
2262
2263                 new->__use = 1;
2264                 new->input = dst_discard;
2265                 new->output = dst_discard_sk;
2266
2267                 new->dev = ort->dst.dev;
2268                 if (new->dev)
2269                         dev_hold(new->dev);
2270
2271                 rt->rt_is_input = ort->rt_is_input;
2272                 rt->rt_iif = ort->rt_iif;
2273                 rt->rt_pmtu = ort->rt_pmtu;
2274
2275                 rt->rt_genid = rt_genid_ipv4(net);
2276                 rt->rt_flags = ort->rt_flags;
2277                 rt->rt_type = ort->rt_type;
2278                 rt->rt_gateway = ort->rt_gateway;
2279                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2280
2281                 INIT_LIST_HEAD(&rt->rt_uncached);
2282                 rt->rt_lwtstate = NULL;
2283                 dst_free(new);
2284         }
2285
2286         dst_release(dst_orig);
2287
2288         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2289 }
2290
2291 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2292                                     struct sock *sk)
2293 {
2294         struct rtable *rt = __ip_route_output_key(net, flp4);
2295
2296         if (IS_ERR(rt))
2297                 return rt;
2298
2299         if (flp4->flowi4_proto)
2300                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2301                                                         flowi4_to_flowi(flp4),
2302                                                         sk, 0);
2303
2304         return rt;
2305 }
2306 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2307
2308 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src,
2309                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2310                         u32 seq, int event, int nowait, unsigned int flags)
2311 {
2312         struct rtable *rt = skb_rtable(skb);
2313         struct rtmsg *r;
2314         struct nlmsghdr *nlh;
2315         unsigned long expires = 0;
2316         u32 error;
2317         u32 metrics[RTAX_MAX];
2318
2319         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2320         if (!nlh)
2321                 return -EMSGSIZE;
2322
2323         r = nlmsg_data(nlh);
2324         r->rtm_family    = AF_INET;
2325         r->rtm_dst_len  = 32;
2326         r->rtm_src_len  = 0;
2327         r->rtm_tos      = fl4->flowi4_tos;
2328         r->rtm_table    = RT_TABLE_MAIN;
2329         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2330                 goto nla_put_failure;
2331         r->rtm_type     = rt->rt_type;
2332         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2333         r->rtm_protocol = RTPROT_UNSPEC;
2334         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2335         if (rt->rt_flags & RTCF_NOTIFY)
2336                 r->rtm_flags |= RTM_F_NOTIFY;
2337         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2338                 r->rtm_flags |= RTCF_DOREDIRECT;
2339
2340         if (nla_put_in_addr(skb, RTA_DST, dst))
2341                 goto nla_put_failure;
2342         if (src) {
2343                 r->rtm_src_len = 32;
2344                 if (nla_put_in_addr(skb, RTA_SRC, src))
2345                         goto nla_put_failure;
2346         }
2347         if (rt->dst.dev &&
2348             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2349                 goto nla_put_failure;
2350 #ifdef CONFIG_IP_ROUTE_CLASSID
2351         if (rt->dst.tclassid &&
2352             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2353                 goto nla_put_failure;
2354 #endif
2355         if (!rt_is_input_route(rt) &&
2356             fl4->saddr != src) {
2357                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2358                         goto nla_put_failure;
2359         }
2360         if (rt->rt_uses_gateway &&
2361             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2362                 goto nla_put_failure;
2363
2364         expires = rt->dst.expires;
2365         if (expires) {
2366                 unsigned long now = jiffies;
2367
2368                 if (time_before(now, expires))
2369                         expires -= now;
2370                 else
2371                         expires = 0;
2372         }
2373
2374         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2375         if (rt->rt_pmtu && expires)
2376                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2377         if (rtnetlink_put_metrics(skb, metrics) < 0)
2378                 goto nla_put_failure;
2379
2380         if (fl4->flowi4_mark &&
2381             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2382                 goto nla_put_failure;
2383
2384         error = rt->dst.error;
2385
2386         if (rt_is_input_route(rt)) {
2387 #ifdef CONFIG_IP_MROUTE
2388                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2389                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2390                         int err = ipmr_get_route(net, skb,
2391                                                  fl4->saddr, fl4->daddr,
2392                                                  r, nowait);
2393                         if (err <= 0) {
2394                                 if (!nowait) {
2395                                         if (err == 0)
2396                                                 return 0;
2397                                         goto nla_put_failure;
2398                                 } else {
2399                                         if (err == -EMSGSIZE)
2400                                                 goto nla_put_failure;
2401                                         error = err;
2402                                 }
2403                         }
2404                 } else
2405 #endif
2406                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2407                                 goto nla_put_failure;
2408         }
2409
2410         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2411                 goto nla_put_failure;
2412
2413         nlmsg_end(skb, nlh);
2414         return 0;
2415
2416 nla_put_failure:
2417         nlmsg_cancel(skb, nlh);
2418         return -EMSGSIZE;
2419 }
2420
2421 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2422 {
2423         struct net *net = sock_net(in_skb->sk);
2424         struct rtmsg *rtm;
2425         struct nlattr *tb[RTA_MAX+1];
2426         struct rtable *rt = NULL;
2427         struct flowi4 fl4;
2428         __be32 dst = 0;
2429         __be32 src = 0;
2430         u32 iif;
2431         int err;
2432         int mark;
2433         struct sk_buff *skb;
2434
2435         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2436         if (err < 0)
2437                 goto errout;
2438
2439         rtm = nlmsg_data(nlh);
2440
2441         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2442         if (!skb) {
2443                 err = -ENOBUFS;
2444                 goto errout;
2445         }
2446
2447         /* Reserve room for dummy headers, this skb can pass
2448            through good chunk of routing engine.
2449          */
2450         skb_reset_mac_header(skb);
2451         skb_reset_network_header(skb);
2452
2453         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2454         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2455         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2456
2457         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2458         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2459         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2460         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2461
2462         memset(&fl4, 0, sizeof(fl4));
2463         fl4.daddr = dst;
2464         fl4.saddr = src;
2465         fl4.flowi4_tos = rtm->rtm_tos;
2466         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2467         fl4.flowi4_mark = mark;
2468
2469         if (iif) {
2470                 struct net_device *dev;
2471
2472                 dev = __dev_get_by_index(net, iif);
2473                 if (!dev) {
2474                         err = -ENODEV;
2475                         goto errout_free;
2476                 }
2477
2478                 skb->protocol   = htons(ETH_P_IP);
2479                 skb->dev        = dev;
2480                 skb->mark       = mark;
2481                 local_bh_disable();
2482                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2483                 local_bh_enable();
2484
2485                 rt = skb_rtable(skb);
2486                 if (err == 0 && rt->dst.error)
2487                         err = -rt->dst.error;
2488         } else {
2489                 rt = ip_route_output_key(net, &fl4);
2490
2491                 err = 0;
2492                 if (IS_ERR(rt))
2493                         err = PTR_ERR(rt);
2494         }
2495
2496         if (err)
2497                 goto errout_free;
2498
2499         skb_dst_set(skb, &rt->dst);
2500         if (rtm->rtm_flags & RTM_F_NOTIFY)
2501                 rt->rt_flags |= RTCF_NOTIFY;
2502
2503         err = rt_fill_info(net, dst, src, &fl4, skb,
2504                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2505                            RTM_NEWROUTE, 0, 0);
2506         if (err < 0)
2507                 goto errout_free;
2508
2509         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2510 errout:
2511         return err;
2512
2513 errout_free:
2514         kfree_skb(skb);
2515         goto errout;
2516 }
2517
2518 void ip_rt_multicast_event(struct in_device *in_dev)
2519 {
2520         rt_cache_flush(dev_net(in_dev->dev));
2521 }
2522
2523 #ifdef CONFIG_SYSCTL
2524 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
2525 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2526 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2527 static int ip_rt_gc_elasticity __read_mostly    = 8;
2528
2529 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2530                                         void __user *buffer,
2531                                         size_t *lenp, loff_t *ppos)
2532 {
2533         struct net *net = (struct net *)__ctl->extra1;
2534
2535         if (write) {
2536                 rt_cache_flush(net);
2537                 fnhe_genid_bump(net);
2538                 return 0;
2539         }
2540
2541         return -EINVAL;
2542 }
2543
2544 static struct ctl_table ipv4_route_table[] = {
2545         {
2546                 .procname       = "gc_thresh",
2547                 .data           = &ipv4_dst_ops.gc_thresh,
2548                 .maxlen         = sizeof(int),
2549                 .mode           = 0644,
2550                 .proc_handler   = proc_dointvec,
2551         },
2552         {
2553                 .procname       = "max_size",
2554                 .data           = &ip_rt_max_size,
2555                 .maxlen         = sizeof(int),
2556                 .mode           = 0644,
2557                 .proc_handler   = proc_dointvec,
2558         },
2559         {
2560                 /*  Deprecated. Use gc_min_interval_ms */
2561
2562                 .procname       = "gc_min_interval",
2563                 .data           = &ip_rt_gc_min_interval,
2564                 .maxlen         = sizeof(int),
2565                 .mode           = 0644,
2566                 .proc_handler   = proc_dointvec_jiffies,
2567         },
2568         {
2569                 .procname       = "gc_min_interval_ms",
2570                 .data           = &ip_rt_gc_min_interval,
2571                 .maxlen         = sizeof(int),
2572                 .mode           = 0644,
2573                 .proc_handler   = proc_dointvec_ms_jiffies,
2574         },
2575         {
2576                 .procname       = "gc_timeout",
2577                 .data           = &ip_rt_gc_timeout,
2578                 .maxlen         = sizeof(int),
2579                 .mode           = 0644,
2580                 .proc_handler   = proc_dointvec_jiffies,
2581         },
2582         {
2583                 .procname       = "gc_interval",
2584                 .data           = &ip_rt_gc_interval,
2585                 .maxlen         = sizeof(int),
2586                 .mode           = 0644,
2587                 .proc_handler   = proc_dointvec_jiffies,
2588         },
2589         {
2590                 .procname       = "redirect_load",
2591                 .data           = &ip_rt_redirect_load,
2592                 .maxlen         = sizeof(int),
2593                 .mode           = 0644,
2594                 .proc_handler   = proc_dointvec,
2595         },
2596         {
2597                 .procname       = "redirect_number",
2598                 .data           = &ip_rt_redirect_number,
2599                 .maxlen         = sizeof(int),
2600                 .mode           = 0644,
2601                 .proc_handler   = proc_dointvec,
2602         },
2603         {
2604                 .procname       = "redirect_silence",
2605                 .data           = &ip_rt_redirect_silence,
2606                 .maxlen         = sizeof(int),
2607                 .mode           = 0644,
2608                 .proc_handler   = proc_dointvec,
2609         },
2610         {
2611                 .procname       = "error_cost",
2612                 .data           = &ip_rt_error_cost,
2613                 .maxlen         = sizeof(int),
2614                 .mode           = 0644,
2615                 .proc_handler   = proc_dointvec,
2616         },
2617         {
2618                 .procname       = "error_burst",
2619                 .data           = &ip_rt_error_burst,
2620                 .maxlen         = sizeof(int),
2621                 .mode           = 0644,
2622                 .proc_handler   = proc_dointvec,
2623         },
2624         {
2625                 .procname       = "gc_elasticity",
2626                 .data           = &ip_rt_gc_elasticity,
2627                 .maxlen         = sizeof(int),
2628                 .mode           = 0644,
2629                 .proc_handler   = proc_dointvec,
2630         },
2631         {
2632                 .procname       = "mtu_expires",
2633                 .data           = &ip_rt_mtu_expires,
2634                 .maxlen         = sizeof(int),
2635                 .mode           = 0644,
2636                 .proc_handler   = proc_dointvec_jiffies,
2637         },
2638         {
2639                 .procname       = "min_pmtu",
2640                 .data           = &ip_rt_min_pmtu,
2641                 .maxlen         = sizeof(int),
2642                 .mode           = 0644,
2643                 .proc_handler   = proc_dointvec,
2644         },
2645         {
2646                 .procname       = "min_adv_mss",
2647                 .data           = &ip_rt_min_advmss,
2648                 .maxlen         = sizeof(int),
2649                 .mode           = 0644,
2650                 .proc_handler   = proc_dointvec,
2651         },
2652         { }
2653 };
2654
2655 static struct ctl_table ipv4_route_flush_table[] = {
2656         {
2657                 .procname       = "flush",
2658                 .maxlen         = sizeof(int),
2659                 .mode           = 0200,
2660                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2661         },
2662         { },
2663 };
2664
2665 static __net_init int sysctl_route_net_init(struct net *net)
2666 {
2667         struct ctl_table *tbl;
2668
2669         tbl = ipv4_route_flush_table;
2670         if (!net_eq(net, &init_net)) {
2671                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2672                 if (!tbl)
2673                         goto err_dup;
2674
2675                 /* Don't export sysctls to unprivileged users */
2676                 if (net->user_ns != &init_user_ns)
2677                         tbl[0].procname = NULL;
2678         }
2679         tbl[0].extra1 = net;
2680
2681         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2682         if (!net->ipv4.route_hdr)
2683                 goto err_reg;
2684         return 0;
2685
2686 err_reg:
2687         if (tbl != ipv4_route_flush_table)
2688                 kfree(tbl);
2689 err_dup:
2690         return -ENOMEM;
2691 }
2692
2693 static __net_exit void sysctl_route_net_exit(struct net *net)
2694 {
2695         struct ctl_table *tbl;
2696
2697         tbl = net->ipv4.route_hdr->ctl_table_arg;
2698         unregister_net_sysctl_table(net->ipv4.route_hdr);
2699         BUG_ON(tbl == ipv4_route_flush_table);
2700         kfree(tbl);
2701 }
2702
2703 static __net_initdata struct pernet_operations sysctl_route_ops = {
2704         .init = sysctl_route_net_init,
2705         .exit = sysctl_route_net_exit,
2706 };
2707 #endif
2708
2709 static __net_init int rt_genid_init(struct net *net)
2710 {
2711         atomic_set(&net->ipv4.rt_genid, 0);
2712         atomic_set(&net->fnhe_genid, 0);
2713         get_random_bytes(&net->ipv4.dev_addr_genid,
2714                          sizeof(net->ipv4.dev_addr_genid));
2715         return 0;
2716 }
2717
2718 static __net_initdata struct pernet_operations rt_genid_ops = {
2719         .init = rt_genid_init,
2720 };
2721
2722 static int __net_init ipv4_inetpeer_init(struct net *net)
2723 {
2724         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2725
2726         if (!bp)
2727                 return -ENOMEM;
2728         inet_peer_base_init(bp);
2729         net->ipv4.peers = bp;
2730         return 0;
2731 }
2732
2733 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2734 {
2735         struct inet_peer_base *bp = net->ipv4.peers;
2736
2737         net->ipv4.peers = NULL;
2738         inetpeer_invalidate_tree(bp);
2739         kfree(bp);
2740 }
2741
2742 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2743         .init   =       ipv4_inetpeer_init,
2744         .exit   =       ipv4_inetpeer_exit,
2745 };
2746
2747 #ifdef CONFIG_IP_ROUTE_CLASSID
2748 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2749 #endif /* CONFIG_IP_ROUTE_CLASSID */
2750
2751 int __init ip_rt_init(void)
2752 {
2753         int rc = 0;
2754         int cpu;
2755
2756         ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
2757         if (!ip_idents)
2758                 panic("IP: failed to allocate ip_idents\n");
2759
2760         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
2761
2762         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
2763         if (!ip_tstamps)
2764                 panic("IP: failed to allocate ip_tstamps\n");
2765
2766         for_each_possible_cpu(cpu) {
2767                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
2768
2769                 INIT_LIST_HEAD(&ul->head);
2770                 spin_lock_init(&ul->lock);
2771         }
2772 #ifdef CONFIG_IP_ROUTE_CLASSID
2773         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2774         if (!ip_rt_acct)
2775                 panic("IP: failed to allocate ip_rt_acct\n");
2776 #endif
2777
2778         ipv4_dst_ops.kmem_cachep =
2779                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2780                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2781
2782         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2783
2784         if (dst_entries_init(&ipv4_dst_ops) < 0)
2785                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2786
2787         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2788                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2789
2790         ipv4_dst_ops.gc_thresh = ~0;
2791         ip_rt_max_size = INT_MAX;
2792
2793         devinet_init();
2794         ip_fib_init();
2795
2796         if (ip_rt_proc_init())
2797                 pr_err("Unable to create route proc files\n");
2798 #ifdef CONFIG_XFRM
2799         xfrm_init();
2800         xfrm4_init();
2801 #endif
2802         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2803
2804 #ifdef CONFIG_SYSCTL
2805         register_pernet_subsys(&sysctl_route_ops);
2806 #endif
2807         register_pernet_subsys(&rt_genid_ops);
2808         register_pernet_subsys(&ipv4_inetpeer_ops);
2809         return rc;
2810 }
2811
2812 #ifdef CONFIG_SYSCTL
2813 /*
2814  * We really need to sanitize the damn ipv4 init order, then all
2815  * this nonsense will go away.
2816  */
2817 void __init ip_static_sysctl_init(void)
2818 {
2819         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2820 }
2821 #endif