datapath/linux/compat/inet_fragment.c

   1 /*
   2  * inet fragments management
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  *              Authors:        Pavel Emelyanov <xemul@openvz.org>
  10  *                              Started as consolidation of ipv4/ip_fragment.c,
  11  *                              ipv6/reassembly. and ipv6 nf conntrack reassembly
  12  */
  13
  14 #include <linux/version.h>
  15
  16 #if !defined(HAVE_CORRECT_MRU_HANDLING) && defined(OVS_FRAGMENT_BACKPORT)
  17
  18 #include <linux/list.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/module.h>
  21 #include <linux/timer.h>
  22 #include <linux/mm.h>
  23 #include <linux/random.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/rtnetlink.h>
  26 #include <linux/slab.h>
  27
  28 #include <net/sock.h>
  29 #include <net/inet_frag.h>
  30 #include <net/inet_ecn.h>
  31
  32 #define INETFRAGS_EVICT_BUCKETS   128
  33 #define INETFRAGS_EVICT_MAX       512
  34
  35 /* don't rebuild inetfrag table with new secret more often than this */
  36 #define INETFRAGS_MIN_REBUILD_INTERVAL (5 * HZ)
  37
  38 /* Given the OR values of all fragments, apply RFC 3168 5.3 requirements
  39  * Value : 0xff if frame should be dropped.
  40  *         0 or INET_ECN_CE value, to be ORed in to final iph->tos field
  41  */
  42 const u8 ip_frag_ecn_table[16] = {
  43         /* at least one fragment had CE, and others ECT_0 or ECT_1 */
  44         [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0]                      = INET_ECN_CE,
  45         [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1]                      = INET_ECN_CE,
  46         [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1]   = INET_ECN_CE,
  47
  48         /* invalid combinations : drop frame */
  49         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE] = 0xff,
  50         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0] = 0xff,
  51         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_1] = 0xff,
  52         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
  53         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0] = 0xff,
  54         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1] = 0xff,
  55         [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
  56 };
  57
  58 static unsigned int
  59 inet_frag_hashfn(const struct inet_frags *f, struct inet_frag_queue *q)
  60 {
  61         return f->hashfn(q) & (INETFRAGS_HASHSZ - 1);
  62 }
  63
  64 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
  65 static bool inet_frag_may_rebuild(struct inet_frags *f)
  66 {
  67         return time_after(jiffies,
  68                f->last_rebuild_jiffies + INETFRAGS_MIN_REBUILD_INTERVAL);
  69 }
  70
  71 static void inet_frag_secret_rebuild(struct inet_frags *f)
  72 {
  73         int i;
  74
  75         write_seqlock_bh(&f->rnd_seqlock);
  76
  77         if (!inet_frag_may_rebuild(f))
  78                 goto out;
  79
  80         get_random_bytes(&f->rnd, sizeof(u32));
  81
  82         for (i = 0; i < INETFRAGS_HASHSZ; i++) {
  83                 struct inet_frag_bucket *hb;
  84                 struct inet_frag_queue *q;
  85                 struct hlist_node *n;
  86
  87                 hb = &f->hash[i];
  88                 spin_lock(&hb->chain_lock);
  89
  90                 hlist_for_each_entry_safe(q, n, &hb->chain, list) {
  91                         unsigned int hval = inet_frag_hashfn(f, q);
  92
  93                         if (hval != i) {
  94                                 struct inet_frag_bucket *hb_dest;
  95
  96                                 hlist_del(&q->list);
  97
  98                                 /* Relink to new hash chain. */
  99                                 hb_dest = &f->hash[hval];
 100
 101                                 /* This is the only place where we take
 102                                  * another chain_lock while already holding
 103                                  * one.  As this will not run concurrently,
 104                                  * we cannot deadlock on hb_dest lock below, if its
 105                                  * already locked it will be released soon since
 106                                  * other caller cannot be waiting for hb lock
 107                                  * that we've taken above.
 108                                  */
 109                                 spin_lock_nested(&hb_dest->chain_lock,
 110                                                  SINGLE_DEPTH_NESTING);
 111                                 hlist_add_head(&q->list, &hb_dest->chain);
 112                                 spin_unlock(&hb_dest->chain_lock);
 113                         }
 114                 }
 115                 spin_unlock(&hb->chain_lock);
 116         }
 117
 118         f->rebuild = false;
 119         f->last_rebuild_jiffies = jiffies;
 120 out:
 121         write_sequnlock_bh(&f->rnd_seqlock);
 122 }
 123
 124 static bool inet_fragq_should_evict(const struct inet_frag_queue *q)
 125 {
 126         return q->net->low_thresh == 0 ||
 127                frag_mem_limit(q->net) >= q->net->low_thresh;
 128 }
 129
 130 static unsigned int
 131 inet_evict_bucket(struct inet_frags *f, struct inet_frag_bucket *hb)
 132 {
 133 #ifndef HAVE_INET_FRAG_QUEUE_WITH_LIST_EVICTOR
 134         struct ovs_inet_frag_queue *ofq;
 135 #endif
 136         struct inet_frag_queue *fq;
 137         struct hlist_node *n;
 138         unsigned int evicted = 0;
 139         HLIST_HEAD(expired);
 140
 141         spin_lock(&hb->chain_lock);
 142
 143         hlist_for_each_entry_safe(fq, n, &hb->chain, list) {
 144                 if (!inet_fragq_should_evict(fq))
 145                         continue;
 146
 147                 if (!del_timer(&fq->timer))
 148                         continue;
 149
 150 #ifdef HAVE_INET_FRAG_QUEUE_WITH_LIST_EVICTOR
 151                 hlist_add_head(&fq->list_evictor, &expired);
 152 #else
 153                 ofq = (struct ovs_inet_frag_queue *)fq;
 154                 hlist_add_head(&ofq->list_evictor, &expired);
 155 #endif
 156                 ++evicted;
 157         }
 158
 159         spin_unlock(&hb->chain_lock);
 160
 161 #ifdef HAVE_INET_FRAG_QUEUE_WITH_LIST_EVICTOR
 162         hlist_for_each_entry_safe(fq, n, &expired, list_evictor)
 163                 f->frag_expire((unsigned long) fq);
 164 #else
 165         hlist_for_each_entry_safe(ofq, n, &expired, list_evictor)
 166                 f->frag_expire((unsigned long) &ofq->fq);
 167 #endif
 168
 169         return evicted;
 170 }
 171
 172 static void inet_frag_worker(struct work_struct *work)
 173 {
 174         unsigned int budget = INETFRAGS_EVICT_BUCKETS;
 175         unsigned int i, evicted = 0;
 176         struct inet_frags *f;
 177
 178         f = container_of(work, struct inet_frags, frags_work);
 179
 180         BUILD_BUG_ON(INETFRAGS_EVICT_BUCKETS >= INETFRAGS_HASHSZ);
 181
 182         local_bh_disable();
 183
 184         for (i = ACCESS_ONCE(f->next_bucket); budget; --budget) {
 185                 evicted += inet_evict_bucket(f, &f->hash[i]);
 186                 i = (i + 1) & (INETFRAGS_HASHSZ - 1);
 187                 if (evicted > INETFRAGS_EVICT_MAX)
 188                         break;
 189         }
 190
 191         f->next_bucket = i;
 192
 193         local_bh_enable();
 194
 195         if (f->rebuild && inet_frag_may_rebuild(f))
 196                 inet_frag_secret_rebuild(f);
 197 }
 198
 199 static void inet_frag_schedule_worker(struct inet_frags *f)
 200 {
 201         if (unlikely(!work_pending(&f->frags_work)))
 202                 schedule_work(&f->frags_work);
 203 }
 204 #endif /* >= 3.17 */
 205
 206 int inet_frags_init(struct inet_frags *f)
 207 {
 208         int i;
 209
 210 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 211         INIT_WORK(&f->frags_work, inet_frag_worker);
 212 #endif
 213
 214         for (i = 0; i < INETFRAGS_HASHSZ; i++) {
 215                 struct inet_frag_bucket *hb = &f->hash[i];
 216
 217                 spin_lock_init(&hb->chain_lock);
 218                 INIT_HLIST_HEAD(&hb->chain);
 219         }
 220
 221 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 222         seqlock_init(&f->rnd_seqlock);
 223         f->last_rebuild_jiffies = 0;
 224         f->frags_cachep = kmem_cache_create(f->frags_cache_name, f->qsize, 0, 0,
 225                                             NULL);
 226         if (!f->frags_cachep)
 227                 return -ENOMEM;
 228 #else
 229         rwlock_init(&f->lock);
 230         f->secret_timer.expires = jiffies + f->secret_interval;
 231 #endif
 232
 233         return 0;
 234 }
 235
 236 void inet_frags_fini(struct inet_frags *f)
 237 {
 238 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 239         cancel_work_sync(&f->frags_work);
 240         kmem_cache_destroy(f->frags_cachep);
 241 #endif
 242 }
 243
 244 int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force);
 245
 246 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 247 void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f)
 248 {
 249         unsigned int seq;
 250
 251 evict_again:
 252         local_bh_disable();
 253         seq = read_seqbegin(&f->rnd_seqlock);
 254
 255         inet_frag_evictor(nf, f, true);
 256
 257         local_bh_enable();
 258         cond_resched();
 259
 260         if (read_seqretry(&f->rnd_seqlock, seq) ||
 261             percpu_counter_sum(&nf->mem))
 262                 goto evict_again;
 263 }
 264 #else
 265 void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f)
 266 {
 267         read_lock_bh(&f->lock);
 268         inet_frag_evictor(nf, f, true);
 269         read_unlock_bh(&f->lock);
 270 }
 271 #endif
 272
 273 static struct inet_frag_bucket *
 274 get_frag_bucket_locked(struct inet_frag_queue *fq, struct inet_frags *f)
 275 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 276 __acquires(f->lock)
 277 #endif
 278 __acquires(hb->chain_lock)
 279 {
 280         struct inet_frag_bucket *hb;
 281         unsigned int hash;
 282
 283 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 284         read_lock(&f->lock);
 285 #else
 286         unsigned int seq;
 287  restart:
 288         seq = read_seqbegin(&f->rnd_seqlock);
 289 #endif
 290
 291         hash = inet_frag_hashfn(f, fq);
 292         hb = &f->hash[hash];
 293
 294         spin_lock(&hb->chain_lock);
 295
 296 #ifndef HAVE_INET_FRAGS_WITH_RWLOCK
 297         if (read_seqretry(&f->rnd_seqlock, seq)) {
 298                 spin_unlock(&hb->chain_lock);
 299                 goto restart;
 300         }
 301 #endif
 302
 303         return hb;
 304 }
 305
 306 static inline void fq_unlink(struct inet_frag_queue *fq, struct inet_frags *f)
 307 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 308 __releases(f->lock)
 309 #endif
 310 __releases(hb->chain_lock)
 311 {
 312         struct inet_frag_bucket *hb;
 313
 314         hb = get_frag_bucket_locked(fq, f);
 315         hlist_del(&fq->list);
 316         q_flags(fq) |= INET_FRAG_COMPLETE;
 317         spin_unlock(&hb->chain_lock);
 318
 319 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 320         read_unlock(&f->lock);
 321 #endif
 322 }
 323
 324 void inet_frag_kill(struct inet_frag_queue *fq, struct inet_frags *f)
 325 {
 326         if (del_timer(&fq->timer))
 327                 atomic_dec(&fq->refcnt);
 328
 329         if (!(q_flags(fq) & INET_FRAG_COMPLETE)) {
 330                 fq_unlink(fq, f);
 331                 atomic_dec(&fq->refcnt);
 332         }
 333 }
 334
 335 static inline void frag_kfree_skb(struct netns_frags *nf, struct inet_frags *f,
 336                                   struct sk_buff *skb)
 337 {
 338         if (f->skb_free)
 339                 f->skb_free(skb);
 340         kfree_skb(skb);
 341 }
 342
 343 void rpl_inet_frag_destroy(struct inet_frag_queue *q, struct inet_frags *f)
 344 {
 345         struct sk_buff *fp;
 346         struct netns_frags *nf;
 347         unsigned int sum, sum_truesize = 0;
 348
 349         WARN_ON(!(q_flags(q) & INET_FRAG_COMPLETE));
 350         WARN_ON(del_timer(&q->timer) != 0);
 351
 352         /* Release all fragment data. */
 353         fp = q->fragments;
 354         nf = q->net;
 355         while (fp) {
 356                 struct sk_buff *xp = fp->next;
 357
 358                 sum_truesize += fp->truesize;
 359                 frag_kfree_skb(nf, f, fp);
 360                 fp = xp;
 361         }
 362         sum = sum_truesize + f->qsize;
 363
 364         if (f->destructor)
 365                 f->destructor(q);
 366 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 367         kmem_cache_free(f->frags_cachep, q);
 368 #else
 369         kfree(q);
 370 #endif
 371
 372         sub_frag_mem_limit(nf, sum);
 373 }
 374
 375 int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force)
 376 {
 377 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 378         int i;
 379
 380         for (i = 0; i < INETFRAGS_HASHSZ ; i++)
 381                 inet_evict_bucket(f, &f->hash[i]);
 382
 383         return 0;
 384 #else
 385         struct inet_frag_queue *q;
 386         int work, evicted = 0;
 387
 388         work = frag_mem_limit(nf) - nf->low_thresh;
 389         while (work > 0 || force) {
 390                 spin_lock(&nf->lru_lock);
 391
 392                 if (list_empty(&nf->lru_list)) {
 393                         spin_unlock(&nf->lru_lock);
 394                         break;
 395                 }
 396
 397                 q = list_first_entry(&nf->lru_list,
 398                                      struct inet_frag_queue, lru_list);
 399                 atomic_inc(&q->refcnt);
 400                 /* Remove q from list to avoid several CPUs grabbing it */
 401                 list_del_init(&q->lru_list);
 402
 403                 spin_unlock(&nf->lru_lock);
 404
 405                 spin_lock(&q->lock);
 406                 if (!(q->last_in & INET_FRAG_COMPLETE))
 407                         inet_frag_kill(q, f);
 408                 spin_unlock(&q->lock);
 409
 410                 if (atomic_dec_and_test(&q->refcnt))
 411                         inet_frag_destroy(q, f, &work);
 412                 evicted++;
 413         }
 414
 415         return evicted;
 416 #endif
 417 }
 418
 419 static struct inet_frag_queue *inet_frag_intern(struct netns_frags *nf,
 420                                                 struct inet_frag_queue *qp_in,
 421                                                 struct inet_frags *f,
 422                                                 void *arg)
 423 {
 424         struct inet_frag_bucket *hb = get_frag_bucket_locked(qp_in, f);
 425         struct inet_frag_queue *qp;
 426
 427 #ifdef CONFIG_SMP
 428         /* With SMP race we have to recheck hash table, because
 429          * such entry could have been created on other cpu before
 430          * we acquired hash bucket lock.
 431          */
 432         hlist_for_each_entry(qp, &hb->chain, list) {
 433                 if (qp->net == nf && f->match(qp, arg)) {
 434                         atomic_inc(&qp->refcnt);
 435                         spin_unlock(&hb->chain_lock);
 436 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 437                         read_unlock(&f->lock);
 438 #endif
 439                         q_flags(qp_in) |= INET_FRAG_COMPLETE;
 440                         inet_frag_put(qp_in, f);
 441                         return qp;
 442                 }
 443         }
 444 #endif /* CONFIG_SMP */
 445         qp = qp_in;
 446         if (!mod_timer(&qp->timer, jiffies + nf->timeout))
 447                 atomic_inc(&qp->refcnt);
 448
 449         atomic_inc(&qp->refcnt);
 450         hlist_add_head(&qp->list, &hb->chain);
 451
 452         spin_unlock(&hb->chain_lock);
 453 #ifdef HAVE_INET_FRAGS_WITH_RWLOCK
 454         read_unlock(&f->lock);
 455 #endif
 456
 457         return qp;
 458 }
 459
 460 static struct inet_frag_queue *inet_frag_alloc(struct netns_frags *nf,
 461                                                struct inet_frags *f,
 462                                                void *arg)
 463 {
 464         struct inet_frag_queue *q;
 465
 466         if (frag_mem_limit(nf) > nf->high_thresh) {
 467 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 468                 inet_frag_schedule_worker(f);
 469 #endif
 470                 return NULL;
 471         }
 472
 473 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 474         q = kmem_cache_zalloc(f->frags_cachep, GFP_ATOMIC);
 475 #else
 476         q = kzalloc(f->qsize, GFP_ATOMIC);
 477 #endif
 478         if (!q)
 479                 return NULL;
 480
 481         q->net = nf;
 482         f->constructor(q, arg);
 483         add_frag_mem_limit(nf, f->qsize);
 484
 485         setup_timer(&q->timer, f->frag_expire, (unsigned long)q);
 486         spin_lock_init(&q->lock);
 487         atomic_set(&q->refcnt, 1);
 488
 489         return q;
 490 }
 491
 492 static struct inet_frag_queue *inet_frag_create(struct netns_frags *nf,
 493                                                 struct inet_frags *f,
 494                                                 void *arg)
 495 {
 496         struct inet_frag_queue *q;
 497
 498         q = inet_frag_alloc(nf, f, arg);
 499         if (!q)
 500                 return NULL;
 501
 502         return inet_frag_intern(nf, q, f, arg);
 503 }
 504
 505 struct inet_frag_queue *inet_frag_find(struct netns_frags *nf,
 506                                        struct inet_frags *f, void *key,
 507                                        unsigned int hash)
 508 {
 509         struct inet_frag_bucket *hb;
 510         struct inet_frag_queue *q;
 511         int depth = 0;
 512
 513 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 514         if (frag_mem_limit(nf) > nf->low_thresh)
 515                 inet_frag_schedule_worker(f);
 516 #else
 517         if (frag_mem_limit(nf) > nf->high_thresh)
 518                 inet_frag_evictor(nf, f, false);
 519 #endif
 520
 521         hash &= (INETFRAGS_HASHSZ - 1);
 522         hb = &f->hash[hash];
 523
 524         spin_lock(&hb->chain_lock);
 525         hlist_for_each_entry(q, &hb->chain, list) {
 526                 if (q->net == nf && f->match(q, key)) {
 527                         atomic_inc(&q->refcnt);
 528                         spin_unlock(&hb->chain_lock);
 529                         return q;
 530                 }
 531                 depth++;
 532         }
 533         spin_unlock(&hb->chain_lock);
 534
 535         if (depth <= INETFRAGS_MAXDEPTH)
 536                 return inet_frag_create(nf, f, key);
 537
 538 #ifdef HAVE_INET_FRAGS_WITH_FRAGS_WORK
 539         if (inet_frag_may_rebuild(f)) {
 540                 if (!f->rebuild)
 541                         f->rebuild = true;
 542                 inet_frag_schedule_worker(f);
 543         }
 544 #endif
 545
 546         return ERR_PTR(-ENOBUFS);
 547 }
 548
 549 void inet_frag_maybe_warn_overflow(struct inet_frag_queue *q,
 550                                    const char *prefix)
 551 {
 552         static const char msg[] = "inet_frag_find: Fragment hash bucket"
 553                 " list length grew over limit " __stringify(INETFRAGS_MAXDEPTH)
 554                 ". Dropping fragment.\n";
 555
 556         if (PTR_ERR(q) == -ENOBUFS)
 557                 net_dbg_ratelimited("%s%s", prefix, msg);
 558 }
 559
 560 #endif /* !HAVE_CORRECT_MRU_HANDLING && OVS_FRAGMENT_BACKPORT */