net/xfrm/xfrm_state.c

   1 /*
   2  * xfrm_state.c
   3  *
   4  * Changes:
   5  *      Mitsuru KANDA @USAGI
   6  *      Kazunori MIYAZAWA @USAGI
   7  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
   8  *              IPv6 support
   9  *      YOSHIFUJI Hideaki @USAGI
  10  *              Split up af-specific functions
  11  *      Derek Atkins <derek@ihtfp.com>
  12  *              Add UDP Encapsulation
  13  *
  14  */
  15
  16 #include <linux/workqueue.h>
  17 #include <net/xfrm.h>
  18 #include <linux/pfkeyv2.h>
  19 #include <linux/ipsec.h>
  20 #include <linux/module.h>
  21 #include <linux/cache.h>
  22 #include <linux/audit.h>
  23 #include <asm/uaccess.h>
  24 #include <linux/ktime.h>
  25 #include <linux/slab.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/kernel.h>
  28
  29 #include "xfrm_hash.h"
  30
  31 #define xfrm_state_deref_prot(table, net) \
  32         rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
  33
  34 /* Each xfrm_state may be linked to two tables:
  35
  36    1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
  37    2. Hash table by (daddr,family,reqid) to find what SAs exist for given
  38       destination/tunnel endpoint. (output)
  39  */
  40
  41 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
  42 static __read_mostly seqcount_t xfrm_state_hash_generation = SEQCNT_ZERO(xfrm_state_hash_generation);
  43
  44 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
  45 {
  46         return atomic_inc_not_zero(&x->refcnt);
  47 }
  48
  49 static inline unsigned int xfrm_dst_hash(struct net *net,
  50                                          const xfrm_address_t *daddr,
  51                                          const xfrm_address_t *saddr,
  52                                          u32 reqid,
  53                                          unsigned short family)
  54 {
  55         return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);
  56 }
  57
  58 static inline unsigned int xfrm_src_hash(struct net *net,
  59                                          const xfrm_address_t *daddr,
  60                                          const xfrm_address_t *saddr,
  61                                          unsigned short family)
  62 {
  63         return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);
  64 }
  65
  66 static inline unsigned int
  67 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
  68               __be32 spi, u8 proto, unsigned short family)
  69 {
  70         return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);
  71 }
  72
  73 static void xfrm_hash_transfer(struct hlist_head *list,
  74                                struct hlist_head *ndsttable,
  75                                struct hlist_head *nsrctable,
  76                                struct hlist_head *nspitable,
  77                                unsigned int nhashmask)
  78 {
  79         struct hlist_node *tmp;
  80         struct xfrm_state *x;
  81
  82         hlist_for_each_entry_safe(x, tmp, list, bydst) {
  83                 unsigned int h;
  84
  85                 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
  86                                     x->props.reqid, x->props.family,
  87                                     nhashmask);
  88                 hlist_add_head_rcu(&x->bydst, ndsttable + h);
  89
  90                 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
  91                                     x->props.family,
  92                                     nhashmask);
  93                 hlist_add_head_rcu(&x->bysrc, nsrctable + h);
  94
  95                 if (x->id.spi) {
  96                         h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
  97                                             x->id.proto, x->props.family,
  98                                             nhashmask);
  99                         hlist_add_head_rcu(&x->byspi, nspitable + h);
 100                 }
 101         }
 102 }
 103
 104 static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
 105 {
 106         return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
 107 }
 108
 109 static void xfrm_hash_resize(struct work_struct *work)
 110 {
 111         struct net *net = container_of(work, struct net, xfrm.state_hash_work);
 112         struct hlist_head *ndst, *nsrc, *nspi, *odst, *osrc, *ospi;
 113         unsigned long nsize, osize;
 114         unsigned int nhashmask, ohashmask;
 115         int i;
 116
 117         nsize = xfrm_hash_new_size(net->xfrm.state_hmask);
 118         ndst = xfrm_hash_alloc(nsize);
 119         if (!ndst)
 120                 return;
 121         nsrc = xfrm_hash_alloc(nsize);
 122         if (!nsrc) {
 123                 xfrm_hash_free(ndst, nsize);
 124                 return;
 125         }
 126         nspi = xfrm_hash_alloc(nsize);
 127         if (!nspi) {
 128                 xfrm_hash_free(ndst, nsize);
 129                 xfrm_hash_free(nsrc, nsize);
 130                 return;
 131         }
 132
 133         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 134         write_seqcount_begin(&xfrm_state_hash_generation);
 135
 136         nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
 137         odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
 138         for (i = net->xfrm.state_hmask; i >= 0; i--)
 139                 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nhashmask);
 140
 141         osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
 142         ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
 143         ohashmask = net->xfrm.state_hmask;
 144
 145         rcu_assign_pointer(net->xfrm.state_bydst, ndst);
 146         rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
 147         rcu_assign_pointer(net->xfrm.state_byspi, nspi);
 148         net->xfrm.state_hmask = nhashmask;
 149
 150         write_seqcount_end(&xfrm_state_hash_generation);
 151         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 152
 153         osize = (ohashmask + 1) * sizeof(struct hlist_head);
 154
 155         synchronize_rcu();
 156
 157         xfrm_hash_free(odst, osize);
 158         xfrm_hash_free(osrc, osize);
 159         xfrm_hash_free(ospi, osize);
 160 }
 161
 162 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
 163 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
 164
 165 static DEFINE_SPINLOCK(xfrm_state_gc_lock);
 166
 167 int __xfrm_state_delete(struct xfrm_state *x);
 168
 169 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
 170 bool km_is_alive(const struct km_event *c);
 171 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
 172
 173 static DEFINE_SPINLOCK(xfrm_type_lock);
 174 int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
 175 {
 176         struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
 177         const struct xfrm_type **typemap;
 178         int err = 0;
 179
 180         if (unlikely(afinfo == NULL))
 181                 return -EAFNOSUPPORT;
 182         typemap = afinfo->type_map;
 183         spin_lock_bh(&xfrm_type_lock);
 184
 185         if (likely(typemap[type->proto] == NULL))
 186                 typemap[type->proto] = type;
 187         else
 188                 err = -EEXIST;
 189         spin_unlock_bh(&xfrm_type_lock);
 190         xfrm_state_put_afinfo(afinfo);
 191         return err;
 192 }
 193 EXPORT_SYMBOL(xfrm_register_type);
 194
 195 int xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
 196 {
 197         struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
 198         const struct xfrm_type **typemap;
 199         int err = 0;
 200
 201         if (unlikely(afinfo == NULL))
 202                 return -EAFNOSUPPORT;
 203         typemap = afinfo->type_map;
 204         spin_lock_bh(&xfrm_type_lock);
 205
 206         if (unlikely(typemap[type->proto] != type))
 207                 err = -ENOENT;
 208         else
 209                 typemap[type->proto] = NULL;
 210         spin_unlock_bh(&xfrm_type_lock);
 211         xfrm_state_put_afinfo(afinfo);
 212         return err;
 213 }
 214 EXPORT_SYMBOL(xfrm_unregister_type);
 215
 216 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
 217 {
 218         struct xfrm_state_afinfo *afinfo;
 219         const struct xfrm_type **typemap;
 220         const struct xfrm_type *type;
 221         int modload_attempted = 0;
 222
 223 retry:
 224         afinfo = xfrm_state_get_afinfo(family);
 225         if (unlikely(afinfo == NULL))
 226                 return NULL;
 227         typemap = afinfo->type_map;
 228
 229         type = typemap[proto];
 230         if (unlikely(type && !try_module_get(type->owner)))
 231                 type = NULL;
 232         if (!type && !modload_attempted) {
 233                 xfrm_state_put_afinfo(afinfo);
 234                 request_module("xfrm-type-%d-%d", family, proto);
 235                 modload_attempted = 1;
 236                 goto retry;
 237         }
 238
 239         xfrm_state_put_afinfo(afinfo);
 240         return type;
 241 }
 242
 243 static void xfrm_put_type(const struct xfrm_type *type)
 244 {
 245         module_put(type->owner);
 246 }
 247
 248 static DEFINE_SPINLOCK(xfrm_mode_lock);
 249 int xfrm_register_mode(struct xfrm_mode *mode, int family)
 250 {
 251         struct xfrm_state_afinfo *afinfo;
 252         struct xfrm_mode **modemap;
 253         int err;
 254
 255         if (unlikely(mode->encap >= XFRM_MODE_MAX))
 256                 return -EINVAL;
 257
 258         afinfo = xfrm_state_get_afinfo(family);
 259         if (unlikely(afinfo == NULL))
 260                 return -EAFNOSUPPORT;
 261
 262         err = -EEXIST;
 263         modemap = afinfo->mode_map;
 264         spin_lock_bh(&xfrm_mode_lock);
 265         if (modemap[mode->encap])
 266                 goto out;
 267
 268         err = -ENOENT;
 269         if (!try_module_get(afinfo->owner))
 270                 goto out;
 271
 272         mode->afinfo = afinfo;
 273         modemap[mode->encap] = mode;
 274         err = 0;
 275
 276 out:
 277         spin_unlock_bh(&xfrm_mode_lock);
 278         xfrm_state_put_afinfo(afinfo);
 279         return err;
 280 }
 281 EXPORT_SYMBOL(xfrm_register_mode);
 282
 283 int xfrm_unregister_mode(struct xfrm_mode *mode, int family)
 284 {
 285         struct xfrm_state_afinfo *afinfo;
 286         struct xfrm_mode **modemap;
 287         int err;
 288
 289         if (unlikely(mode->encap >= XFRM_MODE_MAX))
 290                 return -EINVAL;
 291
 292         afinfo = xfrm_state_get_afinfo(family);
 293         if (unlikely(afinfo == NULL))
 294                 return -EAFNOSUPPORT;
 295
 296         err = -ENOENT;
 297         modemap = afinfo->mode_map;
 298         spin_lock_bh(&xfrm_mode_lock);
 299         if (likely(modemap[mode->encap] == mode)) {
 300                 modemap[mode->encap] = NULL;
 301                 module_put(mode->afinfo->owner);
 302                 err = 0;
 303         }
 304
 305         spin_unlock_bh(&xfrm_mode_lock);
 306         xfrm_state_put_afinfo(afinfo);
 307         return err;
 308 }
 309 EXPORT_SYMBOL(xfrm_unregister_mode);
 310
 311 static struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
 312 {
 313         struct xfrm_state_afinfo *afinfo;
 314         struct xfrm_mode *mode;
 315         int modload_attempted = 0;
 316
 317         if (unlikely(encap >= XFRM_MODE_MAX))
 318                 return NULL;
 319
 320 retry:
 321         afinfo = xfrm_state_get_afinfo(family);
 322         if (unlikely(afinfo == NULL))
 323                 return NULL;
 324
 325         mode = afinfo->mode_map[encap];
 326         if (unlikely(mode && !try_module_get(mode->owner)))
 327                 mode = NULL;
 328         if (!mode && !modload_attempted) {
 329                 xfrm_state_put_afinfo(afinfo);
 330                 request_module("xfrm-mode-%d-%d", family, encap);
 331                 modload_attempted = 1;
 332                 goto retry;
 333         }
 334
 335         xfrm_state_put_afinfo(afinfo);
 336         return mode;
 337 }
 338
 339 static void xfrm_put_mode(struct xfrm_mode *mode)
 340 {
 341         module_put(mode->owner);
 342 }
 343
 344 static void xfrm_state_gc_destroy(struct xfrm_state *x)
 345 {
 346         tasklet_hrtimer_cancel(&x->mtimer);
 347         del_timer_sync(&x->rtimer);
 348         kfree(x->aalg);
 349         kfree(x->ealg);
 350         kfree(x->calg);
 351         kfree(x->encap);
 352         kfree(x->coaddr);
 353         kfree(x->replay_esn);
 354         kfree(x->preplay_esn);
 355         if (x->inner_mode)
 356                 xfrm_put_mode(x->inner_mode);
 357         if (x->inner_mode_iaf)
 358                 xfrm_put_mode(x->inner_mode_iaf);
 359         if (x->outer_mode)
 360                 xfrm_put_mode(x->outer_mode);
 361         if (x->type) {
 362                 x->type->destructor(x);
 363                 xfrm_put_type(x->type);
 364         }
 365         security_xfrm_state_free(x);
 366         kfree(x);
 367 }
 368
 369 static void xfrm_state_gc_task(struct work_struct *work)
 370 {
 371         struct net *net = container_of(work, struct net, xfrm.state_gc_work);
 372         struct xfrm_state *x;
 373         struct hlist_node *tmp;
 374         struct hlist_head gc_list;
 375
 376         spin_lock_bh(&xfrm_state_gc_lock);
 377         hlist_move_list(&net->xfrm.state_gc_list, &gc_list);
 378         spin_unlock_bh(&xfrm_state_gc_lock);
 379
 380         synchronize_rcu();
 381
 382         hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
 383                 xfrm_state_gc_destroy(x);
 384 }
 385
 386 static inline unsigned long make_jiffies(long secs)
 387 {
 388         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
 389                 return MAX_SCHEDULE_TIMEOUT-1;
 390         else
 391                 return secs*HZ;
 392 }
 393
 394 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
 395 {
 396         struct tasklet_hrtimer *thr = container_of(me, struct tasklet_hrtimer, timer);
 397         struct xfrm_state *x = container_of(thr, struct xfrm_state, mtimer);
 398         unsigned long now = get_seconds();
 399         long next = LONG_MAX;
 400         int warn = 0;
 401         int err = 0;
 402
 403         spin_lock(&x->lock);
 404         if (x->km.state == XFRM_STATE_DEAD)
 405                 goto out;
 406         if (x->km.state == XFRM_STATE_EXPIRED)
 407                 goto expired;
 408         if (x->lft.hard_add_expires_seconds) {
 409                 long tmo = x->lft.hard_add_expires_seconds +
 410                         x->curlft.add_time - now;
 411                 if (tmo <= 0) {
 412                         if (x->xflags & XFRM_SOFT_EXPIRE) {
 413                                 /* enter hard expire without soft expire first?!
 414                                  * setting a new date could trigger this.
 415                                  * workarbound: fix x->curflt.add_time by below:
 416                                  */
 417                                 x->curlft.add_time = now - x->saved_tmo - 1;
 418                                 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
 419                         } else
 420                                 goto expired;
 421                 }
 422                 if (tmo < next)
 423                         next = tmo;
 424         }
 425         if (x->lft.hard_use_expires_seconds) {
 426                 long tmo = x->lft.hard_use_expires_seconds +
 427                         (x->curlft.use_time ? : now) - now;
 428                 if (tmo <= 0)
 429                         goto expired;
 430                 if (tmo < next)
 431                         next = tmo;
 432         }
 433         if (x->km.dying)
 434                 goto resched;
 435         if (x->lft.soft_add_expires_seconds) {
 436                 long tmo = x->lft.soft_add_expires_seconds +
 437                         x->curlft.add_time - now;
 438                 if (tmo <= 0) {
 439                         warn = 1;
 440                         x->xflags &= ~XFRM_SOFT_EXPIRE;
 441                 } else if (tmo < next) {
 442                         next = tmo;
 443                         x->xflags |= XFRM_SOFT_EXPIRE;
 444                         x->saved_tmo = tmo;
 445                 }
 446         }
 447         if (x->lft.soft_use_expires_seconds) {
 448                 long tmo = x->lft.soft_use_expires_seconds +
 449                         (x->curlft.use_time ? : now) - now;
 450                 if (tmo <= 0)
 451                         warn = 1;
 452                 else if (tmo < next)
 453                         next = tmo;
 454         }
 455
 456         x->km.dying = warn;
 457         if (warn)
 458                 km_state_expired(x, 0, 0);
 459 resched:
 460         if (next != LONG_MAX) {
 461                 tasklet_hrtimer_start(&x->mtimer, ktime_set(next, 0), HRTIMER_MODE_REL);
 462         }
 463
 464         goto out;
 465
 466 expired:
 467         if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
 468                 x->km.state = XFRM_STATE_EXPIRED;
 469
 470         err = __xfrm_state_delete(x);
 471         if (!err)
 472                 km_state_expired(x, 1, 0);
 473
 474         xfrm_audit_state_delete(x, err ? 0 : 1, true);
 475
 476 out:
 477         spin_unlock(&x->lock);
 478         return HRTIMER_NORESTART;
 479 }
 480
 481 static void xfrm_replay_timer_handler(unsigned long data);
 482
 483 struct xfrm_state *xfrm_state_alloc(struct net *net)
 484 {
 485         struct xfrm_state *x;
 486
 487         x = kzalloc(sizeof(struct xfrm_state), GFP_ATOMIC);
 488
 489         if (x) {
 490                 write_pnet(&x->xs_net, net);
 491                 atomic_set(&x->refcnt, 1);
 492                 atomic_set(&x->tunnel_users, 0);
 493                 INIT_LIST_HEAD(&x->km.all);
 494                 INIT_HLIST_NODE(&x->bydst);
 495                 INIT_HLIST_NODE(&x->bysrc);
 496                 INIT_HLIST_NODE(&x->byspi);
 497                 tasklet_hrtimer_init(&x->mtimer, xfrm_timer_handler,
 498                                         CLOCK_BOOTTIME, HRTIMER_MODE_ABS);
 499                 setup_timer(&x->rtimer, xfrm_replay_timer_handler,
 500                                 (unsigned long)x);
 501                 x->curlft.add_time = get_seconds();
 502                 x->lft.soft_byte_limit = XFRM_INF;
 503                 x->lft.soft_packet_limit = XFRM_INF;
 504                 x->lft.hard_byte_limit = XFRM_INF;
 505                 x->lft.hard_packet_limit = XFRM_INF;
 506                 x->replay_maxage = 0;
 507                 x->replay_maxdiff = 0;
 508                 x->inner_mode = NULL;
 509                 x->inner_mode_iaf = NULL;
 510                 spin_lock_init(&x->lock);
 511         }
 512         return x;
 513 }
 514 EXPORT_SYMBOL(xfrm_state_alloc);
 515
 516 void __xfrm_state_destroy(struct xfrm_state *x)
 517 {
 518         struct net *net = xs_net(x);
 519
 520         WARN_ON(x->km.state != XFRM_STATE_DEAD);
 521
 522         spin_lock_bh(&xfrm_state_gc_lock);
 523         hlist_add_head(&x->gclist, &net->xfrm.state_gc_list);
 524         spin_unlock_bh(&xfrm_state_gc_lock);
 525         schedule_work(&net->xfrm.state_gc_work);
 526 }
 527 EXPORT_SYMBOL(__xfrm_state_destroy);
 528
 529 int __xfrm_state_delete(struct xfrm_state *x)
 530 {
 531         struct net *net = xs_net(x);
 532         int err = -ESRCH;
 533
 534         if (x->km.state != XFRM_STATE_DEAD) {
 535                 x->km.state = XFRM_STATE_DEAD;
 536                 spin_lock(&net->xfrm.xfrm_state_lock);
 537                 list_del(&x->km.all);
 538                 hlist_del_rcu(&x->bydst);
 539                 hlist_del_rcu(&x->bysrc);
 540                 if (x->id.spi)
 541                         hlist_del_rcu(&x->byspi);
 542                 net->xfrm.state_num--;
 543                 spin_unlock(&net->xfrm.xfrm_state_lock);
 544
 545                 /* All xfrm_state objects are created by xfrm_state_alloc.
 546                  * The xfrm_state_alloc call gives a reference, and that
 547                  * is what we are dropping here.
 548                  */
 549                 xfrm_state_put(x);
 550                 err = 0;
 551         }
 552
 553         return err;
 554 }
 555 EXPORT_SYMBOL(__xfrm_state_delete);
 556
 557 int xfrm_state_delete(struct xfrm_state *x)
 558 {
 559         int err;
 560
 561         spin_lock_bh(&x->lock);
 562         err = __xfrm_state_delete(x);
 563         spin_unlock_bh(&x->lock);
 564
 565         return err;
 566 }
 567 EXPORT_SYMBOL(xfrm_state_delete);
 568
 569 #ifdef CONFIG_SECURITY_NETWORK_XFRM
 570 static inline int
 571 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
 572 {
 573         int i, err = 0;
 574
 575         for (i = 0; i <= net->xfrm.state_hmask; i++) {
 576                 struct xfrm_state *x;
 577
 578                 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
 579                         if (xfrm_id_proto_match(x->id.proto, proto) &&
 580                            (err = security_xfrm_state_delete(x)) != 0) {
 581                                 xfrm_audit_state_delete(x, 0, task_valid);
 582                                 return err;
 583                         }
 584                 }
 585         }
 586
 587         return err;
 588 }
 589 #else
 590 static inline int
 591 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
 592 {
 593         return 0;
 594 }
 595 #endif
 596
 597 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
 598 {
 599         int i, err = 0, cnt = 0;
 600
 601         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 602         err = xfrm_state_flush_secctx_check(net, proto, task_valid);
 603         if (err)
 604                 goto out;
 605
 606         err = -ESRCH;
 607         for (i = 0; i <= net->xfrm.state_hmask; i++) {
 608                 struct xfrm_state *x;
 609 restart:
 610                 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
 611                         if (!xfrm_state_kern(x) &&
 612                             xfrm_id_proto_match(x->id.proto, proto)) {
 613                                 xfrm_state_hold(x);
 614                                 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 615
 616                                 err = xfrm_state_delete(x);
 617                                 xfrm_audit_state_delete(x, err ? 0 : 1,
 618                                                         task_valid);
 619                                 xfrm_state_put(x);
 620                                 if (!err)
 621                                         cnt++;
 622
 623                                 spin_lock_bh(&net->xfrm.xfrm_state_lock);
 624                                 goto restart;
 625                         }
 626                 }
 627         }
 628         if (cnt)
 629                 err = 0;
 630
 631 out:
 632         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 633         return err;
 634 }
 635 EXPORT_SYMBOL(xfrm_state_flush);
 636
 637 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
 638 {
 639         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 640         si->sadcnt = net->xfrm.state_num;
 641         si->sadhcnt = net->xfrm.state_hmask;
 642         si->sadhmcnt = xfrm_state_hashmax;
 643         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 644 }
 645 EXPORT_SYMBOL(xfrm_sad_getinfo);
 646
 647 static int
 648 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
 649                     const struct xfrm_tmpl *tmpl,
 650                     const xfrm_address_t *daddr, const xfrm_address_t *saddr,
 651                     unsigned short family)
 652 {
 653         struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
 654         if (!afinfo)
 655                 return -1;
 656         afinfo->init_tempsel(&x->sel, fl);
 657
 658         if (family != tmpl->encap_family) {
 659                 xfrm_state_put_afinfo(afinfo);
 660                 afinfo = xfrm_state_get_afinfo(tmpl->encap_family);
 661                 if (!afinfo)
 662                         return -1;
 663         }
 664         afinfo->init_temprop(x, tmpl, daddr, saddr);
 665         xfrm_state_put_afinfo(afinfo);
 666         return 0;
 667 }
 668
 669 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
 670                                               const xfrm_address_t *daddr,
 671                                               __be32 spi, u8 proto,
 672                                               unsigned short family)
 673 {
 674         unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
 675         struct xfrm_state *x;
 676
 677         hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
 678                 if (x->props.family != family ||
 679                     x->id.spi       != spi ||
 680                     x->id.proto     != proto ||
 681                     !xfrm_addr_equal(&x->id.daddr, daddr, family))
 682                         continue;
 683
 684                 if ((mark & x->mark.m) != x->mark.v)
 685                         continue;
 686                 if (!xfrm_state_hold_rcu(x))
 687                         continue;
 688                 return x;
 689         }
 690
 691         return NULL;
 692 }
 693
 694 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
 695                                                      const xfrm_address_t *daddr,
 696                                                      const xfrm_address_t *saddr,
 697                                                      u8 proto, unsigned short family)
 698 {
 699         unsigned int h = xfrm_src_hash(net, daddr, saddr, family);
 700         struct xfrm_state *x;
 701
 702         hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {
 703                 if (x->props.family != family ||
 704                     x->id.proto     != proto ||
 705                     !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
 706                     !xfrm_addr_equal(&x->props.saddr, saddr, family))
 707                         continue;
 708
 709                 if ((mark & x->mark.m) != x->mark.v)
 710                         continue;
 711                 if (!xfrm_state_hold_rcu(x))
 712                         continue;
 713                 return x;
 714         }
 715
 716         return NULL;
 717 }
 718
 719 static inline struct xfrm_state *
 720 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
 721 {
 722         struct net *net = xs_net(x);
 723         u32 mark = x->mark.v & x->mark.m;
 724
 725         if (use_spi)
 726                 return __xfrm_state_lookup(net, mark, &x->id.daddr,
 727                                            x->id.spi, x->id.proto, family);
 728         else
 729                 return __xfrm_state_lookup_byaddr(net, mark,
 730                                                   &x->id.daddr,
 731                                                   &x->props.saddr,
 732                                                   x->id.proto, family);
 733 }
 734
 735 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
 736 {
 737         if (have_hash_collision &&
 738             (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
 739             net->xfrm.state_num > net->xfrm.state_hmask)
 740                 schedule_work(&net->xfrm.state_hash_work);
 741 }
 742
 743 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
 744                                const struct flowi *fl, unsigned short family,
 745                                struct xfrm_state **best, int *acq_in_progress,
 746                                int *error)
 747 {
 748         /* Resolution logic:
 749          * 1. There is a valid state with matching selector. Done.
 750          * 2. Valid state with inappropriate selector. Skip.
 751          *
 752          * Entering area of "sysdeps".
 753          *
 754          * 3. If state is not valid, selector is temporary, it selects
 755          *    only session which triggered previous resolution. Key
 756          *    manager will do something to install a state with proper
 757          *    selector.
 758          */
 759         if (x->km.state == XFRM_STATE_VALID) {
 760                 if ((x->sel.family &&
 761                      !xfrm_selector_match(&x->sel, fl, x->sel.family)) ||
 762                     !security_xfrm_state_pol_flow_match(x, pol, fl))
 763                         return;
 764
 765                 if (!*best ||
 766                     (*best)->km.dying > x->km.dying ||
 767                     ((*best)->km.dying == x->km.dying &&
 768                      (*best)->curlft.add_time < x->curlft.add_time))
 769                         *best = x;
 770         } else if (x->km.state == XFRM_STATE_ACQ) {
 771                 *acq_in_progress = 1;
 772         } else if (x->km.state == XFRM_STATE_ERROR ||
 773                    x->km.state == XFRM_STATE_EXPIRED) {
 774                 if (xfrm_selector_match(&x->sel, fl, x->sel.family) &&
 775                     security_xfrm_state_pol_flow_match(x, pol, fl))
 776                         *error = -ESRCH;
 777         }
 778 }
 779
 780 struct xfrm_state *
 781 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
 782                 const struct flowi *fl, struct xfrm_tmpl *tmpl,
 783                 struct xfrm_policy *pol, int *err,
 784                 unsigned short family)
 785 {
 786         static xfrm_address_t saddr_wildcard = { };
 787         struct net *net = xp_net(pol);
 788         unsigned int h, h_wildcard;
 789         struct xfrm_state *x, *x0, *to_put;
 790         int acquire_in_progress = 0;
 791         int error = 0;
 792         struct xfrm_state *best = NULL;
 793         u32 mark = pol->mark.v & pol->mark.m;
 794         unsigned short encap_family = tmpl->encap_family;
 795         unsigned int sequence;
 796         struct km_event c;
 797
 798         to_put = NULL;
 799
 800         sequence = read_seqcount_begin(&xfrm_state_hash_generation);
 801
 802         rcu_read_lock();
 803         h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
 804         hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
 805                 if (x->props.family == encap_family &&
 806                     x->props.reqid == tmpl->reqid &&
 807                     (mark & x->mark.m) == x->mark.v &&
 808                     !(x->props.flags & XFRM_STATE_WILDRECV) &&
 809                     xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
 810                     tmpl->mode == x->props.mode &&
 811                     tmpl->id.proto == x->id.proto &&
 812                     (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
 813                         xfrm_state_look_at(pol, x, fl, encap_family,
 814                                            &best, &acquire_in_progress, &error);
 815         }
 816         if (best || acquire_in_progress)
 817                 goto found;
 818
 819         h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family);
 820         hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
 821                 if (x->props.family == encap_family &&
 822                     x->props.reqid == tmpl->reqid &&
 823                     (mark & x->mark.m) == x->mark.v &&
 824                     !(x->props.flags & XFRM_STATE_WILDRECV) &&
 825                     xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
 826                     tmpl->mode == x->props.mode &&
 827                     tmpl->id.proto == x->id.proto &&
 828                     (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
 829                         xfrm_state_look_at(pol, x, fl, encap_family,
 830                                            &best, &acquire_in_progress, &error);
 831         }
 832
 833 found:
 834         x = best;
 835         if (!x && !error && !acquire_in_progress) {
 836                 if (tmpl->id.spi &&
 837                     (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi,
 838                                               tmpl->id.proto, encap_family)) != NULL) {
 839                         to_put = x0;
 840                         error = -EEXIST;
 841                         goto out;
 842                 }
 843
 844                 c.net = net;
 845                 /* If the KMs have no listeners (yet...), avoid allocating an SA
 846                  * for each and every packet - garbage collection might not
 847                  * handle the flood.
 848                  */
 849                 if (!km_is_alive(&c)) {
 850                         error = -ESRCH;
 851                         goto out;
 852                 }
 853
 854                 x = xfrm_state_alloc(net);
 855                 if (x == NULL) {
 856                         error = -ENOMEM;
 857                         goto out;
 858                 }
 859                 /* Initialize temporary state matching only
 860                  * to current session. */
 861                 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
 862                 memcpy(&x->mark, &pol->mark, sizeof(x->mark));
 863
 864                 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);
 865                 if (error) {
 866                         x->km.state = XFRM_STATE_DEAD;
 867                         to_put = x;
 868                         x = NULL;
 869                         goto out;
 870                 }
 871
 872                 if (km_query(x, tmpl, pol) == 0) {
 873                         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 874                         x->km.state = XFRM_STATE_ACQ;
 875                         list_add(&x->km.all, &net->xfrm.state_all);
 876                         hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
 877                         h = xfrm_src_hash(net, daddr, saddr, encap_family);
 878                         hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
 879                         if (x->id.spi) {
 880                                 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
 881                                 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
 882                         }
 883                         x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
 884                         tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);
 885                         net->xfrm.state_num++;
 886                         xfrm_hash_grow_check(net, x->bydst.next != NULL);
 887                         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 888                 } else {
 889                         x->km.state = XFRM_STATE_DEAD;
 890                         to_put = x;
 891                         x = NULL;
 892                         error = -ESRCH;
 893                 }
 894         }
 895 out:
 896         if (x) {
 897                 if (!xfrm_state_hold_rcu(x)) {
 898                         *err = -EAGAIN;
 899                         x = NULL;
 900                 }
 901         } else {
 902                 *err = acquire_in_progress ? -EAGAIN : error;
 903         }
 904         rcu_read_unlock();
 905         if (to_put)
 906                 xfrm_state_put(to_put);
 907
 908         if (read_seqcount_retry(&xfrm_state_hash_generation, sequence)) {
 909                 *err = -EAGAIN;
 910                 if (x) {
 911                         xfrm_state_put(x);
 912                         x = NULL;
 913                 }
 914         }
 915
 916         return x;
 917 }
 918
 919 struct xfrm_state *
 920 xfrm_stateonly_find(struct net *net, u32 mark,
 921                     xfrm_address_t *daddr, xfrm_address_t *saddr,
 922                     unsigned short family, u8 mode, u8 proto, u32 reqid)
 923 {
 924         unsigned int h;
 925         struct xfrm_state *rx = NULL, *x = NULL;
 926
 927         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 928         h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
 929         hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
 930                 if (x->props.family == family &&
 931                     x->props.reqid == reqid &&
 932                     (mark & x->mark.m) == x->mark.v &&
 933                     !(x->props.flags & XFRM_STATE_WILDRECV) &&
 934                     xfrm_state_addr_check(x, daddr, saddr, family) &&
 935                     mode == x->props.mode &&
 936                     proto == x->id.proto &&
 937                     x->km.state == XFRM_STATE_VALID) {
 938                         rx = x;
 939                         break;
 940                 }
 941         }
 942
 943         if (rx)
 944                 xfrm_state_hold(rx);
 945         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 946
 947
 948         return rx;
 949 }
 950 EXPORT_SYMBOL(xfrm_stateonly_find);
 951
 952 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
 953                                               unsigned short family)
 954 {
 955         struct xfrm_state *x;
 956         struct xfrm_state_walk *w;
 957
 958         spin_lock_bh(&net->xfrm.xfrm_state_lock);
 959         list_for_each_entry(w, &net->xfrm.state_all, all) {
 960                 x = container_of(w, struct xfrm_state, km);
 961                 if (x->props.family != family ||
 962                         x->id.spi != spi)
 963                         continue;
 964
 965                 xfrm_state_hold(x);
 966                 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 967                 return x;
 968         }
 969         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
 970         return NULL;
 971 }
 972 EXPORT_SYMBOL(xfrm_state_lookup_byspi);
 973
 974 static void __xfrm_state_insert(struct xfrm_state *x)
 975 {
 976         struct net *net = xs_net(x);
 977         unsigned int h;
 978
 979         list_add(&x->km.all, &net->xfrm.state_all);
 980
 981         h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
 982                           x->props.reqid, x->props.family);
 983         hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
 984
 985         h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
 986         hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
 987
 988         if (x->id.spi) {
 989                 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
 990                                   x->props.family);
 991
 992                 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
 993         }
 994
 995         tasklet_hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL);
 996         if (x->replay_maxage)
 997                 mod_timer(&x->rtimer, jiffies + x->replay_maxage);
 998
 999         net->xfrm.state_num++;
1000
1001         xfrm_hash_grow_check(net, x->bydst.next != NULL);
1002 }
1003
1004 /* net->xfrm.xfrm_state_lock is held */
1005 static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1006 {
1007         struct net *net = xs_net(xnew);
1008         unsigned short family = xnew->props.family;
1009         u32 reqid = xnew->props.reqid;
1010         struct xfrm_state *x;
1011         unsigned int h;
1012         u32 mark = xnew->mark.v & xnew->mark.m;
1013
1014         h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
1015         hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1016                 if (x->props.family     == family &&
1017                     x->props.reqid      == reqid &&
1018                     (mark & x->mark.m) == x->mark.v &&
1019                     xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
1020                     xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
1021                         x->genid++;
1022         }
1023 }
1024
1025 void xfrm_state_insert(struct xfrm_state *x)
1026 {
1027         struct net *net = xs_net(x);
1028
1029         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1030         __xfrm_state_bump_genids(x);
1031         __xfrm_state_insert(x);
1032         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1033 }
1034 EXPORT_SYMBOL(xfrm_state_insert);
1035
1036 /* net->xfrm.xfrm_state_lock is held */
1037 static struct xfrm_state *__find_acq_core(struct net *net,
1038                                           const struct xfrm_mark *m,
1039                                           unsigned short family, u8 mode,
1040                                           u32 reqid, u8 proto,
1041                                           const xfrm_address_t *daddr,
1042                                           const xfrm_address_t *saddr,
1043                                           int create)
1044 {
1045         unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1046         struct xfrm_state *x;
1047         u32 mark = m->v & m->m;
1048
1049         hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1050                 if (x->props.reqid  != reqid ||
1051                     x->props.mode   != mode ||
1052                     x->props.family != family ||
1053                     x->km.state     != XFRM_STATE_ACQ ||
1054                     x->id.spi       != 0 ||
1055                     x->id.proto     != proto ||
1056                     (mark & x->mark.m) != x->mark.v ||
1057                     !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1058                     !xfrm_addr_equal(&x->props.saddr, saddr, family))
1059                         continue;
1060
1061                 xfrm_state_hold(x);
1062                 return x;
1063         }
1064
1065         if (!create)
1066                 return NULL;
1067
1068         x = xfrm_state_alloc(net);
1069         if (likely(x)) {
1070                 switch (family) {
1071                 case AF_INET:
1072                         x->sel.daddr.a4 = daddr->a4;
1073                         x->sel.saddr.a4 = saddr->a4;
1074                         x->sel.prefixlen_d = 32;
1075                         x->sel.prefixlen_s = 32;
1076                         x->props.saddr.a4 = saddr->a4;
1077                         x->id.daddr.a4 = daddr->a4;
1078                         break;
1079
1080                 case AF_INET6:
1081                         x->sel.daddr.in6 = daddr->in6;
1082                         x->sel.saddr.in6 = saddr->in6;
1083                         x->sel.prefixlen_d = 128;
1084                         x->sel.prefixlen_s = 128;
1085                         x->props.saddr.in6 = saddr->in6;
1086                         x->id.daddr.in6 = daddr->in6;
1087                         break;
1088                 }
1089
1090                 x->km.state = XFRM_STATE_ACQ;
1091                 x->id.proto = proto;
1092                 x->props.family = family;
1093                 x->props.mode = mode;
1094                 x->props.reqid = reqid;
1095                 x->mark.v = m->v;
1096                 x->mark.m = m->m;
1097                 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1098                 xfrm_state_hold(x);
1099                 tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);
1100                 list_add(&x->km.all, &net->xfrm.state_all);
1101                 hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
1102                 h = xfrm_src_hash(net, daddr, saddr, family);
1103                 hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
1104
1105                 net->xfrm.state_num++;
1106
1107                 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1108         }
1109
1110         return x;
1111 }
1112
1113 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
1114
1115 int xfrm_state_add(struct xfrm_state *x)
1116 {
1117         struct net *net = xs_net(x);
1118         struct xfrm_state *x1, *to_put;
1119         int family;
1120         int err;
1121         u32 mark = x->mark.v & x->mark.m;
1122         int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1123
1124         family = x->props.family;
1125
1126         to_put = NULL;
1127
1128         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1129
1130         x1 = __xfrm_state_locate(x, use_spi, family);
1131         if (x1) {
1132                 to_put = x1;
1133                 x1 = NULL;
1134                 err = -EEXIST;
1135                 goto out;
1136         }
1137
1138         if (use_spi && x->km.seq) {
1139                 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq);
1140                 if (x1 && ((x1->id.proto != x->id.proto) ||
1141                     !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {
1142                         to_put = x1;
1143                         x1 = NULL;
1144                 }
1145         }
1146
1147         if (use_spi && !x1)
1148                 x1 = __find_acq_core(net, &x->mark, family, x->props.mode,
1149                                      x->props.reqid, x->id.proto,
1150                                      &x->id.daddr, &x->props.saddr, 0);
1151
1152         __xfrm_state_bump_genids(x);
1153         __xfrm_state_insert(x);
1154         err = 0;
1155
1156 out:
1157         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1158
1159         if (x1) {
1160                 xfrm_state_delete(x1);
1161                 xfrm_state_put(x1);
1162         }
1163
1164         if (to_put)
1165                 xfrm_state_put(to_put);
1166
1167         return err;
1168 }
1169 EXPORT_SYMBOL(xfrm_state_add);
1170
1171 #ifdef CONFIG_XFRM_MIGRATE
1172 static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig)
1173 {
1174         struct net *net = xs_net(orig);
1175         struct xfrm_state *x = xfrm_state_alloc(net);
1176         if (!x)
1177                 goto out;
1178
1179         memcpy(&x->id, &orig->id, sizeof(x->id));
1180         memcpy(&x->sel, &orig->sel, sizeof(x->sel));
1181         memcpy(&x->lft, &orig->lft, sizeof(x->lft));
1182         x->props.mode = orig->props.mode;
1183         x->props.replay_window = orig->props.replay_window;
1184         x->props.reqid = orig->props.reqid;
1185         x->props.family = orig->props.family;
1186         x->props.saddr = orig->props.saddr;
1187
1188         if (orig->aalg) {
1189                 x->aalg = xfrm_algo_auth_clone(orig->aalg);
1190                 if (!x->aalg)
1191                         goto error;
1192         }
1193         x->props.aalgo = orig->props.aalgo;
1194
1195         if (orig->aead) {
1196                 x->aead = xfrm_algo_aead_clone(orig->aead);
1197                 if (!x->aead)
1198                         goto error;
1199         }
1200         if (orig->ealg) {
1201                 x->ealg = xfrm_algo_clone(orig->ealg);
1202                 if (!x->ealg)
1203                         goto error;
1204         }
1205         x->props.ealgo = orig->props.ealgo;
1206
1207         if (orig->calg) {
1208                 x->calg = xfrm_algo_clone(orig->calg);
1209                 if (!x->calg)
1210                         goto error;
1211         }
1212         x->props.calgo = orig->props.calgo;
1213
1214         if (orig->encap) {
1215                 x->encap = kmemdup(orig->encap, sizeof(*x->encap), GFP_KERNEL);
1216                 if (!x->encap)
1217                         goto error;
1218         }
1219
1220         if (orig->coaddr) {
1221                 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
1222                                     GFP_KERNEL);
1223                 if (!x->coaddr)
1224                         goto error;
1225         }
1226
1227         if (orig->replay_esn) {
1228                 if (xfrm_replay_clone(x, orig))
1229                         goto error;
1230         }
1231
1232         memcpy(&x->mark, &orig->mark, sizeof(x->mark));
1233
1234         if (xfrm_init_state(x) < 0)
1235                 goto error;
1236
1237         x->props.flags = orig->props.flags;
1238         x->props.extra_flags = orig->props.extra_flags;
1239
1240         x->tfcpad = orig->tfcpad;
1241         x->replay_maxdiff = orig->replay_maxdiff;
1242         x->replay_maxage = orig->replay_maxage;
1243         x->curlft.add_time = orig->curlft.add_time;
1244         x->km.state = orig->km.state;
1245         x->km.seq = orig->km.seq;
1246
1247         return x;
1248
1249  error:
1250         xfrm_state_put(x);
1251 out:
1252         return NULL;
1253 }
1254
1255 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net)
1256 {
1257         unsigned int h;
1258         struct xfrm_state *x = NULL;
1259
1260         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1261
1262         if (m->reqid) {
1263                 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,
1264                                   m->reqid, m->old_family);
1265                 hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1266                         if (x->props.mode != m->mode ||
1267                             x->id.proto != m->proto)
1268                                 continue;
1269                         if (m->reqid && x->props.reqid != m->reqid)
1270                                 continue;
1271                         if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1272                                              m->old_family) ||
1273                             !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
1274                                              m->old_family))
1275                                 continue;
1276                         xfrm_state_hold(x);
1277                         break;
1278                 }
1279         } else {
1280                 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,
1281                                   m->old_family);
1282                 hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
1283                         if (x->props.mode != m->mode ||
1284                             x->id.proto != m->proto)
1285                                 continue;
1286                         if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1287                                              m->old_family) ||
1288                             !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
1289                                              m->old_family))
1290                                 continue;
1291                         xfrm_state_hold(x);
1292                         break;
1293                 }
1294         }
1295
1296         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1297
1298         return x;
1299 }
1300 EXPORT_SYMBOL(xfrm_migrate_state_find);
1301
1302 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
1303                                       struct xfrm_migrate *m)
1304 {
1305         struct xfrm_state *xc;
1306
1307         xc = xfrm_state_clone(x);
1308         if (!xc)
1309                 return NULL;
1310
1311         memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
1312         memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
1313
1314         /* add state */
1315         if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
1316                 /* a care is needed when the destination address of the
1317                    state is to be updated as it is a part of triplet */
1318                 xfrm_state_insert(xc);
1319         } else {
1320                 if (xfrm_state_add(xc) < 0)
1321                         goto error;
1322         }
1323
1324         return xc;
1325 error:
1326         xfrm_state_put(xc);
1327         return NULL;
1328 }
1329 EXPORT_SYMBOL(xfrm_state_migrate);
1330 #endif
1331
1332 int xfrm_state_update(struct xfrm_state *x)
1333 {
1334         struct xfrm_state *x1, *to_put;
1335         int err;
1336         int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1337         struct net *net = xs_net(x);
1338
1339         to_put = NULL;
1340
1341         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1342         x1 = __xfrm_state_locate(x, use_spi, x->props.family);
1343
1344         err = -ESRCH;
1345         if (!x1)
1346                 goto out;
1347
1348         if (xfrm_state_kern(x1)) {
1349                 to_put = x1;
1350                 err = -EEXIST;
1351                 goto out;
1352         }
1353
1354         if (x1->km.state == XFRM_STATE_ACQ) {
1355                 __xfrm_state_insert(x);
1356                 x = NULL;
1357         }
1358         err = 0;
1359
1360 out:
1361         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1362
1363         if (to_put)
1364                 xfrm_state_put(to_put);
1365
1366         if (err)
1367                 return err;
1368
1369         if (!x) {
1370                 xfrm_state_delete(x1);
1371                 xfrm_state_put(x1);
1372                 return 0;
1373         }
1374
1375         err = -EINVAL;
1376         spin_lock_bh(&x1->lock);
1377         if (likely(x1->km.state == XFRM_STATE_VALID)) {
1378                 if (x->encap && x1->encap)
1379                         memcpy(x1->encap, x->encap, sizeof(*x1->encap));
1380                 if (x->coaddr && x1->coaddr) {
1381                         memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
1382                 }
1383                 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))
1384                         memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
1385                 memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
1386                 x1->km.dying = 0;
1387
1388                 tasklet_hrtimer_start(&x1->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL);
1389                 if (x1->curlft.use_time)
1390                         xfrm_state_check_expire(x1);
1391
1392                 err = 0;
1393                 x->km.state = XFRM_STATE_DEAD;
1394                 __xfrm_state_put(x);
1395         }
1396         spin_unlock_bh(&x1->lock);
1397
1398         xfrm_state_put(x1);
1399
1400         return err;
1401 }
1402 EXPORT_SYMBOL(xfrm_state_update);
1403
1404 int xfrm_state_check_expire(struct xfrm_state *x)
1405 {
1406         if (!x->curlft.use_time)
1407                 x->curlft.use_time = get_seconds();
1408
1409         if (x->curlft.bytes >= x->lft.hard_byte_limit ||
1410             x->curlft.packets >= x->lft.hard_packet_limit) {
1411                 x->km.state = XFRM_STATE_EXPIRED;
1412                 tasklet_hrtimer_start(&x->mtimer, ktime_set(0, 0), HRTIMER_MODE_REL);
1413                 return -EINVAL;
1414         }
1415
1416         if (!x->km.dying &&
1417             (x->curlft.bytes >= x->lft.soft_byte_limit ||
1418              x->curlft.packets >= x->lft.soft_packet_limit)) {
1419                 x->km.dying = 1;
1420                 km_state_expired(x, 0, 0);
1421         }
1422         return 0;
1423 }
1424 EXPORT_SYMBOL(xfrm_state_check_expire);
1425
1426 struct xfrm_state *
1427 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
1428                   u8 proto, unsigned short family)
1429 {
1430         struct xfrm_state *x;
1431
1432         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1433         x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);
1434         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1435         return x;
1436 }
1437 EXPORT_SYMBOL(xfrm_state_lookup);
1438
1439 struct xfrm_state *
1440 xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1441                          const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1442                          u8 proto, unsigned short family)
1443 {
1444         struct xfrm_state *x;
1445
1446         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1447         x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);
1448         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1449         return x;
1450 }
1451 EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
1452
1453 struct xfrm_state *
1454 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
1455               u8 proto, const xfrm_address_t *daddr,
1456               const xfrm_address_t *saddr, int create, unsigned short family)
1457 {
1458         struct xfrm_state *x;
1459
1460         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1461         x = __find_acq_core(net, mark, family, mode, reqid, proto, daddr, saddr, create);
1462         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1463
1464         return x;
1465 }
1466 EXPORT_SYMBOL(xfrm_find_acq);
1467
1468 #ifdef CONFIG_XFRM_SUB_POLICY
1469 int
1470 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
1471                unsigned short family, struct net *net)
1472 {
1473         int err = 0;
1474         struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
1475         if (!afinfo)
1476                 return -EAFNOSUPPORT;
1477
1478         spin_lock_bh(&net->xfrm.xfrm_state_lock); /*FIXME*/
1479         if (afinfo->tmpl_sort)
1480                 err = afinfo->tmpl_sort(dst, src, n);
1481         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1482         xfrm_state_put_afinfo(afinfo);
1483         return err;
1484 }
1485 EXPORT_SYMBOL(xfrm_tmpl_sort);
1486
1487 int
1488 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
1489                 unsigned short family)
1490 {
1491         int err = 0;
1492         struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
1493         struct net *net = xs_net(*src);
1494
1495         if (!afinfo)
1496                 return -EAFNOSUPPORT;
1497
1498         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1499         if (afinfo->state_sort)
1500                 err = afinfo->state_sort(dst, src, n);
1501         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1502         xfrm_state_put_afinfo(afinfo);
1503         return err;
1504 }
1505 EXPORT_SYMBOL(xfrm_state_sort);
1506 #endif
1507
1508 /* Silly enough, but I'm lazy to build resolution list */
1509
1510 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
1511 {
1512         int i;
1513
1514         for (i = 0; i <= net->xfrm.state_hmask; i++) {
1515                 struct xfrm_state *x;
1516
1517                 hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
1518                         if (x->km.seq == seq &&
1519                             (mark & x->mark.m) == x->mark.v &&
1520                             x->km.state == XFRM_STATE_ACQ) {
1521                                 xfrm_state_hold(x);
1522                                 return x;
1523                         }
1524                 }
1525         }
1526         return NULL;
1527 }
1528
1529 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
1530 {
1531         struct xfrm_state *x;
1532
1533         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1534         x = __xfrm_find_acq_byseq(net, mark, seq);
1535         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1536         return x;
1537 }
1538 EXPORT_SYMBOL(xfrm_find_acq_byseq);
1539
1540 u32 xfrm_get_acqseq(void)
1541 {
1542         u32 res;
1543         static atomic_t acqseq;
1544
1545         do {
1546                 res = atomic_inc_return(&acqseq);
1547         } while (!res);
1548
1549         return res;
1550 }
1551 EXPORT_SYMBOL(xfrm_get_acqseq);
1552
1553 int verify_spi_info(u8 proto, u32 min, u32 max)
1554 {
1555         switch (proto) {
1556         case IPPROTO_AH:
1557         case IPPROTO_ESP:
1558                 break;
1559
1560         case IPPROTO_COMP:
1561                 /* IPCOMP spi is 16-bits. */
1562                 if (max >= 0x10000)
1563                         return -EINVAL;
1564                 break;
1565
1566         default:
1567                 return -EINVAL;
1568         }
1569
1570         if (min > max)
1571                 return -EINVAL;
1572
1573         return 0;
1574 }
1575 EXPORT_SYMBOL(verify_spi_info);
1576
1577 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high)
1578 {
1579         struct net *net = xs_net(x);
1580         unsigned int h;
1581         struct xfrm_state *x0;
1582         int err = -ENOENT;
1583         __be32 minspi = htonl(low);
1584         __be32 maxspi = htonl(high);
1585         u32 mark = x->mark.v & x->mark.m;
1586
1587         spin_lock_bh(&x->lock);
1588         if (x->km.state == XFRM_STATE_DEAD)
1589                 goto unlock;
1590
1591         err = 0;
1592         if (x->id.spi)
1593                 goto unlock;
1594
1595         err = -ENOENT;
1596
1597         if (minspi == maxspi) {
1598                 x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
1599                 if (x0) {
1600                         xfrm_state_put(x0);
1601                         goto unlock;
1602                 }
1603                 x->id.spi = minspi;
1604         } else {
1605                 u32 spi = 0;
1606                 for (h = 0; h < high-low+1; h++) {
1607                         spi = low + prandom_u32()%(high-low+1);
1608                         x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
1609                         if (x0 == NULL) {
1610                                 x->id.spi = htonl(spi);
1611                                 break;
1612                         }
1613                         xfrm_state_put(x0);
1614                 }
1615         }
1616         if (x->id.spi) {
1617                 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1618                 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
1619                 hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
1620                 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1621
1622                 err = 0;
1623         }
1624
1625 unlock:
1626         spin_unlock_bh(&x->lock);
1627
1628         return err;
1629 }
1630 EXPORT_SYMBOL(xfrm_alloc_spi);
1631
1632 static bool __xfrm_state_filter_match(struct xfrm_state *x,
1633                                       struct xfrm_address_filter *filter)
1634 {
1635         if (filter) {
1636                 if ((filter->family == AF_INET ||
1637                      filter->family == AF_INET6) &&
1638                     x->props.family != filter->family)
1639                         return false;
1640
1641                 return addr_match(&x->props.saddr, &filter->saddr,
1642                                   filter->splen) &&
1643                        addr_match(&x->id.daddr, &filter->daddr,
1644                                   filter->dplen);
1645         }
1646         return true;
1647 }
1648
1649 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
1650                     int (*func)(struct xfrm_state *, int, void*),
1651                     void *data)
1652 {
1653         struct xfrm_state *state;
1654         struct xfrm_state_walk *x;
1655         int err = 0;
1656
1657         if (walk->seq != 0 && list_empty(&walk->all))
1658                 return 0;
1659
1660         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1661         if (list_empty(&walk->all))
1662                 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
1663         else
1664                 x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
1665         list_for_each_entry_from(x, &net->xfrm.state_all, all) {
1666                 if (x->state == XFRM_STATE_DEAD)
1667                         continue;
1668                 state = container_of(x, struct xfrm_state, km);
1669                 if (!xfrm_id_proto_match(state->id.proto, walk->proto))
1670                         continue;
1671                 if (!__xfrm_state_filter_match(state, walk->filter))
1672                         continue;
1673                 err = func(state, walk->seq, data);
1674                 if (err) {
1675                         list_move_tail(&walk->all, &x->all);
1676                         goto out;
1677                 }
1678                 walk->seq++;
1679         }
1680         if (walk->seq == 0) {
1681                 err = -ENOENT;
1682                 goto out;
1683         }
1684         list_del_init(&walk->all);
1685 out:
1686         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1687         return err;
1688 }
1689 EXPORT_SYMBOL(xfrm_state_walk);
1690
1691 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
1692                           struct xfrm_address_filter *filter)
1693 {
1694         INIT_LIST_HEAD(&walk->all);
1695         walk->proto = proto;
1696         walk->state = XFRM_STATE_DEAD;
1697         walk->seq = 0;
1698         walk->filter = filter;
1699 }
1700 EXPORT_SYMBOL(xfrm_state_walk_init);
1701
1702 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
1703 {
1704         kfree(walk->filter);
1705
1706         if (list_empty(&walk->all))
1707                 return;
1708
1709         spin_lock_bh(&net->xfrm.xfrm_state_lock);
1710         list_del(&walk->all);
1711         spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1712 }
1713 EXPORT_SYMBOL(xfrm_state_walk_done);
1714
1715 static void xfrm_replay_timer_handler(unsigned long data)
1716 {
1717         struct xfrm_state *x = (struct xfrm_state *)data;
1718
1719         spin_lock(&x->lock);
1720
1721         if (x->km.state == XFRM_STATE_VALID) {
1722                 if (xfrm_aevent_is_on(xs_net(x)))
1723                         x->repl->notify(x, XFRM_REPLAY_TIMEOUT);
1724                 else
1725                         x->xflags |= XFRM_TIME_DEFER;
1726         }
1727
1728         spin_unlock(&x->lock);
1729 }
1730
1731 static LIST_HEAD(xfrm_km_list);
1732
1733 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
1734 {
1735         struct xfrm_mgr *km;
1736
1737         rcu_read_lock();
1738         list_for_each_entry_rcu(km, &xfrm_km_list, list)
1739                 if (km->notify_policy)
1740                         km->notify_policy(xp, dir, c);
1741         rcu_read_unlock();
1742 }
1743
1744 void km_state_notify(struct xfrm_state *x, const struct km_event *c)
1745 {
1746         struct xfrm_mgr *km;
1747         rcu_read_lock();
1748         list_for_each_entry_rcu(km, &xfrm_km_list, list)
1749                 if (km->notify)
1750                         km->notify(x, c);
1751         rcu_read_unlock();
1752 }
1753
1754 EXPORT_SYMBOL(km_policy_notify);
1755 EXPORT_SYMBOL(km_state_notify);
1756
1757 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
1758 {
1759         struct km_event c;
1760
1761         c.data.hard = hard;
1762         c.portid = portid;
1763         c.event = XFRM_MSG_EXPIRE;
1764         km_state_notify(x, &c);
1765 }
1766
1767 EXPORT_SYMBOL(km_state_expired);
1768 /*
1769  * We send to all registered managers regardless of failure
1770  * We are happy with one success
1771 */
1772 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
1773 {
1774         int err = -EINVAL, acqret;
1775         struct xfrm_mgr *km;
1776
1777         rcu_read_lock();
1778         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1779                 acqret = km->acquire(x, t, pol);
1780                 if (!acqret)
1781                         err = acqret;
1782         }
1783         rcu_read_unlock();
1784         return err;
1785 }
1786 EXPORT_SYMBOL(km_query);
1787
1788 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
1789 {
1790         int err = -EINVAL;
1791         struct xfrm_mgr *km;
1792
1793         rcu_read_lock();
1794         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1795                 if (km->new_mapping)
1796                         err = km->new_mapping(x, ipaddr, sport);
1797                 if (!err)
1798                         break;
1799         }
1800         rcu_read_unlock();
1801         return err;
1802 }
1803 EXPORT_SYMBOL(km_new_mapping);
1804
1805 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
1806 {
1807         struct km_event c;
1808
1809         c.data.hard = hard;
1810         c.portid = portid;
1811         c.event = XFRM_MSG_POLEXPIRE;
1812         km_policy_notify(pol, dir, &c);
1813 }
1814 EXPORT_SYMBOL(km_policy_expired);
1815
1816 #ifdef CONFIG_XFRM_MIGRATE
1817 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
1818                const struct xfrm_migrate *m, int num_migrate,
1819                const struct xfrm_kmaddress *k)
1820 {
1821         int err = -EINVAL;
1822         int ret;
1823         struct xfrm_mgr *km;
1824
1825         rcu_read_lock();
1826         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1827                 if (km->migrate) {
1828                         ret = km->migrate(sel, dir, type, m, num_migrate, k);
1829                         if (!ret)
1830                                 err = ret;
1831                 }
1832         }
1833         rcu_read_unlock();
1834         return err;
1835 }
1836 EXPORT_SYMBOL(km_migrate);
1837 #endif
1838
1839 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
1840 {
1841         int err = -EINVAL;
1842         int ret;
1843         struct xfrm_mgr *km;
1844
1845         rcu_read_lock();
1846         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1847                 if (km->report) {
1848                         ret = km->report(net, proto, sel, addr);
1849                         if (!ret)
1850                                 err = ret;
1851                 }
1852         }
1853         rcu_read_unlock();
1854         return err;
1855 }
1856 EXPORT_SYMBOL(km_report);
1857
1858 bool km_is_alive(const struct km_event *c)
1859 {
1860         struct xfrm_mgr *km;
1861         bool is_alive = false;
1862
1863         rcu_read_lock();
1864         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1865                 if (km->is_alive && km->is_alive(c)) {
1866                         is_alive = true;
1867                         break;
1868                 }
1869         }
1870         rcu_read_unlock();
1871
1872         return is_alive;
1873 }
1874 EXPORT_SYMBOL(km_is_alive);
1875
1876 int xfrm_user_policy(struct sock *sk, int optname, u8 __user *optval, int optlen)
1877 {
1878         int err;
1879         u8 *data;
1880         struct xfrm_mgr *km;
1881         struct xfrm_policy *pol = NULL;
1882
1883         if (optlen <= 0 || optlen > PAGE_SIZE)
1884                 return -EMSGSIZE;
1885
1886         data = kmalloc(optlen, GFP_KERNEL);
1887         if (!data)
1888                 return -ENOMEM;
1889
1890         err = -EFAULT;
1891         if (copy_from_user(data, optval, optlen))
1892                 goto out;
1893
1894         err = -EINVAL;
1895         rcu_read_lock();
1896         list_for_each_entry_rcu(km, &xfrm_km_list, list) {
1897                 pol = km->compile_policy(sk, optname, data,
1898                                          optlen, &err);
1899                 if (err >= 0)
1900                         break;
1901         }
1902         rcu_read_unlock();
1903
1904         if (err >= 0) {
1905                 xfrm_sk_policy_insert(sk, err, pol);
1906                 xfrm_pol_put(pol);
1907                 err = 0;
1908         }
1909
1910 out:
1911         kfree(data);
1912         return err;
1913 }
1914 EXPORT_SYMBOL(xfrm_user_policy);
1915
1916 static DEFINE_SPINLOCK(xfrm_km_lock);
1917
1918 int xfrm_register_km(struct xfrm_mgr *km)
1919 {
1920         spin_lock_bh(&xfrm_km_lock);
1921         list_add_tail_rcu(&km->list, &xfrm_km_list);
1922         spin_unlock_bh(&xfrm_km_lock);
1923         return 0;
1924 }
1925 EXPORT_SYMBOL(xfrm_register_km);
1926
1927 int xfrm_unregister_km(struct xfrm_mgr *km)
1928 {
1929         spin_lock_bh(&xfrm_km_lock);
1930         list_del_rcu(&km->list);
1931         spin_unlock_bh(&xfrm_km_lock);
1932         synchronize_rcu();
1933         return 0;
1934 }
1935 EXPORT_SYMBOL(xfrm_unregister_km);
1936
1937 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
1938 {
1939         int err = 0;
1940         if (unlikely(afinfo == NULL))
1941                 return -EINVAL;
1942         if (unlikely(afinfo->family >= NPROTO))
1943                 return -EAFNOSUPPORT;
1944         spin_lock_bh(&xfrm_state_afinfo_lock);
1945         if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
1946                 err = -EEXIST;
1947         else
1948                 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
1949         spin_unlock_bh(&xfrm_state_afinfo_lock);
1950         return err;
1951 }
1952 EXPORT_SYMBOL(xfrm_state_register_afinfo);
1953
1954 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
1955 {
1956         int err = 0;
1957         if (unlikely(afinfo == NULL))
1958                 return -EINVAL;
1959         if (unlikely(afinfo->family >= NPROTO))
1960                 return -EAFNOSUPPORT;
1961         spin_lock_bh(&xfrm_state_afinfo_lock);
1962         if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
1963                 if (unlikely(xfrm_state_afinfo[afinfo->family] != afinfo))
1964                         err = -EINVAL;
1965                 else
1966                         RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
1967         }
1968         spin_unlock_bh(&xfrm_state_afinfo_lock);
1969         synchronize_rcu();
1970         return err;
1971 }
1972 EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
1973
1974 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
1975 {
1976         struct xfrm_state_afinfo *afinfo;
1977         if (unlikely(family >= NPROTO))
1978                 return NULL;
1979         rcu_read_lock();
1980         afinfo = rcu_dereference(xfrm_state_afinfo[family]);
1981         if (unlikely(!afinfo))
1982                 rcu_read_unlock();
1983         return afinfo;
1984 }
1985
1986 void xfrm_state_put_afinfo(struct xfrm_state_afinfo *afinfo)
1987 {
1988         rcu_read_unlock();
1989 }
1990
1991 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
1992 void xfrm_state_delete_tunnel(struct xfrm_state *x)
1993 {
1994         if (x->tunnel) {
1995                 struct xfrm_state *t = x->tunnel;
1996
1997                 if (atomic_read(&t->tunnel_users) == 2)
1998                         xfrm_state_delete(t);
1999                 atomic_dec(&t->tunnel_users);
2000                 xfrm_state_put(t);
2001                 x->tunnel = NULL;
2002         }
2003 }
2004 EXPORT_SYMBOL(xfrm_state_delete_tunnel);
2005
2006 int xfrm_state_mtu(struct xfrm_state *x, int mtu)
2007 {
2008         int res;
2009
2010         spin_lock_bh(&x->lock);
2011         if (x->km.state == XFRM_STATE_VALID &&
2012             x->type && x->type->get_mtu)
2013                 res = x->type->get_mtu(x, mtu);
2014         else
2015                 res = mtu - x->props.header_len;
2016         spin_unlock_bh(&x->lock);
2017         return res;
2018 }
2019
2020 int __xfrm_init_state(struct xfrm_state *x, bool init_replay)
2021 {
2022         struct xfrm_state_afinfo *afinfo;
2023         struct xfrm_mode *inner_mode;
2024         int family = x->props.family;
2025         int err;
2026
2027         err = -EAFNOSUPPORT;
2028         afinfo = xfrm_state_get_afinfo(family);
2029         if (!afinfo)
2030                 goto error;
2031
2032         err = 0;
2033         if (afinfo->init_flags)
2034                 err = afinfo->init_flags(x);
2035
2036         xfrm_state_put_afinfo(afinfo);
2037
2038         if (err)
2039                 goto error;
2040
2041         err = -EPROTONOSUPPORT;
2042
2043         if (x->sel.family != AF_UNSPEC) {
2044                 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
2045                 if (inner_mode == NULL)
2046                         goto error;
2047
2048                 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2049                     family != x->sel.family) {
2050                         xfrm_put_mode(inner_mode);
2051                         goto error;
2052                 }
2053
2054                 x->inner_mode = inner_mode;
2055         } else {
2056                 struct xfrm_mode *inner_mode_iaf;
2057                 int iafamily = AF_INET;
2058
2059                 inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
2060                 if (inner_mode == NULL)
2061                         goto error;
2062
2063                 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) {
2064                         xfrm_put_mode(inner_mode);
2065                         goto error;
2066                 }
2067                 x->inner_mode = inner_mode;
2068
2069                 if (x->props.family == AF_INET)
2070                         iafamily = AF_INET6;
2071
2072                 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
2073                 if (inner_mode_iaf) {
2074                         if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
2075                                 x->inner_mode_iaf = inner_mode_iaf;
2076                         else
2077                                 xfrm_put_mode(inner_mode_iaf);
2078                 }
2079         }
2080
2081         x->type = xfrm_get_type(x->id.proto, family);
2082         if (x->type == NULL)
2083                 goto error;
2084
2085         err = x->type->init_state(x);
2086         if (err)
2087                 goto error;
2088
2089         x->outer_mode = xfrm_get_mode(x->props.mode, family);
2090         if (x->outer_mode == NULL) {
2091                 err = -EPROTONOSUPPORT;
2092                 goto error;
2093         }
2094
2095         if (init_replay) {
2096                 err = xfrm_init_replay(x);
2097                 if (err)
2098                         goto error;
2099         }
2100
2101         x->km.state = XFRM_STATE_VALID;
2102
2103 error:
2104         return err;
2105 }
2106
2107 EXPORT_SYMBOL(__xfrm_init_state);
2108
2109 int xfrm_init_state(struct xfrm_state *x)
2110 {
2111         return __xfrm_init_state(x, true);
2112 }
2113
2114 EXPORT_SYMBOL(xfrm_init_state);
2115
2116 int __net_init xfrm_state_init(struct net *net)
2117 {
2118         unsigned int sz;
2119
2120         INIT_LIST_HEAD(&net->xfrm.state_all);
2121
2122         sz = sizeof(struct hlist_head) * 8;
2123
2124         net->xfrm.state_bydst = xfrm_hash_alloc(sz);
2125         if (!net->xfrm.state_bydst)
2126                 goto out_bydst;
2127         net->xfrm.state_bysrc = xfrm_hash_alloc(sz);
2128         if (!net->xfrm.state_bysrc)
2129                 goto out_bysrc;
2130         net->xfrm.state_byspi = xfrm_hash_alloc(sz);
2131         if (!net->xfrm.state_byspi)
2132                 goto out_byspi;
2133         net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
2134
2135         net->xfrm.state_num = 0;
2136         INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
2137         INIT_HLIST_HEAD(&net->xfrm.state_gc_list);
2138         INIT_WORK(&net->xfrm.state_gc_work, xfrm_state_gc_task);
2139         spin_lock_init(&net->xfrm.xfrm_state_lock);
2140         return 0;
2141
2142 out_byspi:
2143         xfrm_hash_free(net->xfrm.state_bysrc, sz);
2144 out_bysrc:
2145         xfrm_hash_free(net->xfrm.state_bydst, sz);
2146 out_bydst:
2147         return -ENOMEM;
2148 }
2149
2150 void xfrm_state_fini(struct net *net)
2151 {
2152         unsigned int sz;
2153
2154         flush_work(&net->xfrm.state_hash_work);
2155         xfrm_state_flush(net, IPSEC_PROTO_ANY, false);
2156         flush_work(&net->xfrm.state_gc_work);
2157
2158         WARN_ON(!list_empty(&net->xfrm.state_all));
2159
2160         sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
2161         WARN_ON(!hlist_empty(net->xfrm.state_byspi));
2162         xfrm_hash_free(net->xfrm.state_byspi, sz);
2163         WARN_ON(!hlist_empty(net->xfrm.state_bysrc));
2164         xfrm_hash_free(net->xfrm.state_bysrc, sz);
2165         WARN_ON(!hlist_empty(net->xfrm.state_bydst));
2166         xfrm_hash_free(net->xfrm.state_bydst, sz);
2167 }
2168
2169 #ifdef CONFIG_AUDITSYSCALL
2170 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
2171                                      struct audit_buffer *audit_buf)
2172 {
2173         struct xfrm_sec_ctx *ctx = x->security;
2174         u32 spi = ntohl(x->id.spi);
2175
2176         if (ctx)
2177                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
2178                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2179
2180         switch (x->props.family) {
2181         case AF_INET:
2182                 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
2183                                  &x->props.saddr.a4, &x->id.daddr.a4);
2184                 break;
2185         case AF_INET6:
2186                 audit_log_format(audit_buf, " src=%pI6 dst=%pI6",
2187                                  x->props.saddr.a6, x->id.daddr.a6);
2188                 break;
2189         }
2190
2191         audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
2192 }
2193
2194 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
2195                                       struct audit_buffer *audit_buf)
2196 {
2197         const struct iphdr *iph4;
2198         const struct ipv6hdr *iph6;
2199
2200         switch (family) {
2201         case AF_INET:
2202                 iph4 = ip_hdr(skb);
2203                 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
2204                                  &iph4->saddr, &iph4->daddr);
2205                 break;
2206         case AF_INET6:
2207                 iph6 = ipv6_hdr(skb);
2208                 audit_log_format(audit_buf,
2209                                  " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
2210                                  &iph6->saddr, &iph6->daddr,
2211                                  iph6->flow_lbl[0] & 0x0f,
2212                                  iph6->flow_lbl[1],
2213                                  iph6->flow_lbl[2]);
2214                 break;
2215         }
2216 }
2217
2218 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
2219 {
2220         struct audit_buffer *audit_buf;
2221
2222         audit_buf = xfrm_audit_start("SAD-add");
2223         if (audit_buf == NULL)
2224                 return;
2225         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
2226         xfrm_audit_helper_sainfo(x, audit_buf);
2227         audit_log_format(audit_buf, " res=%u", result);
2228         audit_log_end(audit_buf);
2229 }
2230 EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
2231
2232 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
2233 {
2234         struct audit_buffer *audit_buf;
2235
2236         audit_buf = xfrm_audit_start("SAD-delete");
2237         if (audit_buf == NULL)
2238                 return;
2239         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
2240         xfrm_audit_helper_sainfo(x, audit_buf);
2241         audit_log_format(audit_buf, " res=%u", result);
2242         audit_log_end(audit_buf);
2243 }
2244 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
2245
2246 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
2247                                       struct sk_buff *skb)
2248 {
2249         struct audit_buffer *audit_buf;
2250         u32 spi;
2251
2252         audit_buf = xfrm_audit_start("SA-replay-overflow");
2253         if (audit_buf == NULL)
2254                 return;
2255         xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
2256         /* don't record the sequence number because it's inherent in this kind
2257          * of audit message */
2258         spi = ntohl(x->id.spi);
2259         audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
2260         audit_log_end(audit_buf);
2261 }
2262 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
2263
2264 void xfrm_audit_state_replay(struct xfrm_state *x,
2265                              struct sk_buff *skb, __be32 net_seq)
2266 {
2267         struct audit_buffer *audit_buf;
2268         u32 spi;
2269
2270         audit_buf = xfrm_audit_start("SA-replayed-pkt");
2271         if (audit_buf == NULL)
2272                 return;
2273         xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
2274         spi = ntohl(x->id.spi);
2275         audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
2276                          spi, spi, ntohl(net_seq));
2277         audit_log_end(audit_buf);
2278 }
2279 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
2280
2281 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
2282 {
2283         struct audit_buffer *audit_buf;
2284
2285         audit_buf = xfrm_audit_start("SA-notfound");
2286         if (audit_buf == NULL)
2287                 return;
2288         xfrm_audit_helper_pktinfo(skb, family, audit_buf);
2289         audit_log_end(audit_buf);
2290 }
2291 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
2292
2293 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
2294                                __be32 net_spi, __be32 net_seq)
2295 {
2296         struct audit_buffer *audit_buf;
2297         u32 spi;
2298
2299         audit_buf = xfrm_audit_start("SA-notfound");
2300         if (audit_buf == NULL)
2301                 return;
2302         xfrm_audit_helper_pktinfo(skb, family, audit_buf);
2303         spi = ntohl(net_spi);
2304         audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
2305                          spi, spi, ntohl(net_seq));
2306         audit_log_end(audit_buf);
2307 }
2308 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
2309
2310 void xfrm_audit_state_icvfail(struct xfrm_state *x,
2311                               struct sk_buff *skb, u8 proto)
2312 {
2313         struct audit_buffer *audit_buf;
2314         __be32 net_spi;
2315         __be32 net_seq;
2316
2317         audit_buf = xfrm_audit_start("SA-icv-failure");
2318         if (audit_buf == NULL)
2319                 return;
2320         xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
2321         if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) {
2322                 u32 spi = ntohl(net_spi);
2323                 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
2324                                  spi, spi, ntohl(net_seq));
2325         }
2326         audit_log_end(audit_buf);
2327 }
2328 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
2329 #endif /* CONFIG_AUDITSYSCALL */