drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44 #include <linux/module.h>
  45 #include <net/route.h>
  46
  47 #include <net/tcp.h>
  48 #include <net/ipv6.h>
  49
  50 #include <rdma/rdma_cm.h>
  51 #include <rdma/rdma_cm_ib.h>
  52 #include <rdma/rdma_netlink.h>
  53 #include <rdma/ib.h>
  54 #include <rdma/ib_cache.h>
  55 #include <rdma/ib_cm.h>
  56 #include <rdma/ib_sa.h>
  57 #include <rdma/iw_cm.h>
  58
  59 MODULE_AUTHOR("Sean Hefty");
  60 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  61 MODULE_LICENSE("Dual BSD/GPL");
  62
  63 #define CMA_CM_RESPONSE_TIMEOUT 20
  64 #define CMA_MAX_CM_RETRIES 15
  65 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  66 #define CMA_IBOE_PACKET_LIFETIME 18
  67
  68 static void cma_add_one(struct ib_device *device);
  69 static void cma_remove_one(struct ib_device *device);
  70
  71 static struct ib_client cma_client = {
  72         .name   = "cma",
  73         .add    = cma_add_one,
  74         .remove = cma_remove_one
  75 };
  76
  77 static struct ib_sa_client sa_client;
  78 static struct rdma_addr_client addr_client;
  79 static LIST_HEAD(dev_list);
  80 static LIST_HEAD(listen_any_list);
  81 static DEFINE_MUTEX(lock);
  82 static struct workqueue_struct *cma_wq;
  83 static DEFINE_IDR(sdp_ps);
  84 static DEFINE_IDR(tcp_ps);
  85 static DEFINE_IDR(udp_ps);
  86 static DEFINE_IDR(ipoib_ps);
  87 static DEFINE_IDR(ib_ps);
  88
  89 struct cma_device {
  90         struct list_head        list;
  91         struct ib_device        *device;
  92         struct completion       comp;
  93         atomic_t                refcount;
  94         struct list_head        id_list;
  95 };
  96
  97 struct rdma_bind_list {
  98         struct idr              *ps;
  99         struct hlist_head       owners;
 100         unsigned short          port;
 101 };
 102
 103 enum {
 104         CMA_OPTION_AFONLY,
 105 };
 106
 107 /*
 108  * Device removal can occur at anytime, so we need extra handling to
 109  * serialize notifying the user of device removal with other callbacks.
 110  * We do this by disabling removal notification while a callback is in process,
 111  * and reporting it after the callback completes.
 112  */
 113 struct rdma_id_private {
 114         struct rdma_cm_id       id;
 115
 116         struct rdma_bind_list   *bind_list;
 117         struct hlist_node       node;
 118         struct list_head        list; /* listen_any_list or cma_device.list */
 119         struct list_head        listen_list; /* per device listens */
 120         struct cma_device       *cma_dev;
 121         struct list_head        mc_list;
 122
 123         int                     internal_id;
 124         enum rdma_cm_state      state;
 125         spinlock_t              lock;
 126         struct mutex            qp_mutex;
 127
 128         struct completion       comp;
 129         atomic_t                refcount;
 130         struct mutex            handler_mutex;
 131
 132         int                     backlog;
 133         int                     timeout_ms;
 134         struct ib_sa_query      *query;
 135         int                     query_id;
 136         union {
 137                 struct ib_cm_id *ib;
 138                 struct iw_cm_id *iw;
 139         } cm_id;
 140
 141         u32                     seq_num;
 142         u32                     qkey;
 143         u32                     qp_num;
 144         pid_t                   owner;
 145         u32                     options;
 146         u8                      srq;
 147         u8                      tos;
 148         u8                      reuseaddr;
 149         u8                      afonly;
 150 };
 151
 152 struct cma_multicast {
 153         struct rdma_id_private *id_priv;
 154         union {
 155                 struct ib_sa_multicast *ib;
 156         } multicast;
 157         struct list_head        list;
 158         void                    *context;
 159         struct sockaddr_storage addr;
 160         struct kref             mcref;
 161 };
 162
 163 struct cma_work {
 164         struct work_struct      work;
 165         struct rdma_id_private  *id;
 166         enum rdma_cm_state      old_state;
 167         enum rdma_cm_state      new_state;
 168         struct rdma_cm_event    event;
 169 };
 170
 171 struct cma_ndev_work {
 172         struct work_struct      work;
 173         struct rdma_id_private  *id;
 174         struct rdma_cm_event    event;
 175 };
 176
 177 struct iboe_mcast_work {
 178         struct work_struct       work;
 179         struct rdma_id_private  *id;
 180         struct cma_multicast    *mc;
 181 };
 182
 183 union cma_ip_addr {
 184         struct in6_addr ip6;
 185         struct {
 186                 __be32 pad[3];
 187                 __be32 addr;
 188         } ip4;
 189 };
 190
 191 struct cma_hdr {
 192         u8 cma_version;
 193         u8 ip_version;  /* IP version: 7:4 */
 194         __be16 port;
 195         union cma_ip_addr src_addr;
 196         union cma_ip_addr dst_addr;
 197 };
 198
 199 struct sdp_hh {
 200         u8 bsdh[16];
 201         u8 sdp_version; /* Major version: 7:4 */
 202         u8 ip_version;  /* IP version: 7:4 */
 203         u8 sdp_specific1[10];
 204         __be16 port;
 205         __be16 sdp_specific2;
 206         union cma_ip_addr src_addr;
 207         union cma_ip_addr dst_addr;
 208 };
 209
 210 struct sdp_hah {
 211         u8 bsdh[16];
 212         u8 sdp_version;
 213 };
 214
 215 #define CMA_VERSION 0x00
 216 #define SDP_MAJ_VERSION 0x2
 217
 218 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 219 {
 220         unsigned long flags;
 221         int ret;
 222
 223         spin_lock_irqsave(&id_priv->lock, flags);
 224         ret = (id_priv->state == comp);
 225         spin_unlock_irqrestore(&id_priv->lock, flags);
 226         return ret;
 227 }
 228
 229 static int cma_comp_exch(struct rdma_id_private *id_priv,
 230                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 231 {
 232         unsigned long flags;
 233         int ret;
 234
 235         spin_lock_irqsave(&id_priv->lock, flags);
 236         if ((ret = (id_priv->state == comp)))
 237                 id_priv->state = exch;
 238         spin_unlock_irqrestore(&id_priv->lock, flags);
 239         return ret;
 240 }
 241
 242 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 243                                    enum rdma_cm_state exch)
 244 {
 245         unsigned long flags;
 246         enum rdma_cm_state old;
 247
 248         spin_lock_irqsave(&id_priv->lock, flags);
 249         old = id_priv->state;
 250         id_priv->state = exch;
 251         spin_unlock_irqrestore(&id_priv->lock, flags);
 252         return old;
 253 }
 254
 255 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 256 {
 257         return hdr->ip_version >> 4;
 258 }
 259
 260 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 261 {
 262         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 263 }
 264
 265 static inline u8 sdp_get_majv(u8 sdp_version)
 266 {
 267         return sdp_version >> 4;
 268 }
 269
 270 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 271 {
 272         return hh->ip_version >> 4;
 273 }
 274
 275 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 276 {
 277         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 278 }
 279
 280 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 281                               struct cma_device *cma_dev)
 282 {
 283         atomic_inc(&cma_dev->refcount);
 284         id_priv->cma_dev = cma_dev;
 285         id_priv->id.device = cma_dev->device;
 286         id_priv->id.route.addr.dev_addr.transport =
 287                 rdma_node_get_transport(cma_dev->device->node_type);
 288         list_add_tail(&id_priv->list, &cma_dev->id_list);
 289 }
 290
 291 static inline void cma_deref_dev(struct cma_device *cma_dev)
 292 {
 293         if (atomic_dec_and_test(&cma_dev->refcount))
 294                 complete(&cma_dev->comp);
 295 }
 296
 297 static inline void release_mc(struct kref *kref)
 298 {
 299         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 300
 301         kfree(mc->multicast.ib);
 302         kfree(mc);
 303 }
 304
 305 static void cma_release_dev(struct rdma_id_private *id_priv)
 306 {
 307         mutex_lock(&lock);
 308         list_del(&id_priv->list);
 309         cma_deref_dev(id_priv->cma_dev);
 310         id_priv->cma_dev = NULL;
 311         mutex_unlock(&lock);
 312 }
 313
 314 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 315 {
 316         return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
 317 }
 318
 319 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 320 {
 321         return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
 322 }
 323
 324 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 325 {
 326         return id_priv->id.route.addr.src_addr.ss_family;
 327 }
 328
 329 static int cma_set_qkey(struct rdma_id_private *id_priv)
 330 {
 331         struct ib_sa_mcmember_rec rec;
 332         int ret = 0;
 333
 334         if (id_priv->qkey)
 335                 return 0;
 336
 337         switch (id_priv->id.ps) {
 338         case RDMA_PS_UDP:
 339                 id_priv->qkey = RDMA_UDP_QKEY;
 340                 break;
 341         case RDMA_PS_IPOIB:
 342                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 343                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 344                                              id_priv->id.port_num, &rec.mgid,
 345                                              &rec);
 346                 if (!ret)
 347                         id_priv->qkey = be32_to_cpu(rec.qkey);
 348                 break;
 349         default:
 350                 break;
 351         }
 352         return ret;
 353 }
 354
 355 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 356 {
 357         int i;
 358         int err;
 359         struct ib_port_attr props;
 360         union ib_gid tmp;
 361
 362         err = ib_query_port(device, port_num, &props);
 363         if (err)
 364                 return err;
 365
 366         for (i = 0; i < props.gid_tbl_len; ++i) {
 367                 err = ib_query_gid(device, port_num, i, &tmp);
 368                 if (err)
 369                         return err;
 370                 if (!memcmp(&tmp, gid, sizeof tmp))
 371                         return 0;
 372         }
 373
 374         return -EADDRNOTAVAIL;
 375 }
 376
 377 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 378 {
 379         dev_addr->dev_type = ARPHRD_INFINIBAND;
 380         rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 381         ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 382 }
 383
 384 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 385 {
 386         int ret;
 387
 388         if (addr->sa_family != AF_IB) {
 389                 ret = rdma_translate_ip(addr, dev_addr);
 390         } else {
 391                 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 392                 ret = 0;
 393         }
 394
 395         return ret;
 396 }
 397
 398 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 399 {
 400         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 401         struct cma_device *cma_dev;
 402         union ib_gid gid, iboe_gid;
 403         int ret = -ENODEV;
 404         u8 port;
 405         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 406                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 407
 408         if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
 409             id_priv->id.ps == RDMA_PS_IPOIB)
 410                 return -EINVAL;
 411
 412         mutex_lock(&lock);
 413         iboe_addr_get_sgid(dev_addr, &iboe_gid);
 414         memcpy(&gid, dev_addr->src_dev_addr +
 415                rdma_addr_gid_offset(dev_addr), sizeof gid);
 416         list_for_each_entry(cma_dev, &dev_list, list) {
 417                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 418                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 419                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 420                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 421                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 422                                 else
 423                                         ret = find_gid_port(cma_dev->device, &gid, port);
 424
 425                                 if (!ret) {
 426                                         id_priv->id.port_num = port;
 427                                         goto out;
 428                                 }
 429                         }
 430                 }
 431         }
 432
 433 out:
 434         if (!ret)
 435                 cma_attach_to_dev(id_priv, cma_dev);
 436
 437         mutex_unlock(&lock);
 438         return ret;
 439 }
 440
 441 static void cma_deref_id(struct rdma_id_private *id_priv)
 442 {
 443         if (atomic_dec_and_test(&id_priv->refcount))
 444                 complete(&id_priv->comp);
 445 }
 446
 447 static int cma_disable_callback(struct rdma_id_private *id_priv,
 448                                 enum rdma_cm_state state)
 449 {
 450         mutex_lock(&id_priv->handler_mutex);
 451         if (id_priv->state != state) {
 452                 mutex_unlock(&id_priv->handler_mutex);
 453                 return -EINVAL;
 454         }
 455         return 0;
 456 }
 457
 458 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 459                                   void *context, enum rdma_port_space ps,
 460                                   enum ib_qp_type qp_type)
 461 {
 462         struct rdma_id_private *id_priv;
 463
 464         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 465         if (!id_priv)
 466                 return ERR_PTR(-ENOMEM);
 467
 468         id_priv->owner = task_pid_nr(current);
 469         id_priv->state = RDMA_CM_IDLE;
 470         id_priv->id.context = context;
 471         id_priv->id.event_handler = event_handler;
 472         id_priv->id.ps = ps;
 473         id_priv->id.qp_type = qp_type;
 474         spin_lock_init(&id_priv->lock);
 475         mutex_init(&id_priv->qp_mutex);
 476         init_completion(&id_priv->comp);
 477         atomic_set(&id_priv->refcount, 1);
 478         mutex_init(&id_priv->handler_mutex);
 479         INIT_LIST_HEAD(&id_priv->listen_list);
 480         INIT_LIST_HEAD(&id_priv->mc_list);
 481         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 482
 483         return &id_priv->id;
 484 }
 485 EXPORT_SYMBOL(rdma_create_id);
 486
 487 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 488 {
 489         struct ib_qp_attr qp_attr;
 490         int qp_attr_mask, ret;
 491
 492         qp_attr.qp_state = IB_QPS_INIT;
 493         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 494         if (ret)
 495                 return ret;
 496
 497         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 498         if (ret)
 499                 return ret;
 500
 501         qp_attr.qp_state = IB_QPS_RTR;
 502         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 503         if (ret)
 504                 return ret;
 505
 506         qp_attr.qp_state = IB_QPS_RTS;
 507         qp_attr.sq_psn = 0;
 508         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 509
 510         return ret;
 511 }
 512
 513 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 514 {
 515         struct ib_qp_attr qp_attr;
 516         int qp_attr_mask, ret;
 517
 518         qp_attr.qp_state = IB_QPS_INIT;
 519         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 520         if (ret)
 521                 return ret;
 522
 523         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 524 }
 525
 526 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 527                    struct ib_qp_init_attr *qp_init_attr)
 528 {
 529         struct rdma_id_private *id_priv;
 530         struct ib_qp *qp;
 531         int ret;
 532
 533         id_priv = container_of(id, struct rdma_id_private, id);
 534         if (id->device != pd->device)
 535                 return -EINVAL;
 536
 537         qp = ib_create_qp(pd, qp_init_attr);
 538         if (IS_ERR(qp))
 539                 return PTR_ERR(qp);
 540
 541         if (id->qp_type == IB_QPT_UD)
 542                 ret = cma_init_ud_qp(id_priv, qp);
 543         else
 544                 ret = cma_init_conn_qp(id_priv, qp);
 545         if (ret)
 546                 goto err;
 547
 548         id->qp = qp;
 549         id_priv->qp_num = qp->qp_num;
 550         id_priv->srq = (qp->srq != NULL);
 551         return 0;
 552 err:
 553         ib_destroy_qp(qp);
 554         return ret;
 555 }
 556 EXPORT_SYMBOL(rdma_create_qp);
 557
 558 void rdma_destroy_qp(struct rdma_cm_id *id)
 559 {
 560         struct rdma_id_private *id_priv;
 561
 562         id_priv = container_of(id, struct rdma_id_private, id);
 563         mutex_lock(&id_priv->qp_mutex);
 564         ib_destroy_qp(id_priv->id.qp);
 565         id_priv->id.qp = NULL;
 566         mutex_unlock(&id_priv->qp_mutex);
 567 }
 568 EXPORT_SYMBOL(rdma_destroy_qp);
 569
 570 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 571                              struct rdma_conn_param *conn_param)
 572 {
 573         struct ib_qp_attr qp_attr;
 574         int qp_attr_mask, ret;
 575
 576         mutex_lock(&id_priv->qp_mutex);
 577         if (!id_priv->id.qp) {
 578                 ret = 0;
 579                 goto out;
 580         }
 581
 582         /* Need to update QP attributes from default values. */
 583         qp_attr.qp_state = IB_QPS_INIT;
 584         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 585         if (ret)
 586                 goto out;
 587
 588         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 589         if (ret)
 590                 goto out;
 591
 592         qp_attr.qp_state = IB_QPS_RTR;
 593         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 594         if (ret)
 595                 goto out;
 596
 597         if (conn_param)
 598                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 599         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 600 out:
 601         mutex_unlock(&id_priv->qp_mutex);
 602         return ret;
 603 }
 604
 605 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 606                              struct rdma_conn_param *conn_param)
 607 {
 608         struct ib_qp_attr qp_attr;
 609         int qp_attr_mask, ret;
 610
 611         mutex_lock(&id_priv->qp_mutex);
 612         if (!id_priv->id.qp) {
 613                 ret = 0;
 614                 goto out;
 615         }
 616
 617         qp_attr.qp_state = IB_QPS_RTS;
 618         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 619         if (ret)
 620                 goto out;
 621
 622         if (conn_param)
 623                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 624         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 625 out:
 626         mutex_unlock(&id_priv->qp_mutex);
 627         return ret;
 628 }
 629
 630 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 631 {
 632         struct ib_qp_attr qp_attr;
 633         int ret;
 634
 635         mutex_lock(&id_priv->qp_mutex);
 636         if (!id_priv->id.qp) {
 637                 ret = 0;
 638                 goto out;
 639         }
 640
 641         qp_attr.qp_state = IB_QPS_ERR;
 642         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 643 out:
 644         mutex_unlock(&id_priv->qp_mutex);
 645         return ret;
 646 }
 647
 648 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 649                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 650 {
 651         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 652         int ret;
 653         u16 pkey;
 654
 655         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 656             IB_LINK_LAYER_INFINIBAND)
 657                 pkey = ib_addr_get_pkey(dev_addr);
 658         else
 659                 pkey = 0xffff;
 660
 661         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 662                                   pkey, &qp_attr->pkey_index);
 663         if (ret)
 664                 return ret;
 665
 666         qp_attr->port_num = id_priv->id.port_num;
 667         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 668
 669         if (id_priv->id.qp_type == IB_QPT_UD) {
 670                 ret = cma_set_qkey(id_priv);
 671                 if (ret)
 672                         return ret;
 673
 674                 qp_attr->qkey = id_priv->qkey;
 675                 *qp_attr_mask |= IB_QP_QKEY;
 676         } else {
 677                 qp_attr->qp_access_flags = 0;
 678                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 679         }
 680         return 0;
 681 }
 682
 683 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 684                        int *qp_attr_mask)
 685 {
 686         struct rdma_id_private *id_priv;
 687         int ret = 0;
 688
 689         id_priv = container_of(id, struct rdma_id_private, id);
 690         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 691         case RDMA_TRANSPORT_IB:
 692                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
 693                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 694                 else
 695                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 696                                                  qp_attr_mask);
 697                 if (qp_attr->qp_state == IB_QPS_RTR)
 698                         qp_attr->rq_psn = id_priv->seq_num;
 699                 break;
 700         case RDMA_TRANSPORT_IWARP:
 701                 if (!id_priv->cm_id.iw) {
 702                         qp_attr->qp_access_flags = 0;
 703                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 704                 } else
 705                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 706                                                  qp_attr_mask);
 707                 break;
 708         default:
 709                 ret = -ENOSYS;
 710                 break;
 711         }
 712
 713         return ret;
 714 }
 715 EXPORT_SYMBOL(rdma_init_qp_attr);
 716
 717 static inline int cma_zero_addr(struct sockaddr *addr)
 718 {
 719         switch (addr->sa_family) {
 720         case AF_INET:
 721                 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
 722         case AF_INET6:
 723                 return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr);
 724         case AF_IB:
 725                 return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr);
 726         default:
 727                 return 0;
 728         }
 729 }
 730
 731 static inline int cma_loopback_addr(struct sockaddr *addr)
 732 {
 733         switch (addr->sa_family) {
 734         case AF_INET:
 735                 return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
 736         case AF_INET6:
 737                 return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr);
 738         case AF_IB:
 739                 return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr);
 740         default:
 741                 return 0;
 742         }
 743 }
 744
 745 static inline int cma_any_addr(struct sockaddr *addr)
 746 {
 747         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 748 }
 749
 750 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 751 {
 752         if (src->sa_family != dst->sa_family)
 753                 return -1;
 754
 755         switch (src->sa_family) {
 756         case AF_INET:
 757                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 758                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 759         case AF_INET6:
 760                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 761                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 762         default:
 763                 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
 764                                    &((struct sockaddr_ib *) dst)->sib_addr);
 765         }
 766 }
 767
 768 static __be16 cma_port(struct sockaddr *addr)
 769 {
 770         struct sockaddr_ib *sib;
 771
 772         switch (addr->sa_family) {
 773         case AF_INET:
 774                 return ((struct sockaddr_in *) addr)->sin_port;
 775         case AF_INET6:
 776                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 777         case AF_IB:
 778                 sib = (struct sockaddr_ib *) addr;
 779                 return htons((u16) (be64_to_cpu(sib->sib_sid) &
 780                                     be64_to_cpu(sib->sib_sid_mask)));
 781         default:
 782                 return 0;
 783         }
 784 }
 785
 786 static inline int cma_any_port(struct sockaddr *addr)
 787 {
 788         return !cma_port(addr);
 789 }
 790
 791 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 792                             u8 *ip_ver, __be16 *port,
 793                             union cma_ip_addr **src, union cma_ip_addr **dst)
 794 {
 795         switch (ps) {
 796         case RDMA_PS_SDP:
 797                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 798                     SDP_MAJ_VERSION)
 799                         return -EINVAL;
 800
 801                 *ip_ver = sdp_get_ip_ver(hdr);
 802                 *port   = ((struct sdp_hh *) hdr)->port;
 803                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 804                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 805                 break;
 806         default:
 807                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 808                         return -EINVAL;
 809
 810                 *ip_ver = cma_get_ip_ver(hdr);
 811                 *port   = ((struct cma_hdr *) hdr)->port;
 812                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 813                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 814                 break;
 815         }
 816
 817         if (*ip_ver != 4 && *ip_ver != 6)
 818                 return -EINVAL;
 819         return 0;
 820 }
 821
 822 static void cma_save_net_info(struct rdma_addr *addr,
 823                               struct rdma_addr *listen_addr,
 824                               u8 ip_ver, __be16 port,
 825                               union cma_ip_addr *src, union cma_ip_addr *dst)
 826 {
 827         struct sockaddr_in *listen4, *ip4;
 828         struct sockaddr_in6 *listen6, *ip6;
 829
 830         switch (ip_ver) {
 831         case 4:
 832                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 833                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 834                 ip4->sin_family = listen4->sin_family;
 835                 ip4->sin_addr.s_addr = dst->ip4.addr;
 836                 ip4->sin_port = listen4->sin_port;
 837
 838                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 839                 ip4->sin_family = listen4->sin_family;
 840                 ip4->sin_addr.s_addr = src->ip4.addr;
 841                 ip4->sin_port = port;
 842                 break;
 843         case 6:
 844                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 845                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 846                 ip6->sin6_family = listen6->sin6_family;
 847                 ip6->sin6_addr = dst->ip6;
 848                 ip6->sin6_port = listen6->sin6_port;
 849
 850                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 851                 ip6->sin6_family = listen6->sin6_family;
 852                 ip6->sin6_addr = src->ip6;
 853                 ip6->sin6_port = port;
 854                 break;
 855         default:
 856                 break;
 857         }
 858 }
 859
 860 static inline int cma_user_data_offset(enum rdma_port_space ps)
 861 {
 862         switch (ps) {
 863         case RDMA_PS_SDP:
 864                 return 0;
 865         default:
 866                 return sizeof(struct cma_hdr);
 867         }
 868 }
 869
 870 static void cma_cancel_route(struct rdma_id_private *id_priv)
 871 {
 872         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 873         case IB_LINK_LAYER_INFINIBAND:
 874                 if (id_priv->query)
 875                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 876                 break;
 877         default:
 878                 break;
 879         }
 880 }
 881
 882 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 883 {
 884         struct rdma_id_private *dev_id_priv;
 885
 886         /*
 887          * Remove from listen_any_list to prevent added devices from spawning
 888          * additional listen requests.
 889          */
 890         mutex_lock(&lock);
 891         list_del(&id_priv->list);
 892
 893         while (!list_empty(&id_priv->listen_list)) {
 894                 dev_id_priv = list_entry(id_priv->listen_list.next,
 895                                          struct rdma_id_private, listen_list);
 896                 /* sync with device removal to avoid duplicate destruction */
 897                 list_del_init(&dev_id_priv->list);
 898                 list_del(&dev_id_priv->listen_list);
 899                 mutex_unlock(&lock);
 900
 901                 rdma_destroy_id(&dev_id_priv->id);
 902                 mutex_lock(&lock);
 903         }
 904         mutex_unlock(&lock);
 905 }
 906
 907 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 908                                  enum rdma_cm_state state)
 909 {
 910         switch (state) {
 911         case RDMA_CM_ADDR_QUERY:
 912                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 913                 break;
 914         case RDMA_CM_ROUTE_QUERY:
 915                 cma_cancel_route(id_priv);
 916                 break;
 917         case RDMA_CM_LISTEN:
 918                 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
 919                         cma_cancel_listens(id_priv);
 920                 break;
 921         default:
 922                 break;
 923         }
 924 }
 925
 926 static void cma_release_port(struct rdma_id_private *id_priv)
 927 {
 928         struct rdma_bind_list *bind_list = id_priv->bind_list;
 929
 930         if (!bind_list)
 931                 return;
 932
 933         mutex_lock(&lock);
 934         hlist_del(&id_priv->node);
 935         if (hlist_empty(&bind_list->owners)) {
 936                 idr_remove(bind_list->ps, bind_list->port);
 937                 kfree(bind_list);
 938         }
 939         mutex_unlock(&lock);
 940 }
 941
 942 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 943 {
 944         struct cma_multicast *mc;
 945
 946         while (!list_empty(&id_priv->mc_list)) {
 947                 mc = container_of(id_priv->mc_list.next,
 948                                   struct cma_multicast, list);
 949                 list_del(&mc->list);
 950                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 951                 case IB_LINK_LAYER_INFINIBAND:
 952                         ib_sa_free_multicast(mc->multicast.ib);
 953                         kfree(mc);
 954                         break;
 955                 case IB_LINK_LAYER_ETHERNET:
 956                         kref_put(&mc->mcref, release_mc);
 957                         break;
 958                 default:
 959                         break;
 960                 }
 961         }
 962 }
 963
 964 void rdma_destroy_id(struct rdma_cm_id *id)
 965 {
 966         struct rdma_id_private *id_priv;
 967         enum rdma_cm_state state;
 968
 969         id_priv = container_of(id, struct rdma_id_private, id);
 970         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
 971         cma_cancel_operation(id_priv, state);
 972
 973         /*
 974          * Wait for any active callback to finish.  New callbacks will find
 975          * the id_priv state set to destroying and abort.
 976          */
 977         mutex_lock(&id_priv->handler_mutex);
 978         mutex_unlock(&id_priv->handler_mutex);
 979
 980         if (id_priv->cma_dev) {
 981                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 982                 case RDMA_TRANSPORT_IB:
 983                         if (id_priv->cm_id.ib)
 984                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 985                         break;
 986                 case RDMA_TRANSPORT_IWARP:
 987                         if (id_priv->cm_id.iw)
 988                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 989                         break;
 990                 default:
 991                         break;
 992                 }
 993                 cma_leave_mc_groups(id_priv);
 994                 cma_release_dev(id_priv);
 995         }
 996
 997         cma_release_port(id_priv);
 998         cma_deref_id(id_priv);
 999         wait_for_completion(&id_priv->comp);
1000
1001         if (id_priv->internal_id)
1002                 cma_deref_id(id_priv->id.context);
1003
1004         kfree(id_priv->id.route.path_rec);
1005         kfree(id_priv);
1006 }
1007 EXPORT_SYMBOL(rdma_destroy_id);
1008
1009 static int cma_rep_recv(struct rdma_id_private *id_priv)
1010 {
1011         int ret;
1012
1013         ret = cma_modify_qp_rtr(id_priv, NULL);
1014         if (ret)
1015                 goto reject;
1016
1017         ret = cma_modify_qp_rts(id_priv, NULL);
1018         if (ret)
1019                 goto reject;
1020
1021         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1022         if (ret)
1023                 goto reject;
1024
1025         return 0;
1026 reject:
1027         cma_modify_qp_err(id_priv);
1028         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1029                        NULL, 0, NULL, 0);
1030         return ret;
1031 }
1032
1033 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
1034 {
1035         if (id_priv->id.ps == RDMA_PS_SDP &&
1036             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
1037             SDP_MAJ_VERSION)
1038                 return -EINVAL;
1039
1040         return 0;
1041 }
1042
1043 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1044                                    struct ib_cm_rep_event_param *rep_data,
1045                                    void *private_data)
1046 {
1047         event->param.conn.private_data = private_data;
1048         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1049         event->param.conn.responder_resources = rep_data->responder_resources;
1050         event->param.conn.initiator_depth = rep_data->initiator_depth;
1051         event->param.conn.flow_control = rep_data->flow_control;
1052         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1053         event->param.conn.srq = rep_data->srq;
1054         event->param.conn.qp_num = rep_data->remote_qpn;
1055 }
1056
1057 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1058 {
1059         struct rdma_id_private *id_priv = cm_id->context;
1060         struct rdma_cm_event event;
1061         int ret = 0;
1062
1063         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1064                 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1065             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1066                 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1067                 return 0;
1068
1069         memset(&event, 0, sizeof event);
1070         switch (ib_event->event) {
1071         case IB_CM_REQ_ERROR:
1072         case IB_CM_REP_ERROR:
1073                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1074                 event.status = -ETIMEDOUT;
1075                 break;
1076         case IB_CM_REP_RECEIVED:
1077                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1078                 if (event.status)
1079                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1080                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1081                         event.status = cma_rep_recv(id_priv);
1082                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1083                                                      RDMA_CM_EVENT_ESTABLISHED;
1084                 } else
1085                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1086                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1087                                        ib_event->private_data);
1088                 break;
1089         case IB_CM_RTU_RECEIVED:
1090         case IB_CM_USER_ESTABLISHED:
1091                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1092                 break;
1093         case IB_CM_DREQ_ERROR:
1094                 event.status = -ETIMEDOUT; /* fall through */
1095         case IB_CM_DREQ_RECEIVED:
1096         case IB_CM_DREP_RECEIVED:
1097                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1098                                    RDMA_CM_DISCONNECT))
1099                         goto out;
1100                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1101                 break;
1102         case IB_CM_TIMEWAIT_EXIT:
1103                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1104                 break;
1105         case IB_CM_MRA_RECEIVED:
1106                 /* ignore event */
1107                 goto out;
1108         case IB_CM_REJ_RECEIVED:
1109                 cma_modify_qp_err(id_priv);
1110                 event.status = ib_event->param.rej_rcvd.reason;
1111                 event.event = RDMA_CM_EVENT_REJECTED;
1112                 event.param.conn.private_data = ib_event->private_data;
1113                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1114                 break;
1115         default:
1116                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1117                        ib_event->event);
1118                 goto out;
1119         }
1120
1121         ret = id_priv->id.event_handler(&id_priv->id, &event);
1122         if (ret) {
1123                 /* Destroy the CM ID by returning a non-zero value. */
1124                 id_priv->cm_id.ib = NULL;
1125                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1126                 mutex_unlock(&id_priv->handler_mutex);
1127                 rdma_destroy_id(&id_priv->id);
1128                 return ret;
1129         }
1130 out:
1131         mutex_unlock(&id_priv->handler_mutex);
1132         return ret;
1133 }
1134
1135 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1136                                                struct ib_cm_event *ib_event)
1137 {
1138         struct rdma_id_private *id_priv;
1139         struct rdma_cm_id *id;
1140         struct rdma_route *rt;
1141         union cma_ip_addr *src, *dst;
1142         __be16 port;
1143         u8 ip_ver;
1144         int ret;
1145
1146         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1147                              &ip_ver, &port, &src, &dst))
1148                 return NULL;
1149
1150         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1151                             listen_id->ps, ib_event->param.req_rcvd.qp_type);
1152         if (IS_ERR(id))
1153                 return NULL;
1154
1155         id_priv = container_of(id, struct rdma_id_private, id);
1156         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1157                           ip_ver, port, src, dst);
1158
1159         rt = &id->route;
1160         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1161         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1162                                GFP_KERNEL);
1163         if (!rt->path_rec)
1164                 goto err;
1165
1166         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1167         if (rt->num_paths == 2)
1168                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1169
1170         if (cma_any_addr(cma_src_addr(id_priv))) {
1171                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1172                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1173                 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1174         } else {
1175                 ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
1176                 if (ret)
1177                         goto err;
1178         }
1179         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1180
1181         id_priv->state = RDMA_CM_CONNECT;
1182         return id_priv;
1183
1184 err:
1185         rdma_destroy_id(id);
1186         return NULL;
1187 }
1188
1189 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1190                                               struct ib_cm_event *ib_event)
1191 {
1192         struct rdma_id_private *id_priv;
1193         struct rdma_cm_id *id;
1194         union cma_ip_addr *src, *dst;
1195         __be16 port;
1196         u8 ip_ver;
1197         int ret;
1198
1199         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1200                             listen_id->ps, IB_QPT_UD);
1201         if (IS_ERR(id))
1202                 return NULL;
1203
1204         id_priv = container_of(id, struct rdma_id_private, id);
1205         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1206                              &ip_ver, &port, &src, &dst))
1207                 goto err;
1208
1209         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1210                           ip_ver, port, src, dst);
1211
1212         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1213                 ret = cma_translate_addr(cma_src_addr(id_priv), &id->route.addr.dev_addr);
1214                 if (ret)
1215                         goto err;
1216         }
1217
1218         id_priv->state = RDMA_CM_CONNECT;
1219         return id_priv;
1220 err:
1221         rdma_destroy_id(id);
1222         return NULL;
1223 }
1224
1225 static void cma_set_req_event_data(struct rdma_cm_event *event,
1226                                    struct ib_cm_req_event_param *req_data,
1227                                    void *private_data, int offset)
1228 {
1229         event->param.conn.private_data = private_data + offset;
1230         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1231         event->param.conn.responder_resources = req_data->responder_resources;
1232         event->param.conn.initiator_depth = req_data->initiator_depth;
1233         event->param.conn.flow_control = req_data->flow_control;
1234         event->param.conn.retry_count = req_data->retry_count;
1235         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1236         event->param.conn.srq = req_data->srq;
1237         event->param.conn.qp_num = req_data->remote_qpn;
1238 }
1239
1240 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1241 {
1242         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1243                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1244                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1245                  (id->qp_type == IB_QPT_UD)) ||
1246                 (!id->qp_type));
1247 }
1248
1249 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1250 {
1251         struct rdma_id_private *listen_id, *conn_id;
1252         struct rdma_cm_event event;
1253         int offset, ret;
1254
1255         listen_id = cm_id->context;
1256         if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1257                 return -EINVAL;
1258
1259         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1260                 return -ECONNABORTED;
1261
1262         memset(&event, 0, sizeof event);
1263         offset = cma_user_data_offset(listen_id->id.ps);
1264         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1265         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1266                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1267                 event.param.ud.private_data = ib_event->private_data + offset;
1268                 event.param.ud.private_data_len =
1269                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1270         } else {
1271                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1272                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1273                                        ib_event->private_data, offset);
1274         }
1275         if (!conn_id) {
1276                 ret = -ENOMEM;
1277                 goto err1;
1278         }
1279
1280         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1281         ret = cma_acquire_dev(conn_id);
1282         if (ret)
1283                 goto err2;
1284
1285         conn_id->cm_id.ib = cm_id;
1286         cm_id->context = conn_id;
1287         cm_id->cm_handler = cma_ib_handler;
1288
1289         /*
1290          * Protect against the user destroying conn_id from another thread
1291          * until we're done accessing it.
1292          */
1293         atomic_inc(&conn_id->refcount);
1294         ret = conn_id->id.event_handler(&conn_id->id, &event);
1295         if (ret)
1296                 goto err3;
1297
1298         /*
1299          * Acquire mutex to prevent user executing rdma_destroy_id()
1300          * while we're accessing the cm_id.
1301          */
1302         mutex_lock(&lock);
1303         if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
1304                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1305         mutex_unlock(&lock);
1306         mutex_unlock(&conn_id->handler_mutex);
1307         mutex_unlock(&listen_id->handler_mutex);
1308         cma_deref_id(conn_id);
1309         return 0;
1310
1311 err3:
1312         cma_deref_id(conn_id);
1313         /* Destroy the CM ID by returning a non-zero value. */
1314         conn_id->cm_id.ib = NULL;
1315 err2:
1316         cma_exch(conn_id, RDMA_CM_DESTROYING);
1317         mutex_unlock(&conn_id->handler_mutex);
1318 err1:
1319         mutex_unlock(&listen_id->handler_mutex);
1320         if (conn_id)
1321                 rdma_destroy_id(&conn_id->id);
1322         return ret;
1323 }
1324
1325 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1326 {
1327         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1328 }
1329
1330 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1331                                  struct ib_cm_compare_data *compare)
1332 {
1333         struct cma_hdr *cma_data, *cma_mask;
1334         struct sdp_hh *sdp_data, *sdp_mask;
1335         __be32 ip4_addr;
1336         struct in6_addr ip6_addr;
1337
1338         memset(compare, 0, sizeof *compare);
1339         cma_data = (void *) compare->data;
1340         cma_mask = (void *) compare->mask;
1341         sdp_data = (void *) compare->data;
1342         sdp_mask = (void *) compare->mask;
1343
1344         switch (addr->sa_family) {
1345         case AF_INET:
1346                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1347                 if (ps == RDMA_PS_SDP) {
1348                         sdp_set_ip_ver(sdp_data, 4);
1349                         sdp_set_ip_ver(sdp_mask, 0xF);
1350                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1351                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1352                 } else {
1353                         cma_set_ip_ver(cma_data, 4);
1354                         cma_set_ip_ver(cma_mask, 0xF);
1355                         if (!cma_any_addr(addr)) {
1356                                 cma_data->dst_addr.ip4.addr = ip4_addr;
1357                                 cma_mask->dst_addr.ip4.addr = htonl(~0);
1358                         }
1359                 }
1360                 break;
1361         case AF_INET6:
1362                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1363                 if (ps == RDMA_PS_SDP) {
1364                         sdp_set_ip_ver(sdp_data, 6);
1365                         sdp_set_ip_ver(sdp_mask, 0xF);
1366                         sdp_data->dst_addr.ip6 = ip6_addr;
1367                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1368                                sizeof sdp_mask->dst_addr.ip6);
1369                 } else {
1370                         cma_set_ip_ver(cma_data, 6);
1371                         cma_set_ip_ver(cma_mask, 0xF);
1372                         if (!cma_any_addr(addr)) {
1373                                 cma_data->dst_addr.ip6 = ip6_addr;
1374                                 memset(&cma_mask->dst_addr.ip6, 0xFF,
1375                                        sizeof cma_mask->dst_addr.ip6);
1376                         }
1377                 }
1378                 break;
1379         default:
1380                 break;
1381         }
1382 }
1383
1384 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1385 {
1386         struct rdma_id_private *id_priv = iw_id->context;
1387         struct rdma_cm_event event;
1388         struct sockaddr_in *sin;
1389         int ret = 0;
1390
1391         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1392                 return 0;
1393
1394         memset(&event, 0, sizeof event);
1395         switch (iw_event->event) {
1396         case IW_CM_EVENT_CLOSE:
1397                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1398                 break;
1399         case IW_CM_EVENT_CONNECT_REPLY:
1400                 sin = (struct sockaddr_in *) cma_src_addr(id_priv);
1401                 *sin = iw_event->local_addr;
1402                 sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
1403                 *sin = iw_event->remote_addr;
1404                 switch (iw_event->status) {
1405                 case 0:
1406                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1407                         event.param.conn.initiator_depth = iw_event->ird;
1408                         event.param.conn.responder_resources = iw_event->ord;
1409                         break;
1410                 case -ECONNRESET:
1411                 case -ECONNREFUSED:
1412                         event.event = RDMA_CM_EVENT_REJECTED;
1413                         break;
1414                 case -ETIMEDOUT:
1415                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1416                         break;
1417                 default:
1418                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1419                         break;
1420                 }
1421                 break;
1422         case IW_CM_EVENT_ESTABLISHED:
1423                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1424                 event.param.conn.initiator_depth = iw_event->ird;
1425                 event.param.conn.responder_resources = iw_event->ord;
1426                 break;
1427         default:
1428                 BUG_ON(1);
1429         }
1430
1431         event.status = iw_event->status;
1432         event.param.conn.private_data = iw_event->private_data;
1433         event.param.conn.private_data_len = iw_event->private_data_len;
1434         ret = id_priv->id.event_handler(&id_priv->id, &event);
1435         if (ret) {
1436                 /* Destroy the CM ID by returning a non-zero value. */
1437                 id_priv->cm_id.iw = NULL;
1438                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1439                 mutex_unlock(&id_priv->handler_mutex);
1440                 rdma_destroy_id(&id_priv->id);
1441                 return ret;
1442         }
1443
1444         mutex_unlock(&id_priv->handler_mutex);
1445         return ret;
1446 }
1447
1448 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1449                                struct iw_cm_event *iw_event)
1450 {
1451         struct rdma_cm_id *new_cm_id;
1452         struct rdma_id_private *listen_id, *conn_id;
1453         struct sockaddr_in *sin;
1454         struct net_device *dev = NULL;
1455         struct rdma_cm_event event;
1456         int ret;
1457         struct ib_device_attr attr;
1458
1459         listen_id = cm_id->context;
1460         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1461                 return -ECONNABORTED;
1462
1463         /* Create a new RDMA id for the new IW CM ID */
1464         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1465                                    listen_id->id.context,
1466                                    RDMA_PS_TCP, IB_QPT_RC);
1467         if (IS_ERR(new_cm_id)) {
1468                 ret = -ENOMEM;
1469                 goto out;
1470         }
1471         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1472         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1473         conn_id->state = RDMA_CM_CONNECT;
1474
1475         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1476         if (!dev) {
1477                 ret = -EADDRNOTAVAIL;
1478                 mutex_unlock(&conn_id->handler_mutex);
1479                 rdma_destroy_id(new_cm_id);
1480                 goto out;
1481         }
1482         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1483         if (ret) {
1484                 mutex_unlock(&conn_id->handler_mutex);
1485                 rdma_destroy_id(new_cm_id);
1486                 goto out;
1487         }
1488
1489         ret = cma_acquire_dev(conn_id);
1490         if (ret) {
1491                 mutex_unlock(&conn_id->handler_mutex);
1492                 rdma_destroy_id(new_cm_id);
1493                 goto out;
1494         }
1495
1496         conn_id->cm_id.iw = cm_id;
1497         cm_id->context = conn_id;
1498         cm_id->cm_handler = cma_iw_handler;
1499
1500         sin = (struct sockaddr_in *) cma_src_addr(conn_id);
1501         *sin = iw_event->local_addr;
1502         sin = (struct sockaddr_in *) cma_dst_addr(conn_id);
1503         *sin = iw_event->remote_addr;
1504
1505         ret = ib_query_device(conn_id->id.device, &attr);
1506         if (ret) {
1507                 mutex_unlock(&conn_id->handler_mutex);
1508                 rdma_destroy_id(new_cm_id);
1509                 goto out;
1510         }
1511
1512         memset(&event, 0, sizeof event);
1513         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1514         event.param.conn.private_data = iw_event->private_data;
1515         event.param.conn.private_data_len = iw_event->private_data_len;
1516         event.param.conn.initiator_depth = iw_event->ird;
1517         event.param.conn.responder_resources = iw_event->ord;
1518
1519         /*
1520          * Protect against the user destroying conn_id from another thread
1521          * until we're done accessing it.
1522          */
1523         atomic_inc(&conn_id->refcount);
1524         ret = conn_id->id.event_handler(&conn_id->id, &event);
1525         if (ret) {
1526                 /* User wants to destroy the CM ID */
1527                 conn_id->cm_id.iw = NULL;
1528                 cma_exch(conn_id, RDMA_CM_DESTROYING);
1529                 mutex_unlock(&conn_id->handler_mutex);
1530                 cma_deref_id(conn_id);
1531                 rdma_destroy_id(&conn_id->id);
1532                 goto out;
1533         }
1534
1535         mutex_unlock(&conn_id->handler_mutex);
1536         cma_deref_id(conn_id);
1537
1538 out:
1539         if (dev)
1540                 dev_put(dev);
1541         mutex_unlock(&listen_id->handler_mutex);
1542         return ret;
1543 }
1544
1545 static int cma_ib_listen(struct rdma_id_private *id_priv)
1546 {
1547         struct ib_cm_compare_data compare_data;
1548         struct sockaddr *addr;
1549         struct ib_cm_id *id;
1550         __be64 svc_id;
1551         int ret;
1552
1553         id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1554         if (IS_ERR(id))
1555                 return PTR_ERR(id);
1556
1557         id_priv->cm_id.ib = id;
1558
1559         addr = cma_src_addr(id_priv);
1560         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1561         if (cma_any_addr(addr) && !id_priv->afonly)
1562                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1563         else {
1564                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1565                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1566         }
1567
1568         if (ret) {
1569                 ib_destroy_cm_id(id_priv->cm_id.ib);
1570                 id_priv->cm_id.ib = NULL;
1571         }
1572
1573         return ret;
1574 }
1575
1576 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1577 {
1578         int ret;
1579         struct sockaddr_in *sin;
1580         struct iw_cm_id *id;
1581
1582         id = iw_create_cm_id(id_priv->id.device,
1583                              iw_conn_req_handler,
1584                              id_priv);
1585         if (IS_ERR(id))
1586                 return PTR_ERR(id);
1587
1588         id_priv->cm_id.iw = id;
1589
1590         sin = (struct sockaddr_in *) cma_src_addr(id_priv);
1591         id_priv->cm_id.iw->local_addr = *sin;
1592
1593         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1594
1595         if (ret) {
1596                 iw_destroy_cm_id(id_priv->cm_id.iw);
1597                 id_priv->cm_id.iw = NULL;
1598         }
1599
1600         return ret;
1601 }
1602
1603 static int cma_listen_handler(struct rdma_cm_id *id,
1604                               struct rdma_cm_event *event)
1605 {
1606         struct rdma_id_private *id_priv = id->context;
1607
1608         id->context = id_priv->id.context;
1609         id->event_handler = id_priv->id.event_handler;
1610         return id_priv->id.event_handler(id, event);
1611 }
1612
1613 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1614                               struct cma_device *cma_dev)
1615 {
1616         struct rdma_id_private *dev_id_priv;
1617         struct rdma_cm_id *id;
1618         int ret;
1619
1620         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1621                             id_priv->id.qp_type);
1622         if (IS_ERR(id))
1623                 return;
1624
1625         dev_id_priv = container_of(id, struct rdma_id_private, id);
1626
1627         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1628         memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
1629                rdma_addr_size(cma_src_addr(id_priv)));
1630
1631         cma_attach_to_dev(dev_id_priv, cma_dev);
1632         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1633         atomic_inc(&id_priv->refcount);
1634         dev_id_priv->internal_id = 1;
1635         dev_id_priv->afonly = id_priv->afonly;
1636
1637         ret = rdma_listen(id, id_priv->backlog);
1638         if (ret)
1639                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1640                        "listening on device %s\n", ret, cma_dev->device->name);
1641 }
1642
1643 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1644 {
1645         struct cma_device *cma_dev;
1646
1647         mutex_lock(&lock);
1648         list_add_tail(&id_priv->list, &listen_any_list);
1649         list_for_each_entry(cma_dev, &dev_list, list)
1650                 cma_listen_on_dev(id_priv, cma_dev);
1651         mutex_unlock(&lock);
1652 }
1653
1654 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1655 {
1656         struct rdma_id_private *id_priv;
1657
1658         id_priv = container_of(id, struct rdma_id_private, id);
1659         id_priv->tos = (u8) tos;
1660 }
1661 EXPORT_SYMBOL(rdma_set_service_type);
1662
1663 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1664                               void *context)
1665 {
1666         struct cma_work *work = context;
1667         struct rdma_route *route;
1668
1669         route = &work->id->id.route;
1670
1671         if (!status) {
1672                 route->num_paths = 1;
1673                 *route->path_rec = *path_rec;
1674         } else {
1675                 work->old_state = RDMA_CM_ROUTE_QUERY;
1676                 work->new_state = RDMA_CM_ADDR_RESOLVED;
1677                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1678                 work->event.status = status;
1679         }
1680
1681         queue_work(cma_wq, &work->work);
1682 }
1683
1684 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1685                               struct cma_work *work)
1686 {
1687         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1688         struct ib_sa_path_rec path_rec;
1689         ib_sa_comp_mask comp_mask;
1690         struct sockaddr_in6 *sin6;
1691
1692         memset(&path_rec, 0, sizeof path_rec);
1693         rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
1694         rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
1695         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
1696         path_rec.numb_path = 1;
1697         path_rec.reversible = 1;
1698         path_rec.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
1699
1700         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1701                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1702                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1703
1704         if (cma_family(id_priv) == AF_INET) {
1705                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1706                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1707         } else {
1708                 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
1709                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1710                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1711         }
1712
1713         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1714                                                id_priv->id.port_num, &path_rec,
1715                                                comp_mask, timeout_ms,
1716                                                GFP_KERNEL, cma_query_handler,
1717                                                work, &id_priv->query);
1718
1719         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1720 }
1721
1722 static void cma_work_handler(struct work_struct *_work)
1723 {
1724         struct cma_work *work = container_of(_work, struct cma_work, work);
1725         struct rdma_id_private *id_priv = work->id;
1726         int destroy = 0;
1727
1728         mutex_lock(&id_priv->handler_mutex);
1729         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1730                 goto out;
1731
1732         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1733                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1734                 destroy = 1;
1735         }
1736 out:
1737         mutex_unlock(&id_priv->handler_mutex);
1738         cma_deref_id(id_priv);
1739         if (destroy)
1740                 rdma_destroy_id(&id_priv->id);
1741         kfree(work);
1742 }
1743
1744 static void cma_ndev_work_handler(struct work_struct *_work)
1745 {
1746         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1747         struct rdma_id_private *id_priv = work->id;
1748         int destroy = 0;
1749
1750         mutex_lock(&id_priv->handler_mutex);
1751         if (id_priv->state == RDMA_CM_DESTROYING ||
1752             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1753                 goto out;
1754
1755         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1756                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1757                 destroy = 1;
1758         }
1759
1760 out:
1761         mutex_unlock(&id_priv->handler_mutex);
1762         cma_deref_id(id_priv);
1763         if (destroy)
1764                 rdma_destroy_id(&id_priv->id);
1765         kfree(work);
1766 }
1767
1768 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1769 {
1770         struct rdma_route *route = &id_priv->id.route;
1771         struct cma_work *work;
1772         int ret;
1773
1774         work = kzalloc(sizeof *work, GFP_KERNEL);
1775         if (!work)
1776                 return -ENOMEM;
1777
1778         work->id = id_priv;
1779         INIT_WORK(&work->work, cma_work_handler);
1780         work->old_state = RDMA_CM_ROUTE_QUERY;
1781         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1782         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1783
1784         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1785         if (!route->path_rec) {
1786                 ret = -ENOMEM;
1787                 goto err1;
1788         }
1789
1790         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1791         if (ret)
1792                 goto err2;
1793
1794         return 0;
1795 err2:
1796         kfree(route->path_rec);
1797         route->path_rec = NULL;
1798 err1:
1799         kfree(work);
1800         return ret;
1801 }
1802
1803 int rdma_set_ib_paths(struct rdma_cm_id *id,
1804                       struct ib_sa_path_rec *path_rec, int num_paths)
1805 {
1806         struct rdma_id_private *id_priv;
1807         int ret;
1808
1809         id_priv = container_of(id, struct rdma_id_private, id);
1810         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1811                            RDMA_CM_ROUTE_RESOLVED))
1812                 return -EINVAL;
1813
1814         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1815                                      GFP_KERNEL);
1816         if (!id->route.path_rec) {
1817                 ret = -ENOMEM;
1818                 goto err;
1819         }
1820
1821         id->route.num_paths = num_paths;
1822         return 0;
1823 err:
1824         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1825         return ret;
1826 }
1827 EXPORT_SYMBOL(rdma_set_ib_paths);
1828
1829 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1830 {
1831         struct cma_work *work;
1832
1833         work = kzalloc(sizeof *work, GFP_KERNEL);
1834         if (!work)
1835                 return -ENOMEM;
1836
1837         work->id = id_priv;
1838         INIT_WORK(&work->work, cma_work_handler);
1839         work->old_state = RDMA_CM_ROUTE_QUERY;
1840         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1841         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1842         queue_work(cma_wq, &work->work);
1843         return 0;
1844 }
1845
1846 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1847 {
1848         struct rdma_route *route = &id_priv->id.route;
1849         struct rdma_addr *addr = &route->addr;
1850         struct cma_work *work;
1851         int ret;
1852         struct net_device *ndev = NULL;
1853         u16 vid;
1854
1855         work = kzalloc(sizeof *work, GFP_KERNEL);
1856         if (!work)
1857                 return -ENOMEM;
1858
1859         work->id = id_priv;
1860         INIT_WORK(&work->work, cma_work_handler);
1861
1862         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1863         if (!route->path_rec) {
1864                 ret = -ENOMEM;
1865                 goto err1;
1866         }
1867
1868         route->num_paths = 1;
1869
1870         if (addr->dev_addr.bound_dev_if)
1871                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1872         if (!ndev) {
1873                 ret = -ENODEV;
1874                 goto err2;
1875         }
1876
1877         vid = rdma_vlan_dev_vlan_id(ndev);
1878
1879         iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1880         iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1881
1882         route->path_rec->hop_limit = 1;
1883         route->path_rec->reversible = 1;
1884         route->path_rec->pkey = cpu_to_be16(0xffff);
1885         route->path_rec->mtu_selector = IB_SA_EQ;
1886         route->path_rec->sl = netdev_get_prio_tc_map(
1887                         ndev->priv_flags & IFF_802_1Q_VLAN ?
1888                                 vlan_dev_real_dev(ndev) : ndev,
1889                         rt_tos2priority(id_priv->tos));
1890
1891         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1892         route->path_rec->rate_selector = IB_SA_EQ;
1893         route->path_rec->rate = iboe_get_rate(ndev);
1894         dev_put(ndev);
1895         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1896         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1897         if (!route->path_rec->mtu) {
1898                 ret = -EINVAL;
1899                 goto err2;
1900         }
1901
1902         work->old_state = RDMA_CM_ROUTE_QUERY;
1903         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1904         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1905         work->event.status = 0;
1906
1907         queue_work(cma_wq, &work->work);
1908
1909         return 0;
1910
1911 err2:
1912         kfree(route->path_rec);
1913         route->path_rec = NULL;
1914 err1:
1915         kfree(work);
1916         return ret;
1917 }
1918
1919 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1920 {
1921         struct rdma_id_private *id_priv;
1922         int ret;
1923
1924         id_priv = container_of(id, struct rdma_id_private, id);
1925         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
1926                 return -EINVAL;
1927
1928         atomic_inc(&id_priv->refcount);
1929         switch (rdma_node_get_transport(id->device->node_type)) {
1930         case RDMA_TRANSPORT_IB:
1931                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1932                 case IB_LINK_LAYER_INFINIBAND:
1933                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
1934                         break;
1935                 case IB_LINK_LAYER_ETHERNET:
1936                         ret = cma_resolve_iboe_route(id_priv);
1937                         break;
1938                 default:
1939                         ret = -ENOSYS;
1940                 }
1941                 break;
1942         case RDMA_TRANSPORT_IWARP:
1943                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1944                 break;
1945         default:
1946                 ret = -ENOSYS;
1947                 break;
1948         }
1949         if (ret)
1950                 goto err;
1951
1952         return 0;
1953 err:
1954         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
1955         cma_deref_id(id_priv);
1956         return ret;
1957 }
1958 EXPORT_SYMBOL(rdma_resolve_route);
1959
1960 static void cma_set_loopback(struct sockaddr *addr)
1961 {
1962         switch (addr->sa_family) {
1963         case AF_INET:
1964                 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
1965                 break;
1966         case AF_INET6:
1967                 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
1968                               0, 0, 0, htonl(1));
1969                 break;
1970         default:
1971                 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
1972                             0, 0, 0, htonl(1));
1973                 break;
1974         }
1975 }
1976
1977 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1978 {
1979         struct cma_device *cma_dev, *cur_dev;
1980         struct ib_port_attr port_attr;
1981         union ib_gid gid;
1982         u16 pkey;
1983         int ret;
1984         u8 p;
1985
1986         cma_dev = NULL;
1987         mutex_lock(&lock);
1988         list_for_each_entry(cur_dev, &dev_list, list) {
1989                 if (cma_family(id_priv) == AF_IB &&
1990                     rdma_node_get_transport(cur_dev->device->node_type) != RDMA_TRANSPORT_IB)
1991                         continue;
1992
1993                 if (!cma_dev)
1994                         cma_dev = cur_dev;
1995
1996                 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
1997                         if (!ib_query_port(cur_dev->device, p, &port_attr) &&
1998                             port_attr.state == IB_PORT_ACTIVE) {
1999                                 cma_dev = cur_dev;
2000                                 goto port_found;
2001                         }
2002                 }
2003         }
2004
2005         if (!cma_dev) {
2006                 ret = -ENODEV;
2007                 goto out;
2008         }
2009
2010         p = 1;
2011
2012 port_found:
2013         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
2014         if (ret)
2015                 goto out;
2016
2017         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
2018         if (ret)
2019                 goto out;
2020
2021         id_priv->id.route.addr.dev_addr.dev_type =
2022                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
2023                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
2024
2025         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2026         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
2027         id_priv->id.port_num = p;
2028         cma_attach_to_dev(id_priv, cma_dev);
2029         cma_set_loopback(cma_src_addr(id_priv));
2030 out:
2031         mutex_unlock(&lock);
2032         return ret;
2033 }
2034
2035 static void addr_handler(int status, struct sockaddr *src_addr,
2036                          struct rdma_dev_addr *dev_addr, void *context)
2037 {
2038         struct rdma_id_private *id_priv = context;
2039         struct rdma_cm_event event;
2040
2041         memset(&event, 0, sizeof event);
2042         mutex_lock(&id_priv->handler_mutex);
2043         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
2044                            RDMA_CM_ADDR_RESOLVED))
2045                 goto out;
2046
2047         if (!status && !id_priv->cma_dev)
2048                 status = cma_acquire_dev(id_priv);
2049
2050         if (status) {
2051                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2052                                    RDMA_CM_ADDR_BOUND))
2053                         goto out;
2054                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2055                 event.status = status;
2056         } else {
2057                 memcpy(cma_src_addr(id_priv), src_addr, rdma_addr_size(src_addr));
2058                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2059         }
2060
2061         if (id_priv->id.event_handler(&id_priv->id, &event)) {
2062                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2063                 mutex_unlock(&id_priv->handler_mutex);
2064                 cma_deref_id(id_priv);
2065                 rdma_destroy_id(&id_priv->id);
2066                 return;
2067         }
2068 out:
2069         mutex_unlock(&id_priv->handler_mutex);
2070         cma_deref_id(id_priv);
2071 }
2072
2073 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2074 {
2075         struct cma_work *work;
2076         union ib_gid gid;
2077         int ret;
2078
2079         work = kzalloc(sizeof *work, GFP_KERNEL);
2080         if (!work)
2081                 return -ENOMEM;
2082
2083         if (!id_priv->cma_dev) {
2084                 ret = cma_bind_loopback(id_priv);
2085                 if (ret)
2086                         goto err;
2087         }
2088
2089         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2090         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2091
2092         work->id = id_priv;
2093         INIT_WORK(&work->work, cma_work_handler);
2094         work->old_state = RDMA_CM_ADDR_QUERY;
2095         work->new_state = RDMA_CM_ADDR_RESOLVED;
2096         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2097         queue_work(cma_wq, &work->work);
2098         return 0;
2099 err:
2100         kfree(work);
2101         return ret;
2102 }
2103
2104 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2105                          struct sockaddr *dst_addr)
2106 {
2107         if (!src_addr || !src_addr->sa_family) {
2108                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2109                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2110                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2111                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2112                 }
2113         }
2114         return rdma_bind_addr(id, src_addr);
2115 }
2116
2117 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2118                       struct sockaddr *dst_addr, int timeout_ms)
2119 {
2120         struct rdma_id_private *id_priv;
2121         int ret;
2122
2123         id_priv = container_of(id, struct rdma_id_private, id);
2124         if (id_priv->state == RDMA_CM_IDLE) {
2125                 ret = cma_bind_addr(id, src_addr, dst_addr);
2126                 if (ret)
2127                         return ret;
2128         }
2129
2130         if (cma_family(id_priv) != dst_addr->sa_family)
2131                 return -EINVAL;
2132
2133         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2134                 return -EINVAL;
2135
2136         atomic_inc(&id_priv->refcount);
2137         memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
2138         if (cma_any_addr(dst_addr))
2139                 ret = cma_resolve_loopback(id_priv);
2140         else
2141                 ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv),
2142                                       dst_addr, &id->route.addr.dev_addr,
2143                                       timeout_ms, addr_handler, id_priv);
2144         if (ret)
2145                 goto err;
2146
2147         return 0;
2148 err:
2149         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2150         cma_deref_id(id_priv);
2151         return ret;
2152 }
2153 EXPORT_SYMBOL(rdma_resolve_addr);
2154
2155 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2156 {
2157         struct rdma_id_private *id_priv;
2158         unsigned long flags;
2159         int ret;
2160
2161         id_priv = container_of(id, struct rdma_id_private, id);
2162         spin_lock_irqsave(&id_priv->lock, flags);
2163         if (reuse || id_priv->state == RDMA_CM_IDLE) {
2164                 id_priv->reuseaddr = reuse;
2165                 ret = 0;
2166         } else {
2167                 ret = -EINVAL;
2168         }
2169         spin_unlock_irqrestore(&id_priv->lock, flags);
2170         return ret;
2171 }
2172 EXPORT_SYMBOL(rdma_set_reuseaddr);
2173
2174 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2175 {
2176         struct rdma_id_private *id_priv;
2177         unsigned long flags;
2178         int ret;
2179
2180         id_priv = container_of(id, struct rdma_id_private, id);
2181         spin_lock_irqsave(&id_priv->lock, flags);
2182         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2183                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2184                 id_priv->afonly = afonly;
2185                 ret = 0;
2186         } else {
2187                 ret = -EINVAL;
2188         }
2189         spin_unlock_irqrestore(&id_priv->lock, flags);
2190         return ret;
2191 }
2192 EXPORT_SYMBOL(rdma_set_afonly);
2193
2194 static void cma_bind_port(struct rdma_bind_list *bind_list,
2195                           struct rdma_id_private *id_priv)
2196 {
2197         struct sockaddr *addr;
2198         struct sockaddr_ib *sib;
2199         u64 sid, mask;
2200         __be16 port;
2201
2202         addr = cma_src_addr(id_priv);
2203         port = htons(bind_list->port);
2204
2205         switch (addr->sa_family) {
2206         case AF_INET:
2207                 ((struct sockaddr_in *) addr)->sin_port = port;
2208                 break;
2209         case AF_INET6:
2210                 ((struct sockaddr_in6 *) addr)->sin6_port = port;
2211                 break;
2212         case AF_IB:
2213                 sib = (struct sockaddr_ib *) addr;
2214                 sid = be64_to_cpu(sib->sib_sid);
2215                 mask = be64_to_cpu(sib->sib_sid_mask);
2216                 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
2217                 sib->sib_sid_mask = cpu_to_be64(~0ULL);
2218                 break;
2219         }
2220         id_priv->bind_list = bind_list;
2221         hlist_add_head(&id_priv->node, &bind_list->owners);
2222 }
2223
2224 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2225                           unsigned short snum)
2226 {
2227         struct rdma_bind_list *bind_list;
2228         int ret;
2229
2230         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2231         if (!bind_list)
2232                 return -ENOMEM;
2233
2234         ret = idr_alloc(ps, bind_list, snum, snum + 1, GFP_KERNEL);
2235         if (ret < 0)
2236                 goto err;
2237
2238         bind_list->ps = ps;
2239         bind_list->port = (unsigned short)ret;
2240         cma_bind_port(bind_list, id_priv);
2241         return 0;
2242 err:
2243         kfree(bind_list);
2244         return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
2245 }
2246
2247 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2248 {
2249         static unsigned int last_used_port;
2250         int low, high, remaining;
2251         unsigned int rover;
2252
2253         inet_get_local_port_range(&low, &high);
2254         remaining = (high - low) + 1;
2255         rover = net_random() % remaining + low;
2256 retry:
2257         if (last_used_port != rover &&
2258             !idr_find(ps, (unsigned short) rover)) {
2259                 int ret = cma_alloc_port(ps, id_priv, rover);
2260                 /*
2261                  * Remember previously used port number in order to avoid
2262                  * re-using same port immediately after it is closed.
2263                  */
2264                 if (!ret)
2265                         last_used_port = rover;
2266                 if (ret != -EADDRNOTAVAIL)
2267                         return ret;
2268         }
2269         if (--remaining) {
2270                 rover++;
2271                 if ((rover < low) || (rover > high))
2272                         rover = low;
2273                 goto retry;
2274         }
2275         return -EADDRNOTAVAIL;
2276 }
2277
2278 /*
2279  * Check that the requested port is available.  This is called when trying to
2280  * bind to a specific port, or when trying to listen on a bound port.  In
2281  * the latter case, the provided id_priv may already be on the bind_list, but
2282  * we still need to check that it's okay to start listening.
2283  */
2284 static int cma_check_port(struct rdma_bind_list *bind_list,
2285                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
2286 {
2287         struct rdma_id_private *cur_id;
2288         struct sockaddr *addr, *cur_addr;
2289
2290         addr = cma_src_addr(id_priv);
2291         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
2292                 if (id_priv == cur_id)
2293                         continue;
2294
2295                 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2296                     cur_id->reuseaddr)
2297                         continue;
2298
2299                 cur_addr = cma_src_addr(cur_id);
2300                 if (id_priv->afonly && cur_id->afonly &&
2301                     (addr->sa_family != cur_addr->sa_family))
2302                         continue;
2303
2304                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2305                         return -EADDRNOTAVAIL;
2306
2307                 if (!cma_addr_cmp(addr, cur_addr))
2308                         return -EADDRINUSE;
2309         }
2310         return 0;
2311 }
2312
2313 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2314 {
2315         struct rdma_bind_list *bind_list;
2316         unsigned short snum;
2317         int ret;
2318
2319         snum = ntohs(cma_port(cma_src_addr(id_priv)));
2320         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2321                 return -EACCES;
2322
2323         bind_list = idr_find(ps, snum);
2324         if (!bind_list) {
2325                 ret = cma_alloc_port(ps, id_priv, snum);
2326         } else {
2327                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2328                 if (!ret)
2329                         cma_bind_port(bind_list, id_priv);
2330         }
2331         return ret;
2332 }
2333
2334 static int cma_bind_listen(struct rdma_id_private *id_priv)
2335 {
2336         struct rdma_bind_list *bind_list = id_priv->bind_list;
2337         int ret = 0;
2338
2339         mutex_lock(&lock);
2340         if (bind_list->owners.first->next)
2341                 ret = cma_check_port(bind_list, id_priv, 0);
2342         mutex_unlock(&lock);
2343         return ret;
2344 }
2345
2346 static struct idr *cma_select_inet_ps(struct rdma_id_private *id_priv)
2347 {
2348         switch (id_priv->id.ps) {
2349         case RDMA_PS_SDP:
2350                 return &sdp_ps;
2351         case RDMA_PS_TCP:
2352                 return &tcp_ps;
2353         case RDMA_PS_UDP:
2354                 return &udp_ps;
2355         case RDMA_PS_IPOIB:
2356                 return &ipoib_ps;
2357         case RDMA_PS_IB:
2358                 return &ib_ps;
2359         default:
2360                 return NULL;
2361         }
2362 }
2363
2364 static struct idr *cma_select_ib_ps(struct rdma_id_private *id_priv)
2365 {
2366         struct idr *ps = NULL;
2367         struct sockaddr_ib *sib;
2368         u64 sid_ps, mask, sid;
2369
2370         sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2371         mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
2372         sid = be64_to_cpu(sib->sib_sid) & mask;
2373
2374         if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
2375                 sid_ps = RDMA_IB_IP_PS_IB;
2376                 ps = &ib_ps;
2377         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
2378                    (sid == (RDMA_IB_IP_PS_TCP & mask))) {
2379                 sid_ps = RDMA_IB_IP_PS_TCP;
2380                 ps = &tcp_ps;
2381         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
2382                    (sid == (RDMA_IB_IP_PS_UDP & mask))) {
2383                 sid_ps = RDMA_IB_IP_PS_UDP;
2384                 ps = &udp_ps;
2385         }
2386
2387         if (ps) {
2388                 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
2389                 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
2390                                                 be64_to_cpu(sib->sib_sid_mask));
2391         }
2392         return ps;
2393 }
2394
2395 static int cma_get_port(struct rdma_id_private *id_priv)
2396 {
2397         struct idr *ps;
2398         int ret;
2399
2400         if (cma_family(id_priv) != AF_IB)
2401                 ps = cma_select_inet_ps(id_priv);
2402         else
2403                 ps = cma_select_ib_ps(id_priv);
2404         if (!ps)
2405                 return -EPROTONOSUPPORT;
2406
2407         mutex_lock(&lock);
2408         if (cma_any_port(cma_src_addr(id_priv)))
2409                 ret = cma_alloc_any_port(ps, id_priv);
2410         else
2411                 ret = cma_use_port(ps, id_priv);
2412         mutex_unlock(&lock);
2413
2414         return ret;
2415 }
2416
2417 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2418                                struct sockaddr *addr)
2419 {
2420 #if IS_ENABLED(CONFIG_IPV6)
2421         struct sockaddr_in6 *sin6;
2422
2423         if (addr->sa_family != AF_INET6)
2424                 return 0;
2425
2426         sin6 = (struct sockaddr_in6 *) addr;
2427         if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2428             !sin6->sin6_scope_id)
2429                         return -EINVAL;
2430
2431         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2432 #endif
2433         return 0;
2434 }
2435
2436 int rdma_listen(struct rdma_cm_id *id, int backlog)
2437 {
2438         struct rdma_id_private *id_priv;
2439         int ret;
2440
2441         id_priv = container_of(id, struct rdma_id_private, id);
2442         if (id_priv->state == RDMA_CM_IDLE) {
2443                 id->route.addr.src_addr.ss_family = AF_INET;
2444                 ret = rdma_bind_addr(id, cma_src_addr(id_priv));
2445                 if (ret)
2446                         return ret;
2447         }
2448
2449         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2450                 return -EINVAL;
2451
2452         if (id_priv->reuseaddr) {
2453                 ret = cma_bind_listen(id_priv);
2454                 if (ret)
2455                         goto err;
2456         }
2457
2458         id_priv->backlog = backlog;
2459         if (id->device) {
2460                 switch (rdma_node_get_transport(id->device->node_type)) {
2461                 case RDMA_TRANSPORT_IB:
2462                         ret = cma_ib_listen(id_priv);
2463                         if (ret)
2464                                 goto err;
2465                         break;
2466                 case RDMA_TRANSPORT_IWARP:
2467                         ret = cma_iw_listen(id_priv, backlog);
2468                         if (ret)
2469                                 goto err;
2470                         break;
2471                 default:
2472                         ret = -ENOSYS;
2473                         goto err;
2474                 }
2475         } else
2476                 cma_listen_on_all(id_priv);
2477
2478         return 0;
2479 err:
2480         id_priv->backlog = 0;
2481         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2482         return ret;
2483 }
2484 EXPORT_SYMBOL(rdma_listen);
2485
2486 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2487 {
2488         struct rdma_id_private *id_priv;
2489         int ret;
2490
2491         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
2492             addr->sa_family != AF_IB)
2493                 return -EAFNOSUPPORT;
2494
2495         id_priv = container_of(id, struct rdma_id_private, id);
2496         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2497                 return -EINVAL;
2498
2499         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2500         if (ret)
2501                 goto err1;
2502
2503         if (!cma_any_addr(addr)) {
2504                 ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
2505                 if (ret)
2506                         goto err1;
2507
2508                 ret = cma_acquire_dev(id_priv);
2509                 if (ret)
2510                         goto err1;
2511         }
2512
2513         memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
2514         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2515                 if (addr->sa_family == AF_INET)
2516                         id_priv->afonly = 1;
2517 #if IS_ENABLED(CONFIG_IPV6)
2518                 else if (addr->sa_family == AF_INET6)
2519                         id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
2520 #endif
2521         }
2522         ret = cma_get_port(id_priv);
2523         if (ret)
2524                 goto err2;
2525
2526         return 0;
2527 err2:
2528         if (id_priv->cma_dev)
2529                 cma_release_dev(id_priv);
2530 err1:
2531         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2532         return ret;
2533 }
2534 EXPORT_SYMBOL(rdma_bind_addr);
2535
2536 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
2537 {
2538         struct cma_hdr *cma_hdr;
2539         struct sdp_hh *sdp_hdr;
2540
2541         if (cma_family(id_priv) == AF_INET) {
2542                 struct sockaddr_in *src4, *dst4;
2543
2544                 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
2545                 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
2546
2547                 switch (id_priv->id.ps) {
2548                 case RDMA_PS_SDP:
2549                         sdp_hdr = hdr;
2550                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2551                                 return -EINVAL;
2552                         sdp_set_ip_ver(sdp_hdr, 4);
2553                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2554                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2555                         sdp_hdr->port = src4->sin_port;
2556                         break;
2557                 default:
2558                         cma_hdr = hdr;
2559                         cma_hdr->cma_version = CMA_VERSION;
2560                         cma_set_ip_ver(cma_hdr, 4);
2561                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2562                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2563                         cma_hdr->port = src4->sin_port;
2564                         break;
2565                 }
2566         } else {
2567                 struct sockaddr_in6 *src6, *dst6;
2568
2569                 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2570                 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
2571
2572                 switch (id_priv->id.ps) {
2573                 case RDMA_PS_SDP:
2574                         sdp_hdr = hdr;
2575                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2576                                 return -EINVAL;
2577                         sdp_set_ip_ver(sdp_hdr, 6);
2578                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2579                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2580                         sdp_hdr->port = src6->sin6_port;
2581                         break;
2582                 default:
2583                         cma_hdr = hdr;
2584                         cma_hdr->cma_version = CMA_VERSION;
2585                         cma_set_ip_ver(cma_hdr, 6);
2586                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2587                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2588                         cma_hdr->port = src6->sin6_port;
2589                         break;
2590                 }
2591         }
2592         return 0;
2593 }
2594
2595 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2596                                 struct ib_cm_event *ib_event)
2597 {
2598         struct rdma_id_private *id_priv = cm_id->context;
2599         struct rdma_cm_event event;
2600         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2601         int ret = 0;
2602
2603         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2604                 return 0;
2605
2606         memset(&event, 0, sizeof event);
2607         switch (ib_event->event) {
2608         case IB_CM_SIDR_REQ_ERROR:
2609                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2610                 event.status = -ETIMEDOUT;
2611                 break;
2612         case IB_CM_SIDR_REP_RECEIVED:
2613                 event.param.ud.private_data = ib_event->private_data;
2614                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2615                 if (rep->status != IB_SIDR_SUCCESS) {
2616                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2617                         event.status = ib_event->param.sidr_rep_rcvd.status;
2618                         break;
2619                 }
2620                 ret = cma_set_qkey(id_priv);
2621                 if (ret) {
2622                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2623                         event.status = -EINVAL;
2624                         break;
2625                 }
2626                 if (id_priv->qkey != rep->qkey) {
2627                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2628                         event.status = -EINVAL;
2629                         break;
2630                 }
2631                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2632                                      id_priv->id.route.path_rec,
2633                                      &event.param.ud.ah_attr);
2634                 event.param.ud.qp_num = rep->qpn;
2635                 event.param.ud.qkey = rep->qkey;
2636                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2637                 event.status = 0;
2638                 break;
2639         default:
2640                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2641                        ib_event->event);
2642                 goto out;
2643         }
2644
2645         ret = id_priv->id.event_handler(&id_priv->id, &event);
2646         if (ret) {
2647                 /* Destroy the CM ID by returning a non-zero value. */
2648                 id_priv->cm_id.ib = NULL;
2649                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2650                 mutex_unlock(&id_priv->handler_mutex);
2651                 rdma_destroy_id(&id_priv->id);
2652                 return ret;
2653         }
2654 out:
2655         mutex_unlock(&id_priv->handler_mutex);
2656         return ret;
2657 }
2658
2659 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2660                               struct rdma_conn_param *conn_param)
2661 {
2662         struct ib_cm_sidr_req_param req;
2663         struct ib_cm_id *id;
2664         int ret;
2665
2666         req.private_data_len = sizeof(struct cma_hdr) +
2667                                conn_param->private_data_len;
2668         if (req.private_data_len < conn_param->private_data_len)
2669                 return -EINVAL;
2670
2671         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2672         if (!req.private_data)
2673                 return -ENOMEM;
2674
2675         if (conn_param->private_data && conn_param->private_data_len)
2676                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2677                        conn_param->private_data, conn_param->private_data_len);
2678
2679         ret = cma_format_hdr((void *) req.private_data, id_priv);
2680         if (ret)
2681                 goto out;
2682
2683         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2684                              id_priv);
2685         if (IS_ERR(id)) {
2686                 ret = PTR_ERR(id);
2687                 goto out;
2688         }
2689         id_priv->cm_id.ib = id;
2690
2691         req.path = id_priv->id.route.path_rec;
2692         req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
2693         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2694         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2695
2696         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2697         if (ret) {
2698                 ib_destroy_cm_id(id_priv->cm_id.ib);
2699                 id_priv->cm_id.ib = NULL;
2700         }
2701 out:
2702         kfree(req.private_data);
2703         return ret;
2704 }
2705
2706 static int cma_connect_ib(struct rdma_id_private *id_priv,
2707                           struct rdma_conn_param *conn_param)
2708 {
2709         struct ib_cm_req_param req;
2710         struct rdma_route *route;
2711         void *private_data;
2712         struct ib_cm_id *id;
2713         int offset, ret;
2714
2715         memset(&req, 0, sizeof req);
2716         offset = cma_user_data_offset(id_priv->id.ps);
2717         req.private_data_len = offset + conn_param->private_data_len;
2718         if (req.private_data_len < conn_param->private_data_len)
2719                 return -EINVAL;
2720
2721         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2722         if (!private_data)
2723                 return -ENOMEM;
2724
2725         if (conn_param->private_data && conn_param->private_data_len)
2726                 memcpy(private_data + offset, conn_param->private_data,
2727                        conn_param->private_data_len);
2728
2729         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2730         if (IS_ERR(id)) {
2731                 ret = PTR_ERR(id);
2732                 goto out;
2733         }
2734         id_priv->cm_id.ib = id;
2735
2736         route = &id_priv->id.route;
2737         ret = cma_format_hdr(private_data, id_priv);
2738         if (ret)
2739                 goto out;
2740         req.private_data = private_data;
2741
2742         req.primary_path = &route->path_rec[0];
2743         if (route->num_paths == 2)
2744                 req.alternate_path = &route->path_rec[1];
2745
2746         req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
2747         req.qp_num = id_priv->qp_num;
2748         req.qp_type = id_priv->id.qp_type;
2749         req.starting_psn = id_priv->seq_num;
2750         req.responder_resources = conn_param->responder_resources;
2751         req.initiator_depth = conn_param->initiator_depth;
2752         req.flow_control = conn_param->flow_control;
2753         req.retry_count = min_t(u8, 7, conn_param->retry_count);
2754         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2755         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2756         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2757         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2758         req.srq = id_priv->srq ? 1 : 0;
2759
2760         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2761 out:
2762         if (ret && !IS_ERR(id)) {
2763                 ib_destroy_cm_id(id);
2764                 id_priv->cm_id.ib = NULL;
2765         }
2766
2767         kfree(private_data);
2768         return ret;
2769 }
2770
2771 static int cma_connect_iw(struct rdma_id_private *id_priv,
2772                           struct rdma_conn_param *conn_param)
2773 {
2774         struct iw_cm_id *cm_id;
2775         struct sockaddr_in* sin;
2776         int ret;
2777         struct iw_cm_conn_param iw_param;
2778
2779         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2780         if (IS_ERR(cm_id))
2781                 return PTR_ERR(cm_id);
2782
2783         id_priv->cm_id.iw = cm_id;
2784
2785         sin = (struct sockaddr_in *) cma_src_addr(id_priv);
2786         cm_id->local_addr = *sin;
2787
2788         sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
2789         cm_id->remote_addr = *sin;
2790
2791         ret = cma_modify_qp_rtr(id_priv, conn_param);
2792         if (ret)
2793                 goto out;
2794
2795         if (conn_param) {
2796                 iw_param.ord = conn_param->initiator_depth;
2797                 iw_param.ird = conn_param->responder_resources;
2798                 iw_param.private_data = conn_param->private_data;
2799                 iw_param.private_data_len = conn_param->private_data_len;
2800                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2801         } else {
2802                 memset(&iw_param, 0, sizeof iw_param);
2803                 iw_param.qpn = id_priv->qp_num;
2804         }
2805         ret = iw_cm_connect(cm_id, &iw_param);
2806 out:
2807         if (ret) {
2808                 iw_destroy_cm_id(cm_id);
2809                 id_priv->cm_id.iw = NULL;
2810         }
2811         return ret;
2812 }
2813
2814 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2815 {
2816         struct rdma_id_private *id_priv;
2817         int ret;
2818
2819         id_priv = container_of(id, struct rdma_id_private, id);
2820         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2821                 return -EINVAL;
2822
2823         if (!id->qp) {
2824                 id_priv->qp_num = conn_param->qp_num;
2825                 id_priv->srq = conn_param->srq;
2826         }
2827
2828         switch (rdma_node_get_transport(id->device->node_type)) {
2829         case RDMA_TRANSPORT_IB:
2830                 if (id->qp_type == IB_QPT_UD)
2831                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2832                 else
2833                         ret = cma_connect_ib(id_priv, conn_param);
2834                 break;
2835         case RDMA_TRANSPORT_IWARP:
2836                 ret = cma_connect_iw(id_priv, conn_param);
2837                 break;
2838         default:
2839                 ret = -ENOSYS;
2840                 break;
2841         }
2842         if (ret)
2843                 goto err;
2844
2845         return 0;
2846 err:
2847         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2848         return ret;
2849 }
2850 EXPORT_SYMBOL(rdma_connect);
2851
2852 static int cma_accept_ib(struct rdma_id_private *id_priv,
2853                          struct rdma_conn_param *conn_param)
2854 {
2855         struct ib_cm_rep_param rep;
2856         int ret;
2857
2858         ret = cma_modify_qp_rtr(id_priv, conn_param);
2859         if (ret)
2860                 goto out;
2861
2862         ret = cma_modify_qp_rts(id_priv, conn_param);
2863         if (ret)
2864                 goto out;
2865
2866         memset(&rep, 0, sizeof rep);
2867         rep.qp_num = id_priv->qp_num;
2868         rep.starting_psn = id_priv->seq_num;
2869         rep.private_data = conn_param->private_data;
2870         rep.private_data_len = conn_param->private_data_len;
2871         rep.responder_resources = conn_param->responder_resources;
2872         rep.initiator_depth = conn_param->initiator_depth;
2873         rep.failover_accepted = 0;
2874         rep.flow_control = conn_param->flow_control;
2875         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2876         rep.srq = id_priv->srq ? 1 : 0;
2877
2878         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2879 out:
2880         return ret;
2881 }
2882
2883 static int cma_accept_iw(struct rdma_id_private *id_priv,
2884                   struct rdma_conn_param *conn_param)
2885 {
2886         struct iw_cm_conn_param iw_param;
2887         int ret;
2888
2889         ret = cma_modify_qp_rtr(id_priv, conn_param);
2890         if (ret)
2891                 return ret;
2892
2893         iw_param.ord = conn_param->initiator_depth;
2894         iw_param.ird = conn_param->responder_resources;
2895         iw_param.private_data = conn_param->private_data;
2896         iw_param.private_data_len = conn_param->private_data_len;
2897         if (id_priv->id.qp) {
2898                 iw_param.qpn = id_priv->qp_num;
2899         } else
2900                 iw_param.qpn = conn_param->qp_num;
2901
2902         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2903 }
2904
2905 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2906                              enum ib_cm_sidr_status status,
2907                              const void *private_data, int private_data_len)
2908 {
2909         struct ib_cm_sidr_rep_param rep;
2910         int ret;
2911
2912         memset(&rep, 0, sizeof rep);
2913         rep.status = status;
2914         if (status == IB_SIDR_SUCCESS) {
2915                 ret = cma_set_qkey(id_priv);
2916                 if (ret)
2917                         return ret;
2918                 rep.qp_num = id_priv->qp_num;
2919                 rep.qkey = id_priv->qkey;
2920         }
2921         rep.private_data = private_data;
2922         rep.private_data_len = private_data_len;
2923
2924         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2925 }
2926
2927 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2928 {
2929         struct rdma_id_private *id_priv;
2930         int ret;
2931
2932         id_priv = container_of(id, struct rdma_id_private, id);
2933
2934         id_priv->owner = task_pid_nr(current);
2935
2936         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
2937                 return -EINVAL;
2938
2939         if (!id->qp && conn_param) {
2940                 id_priv->qp_num = conn_param->qp_num;
2941                 id_priv->srq = conn_param->srq;
2942         }
2943
2944         switch (rdma_node_get_transport(id->device->node_type)) {
2945         case RDMA_TRANSPORT_IB:
2946                 if (id->qp_type == IB_QPT_UD) {
2947                         if (conn_param)
2948                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2949                                                         conn_param->private_data,
2950                                                         conn_param->private_data_len);
2951                         else
2952                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2953                                                         NULL, 0);
2954                 } else {
2955                         if (conn_param)
2956                                 ret = cma_accept_ib(id_priv, conn_param);
2957                         else
2958                                 ret = cma_rep_recv(id_priv);
2959                 }
2960                 break;
2961         case RDMA_TRANSPORT_IWARP:
2962                 ret = cma_accept_iw(id_priv, conn_param);
2963                 break;
2964         default:
2965                 ret = -ENOSYS;
2966                 break;
2967         }
2968
2969         if (ret)
2970                 goto reject;
2971
2972         return 0;
2973 reject:
2974         cma_modify_qp_err(id_priv);
2975         rdma_reject(id, NULL, 0);
2976         return ret;
2977 }
2978 EXPORT_SYMBOL(rdma_accept);
2979
2980 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2981 {
2982         struct rdma_id_private *id_priv;
2983         int ret;
2984
2985         id_priv = container_of(id, struct rdma_id_private, id);
2986         if (!id_priv->cm_id.ib)
2987                 return -EINVAL;
2988
2989         switch (id->device->node_type) {
2990         case RDMA_NODE_IB_CA:
2991                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2992                 break;
2993         default:
2994                 ret = 0;
2995                 break;
2996         }
2997         return ret;
2998 }
2999 EXPORT_SYMBOL(rdma_notify);
3000
3001 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
3002                 u8 private_data_len)
3003 {
3004         struct rdma_id_private *id_priv;
3005         int ret;
3006
3007         id_priv = container_of(id, struct rdma_id_private, id);
3008         if (!id_priv->cm_id.ib)
3009                 return -EINVAL;
3010
3011         switch (rdma_node_get_transport(id->device->node_type)) {
3012         case RDMA_TRANSPORT_IB:
3013                 if (id->qp_type == IB_QPT_UD)
3014                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
3015                                                 private_data, private_data_len);
3016                 else
3017                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
3018                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
3019                                              0, private_data, private_data_len);
3020                 break;
3021         case RDMA_TRANSPORT_IWARP:
3022                 ret = iw_cm_reject(id_priv->cm_id.iw,
3023                                    private_data, private_data_len);
3024                 break;
3025         default:
3026                 ret = -ENOSYS;
3027                 break;
3028         }
3029         return ret;
3030 }
3031 EXPORT_SYMBOL(rdma_reject);
3032
3033 int rdma_disconnect(struct rdma_cm_id *id)
3034 {
3035         struct rdma_id_private *id_priv;
3036         int ret;
3037
3038         id_priv = container_of(id, struct rdma_id_private, id);
3039         if (!id_priv->cm_id.ib)
3040                 return -EINVAL;
3041
3042         switch (rdma_node_get_transport(id->device->node_type)) {
3043         case RDMA_TRANSPORT_IB:
3044                 ret = cma_modify_qp_err(id_priv);
3045                 if (ret)
3046                         goto out;
3047                 /* Initiate or respond to a disconnect. */
3048                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
3049                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
3050                 break;
3051         case RDMA_TRANSPORT_IWARP:
3052                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
3053                 break;
3054         default:
3055                 ret = -EINVAL;
3056                 break;
3057         }
3058 out:
3059         return ret;
3060 }
3061 EXPORT_SYMBOL(rdma_disconnect);
3062
3063 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
3064 {
3065         struct rdma_id_private *id_priv;
3066         struct cma_multicast *mc = multicast->context;
3067         struct rdma_cm_event event;
3068         int ret;
3069
3070         id_priv = mc->id_priv;
3071         if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
3072             cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
3073                 return 0;
3074
3075         mutex_lock(&id_priv->qp_mutex);
3076         if (!status && id_priv->id.qp)
3077                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
3078                                          be16_to_cpu(multicast->rec.mlid));
3079         mutex_unlock(&id_priv->qp_mutex);
3080
3081         memset(&event, 0, sizeof event);
3082         event.status = status;
3083         event.param.ud.private_data = mc->context;
3084         if (!status) {
3085                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
3086                 ib_init_ah_from_mcmember(id_priv->id.device,
3087                                          id_priv->id.port_num, &multicast->rec,
3088                                          &event.param.ud.ah_attr);
3089                 event.param.ud.qp_num = 0xFFFFFF;
3090                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
3091         } else
3092                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
3093
3094         ret = id_priv->id.event_handler(&id_priv->id, &event);
3095         if (ret) {
3096                 cma_exch(id_priv, RDMA_CM_DESTROYING);
3097                 mutex_unlock(&id_priv->handler_mutex);
3098                 rdma_destroy_id(&id_priv->id);
3099                 return 0;
3100         }
3101
3102         mutex_unlock(&id_priv->handler_mutex);
3103         return 0;
3104 }
3105
3106 static void cma_set_mgid(struct rdma_id_private *id_priv,
3107                          struct sockaddr *addr, union ib_gid *mgid)
3108 {
3109         unsigned char mc_map[MAX_ADDR_LEN];
3110         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3111         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3112         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3113
3114         if (cma_any_addr(addr)) {
3115                 memset(mgid, 0, sizeof *mgid);
3116         } else if ((addr->sa_family == AF_INET6) &&
3117                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3118                                                                  0xFF10A01B)) {
3119                 /* IPv6 address is an SA assigned MGID. */
3120                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3121         } else if ((addr->sa_family == AF_INET6)) {
3122                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3123                 if (id_priv->id.ps == RDMA_PS_UDP)
3124                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3125                 *mgid = *(union ib_gid *) (mc_map + 4);
3126         } else {
3127                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3128                 if (id_priv->id.ps == RDMA_PS_UDP)
3129                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3130                 *mgid = *(union ib_gid *) (mc_map + 4);
3131         }
3132 }
3133
3134 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3135                                  struct cma_multicast *mc)
3136 {
3137         struct ib_sa_mcmember_rec rec;
3138         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3139         ib_sa_comp_mask comp_mask;
3140         int ret;
3141
3142         ib_addr_get_mgid(dev_addr, &rec.mgid);
3143         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
3144                                      &rec.mgid, &rec);
3145         if (ret)
3146                 return ret;
3147
3148         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3149         if (id_priv->id.ps == RDMA_PS_UDP)
3150                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3151         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3152         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3153         rec.join_state = 1;
3154
3155         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3156                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3157                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3158                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
3159                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3160
3161         if (id_priv->id.ps == RDMA_PS_IPOIB)
3162                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3163                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3164                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3165                              IB_SA_MCMEMBER_REC_MTU |
3166                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
3167
3168         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3169                                                 id_priv->id.port_num, &rec,
3170                                                 comp_mask, GFP_KERNEL,
3171                                                 cma_ib_mc_handler, mc);
3172         return PTR_RET(mc->multicast.ib);
3173 }
3174
3175 static void iboe_mcast_work_handler(struct work_struct *work)
3176 {
3177         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3178         struct cma_multicast *mc = mw->mc;
3179         struct ib_sa_multicast *m = mc->multicast.ib;
3180
3181         mc->multicast.ib->context = mc;
3182         cma_ib_mc_handler(0, m);
3183         kref_put(&mc->mcref, release_mc);
3184         kfree(mw);
3185 }
3186
3187 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3188 {
3189         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3190         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3191
3192         if (cma_any_addr(addr)) {
3193                 memset(mgid, 0, sizeof *mgid);
3194         } else if (addr->sa_family == AF_INET6) {
3195                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3196         } else {
3197                 mgid->raw[0] = 0xff;
3198                 mgid->raw[1] = 0x0e;
3199                 mgid->raw[2] = 0;
3200                 mgid->raw[3] = 0;
3201                 mgid->raw[4] = 0;
3202                 mgid->raw[5] = 0;
3203                 mgid->raw[6] = 0;
3204                 mgid->raw[7] = 0;
3205                 mgid->raw[8] = 0;
3206                 mgid->raw[9] = 0;
3207                 mgid->raw[10] = 0xff;
3208                 mgid->raw[11] = 0xff;
3209                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3210         }
3211 }
3212
3213 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3214                                    struct cma_multicast *mc)
3215 {
3216         struct iboe_mcast_work *work;
3217         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3218         int err;
3219         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3220         struct net_device *ndev = NULL;
3221
3222         if (cma_zero_addr((struct sockaddr *)&mc->addr))
3223                 return -EINVAL;
3224
3225         work = kzalloc(sizeof *work, GFP_KERNEL);
3226         if (!work)
3227                 return -ENOMEM;
3228
3229         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3230         if (!mc->multicast.ib) {
3231                 err = -ENOMEM;
3232                 goto out1;
3233         }
3234
3235         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3236
3237         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3238         if (id_priv->id.ps == RDMA_PS_UDP)
3239                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3240
3241         if (dev_addr->bound_dev_if)
3242                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3243         if (!ndev) {
3244                 err = -ENODEV;
3245                 goto out2;
3246         }
3247         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3248         mc->multicast.ib->rec.hop_limit = 1;
3249         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3250         dev_put(ndev);
3251         if (!mc->multicast.ib->rec.mtu) {
3252                 err = -EINVAL;
3253                 goto out2;
3254         }
3255         iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3256         work->id = id_priv;
3257         work->mc = mc;
3258         INIT_WORK(&work->work, iboe_mcast_work_handler);
3259         kref_get(&mc->mcref);
3260         queue_work(cma_wq, &work->work);
3261
3262         return 0;
3263
3264 out2:
3265         kfree(mc->multicast.ib);
3266 out1:
3267         kfree(work);
3268         return err;
3269 }
3270
3271 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3272                         void *context)
3273 {
3274         struct rdma_id_private *id_priv;
3275         struct cma_multicast *mc;
3276         int ret;
3277
3278         id_priv = container_of(id, struct rdma_id_private, id);
3279         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3280             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3281                 return -EINVAL;
3282
3283         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3284         if (!mc)
3285                 return -ENOMEM;
3286
3287         memcpy(&mc->addr, addr, rdma_addr_size(addr));
3288         mc->context = context;
3289         mc->id_priv = id_priv;
3290
3291         spin_lock(&id_priv->lock);
3292         list_add(&mc->list, &id_priv->mc_list);
3293         spin_unlock(&id_priv->lock);
3294
3295         switch (rdma_node_get_transport(id->device->node_type)) {
3296         case RDMA_TRANSPORT_IB:
3297                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3298                 case IB_LINK_LAYER_INFINIBAND:
3299                         ret = cma_join_ib_multicast(id_priv, mc);
3300                         break;
3301                 case IB_LINK_LAYER_ETHERNET:
3302                         kref_init(&mc->mcref);
3303                         ret = cma_iboe_join_multicast(id_priv, mc);
3304                         break;
3305                 default:
3306                         ret = -EINVAL;
3307                 }
3308                 break;
3309         default:
3310                 ret = -ENOSYS;
3311                 break;
3312         }
3313
3314         if (ret) {
3315                 spin_lock_irq(&id_priv->lock);
3316                 list_del(&mc->list);
3317                 spin_unlock_irq(&id_priv->lock);
3318                 kfree(mc);
3319         }
3320         return ret;
3321 }
3322 EXPORT_SYMBOL(rdma_join_multicast);
3323
3324 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3325 {
3326         struct rdma_id_private *id_priv;
3327         struct cma_multicast *mc;
3328
3329         id_priv = container_of(id, struct rdma_id_private, id);
3330         spin_lock_irq(&id_priv->lock);
3331         list_for_each_entry(mc, &id_priv->mc_list, list) {
3332                 if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
3333                         list_del(&mc->list);
3334                         spin_unlock_irq(&id_priv->lock);
3335
3336                         if (id->qp)
3337                                 ib_detach_mcast(id->qp,
3338                                                 &mc->multicast.ib->rec.mgid,
3339                                                 be16_to_cpu(mc->multicast.ib->rec.mlid));
3340                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3341                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3342                                 case IB_LINK_LAYER_INFINIBAND:
3343                                         ib_sa_free_multicast(mc->multicast.ib);
3344                                         kfree(mc);
3345                                         break;
3346                                 case IB_LINK_LAYER_ETHERNET:
3347                                         kref_put(&mc->mcref, release_mc);
3348                                         break;
3349                                 default:
3350                                         break;
3351                                 }
3352                         }
3353                         return;
3354                 }
3355         }
3356         spin_unlock_irq(&id_priv->lock);
3357 }
3358 EXPORT_SYMBOL(rdma_leave_multicast);
3359
3360 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3361 {
3362         struct rdma_dev_addr *dev_addr;
3363         struct cma_ndev_work *work;
3364
3365         dev_addr = &id_priv->id.route.addr.dev_addr;
3366
3367         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3368             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3369                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3370                        ndev->name, &id_priv->id);
3371                 work = kzalloc(sizeof *work, GFP_KERNEL);
3372                 if (!work)
3373                         return -ENOMEM;
3374
3375                 INIT_WORK(&work->work, cma_ndev_work_handler);
3376                 work->id = id_priv;
3377                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3378                 atomic_inc(&id_priv->refcount);
3379                 queue_work(cma_wq, &work->work);
3380         }
3381
3382         return 0;
3383 }
3384
3385 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3386                                void *ptr)
3387 {
3388         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3389         struct cma_device *cma_dev;
3390         struct rdma_id_private *id_priv;
3391         int ret = NOTIFY_DONE;
3392
3393         if (dev_net(ndev) != &init_net)
3394                 return NOTIFY_DONE;
3395
3396         if (event != NETDEV_BONDING_FAILOVER)
3397                 return NOTIFY_DONE;
3398
3399         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3400                 return NOTIFY_DONE;
3401
3402         mutex_lock(&lock);
3403         list_for_each_entry(cma_dev, &dev_list, list)
3404                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3405                         ret = cma_netdev_change(ndev, id_priv);
3406                         if (ret)
3407                                 goto out;
3408                 }
3409
3410 out:
3411         mutex_unlock(&lock);
3412         return ret;
3413 }
3414
3415 static struct notifier_block cma_nb = {
3416         .notifier_call = cma_netdev_callback
3417 };
3418
3419 static void cma_add_one(struct ib_device *device)
3420 {
3421         struct cma_device *cma_dev;
3422         struct rdma_id_private *id_priv;
3423
3424         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3425         if (!cma_dev)
3426                 return;
3427
3428         cma_dev->device = device;
3429
3430         init_completion(&cma_dev->comp);
3431         atomic_set(&cma_dev->refcount, 1);
3432         INIT_LIST_HEAD(&cma_dev->id_list);
3433         ib_set_client_data(device, &cma_client, cma_dev);
3434
3435         mutex_lock(&lock);
3436         list_add_tail(&cma_dev->list, &dev_list);
3437         list_for_each_entry(id_priv, &listen_any_list, list)
3438                 cma_listen_on_dev(id_priv, cma_dev);
3439         mutex_unlock(&lock);
3440 }
3441
3442 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3443 {
3444         struct rdma_cm_event event;
3445         enum rdma_cm_state state;
3446         int ret = 0;
3447
3448         /* Record that we want to remove the device */
3449         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3450         if (state == RDMA_CM_DESTROYING)
3451                 return 0;
3452
3453         cma_cancel_operation(id_priv, state);
3454         mutex_lock(&id_priv->handler_mutex);
3455
3456         /* Check for destruction from another callback. */
3457         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3458                 goto out;
3459
3460         memset(&event, 0, sizeof event);
3461         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3462         ret = id_priv->id.event_handler(&id_priv->id, &event);
3463 out:
3464         mutex_unlock(&id_priv->handler_mutex);
3465         return ret;
3466 }
3467
3468 static void cma_process_remove(struct cma_device *cma_dev)
3469 {
3470         struct rdma_id_private *id_priv;
3471         int ret;
3472
3473         mutex_lock(&lock);
3474         while (!list_empty(&cma_dev->id_list)) {
3475                 id_priv = list_entry(cma_dev->id_list.next,
3476                                      struct rdma_id_private, list);
3477
3478                 list_del(&id_priv->listen_list);
3479                 list_del_init(&id_priv->list);
3480                 atomic_inc(&id_priv->refcount);
3481                 mutex_unlock(&lock);
3482
3483                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3484                 cma_deref_id(id_priv);
3485                 if (ret)
3486                         rdma_destroy_id(&id_priv->id);
3487
3488                 mutex_lock(&lock);
3489         }
3490         mutex_unlock(&lock);
3491
3492         cma_deref_dev(cma_dev);
3493         wait_for_completion(&cma_dev->comp);
3494 }
3495
3496 static void cma_remove_one(struct ib_device *device)
3497 {
3498         struct cma_device *cma_dev;
3499
3500         cma_dev = ib_get_client_data(device, &cma_client);
3501         if (!cma_dev)
3502                 return;
3503
3504         mutex_lock(&lock);
3505         list_del(&cma_dev->list);
3506         mutex_unlock(&lock);
3507
3508         cma_process_remove(cma_dev);
3509         kfree(cma_dev);
3510 }
3511
3512 static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
3513 {
3514         struct nlmsghdr *nlh;
3515         struct rdma_cm_id_stats *id_stats;
3516         struct rdma_id_private *id_priv;
3517         struct rdma_cm_id *id = NULL;
3518         struct cma_device *cma_dev;
3519         int i_dev = 0, i_id = 0;
3520
3521         /*
3522          * We export all of the IDs as a sequence of messages.  Each
3523          * ID gets its own netlink message.
3524          */
3525         mutex_lock(&lock);
3526
3527         list_for_each_entry(cma_dev, &dev_list, list) {
3528                 if (i_dev < cb->args[0]) {
3529                         i_dev++;
3530                         continue;
3531                 }
3532
3533                 i_id = 0;
3534                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3535                         if (i_id < cb->args[1]) {
3536                                 i_id++;
3537                                 continue;
3538                         }
3539
3540                         id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
3541                                                 sizeof *id_stats, RDMA_NL_RDMA_CM,
3542                                                 RDMA_NL_RDMA_CM_ID_STATS);
3543                         if (!id_stats)
3544                                 goto out;
3545
3546                         memset(id_stats, 0, sizeof *id_stats);
3547                         id = &id_priv->id;
3548                         id_stats->node_type = id->route.addr.dev_addr.dev_type;
3549                         id_stats->port_num = id->port_num;
3550                         id_stats->bound_dev_if =
3551                                 id->route.addr.dev_addr.bound_dev_if;
3552
3553                         if (cma_family(id_priv) == AF_INET) {
3554                                 if (ibnl_put_attr(skb, nlh,
3555                                                   sizeof(struct sockaddr_in),
3556                                                   cma_src_addr(id_priv),
3557                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3558                                         goto out;
3559                                 }
3560                                 if (ibnl_put_attr(skb, nlh,
3561                                                   sizeof(struct sockaddr_in),
3562                                                   cma_dst_addr(id_priv),
3563                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3564                                         goto out;
3565                                 }
3566                         } else if (cma_family(id_priv) == AF_INET6) {
3567                                 if (ibnl_put_attr(skb, nlh,
3568                                                   sizeof(struct sockaddr_in6),
3569                                                   cma_src_addr(id_priv),
3570                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3571                                         goto out;
3572                                 }
3573                                 if (ibnl_put_attr(skb, nlh,
3574                                                   sizeof(struct sockaddr_in6),
3575                                                   cma_dst_addr(id_priv),
3576                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3577                                         goto out;
3578                                 }
3579                         }
3580
3581                         id_stats->pid           = id_priv->owner;
3582                         id_stats->port_space    = id->ps;
3583                         id_stats->cm_state      = id_priv->state;
3584                         id_stats->qp_num        = id_priv->qp_num;
3585                         id_stats->qp_type       = id->qp_type;
3586
3587                         i_id++;
3588                 }
3589
3590                 cb->args[1] = 0;
3591                 i_dev++;
3592         }
3593
3594 out:
3595         mutex_unlock(&lock);
3596         cb->args[0] = i_dev;
3597         cb->args[1] = i_id;
3598
3599         return skb->len;
3600 }
3601
3602 static const struct ibnl_client_cbs cma_cb_table[] = {
3603         [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
3604                                        .module = THIS_MODULE },
3605 };
3606
3607 static int __init cma_init(void)
3608 {
3609         int ret;
3610
3611         cma_wq = create_singlethread_workqueue("rdma_cm");
3612         if (!cma_wq)
3613                 return -ENOMEM;
3614
3615         ib_sa_register_client(&sa_client);
3616         rdma_addr_register_client(&addr_client);
3617         register_netdevice_notifier(&cma_nb);
3618
3619         ret = ib_register_client(&cma_client);
3620         if (ret)
3621                 goto err;
3622
3623         if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
3624                 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
3625
3626         return 0;
3627
3628 err:
3629         unregister_netdevice_notifier(&cma_nb);
3630         rdma_addr_unregister_client(&addr_client);
3631         ib_sa_unregister_client(&sa_client);
3632         destroy_workqueue(cma_wq);
3633         return ret;
3634 }
3635
3636 static void __exit cma_cleanup(void)
3637 {
3638         ibnl_remove_client(RDMA_NL_RDMA_CM);
3639         ib_unregister_client(&cma_client);
3640         unregister_netdevice_notifier(&cma_nb);
3641         rdma_addr_unregister_client(&addr_client);
3642         ib_sa_unregister_client(&sa_client);
3643         destroy_workqueue(cma_wq);
3644         idr_destroy(&sdp_ps);
3645         idr_destroy(&tcp_ps);
3646         idr_destroy(&udp_ps);
3647         idr_destroy(&ipoib_ps);
3648         idr_destroy(&ib_ps);
3649 }
3650
3651 module_init(cma_init);
3652 module_exit(cma_cleanup);