drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44 #include <linux/module.h>
  45 #include <net/route.h>
  46
  47 #include <net/tcp.h>
  48 #include <net/ipv6.h>
  49
  50 #include <rdma/rdma_cm.h>
  51 #include <rdma/rdma_cm_ib.h>
  52 #include <rdma/rdma_netlink.h>
  53 #include <rdma/ib.h>
  54 #include <rdma/ib_cache.h>
  55 #include <rdma/ib_cm.h>
  56 #include <rdma/ib_sa.h>
  57 #include <rdma/iw_cm.h>
  58
  59 MODULE_AUTHOR("Sean Hefty");
  60 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  61 MODULE_LICENSE("Dual BSD/GPL");
  62
  63 #define CMA_CM_RESPONSE_TIMEOUT 20
  64 #define CMA_MAX_CM_RETRIES 15
  65 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  66 #define CMA_IBOE_PACKET_LIFETIME 18
  67
  68 static void cma_add_one(struct ib_device *device);
  69 static void cma_remove_one(struct ib_device *device);
  70
  71 static struct ib_client cma_client = {
  72         .name   = "cma",
  73         .add    = cma_add_one,
  74         .remove = cma_remove_one
  75 };
  76
  77 static struct ib_sa_client sa_client;
  78 static struct rdma_addr_client addr_client;
  79 static LIST_HEAD(dev_list);
  80 static LIST_HEAD(listen_any_list);
  81 static DEFINE_MUTEX(lock);
  82 static struct workqueue_struct *cma_wq;
  83 static DEFINE_IDR(sdp_ps);
  84 static DEFINE_IDR(tcp_ps);
  85 static DEFINE_IDR(udp_ps);
  86 static DEFINE_IDR(ipoib_ps);
  87 static DEFINE_IDR(ib_ps);
  88
  89 struct cma_device {
  90         struct list_head        list;
  91         struct ib_device        *device;
  92         struct completion       comp;
  93         atomic_t                refcount;
  94         struct list_head        id_list;
  95 };
  96
  97 struct rdma_bind_list {
  98         struct idr              *ps;
  99         struct hlist_head       owners;
 100         unsigned short          port;
 101 };
 102
 103 enum {
 104         CMA_OPTION_AFONLY,
 105 };
 106
 107 /*
 108  * Device removal can occur at anytime, so we need extra handling to
 109  * serialize notifying the user of device removal with other callbacks.
 110  * We do this by disabling removal notification while a callback is in process,
 111  * and reporting it after the callback completes.
 112  */
 113 struct rdma_id_private {
 114         struct rdma_cm_id       id;
 115
 116         struct rdma_bind_list   *bind_list;
 117         struct hlist_node       node;
 118         struct list_head        list; /* listen_any_list or cma_device.list */
 119         struct list_head        listen_list; /* per device listens */
 120         struct cma_device       *cma_dev;
 121         struct list_head        mc_list;
 122
 123         int                     internal_id;
 124         enum rdma_cm_state      state;
 125         spinlock_t              lock;
 126         struct mutex            qp_mutex;
 127
 128         struct completion       comp;
 129         atomic_t                refcount;
 130         struct mutex            handler_mutex;
 131
 132         int                     backlog;
 133         int                     timeout_ms;
 134         struct ib_sa_query      *query;
 135         int                     query_id;
 136         union {
 137                 struct ib_cm_id *ib;
 138                 struct iw_cm_id *iw;
 139         } cm_id;
 140
 141         u32                     seq_num;
 142         u32                     qkey;
 143         u32                     qp_num;
 144         pid_t                   owner;
 145         u32                     options;
 146         u8                      srq;
 147         u8                      tos;
 148         u8                      reuseaddr;
 149         u8                      afonly;
 150 };
 151
 152 struct cma_multicast {
 153         struct rdma_id_private *id_priv;
 154         union {
 155                 struct ib_sa_multicast *ib;
 156         } multicast;
 157         struct list_head        list;
 158         void                    *context;
 159         struct sockaddr_storage addr;
 160         struct kref             mcref;
 161 };
 162
 163 struct cma_work {
 164         struct work_struct      work;
 165         struct rdma_id_private  *id;
 166         enum rdma_cm_state      old_state;
 167         enum rdma_cm_state      new_state;
 168         struct rdma_cm_event    event;
 169 };
 170
 171 struct cma_ndev_work {
 172         struct work_struct      work;
 173         struct rdma_id_private  *id;
 174         struct rdma_cm_event    event;
 175 };
 176
 177 struct iboe_mcast_work {
 178         struct work_struct       work;
 179         struct rdma_id_private  *id;
 180         struct cma_multicast    *mc;
 181 };
 182
 183 union cma_ip_addr {
 184         struct in6_addr ip6;
 185         struct {
 186                 __be32 pad[3];
 187                 __be32 addr;
 188         } ip4;
 189 };
 190
 191 struct cma_hdr {
 192         u8 cma_version;
 193         u8 ip_version;  /* IP version: 7:4 */
 194         __be16 port;
 195         union cma_ip_addr src_addr;
 196         union cma_ip_addr dst_addr;
 197 };
 198
 199 struct sdp_hh {
 200         u8 bsdh[16];
 201         u8 sdp_version; /* Major version: 7:4 */
 202         u8 ip_version;  /* IP version: 7:4 */
 203         u8 sdp_specific1[10];
 204         __be16 port;
 205         __be16 sdp_specific2;
 206         union cma_ip_addr src_addr;
 207         union cma_ip_addr dst_addr;
 208 };
 209
 210 struct sdp_hah {
 211         u8 bsdh[16];
 212         u8 sdp_version;
 213 };
 214
 215 #define CMA_VERSION 0x00
 216 #define SDP_MAJ_VERSION 0x2
 217
 218 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 219 {
 220         unsigned long flags;
 221         int ret;
 222
 223         spin_lock_irqsave(&id_priv->lock, flags);
 224         ret = (id_priv->state == comp);
 225         spin_unlock_irqrestore(&id_priv->lock, flags);
 226         return ret;
 227 }
 228
 229 static int cma_comp_exch(struct rdma_id_private *id_priv,
 230                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 231 {
 232         unsigned long flags;
 233         int ret;
 234
 235         spin_lock_irqsave(&id_priv->lock, flags);
 236         if ((ret = (id_priv->state == comp)))
 237                 id_priv->state = exch;
 238         spin_unlock_irqrestore(&id_priv->lock, flags);
 239         return ret;
 240 }
 241
 242 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 243                                    enum rdma_cm_state exch)
 244 {
 245         unsigned long flags;
 246         enum rdma_cm_state old;
 247
 248         spin_lock_irqsave(&id_priv->lock, flags);
 249         old = id_priv->state;
 250         id_priv->state = exch;
 251         spin_unlock_irqrestore(&id_priv->lock, flags);
 252         return old;
 253 }
 254
 255 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 256 {
 257         return hdr->ip_version >> 4;
 258 }
 259
 260 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 261 {
 262         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 263 }
 264
 265 static inline u8 sdp_get_majv(u8 sdp_version)
 266 {
 267         return sdp_version >> 4;
 268 }
 269
 270 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 271 {
 272         return hh->ip_version >> 4;
 273 }
 274
 275 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 276 {
 277         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 278 }
 279
 280 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 281                               struct cma_device *cma_dev)
 282 {
 283         atomic_inc(&cma_dev->refcount);
 284         id_priv->cma_dev = cma_dev;
 285         id_priv->id.device = cma_dev->device;
 286         id_priv->id.route.addr.dev_addr.transport =
 287                 rdma_node_get_transport(cma_dev->device->node_type);
 288         list_add_tail(&id_priv->list, &cma_dev->id_list);
 289 }
 290
 291 static inline void cma_deref_dev(struct cma_device *cma_dev)
 292 {
 293         if (atomic_dec_and_test(&cma_dev->refcount))
 294                 complete(&cma_dev->comp);
 295 }
 296
 297 static inline void release_mc(struct kref *kref)
 298 {
 299         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 300
 301         kfree(mc->multicast.ib);
 302         kfree(mc);
 303 }
 304
 305 static void cma_release_dev(struct rdma_id_private *id_priv)
 306 {
 307         mutex_lock(&lock);
 308         list_del(&id_priv->list);
 309         cma_deref_dev(id_priv->cma_dev);
 310         id_priv->cma_dev = NULL;
 311         mutex_unlock(&lock);
 312 }
 313
 314 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 315 {
 316         return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
 317 }
 318
 319 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 320 {
 321         return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
 322 }
 323
 324 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 325 {
 326         return id_priv->id.route.addr.src_addr.ss_family;
 327 }
 328
 329 static int cma_set_qkey(struct rdma_id_private *id_priv)
 330 {
 331         struct ib_sa_mcmember_rec rec;
 332         int ret = 0;
 333
 334         if (id_priv->qkey)
 335                 return 0;
 336
 337         switch (id_priv->id.ps) {
 338         case RDMA_PS_UDP:
 339                 id_priv->qkey = RDMA_UDP_QKEY;
 340                 break;
 341         case RDMA_PS_IPOIB:
 342                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 343                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 344                                              id_priv->id.port_num, &rec.mgid,
 345                                              &rec);
 346                 if (!ret)
 347                         id_priv->qkey = be32_to_cpu(rec.qkey);
 348                 break;
 349         default:
 350                 break;
 351         }
 352         return ret;
 353 }
 354
 355 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 356 {
 357         int i;
 358         int err;
 359         struct ib_port_attr props;
 360         union ib_gid tmp;
 361
 362         err = ib_query_port(device, port_num, &props);
 363         if (err)
 364                 return err;
 365
 366         for (i = 0; i < props.gid_tbl_len; ++i) {
 367                 err = ib_query_gid(device, port_num, i, &tmp);
 368                 if (err)
 369                         return err;
 370                 if (!memcmp(&tmp, gid, sizeof tmp))
 371                         return 0;
 372         }
 373
 374         return -EADDRNOTAVAIL;
 375 }
 376
 377 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 378 {
 379         dev_addr->dev_type = ARPHRD_INFINIBAND;
 380         rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 381         ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 382 }
 383
 384 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 385 {
 386         int ret;
 387
 388         if (addr->sa_family != AF_IB) {
 389                 ret = rdma_translate_ip(addr, dev_addr);
 390         } else {
 391                 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 392                 ret = 0;
 393         }
 394
 395         return ret;
 396 }
 397
 398 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 399 {
 400         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 401         struct cma_device *cma_dev;
 402         union ib_gid gid, iboe_gid;
 403         int ret = -ENODEV;
 404         u8 port;
 405         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 406                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 407
 408         if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
 409             id_priv->id.ps == RDMA_PS_IPOIB)
 410                 return -EINVAL;
 411
 412         mutex_lock(&lock);
 413         iboe_addr_get_sgid(dev_addr, &iboe_gid);
 414         memcpy(&gid, dev_addr->src_dev_addr +
 415                rdma_addr_gid_offset(dev_addr), sizeof gid);
 416         list_for_each_entry(cma_dev, &dev_list, list) {
 417                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 418                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 419                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 420                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 421                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 422                                 else
 423                                         ret = find_gid_port(cma_dev->device, &gid, port);
 424
 425                                 if (!ret) {
 426                                         id_priv->id.port_num = port;
 427                                         goto out;
 428                                 }
 429                         }
 430                 }
 431         }
 432
 433 out:
 434         if (!ret)
 435                 cma_attach_to_dev(id_priv, cma_dev);
 436
 437         mutex_unlock(&lock);
 438         return ret;
 439 }
 440
 441 static void cma_deref_id(struct rdma_id_private *id_priv)
 442 {
 443         if (atomic_dec_and_test(&id_priv->refcount))
 444                 complete(&id_priv->comp);
 445 }
 446
 447 static int cma_disable_callback(struct rdma_id_private *id_priv,
 448                                 enum rdma_cm_state state)
 449 {
 450         mutex_lock(&id_priv->handler_mutex);
 451         if (id_priv->state != state) {
 452                 mutex_unlock(&id_priv->handler_mutex);
 453                 return -EINVAL;
 454         }
 455         return 0;
 456 }
 457
 458 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 459                                   void *context, enum rdma_port_space ps,
 460                                   enum ib_qp_type qp_type)
 461 {
 462         struct rdma_id_private *id_priv;
 463
 464         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 465         if (!id_priv)
 466                 return ERR_PTR(-ENOMEM);
 467
 468         id_priv->owner = task_pid_nr(current);
 469         id_priv->state = RDMA_CM_IDLE;
 470         id_priv->id.context = context;
 471         id_priv->id.event_handler = event_handler;
 472         id_priv->id.ps = ps;
 473         id_priv->id.qp_type = qp_type;
 474         spin_lock_init(&id_priv->lock);
 475         mutex_init(&id_priv->qp_mutex);
 476         init_completion(&id_priv->comp);
 477         atomic_set(&id_priv->refcount, 1);
 478         mutex_init(&id_priv->handler_mutex);
 479         INIT_LIST_HEAD(&id_priv->listen_list);
 480         INIT_LIST_HEAD(&id_priv->mc_list);
 481         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 482
 483         return &id_priv->id;
 484 }
 485 EXPORT_SYMBOL(rdma_create_id);
 486
 487 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 488 {
 489         struct ib_qp_attr qp_attr;
 490         int qp_attr_mask, ret;
 491
 492         qp_attr.qp_state = IB_QPS_INIT;
 493         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 494         if (ret)
 495                 return ret;
 496
 497         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 498         if (ret)
 499                 return ret;
 500
 501         qp_attr.qp_state = IB_QPS_RTR;
 502         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 503         if (ret)
 504                 return ret;
 505
 506         qp_attr.qp_state = IB_QPS_RTS;
 507         qp_attr.sq_psn = 0;
 508         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 509
 510         return ret;
 511 }
 512
 513 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 514 {
 515         struct ib_qp_attr qp_attr;
 516         int qp_attr_mask, ret;
 517
 518         qp_attr.qp_state = IB_QPS_INIT;
 519         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 520         if (ret)
 521                 return ret;
 522
 523         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 524 }
 525
 526 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 527                    struct ib_qp_init_attr *qp_init_attr)
 528 {
 529         struct rdma_id_private *id_priv;
 530         struct ib_qp *qp;
 531         int ret;
 532
 533         id_priv = container_of(id, struct rdma_id_private, id);
 534         if (id->device != pd->device)
 535                 return -EINVAL;
 536
 537         qp = ib_create_qp(pd, qp_init_attr);
 538         if (IS_ERR(qp))
 539                 return PTR_ERR(qp);
 540
 541         if (id->qp_type == IB_QPT_UD)
 542                 ret = cma_init_ud_qp(id_priv, qp);
 543         else
 544                 ret = cma_init_conn_qp(id_priv, qp);
 545         if (ret)
 546                 goto err;
 547
 548         id->qp = qp;
 549         id_priv->qp_num = qp->qp_num;
 550         id_priv->srq = (qp->srq != NULL);
 551         return 0;
 552 err:
 553         ib_destroy_qp(qp);
 554         return ret;
 555 }
 556 EXPORT_SYMBOL(rdma_create_qp);
 557
 558 void rdma_destroy_qp(struct rdma_cm_id *id)
 559 {
 560         struct rdma_id_private *id_priv;
 561
 562         id_priv = container_of(id, struct rdma_id_private, id);
 563         mutex_lock(&id_priv->qp_mutex);
 564         ib_destroy_qp(id_priv->id.qp);
 565         id_priv->id.qp = NULL;
 566         mutex_unlock(&id_priv->qp_mutex);
 567 }
 568 EXPORT_SYMBOL(rdma_destroy_qp);
 569
 570 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 571                              struct rdma_conn_param *conn_param)
 572 {
 573         struct ib_qp_attr qp_attr;
 574         int qp_attr_mask, ret;
 575
 576         mutex_lock(&id_priv->qp_mutex);
 577         if (!id_priv->id.qp) {
 578                 ret = 0;
 579                 goto out;
 580         }
 581
 582         /* Need to update QP attributes from default values. */
 583         qp_attr.qp_state = IB_QPS_INIT;
 584         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 585         if (ret)
 586                 goto out;
 587
 588         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 589         if (ret)
 590                 goto out;
 591
 592         qp_attr.qp_state = IB_QPS_RTR;
 593         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 594         if (ret)
 595                 goto out;
 596
 597         if (conn_param)
 598                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 599         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 600 out:
 601         mutex_unlock(&id_priv->qp_mutex);
 602         return ret;
 603 }
 604
 605 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 606                              struct rdma_conn_param *conn_param)
 607 {
 608         struct ib_qp_attr qp_attr;
 609         int qp_attr_mask, ret;
 610
 611         mutex_lock(&id_priv->qp_mutex);
 612         if (!id_priv->id.qp) {
 613                 ret = 0;
 614                 goto out;
 615         }
 616
 617         qp_attr.qp_state = IB_QPS_RTS;
 618         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 619         if (ret)
 620                 goto out;
 621
 622         if (conn_param)
 623                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 624         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 625 out:
 626         mutex_unlock(&id_priv->qp_mutex);
 627         return ret;
 628 }
 629
 630 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 631 {
 632         struct ib_qp_attr qp_attr;
 633         int ret;
 634
 635         mutex_lock(&id_priv->qp_mutex);
 636         if (!id_priv->id.qp) {
 637                 ret = 0;
 638                 goto out;
 639         }
 640
 641         qp_attr.qp_state = IB_QPS_ERR;
 642         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 643 out:
 644         mutex_unlock(&id_priv->qp_mutex);
 645         return ret;
 646 }
 647
 648 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 649                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 650 {
 651         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 652         int ret;
 653         u16 pkey;
 654
 655         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 656             IB_LINK_LAYER_INFINIBAND)
 657                 pkey = ib_addr_get_pkey(dev_addr);
 658         else
 659                 pkey = 0xffff;
 660
 661         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 662                                   pkey, &qp_attr->pkey_index);
 663         if (ret)
 664                 return ret;
 665
 666         qp_attr->port_num = id_priv->id.port_num;
 667         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 668
 669         if (id_priv->id.qp_type == IB_QPT_UD) {
 670                 ret = cma_set_qkey(id_priv);
 671                 if (ret)
 672                         return ret;
 673
 674                 qp_attr->qkey = id_priv->qkey;
 675                 *qp_attr_mask |= IB_QP_QKEY;
 676         } else {
 677                 qp_attr->qp_access_flags = 0;
 678                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 679         }
 680         return 0;
 681 }
 682
 683 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 684                        int *qp_attr_mask)
 685 {
 686         struct rdma_id_private *id_priv;
 687         int ret = 0;
 688
 689         id_priv = container_of(id, struct rdma_id_private, id);
 690         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 691         case RDMA_TRANSPORT_IB:
 692                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
 693                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 694                 else
 695                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 696                                                  qp_attr_mask);
 697                 if (qp_attr->qp_state == IB_QPS_RTR)
 698                         qp_attr->rq_psn = id_priv->seq_num;
 699                 break;
 700         case RDMA_TRANSPORT_IWARP:
 701                 if (!id_priv->cm_id.iw) {
 702                         qp_attr->qp_access_flags = 0;
 703                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 704                 } else
 705                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 706                                                  qp_attr_mask);
 707                 break;
 708         default:
 709                 ret = -ENOSYS;
 710                 break;
 711         }
 712
 713         return ret;
 714 }
 715 EXPORT_SYMBOL(rdma_init_qp_attr);
 716
 717 static inline int cma_zero_addr(struct sockaddr *addr)
 718 {
 719         switch (addr->sa_family) {
 720         case AF_INET:
 721                 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
 722         case AF_INET6:
 723                 return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr);
 724         case AF_IB:
 725                 return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr);
 726         default:
 727                 return 0;
 728         }
 729 }
 730
 731 static inline int cma_loopback_addr(struct sockaddr *addr)
 732 {
 733         switch (addr->sa_family) {
 734         case AF_INET:
 735                 return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
 736         case AF_INET6:
 737                 return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr);
 738         case AF_IB:
 739                 return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr);
 740         default:
 741                 return 0;
 742         }
 743 }
 744
 745 static inline int cma_any_addr(struct sockaddr *addr)
 746 {
 747         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 748 }
 749
 750 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 751 {
 752         if (src->sa_family != dst->sa_family)
 753                 return -1;
 754
 755         switch (src->sa_family) {
 756         case AF_INET:
 757                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 758                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 759         case AF_INET6:
 760                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 761                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 762         default:
 763                 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
 764                                    &((struct sockaddr_ib *) dst)->sib_addr);
 765         }
 766 }
 767
 768 static __be16 cma_port(struct sockaddr *addr)
 769 {
 770         struct sockaddr_ib *sib;
 771
 772         switch (addr->sa_family) {
 773         case AF_INET:
 774                 return ((struct sockaddr_in *) addr)->sin_port;
 775         case AF_INET6:
 776                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 777         case AF_IB:
 778                 sib = (struct sockaddr_ib *) addr;
 779                 return htons((u16) (be64_to_cpu(sib->sib_sid) &
 780                                     be64_to_cpu(sib->sib_sid_mask)));
 781         default:
 782                 return 0;
 783         }
 784 }
 785
 786 static inline int cma_any_port(struct sockaddr *addr)
 787 {
 788         return !cma_port(addr);
 789 }
 790
 791 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 792                             u8 *ip_ver, __be16 *port,
 793                             union cma_ip_addr **src, union cma_ip_addr **dst)
 794 {
 795         switch (ps) {
 796         case RDMA_PS_SDP:
 797                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 798                     SDP_MAJ_VERSION)
 799                         return -EINVAL;
 800
 801                 *ip_ver = sdp_get_ip_ver(hdr);
 802                 *port   = ((struct sdp_hh *) hdr)->port;
 803                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 804                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 805                 break;
 806         default:
 807                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 808                         return -EINVAL;
 809
 810                 *ip_ver = cma_get_ip_ver(hdr);
 811                 *port   = ((struct cma_hdr *) hdr)->port;
 812                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 813                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 814                 break;
 815         }
 816
 817         if (*ip_ver != 4 && *ip_ver != 6)
 818                 return -EINVAL;
 819         return 0;
 820 }
 821
 822 static void cma_save_net_info(struct rdma_addr *addr,
 823                               struct rdma_addr *listen_addr,
 824                               u8 ip_ver, __be16 port,
 825                               union cma_ip_addr *src, union cma_ip_addr *dst)
 826 {
 827         struct sockaddr_in *listen4, *ip4;
 828         struct sockaddr_in6 *listen6, *ip6;
 829
 830         switch (ip_ver) {
 831         case 4:
 832                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 833                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 834                 ip4->sin_family = listen4->sin_family;
 835                 ip4->sin_addr.s_addr = dst->ip4.addr;
 836                 ip4->sin_port = listen4->sin_port;
 837
 838                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 839                 ip4->sin_family = listen4->sin_family;
 840                 ip4->sin_addr.s_addr = src->ip4.addr;
 841                 ip4->sin_port = port;
 842                 break;
 843         case 6:
 844                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 845                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 846                 ip6->sin6_family = listen6->sin6_family;
 847                 ip6->sin6_addr = dst->ip6;
 848                 ip6->sin6_port = listen6->sin6_port;
 849
 850                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 851                 ip6->sin6_family = listen6->sin6_family;
 852                 ip6->sin6_addr = src->ip6;
 853                 ip6->sin6_port = port;
 854                 break;
 855         default:
 856                 break;
 857         }
 858 }
 859
 860 static inline int cma_user_data_offset(enum rdma_port_space ps)
 861 {
 862         switch (ps) {
 863         case RDMA_PS_SDP:
 864                 return 0;
 865         default:
 866                 return sizeof(struct cma_hdr);
 867         }
 868 }
 869
 870 static void cma_cancel_route(struct rdma_id_private *id_priv)
 871 {
 872         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 873         case IB_LINK_LAYER_INFINIBAND:
 874                 if (id_priv->query)
 875                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 876                 break;
 877         default:
 878                 break;
 879         }
 880 }
 881
 882 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 883 {
 884         struct rdma_id_private *dev_id_priv;
 885
 886         /*
 887          * Remove from listen_any_list to prevent added devices from spawning
 888          * additional listen requests.
 889          */
 890         mutex_lock(&lock);
 891         list_del(&id_priv->list);
 892
 893         while (!list_empty(&id_priv->listen_list)) {
 894                 dev_id_priv = list_entry(id_priv->listen_list.next,
 895                                          struct rdma_id_private, listen_list);
 896                 /* sync with device removal to avoid duplicate destruction */
 897                 list_del_init(&dev_id_priv->list);
 898                 list_del(&dev_id_priv->listen_list);
 899                 mutex_unlock(&lock);
 900
 901                 rdma_destroy_id(&dev_id_priv->id);
 902                 mutex_lock(&lock);
 903         }
 904         mutex_unlock(&lock);
 905 }
 906
 907 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 908                                  enum rdma_cm_state state)
 909 {
 910         switch (state) {
 911         case RDMA_CM_ADDR_QUERY:
 912                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 913                 break;
 914         case RDMA_CM_ROUTE_QUERY:
 915                 cma_cancel_route(id_priv);
 916                 break;
 917         case RDMA_CM_LISTEN:
 918                 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
 919                         cma_cancel_listens(id_priv);
 920                 break;
 921         default:
 922                 break;
 923         }
 924 }
 925
 926 static void cma_release_port(struct rdma_id_private *id_priv)
 927 {
 928         struct rdma_bind_list *bind_list = id_priv->bind_list;
 929
 930         if (!bind_list)
 931                 return;
 932
 933         mutex_lock(&lock);
 934         hlist_del(&id_priv->node);
 935         if (hlist_empty(&bind_list->owners)) {
 936                 idr_remove(bind_list->ps, bind_list->port);
 937                 kfree(bind_list);
 938         }
 939         mutex_unlock(&lock);
 940 }
 941
 942 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 943 {
 944         struct cma_multicast *mc;
 945
 946         while (!list_empty(&id_priv->mc_list)) {
 947                 mc = container_of(id_priv->mc_list.next,
 948                                   struct cma_multicast, list);
 949                 list_del(&mc->list);
 950                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 951                 case IB_LINK_LAYER_INFINIBAND:
 952                         ib_sa_free_multicast(mc->multicast.ib);
 953                         kfree(mc);
 954                         break;
 955                 case IB_LINK_LAYER_ETHERNET:
 956                         kref_put(&mc->mcref, release_mc);
 957                         break;
 958                 default:
 959                         break;
 960                 }
 961         }
 962 }
 963
 964 void rdma_destroy_id(struct rdma_cm_id *id)
 965 {
 966         struct rdma_id_private *id_priv;
 967         enum rdma_cm_state state;
 968
 969         id_priv = container_of(id, struct rdma_id_private, id);
 970         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
 971         cma_cancel_operation(id_priv, state);
 972
 973         /*
 974          * Wait for any active callback to finish.  New callbacks will find
 975          * the id_priv state set to destroying and abort.
 976          */
 977         mutex_lock(&id_priv->handler_mutex);
 978         mutex_unlock(&id_priv->handler_mutex);
 979
 980         if (id_priv->cma_dev) {
 981                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 982                 case RDMA_TRANSPORT_IB:
 983                         if (id_priv->cm_id.ib)
 984                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 985                         break;
 986                 case RDMA_TRANSPORT_IWARP:
 987                         if (id_priv->cm_id.iw)
 988                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 989                         break;
 990                 default:
 991                         break;
 992                 }
 993                 cma_leave_mc_groups(id_priv);
 994                 cma_release_dev(id_priv);
 995         }
 996
 997         cma_release_port(id_priv);
 998         cma_deref_id(id_priv);
 999         wait_for_completion(&id_priv->comp);
1000
1001         if (id_priv->internal_id)
1002                 cma_deref_id(id_priv->id.context);
1003
1004         kfree(id_priv->id.route.path_rec);
1005         kfree(id_priv);
1006 }
1007 EXPORT_SYMBOL(rdma_destroy_id);
1008
1009 static int cma_rep_recv(struct rdma_id_private *id_priv)
1010 {
1011         int ret;
1012
1013         ret = cma_modify_qp_rtr(id_priv, NULL);
1014         if (ret)
1015                 goto reject;
1016
1017         ret = cma_modify_qp_rts(id_priv, NULL);
1018         if (ret)
1019                 goto reject;
1020
1021         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1022         if (ret)
1023                 goto reject;
1024
1025         return 0;
1026 reject:
1027         cma_modify_qp_err(id_priv);
1028         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1029                        NULL, 0, NULL, 0);
1030         return ret;
1031 }
1032
1033 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
1034 {
1035         if (id_priv->id.ps == RDMA_PS_SDP &&
1036             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
1037             SDP_MAJ_VERSION)
1038                 return -EINVAL;
1039
1040         return 0;
1041 }
1042
1043 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1044                                    struct ib_cm_rep_event_param *rep_data,
1045                                    void *private_data)
1046 {
1047         event->param.conn.private_data = private_data;
1048         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1049         event->param.conn.responder_resources = rep_data->responder_resources;
1050         event->param.conn.initiator_depth = rep_data->initiator_depth;
1051         event->param.conn.flow_control = rep_data->flow_control;
1052         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1053         event->param.conn.srq = rep_data->srq;
1054         event->param.conn.qp_num = rep_data->remote_qpn;
1055 }
1056
1057 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1058 {
1059         struct rdma_id_private *id_priv = cm_id->context;
1060         struct rdma_cm_event event;
1061         int ret = 0;
1062
1063         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1064                 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1065             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1066                 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1067                 return 0;
1068
1069         memset(&event, 0, sizeof event);
1070         switch (ib_event->event) {
1071         case IB_CM_REQ_ERROR:
1072         case IB_CM_REP_ERROR:
1073                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1074                 event.status = -ETIMEDOUT;
1075                 break;
1076         case IB_CM_REP_RECEIVED:
1077                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1078                 if (event.status)
1079                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1080                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1081                         event.status = cma_rep_recv(id_priv);
1082                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1083                                                      RDMA_CM_EVENT_ESTABLISHED;
1084                 } else
1085                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1086                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1087                                        ib_event->private_data);
1088                 break;
1089         case IB_CM_RTU_RECEIVED:
1090         case IB_CM_USER_ESTABLISHED:
1091                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1092                 break;
1093         case IB_CM_DREQ_ERROR:
1094                 event.status = -ETIMEDOUT; /* fall through */
1095         case IB_CM_DREQ_RECEIVED:
1096         case IB_CM_DREP_RECEIVED:
1097                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1098                                    RDMA_CM_DISCONNECT))
1099                         goto out;
1100                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1101                 break;
1102         case IB_CM_TIMEWAIT_EXIT:
1103                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1104                 break;
1105         case IB_CM_MRA_RECEIVED:
1106                 /* ignore event */
1107                 goto out;
1108         case IB_CM_REJ_RECEIVED:
1109                 cma_modify_qp_err(id_priv);
1110                 event.status = ib_event->param.rej_rcvd.reason;
1111                 event.event = RDMA_CM_EVENT_REJECTED;
1112                 event.param.conn.private_data = ib_event->private_data;
1113                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1114                 break;
1115         default:
1116                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1117                        ib_event->event);
1118                 goto out;
1119         }
1120
1121         ret = id_priv->id.event_handler(&id_priv->id, &event);
1122         if (ret) {
1123                 /* Destroy the CM ID by returning a non-zero value. */
1124                 id_priv->cm_id.ib = NULL;
1125                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1126                 mutex_unlock(&id_priv->handler_mutex);
1127                 rdma_destroy_id(&id_priv->id);
1128                 return ret;
1129         }
1130 out:
1131         mutex_unlock(&id_priv->handler_mutex);
1132         return ret;
1133 }
1134
1135 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1136                                                struct ib_cm_event *ib_event)
1137 {
1138         struct rdma_id_private *id_priv;
1139         struct rdma_cm_id *id;
1140         struct rdma_route *rt;
1141         union cma_ip_addr *src, *dst;
1142         __be16 port;
1143         u8 ip_ver;
1144         int ret;
1145
1146         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1147                              &ip_ver, &port, &src, &dst))
1148                 return NULL;
1149
1150         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1151                             listen_id->ps, ib_event->param.req_rcvd.qp_type);
1152         if (IS_ERR(id))
1153                 return NULL;
1154
1155         id_priv = container_of(id, struct rdma_id_private, id);
1156         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1157                           ip_ver, port, src, dst);
1158
1159         rt = &id->route;
1160         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1161         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1162                                GFP_KERNEL);
1163         if (!rt->path_rec)
1164                 goto err;
1165
1166         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1167         if (rt->num_paths == 2)
1168                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1169
1170         if (cma_any_addr(cma_src_addr(id_priv))) {
1171                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1172                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1173                 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1174         } else {
1175                 ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
1176                 if (ret)
1177                         goto err;
1178         }
1179         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1180
1181         id_priv->state = RDMA_CM_CONNECT;
1182         return id_priv;
1183
1184 err:
1185         rdma_destroy_id(id);
1186         return NULL;
1187 }
1188
1189 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1190                                               struct ib_cm_event *ib_event)
1191 {
1192         struct rdma_id_private *id_priv;
1193         struct rdma_cm_id *id;
1194         union cma_ip_addr *src, *dst;
1195         __be16 port;
1196         u8 ip_ver;
1197         int ret;
1198
1199         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1200                             listen_id->ps, IB_QPT_UD);
1201         if (IS_ERR(id))
1202                 return NULL;
1203
1204         id_priv = container_of(id, struct rdma_id_private, id);
1205         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1206                              &ip_ver, &port, &src, &dst))
1207                 goto err;
1208
1209         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1210                           ip_ver, port, src, dst);
1211
1212         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1213                 ret = cma_translate_addr(cma_src_addr(id_priv), &id->route.addr.dev_addr);
1214                 if (ret)
1215                         goto err;
1216         }
1217
1218         id_priv->state = RDMA_CM_CONNECT;
1219         return id_priv;
1220 err:
1221         rdma_destroy_id(id);
1222         return NULL;
1223 }
1224
1225 static void cma_set_req_event_data(struct rdma_cm_event *event,
1226                                    struct ib_cm_req_event_param *req_data,
1227                                    void *private_data, int offset)
1228 {
1229         event->param.conn.private_data = private_data + offset;
1230         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1231         event->param.conn.responder_resources = req_data->responder_resources;
1232         event->param.conn.initiator_depth = req_data->initiator_depth;
1233         event->param.conn.flow_control = req_data->flow_control;
1234         event->param.conn.retry_count = req_data->retry_count;
1235         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1236         event->param.conn.srq = req_data->srq;
1237         event->param.conn.qp_num = req_data->remote_qpn;
1238 }
1239
1240 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1241 {
1242         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1243                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1244                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1245                  (id->qp_type == IB_QPT_UD)) ||
1246                 (!id->qp_type));
1247 }
1248
1249 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1250 {
1251         struct rdma_id_private *listen_id, *conn_id;
1252         struct rdma_cm_event event;
1253         int offset, ret;
1254
1255         listen_id = cm_id->context;
1256         if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1257                 return -EINVAL;
1258
1259         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1260                 return -ECONNABORTED;
1261
1262         memset(&event, 0, sizeof event);
1263         offset = cma_user_data_offset(listen_id->id.ps);
1264         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1265         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1266                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1267                 event.param.ud.private_data = ib_event->private_data + offset;
1268                 event.param.ud.private_data_len =
1269                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1270         } else {
1271                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1272                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1273                                        ib_event->private_data, offset);
1274         }
1275         if (!conn_id) {
1276                 ret = -ENOMEM;
1277                 goto err1;
1278         }
1279
1280         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1281         ret = cma_acquire_dev(conn_id);
1282         if (ret)
1283                 goto err2;
1284
1285         conn_id->cm_id.ib = cm_id;
1286         cm_id->context = conn_id;
1287         cm_id->cm_handler = cma_ib_handler;
1288
1289         /*
1290          * Protect against the user destroying conn_id from another thread
1291          * until we're done accessing it.
1292          */
1293         atomic_inc(&conn_id->refcount);
1294         ret = conn_id->id.event_handler(&conn_id->id, &event);
1295         if (ret)
1296                 goto err3;
1297
1298         /*
1299          * Acquire mutex to prevent user executing rdma_destroy_id()
1300          * while we're accessing the cm_id.
1301          */
1302         mutex_lock(&lock);
1303         if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
1304                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1305         mutex_unlock(&lock);
1306         mutex_unlock(&conn_id->handler_mutex);
1307         mutex_unlock(&listen_id->handler_mutex);
1308         cma_deref_id(conn_id);
1309         return 0;
1310
1311 err3:
1312         cma_deref_id(conn_id);
1313         /* Destroy the CM ID by returning a non-zero value. */
1314         conn_id->cm_id.ib = NULL;
1315 err2:
1316         cma_exch(conn_id, RDMA_CM_DESTROYING);
1317         mutex_unlock(&conn_id->handler_mutex);
1318 err1:
1319         mutex_unlock(&listen_id->handler_mutex);
1320         if (conn_id)
1321                 rdma_destroy_id(&conn_id->id);
1322         return ret;
1323 }
1324
1325 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1326 {
1327         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1328 }
1329
1330 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1331                                  struct ib_cm_compare_data *compare)
1332 {
1333         struct cma_hdr *cma_data, *cma_mask;
1334         struct sdp_hh *sdp_data, *sdp_mask;
1335         __be32 ip4_addr;
1336         struct in6_addr ip6_addr;
1337
1338         memset(compare, 0, sizeof *compare);
1339         cma_data = (void *) compare->data;
1340         cma_mask = (void *) compare->mask;
1341         sdp_data = (void *) compare->data;
1342         sdp_mask = (void *) compare->mask;
1343
1344         switch (addr->sa_family) {
1345         case AF_INET:
1346                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1347                 if (ps == RDMA_PS_SDP) {
1348                         sdp_set_ip_ver(sdp_data, 4);
1349                         sdp_set_ip_ver(sdp_mask, 0xF);
1350                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1351                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1352                 } else {
1353                         cma_set_ip_ver(cma_data, 4);
1354                         cma_set_ip_ver(cma_mask, 0xF);
1355                         if (!cma_any_addr(addr)) {
1356                                 cma_data->dst_addr.ip4.addr = ip4_addr;
1357                                 cma_mask->dst_addr.ip4.addr = htonl(~0);
1358                         }
1359                 }
1360                 break;
1361         case AF_INET6:
1362                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1363                 if (ps == RDMA_PS_SDP) {
1364                         sdp_set_ip_ver(sdp_data, 6);
1365                         sdp_set_ip_ver(sdp_mask, 0xF);
1366                         sdp_data->dst_addr.ip6 = ip6_addr;
1367                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1368                                sizeof sdp_mask->dst_addr.ip6);
1369                 } else {
1370                         cma_set_ip_ver(cma_data, 6);
1371                         cma_set_ip_ver(cma_mask, 0xF);
1372                         if (!cma_any_addr(addr)) {
1373                                 cma_data->dst_addr.ip6 = ip6_addr;
1374                                 memset(&cma_mask->dst_addr.ip6, 0xFF,
1375                                        sizeof cma_mask->dst_addr.ip6);
1376                         }
1377                 }
1378                 break;
1379         default:
1380                 break;
1381         }
1382 }
1383
1384 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1385 {
1386         struct rdma_id_private *id_priv = iw_id->context;
1387         struct rdma_cm_event event;
1388         struct sockaddr_in *sin;
1389         int ret = 0;
1390
1391         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1392                 return 0;
1393
1394         memset(&event, 0, sizeof event);
1395         switch (iw_event->event) {
1396         case IW_CM_EVENT_CLOSE:
1397                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1398                 break;
1399         case IW_CM_EVENT_CONNECT_REPLY:
1400                 sin = (struct sockaddr_in *) cma_src_addr(id_priv);
1401                 *sin = iw_event->local_addr;
1402                 sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
1403                 *sin = iw_event->remote_addr;
1404                 switch (iw_event->status) {
1405                 case 0:
1406                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1407                         event.param.conn.initiator_depth = iw_event->ird;
1408                         event.param.conn.responder_resources = iw_event->ord;
1409                         break;
1410                 case -ECONNRESET:
1411                 case -ECONNREFUSED:
1412                         event.event = RDMA_CM_EVENT_REJECTED;
1413                         break;
1414                 case -ETIMEDOUT:
1415                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1416                         break;
1417                 default:
1418                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1419                         break;
1420                 }
1421                 break;
1422         case IW_CM_EVENT_ESTABLISHED:
1423                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1424                 event.param.conn.initiator_depth = iw_event->ird;
1425                 event.param.conn.responder_resources = iw_event->ord;
1426                 break;
1427         default:
1428                 BUG_ON(1);
1429         }
1430
1431         event.status = iw_event->status;
1432         event.param.conn.private_data = iw_event->private_data;
1433         event.param.conn.private_data_len = iw_event->private_data_len;
1434         ret = id_priv->id.event_handler(&id_priv->id, &event);
1435         if (ret) {
1436                 /* Destroy the CM ID by returning a non-zero value. */
1437                 id_priv->cm_id.iw = NULL;
1438                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1439                 mutex_unlock(&id_priv->handler_mutex);
1440                 rdma_destroy_id(&id_priv->id);
1441                 return ret;
1442         }
1443
1444         mutex_unlock(&id_priv->handler_mutex);
1445         return ret;
1446 }
1447
1448 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1449                                struct iw_cm_event *iw_event)
1450 {
1451         struct rdma_cm_id *new_cm_id;
1452         struct rdma_id_private *listen_id, *conn_id;
1453         struct sockaddr_in *sin;
1454         struct net_device *dev = NULL;
1455         struct rdma_cm_event event;
1456         int ret;
1457         struct ib_device_attr attr;
1458
1459         listen_id = cm_id->context;
1460         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1461                 return -ECONNABORTED;
1462
1463         /* Create a new RDMA id for the new IW CM ID */
1464         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1465                                    listen_id->id.context,
1466                                    RDMA_PS_TCP, IB_QPT_RC);
1467         if (IS_ERR(new_cm_id)) {
1468                 ret = -ENOMEM;
1469                 goto out;
1470         }
1471         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1472         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1473         conn_id->state = RDMA_CM_CONNECT;
1474
1475         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1476         if (!dev) {
1477                 ret = -EADDRNOTAVAIL;
1478                 mutex_unlock(&conn_id->handler_mutex);
1479                 rdma_destroy_id(new_cm_id);
1480                 goto out;
1481         }
1482         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1483         if (ret) {
1484                 mutex_unlock(&conn_id->handler_mutex);
1485                 rdma_destroy_id(new_cm_id);
1486                 goto out;
1487         }
1488
1489         ret = cma_acquire_dev(conn_id);
1490         if (ret) {
1491                 mutex_unlock(&conn_id->handler_mutex);
1492                 rdma_destroy_id(new_cm_id);
1493                 goto out;
1494         }
1495
1496         conn_id->cm_id.iw = cm_id;
1497         cm_id->context = conn_id;
1498         cm_id->cm_handler = cma_iw_handler;
1499
1500         sin = (struct sockaddr_in *) cma_src_addr(conn_id);
1501         *sin = iw_event->local_addr;
1502         sin = (struct sockaddr_in *) cma_dst_addr(conn_id);
1503         *sin = iw_event->remote_addr;
1504
1505         ret = ib_query_device(conn_id->id.device, &attr);
1506         if (ret) {
1507                 mutex_unlock(&conn_id->handler_mutex);
1508                 rdma_destroy_id(new_cm_id);
1509                 goto out;
1510         }
1511
1512         memset(&event, 0, sizeof event);
1513         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1514         event.param.conn.private_data = iw_event->private_data;
1515         event.param.conn.private_data_len = iw_event->private_data_len;
1516         event.param.conn.initiator_depth = iw_event->ird;
1517         event.param.conn.responder_resources = iw_event->ord;
1518
1519         /*
1520          * Protect against the user destroying conn_id from another thread
1521          * until we're done accessing it.
1522          */
1523         atomic_inc(&conn_id->refcount);
1524         ret = conn_id->id.event_handler(&conn_id->id, &event);
1525         if (ret) {
1526                 /* User wants to destroy the CM ID */
1527                 conn_id->cm_id.iw = NULL;
1528                 cma_exch(conn_id, RDMA_CM_DESTROYING);
1529                 mutex_unlock(&conn_id->handler_mutex);
1530                 cma_deref_id(conn_id);
1531                 rdma_destroy_id(&conn_id->id);
1532                 goto out;
1533         }
1534
1535         mutex_unlock(&conn_id->handler_mutex);
1536         cma_deref_id(conn_id);
1537
1538 out:
1539         if (dev)
1540                 dev_put(dev);
1541         mutex_unlock(&listen_id->handler_mutex);
1542         return ret;
1543 }
1544
1545 static int cma_ib_listen(struct rdma_id_private *id_priv)
1546 {
1547         struct ib_cm_compare_data compare_data;
1548         struct sockaddr *addr;
1549         struct ib_cm_id *id;
1550         __be64 svc_id;
1551         int ret;
1552
1553         id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1554         if (IS_ERR(id))
1555                 return PTR_ERR(id);
1556
1557         id_priv->cm_id.ib = id;
1558
1559         addr = cma_src_addr(id_priv);
1560         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1561         if (cma_any_addr(addr) && !id_priv->afonly)
1562                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1563         else {
1564                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1565                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1566         }
1567
1568         if (ret) {
1569                 ib_destroy_cm_id(id_priv->cm_id.ib);
1570                 id_priv->cm_id.ib = NULL;
1571         }
1572
1573         return ret;
1574 }
1575
1576 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1577 {
1578         int ret;
1579         struct sockaddr_in *sin;
1580         struct iw_cm_id *id;
1581
1582         id = iw_create_cm_id(id_priv->id.device,
1583                              iw_conn_req_handler,
1584                              id_priv);
1585         if (IS_ERR(id))
1586                 return PTR_ERR(id);
1587
1588         id_priv->cm_id.iw = id;
1589
1590         sin = (struct sockaddr_in *) cma_src_addr(id_priv);
1591         id_priv->cm_id.iw->local_addr = *sin;
1592
1593         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1594
1595         if (ret) {
1596                 iw_destroy_cm_id(id_priv->cm_id.iw);
1597                 id_priv->cm_id.iw = NULL;
1598         }
1599
1600         return ret;
1601 }
1602
1603 static int cma_listen_handler(struct rdma_cm_id *id,
1604                               struct rdma_cm_event *event)
1605 {
1606         struct rdma_id_private *id_priv = id->context;
1607
1608         id->context = id_priv->id.context;
1609         id->event_handler = id_priv->id.event_handler;
1610         return id_priv->id.event_handler(id, event);
1611 }
1612
1613 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1614                               struct cma_device *cma_dev)
1615 {
1616         struct rdma_id_private *dev_id_priv;
1617         struct rdma_cm_id *id;
1618         int ret;
1619
1620         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1621                             id_priv->id.qp_type);
1622         if (IS_ERR(id))
1623                 return;
1624
1625         dev_id_priv = container_of(id, struct rdma_id_private, id);
1626
1627         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1628         memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
1629                rdma_addr_size(cma_src_addr(id_priv)));
1630
1631         cma_attach_to_dev(dev_id_priv, cma_dev);
1632         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1633         atomic_inc(&id_priv->refcount);
1634         dev_id_priv->internal_id = 1;
1635         dev_id_priv->afonly = id_priv->afonly;
1636
1637         ret = rdma_listen(id, id_priv->backlog);
1638         if (ret)
1639                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1640                        "listening on device %s\n", ret, cma_dev->device->name);
1641 }
1642
1643 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1644 {
1645         struct cma_device *cma_dev;
1646
1647         mutex_lock(&lock);
1648         list_add_tail(&id_priv->list, &listen_any_list);
1649         list_for_each_entry(cma_dev, &dev_list, list)
1650                 cma_listen_on_dev(id_priv, cma_dev);
1651         mutex_unlock(&lock);
1652 }
1653
1654 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1655 {
1656         struct rdma_id_private *id_priv;
1657
1658         id_priv = container_of(id, struct rdma_id_private, id);
1659         id_priv->tos = (u8) tos;
1660 }
1661 EXPORT_SYMBOL(rdma_set_service_type);
1662
1663 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1664                               void *context)
1665 {
1666         struct cma_work *work = context;
1667         struct rdma_route *route;
1668
1669         route = &work->id->id.route;
1670
1671         if (!status) {
1672                 route->num_paths = 1;
1673                 *route->path_rec = *path_rec;
1674         } else {
1675                 work->old_state = RDMA_CM_ROUTE_QUERY;
1676                 work->new_state = RDMA_CM_ADDR_RESOLVED;
1677                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1678                 work->event.status = status;
1679         }
1680
1681         queue_work(cma_wq, &work->work);
1682 }
1683
1684 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1685                               struct cma_work *work)
1686 {
1687         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1688         struct ib_sa_path_rec path_rec;
1689         ib_sa_comp_mask comp_mask;
1690         struct sockaddr_in6 *sin6;
1691
1692         memset(&path_rec, 0, sizeof path_rec);
1693         rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
1694         rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
1695         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
1696         path_rec.numb_path = 1;
1697         path_rec.reversible = 1;
1698         path_rec.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
1699
1700         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1701                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1702                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1703
1704         if (cma_family(id_priv) == AF_INET) {
1705                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1706                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1707         } else {
1708                 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
1709                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1710                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1711         }
1712
1713         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1714                                                id_priv->id.port_num, &path_rec,
1715                                                comp_mask, timeout_ms,
1716                                                GFP_KERNEL, cma_query_handler,
1717                                                work, &id_priv->query);
1718
1719         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1720 }
1721
1722 static void cma_work_handler(struct work_struct *_work)
1723 {
1724         struct cma_work *work = container_of(_work, struct cma_work, work);
1725         struct rdma_id_private *id_priv = work->id;
1726         int destroy = 0;
1727
1728         mutex_lock(&id_priv->handler_mutex);
1729         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1730                 goto out;
1731
1732         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1733                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1734                 destroy = 1;
1735         }
1736 out:
1737         mutex_unlock(&id_priv->handler_mutex);
1738         cma_deref_id(id_priv);
1739         if (destroy)
1740                 rdma_destroy_id(&id_priv->id);
1741         kfree(work);
1742 }
1743
1744 static void cma_ndev_work_handler(struct work_struct *_work)
1745 {
1746         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1747         struct rdma_id_private *id_priv = work->id;
1748         int destroy = 0;
1749
1750         mutex_lock(&id_priv->handler_mutex);
1751         if (id_priv->state == RDMA_CM_DESTROYING ||
1752             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1753                 goto out;
1754
1755         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1756                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1757                 destroy = 1;
1758         }
1759
1760 out:
1761         mutex_unlock(&id_priv->handler_mutex);
1762         cma_deref_id(id_priv);
1763         if (destroy)
1764                 rdma_destroy_id(&id_priv->id);
1765         kfree(work);
1766 }
1767
1768 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1769 {
1770         struct rdma_route *route = &id_priv->id.route;
1771         struct cma_work *work;
1772         int ret;
1773
1774         work = kzalloc(sizeof *work, GFP_KERNEL);
1775         if (!work)
1776                 return -ENOMEM;
1777
1778         work->id = id_priv;
1779         INIT_WORK(&work->work, cma_work_handler);
1780         work->old_state = RDMA_CM_ROUTE_QUERY;
1781         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1782         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1783
1784         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1785         if (!route->path_rec) {
1786                 ret = -ENOMEM;
1787                 goto err1;
1788         }
1789
1790         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1791         if (ret)
1792                 goto err2;
1793
1794         return 0;
1795 err2:
1796         kfree(route->path_rec);
1797         route->path_rec = NULL;
1798 err1:
1799         kfree(work);
1800         return ret;
1801 }
1802
1803 int rdma_set_ib_paths(struct rdma_cm_id *id,
1804                       struct ib_sa_path_rec *path_rec, int num_paths)
1805 {
1806         struct rdma_id_private *id_priv;
1807         int ret;
1808
1809         id_priv = container_of(id, struct rdma_id_private, id);
1810         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1811                            RDMA_CM_ROUTE_RESOLVED))
1812                 return -EINVAL;
1813
1814         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1815                                      GFP_KERNEL);
1816         if (!id->route.path_rec) {
1817                 ret = -ENOMEM;
1818                 goto err;
1819         }
1820
1821         id->route.num_paths = num_paths;
1822         return 0;
1823 err:
1824         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1825         return ret;
1826 }
1827 EXPORT_SYMBOL(rdma_set_ib_paths);
1828
1829 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1830 {
1831         struct cma_work *work;
1832
1833         work = kzalloc(sizeof *work, GFP_KERNEL);
1834         if (!work)
1835                 return -ENOMEM;
1836
1837         work->id = id_priv;
1838         INIT_WORK(&work->work, cma_work_handler);
1839         work->old_state = RDMA_CM_ROUTE_QUERY;
1840         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1841         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1842         queue_work(cma_wq, &work->work);
1843         return 0;
1844 }
1845
1846 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1847 {
1848         struct rdma_route *route = &id_priv->id.route;
1849         struct rdma_addr *addr = &route->addr;
1850         struct cma_work *work;
1851         int ret;
1852         struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1853         struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1854         struct net_device *ndev = NULL;
1855         u16 vid;
1856
1857         if (src_addr->sin_family != dst_addr->sin_family)
1858                 return -EINVAL;
1859
1860         work = kzalloc(sizeof *work, GFP_KERNEL);
1861         if (!work)
1862                 return -ENOMEM;
1863
1864         work->id = id_priv;
1865         INIT_WORK(&work->work, cma_work_handler);
1866
1867         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1868         if (!route->path_rec) {
1869                 ret = -ENOMEM;
1870                 goto err1;
1871         }
1872
1873         route->num_paths = 1;
1874
1875         if (addr->dev_addr.bound_dev_if)
1876                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1877         if (!ndev) {
1878                 ret = -ENODEV;
1879                 goto err2;
1880         }
1881
1882         vid = rdma_vlan_dev_vlan_id(ndev);
1883
1884         iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1885         iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1886
1887         route->path_rec->hop_limit = 1;
1888         route->path_rec->reversible = 1;
1889         route->path_rec->pkey = cpu_to_be16(0xffff);
1890         route->path_rec->mtu_selector = IB_SA_EQ;
1891         route->path_rec->sl = netdev_get_prio_tc_map(
1892                         ndev->priv_flags & IFF_802_1Q_VLAN ?
1893                                 vlan_dev_real_dev(ndev) : ndev,
1894                         rt_tos2priority(id_priv->tos));
1895
1896         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1897         route->path_rec->rate_selector = IB_SA_EQ;
1898         route->path_rec->rate = iboe_get_rate(ndev);
1899         dev_put(ndev);
1900         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1901         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1902         if (!route->path_rec->mtu) {
1903                 ret = -EINVAL;
1904                 goto err2;
1905         }
1906
1907         work->old_state = RDMA_CM_ROUTE_QUERY;
1908         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1909         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1910         work->event.status = 0;
1911
1912         queue_work(cma_wq, &work->work);
1913
1914         return 0;
1915
1916 err2:
1917         kfree(route->path_rec);
1918         route->path_rec = NULL;
1919 err1:
1920         kfree(work);
1921         return ret;
1922 }
1923
1924 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1925 {
1926         struct rdma_id_private *id_priv;
1927         int ret;
1928
1929         id_priv = container_of(id, struct rdma_id_private, id);
1930         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
1931                 return -EINVAL;
1932
1933         atomic_inc(&id_priv->refcount);
1934         switch (rdma_node_get_transport(id->device->node_type)) {
1935         case RDMA_TRANSPORT_IB:
1936                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1937                 case IB_LINK_LAYER_INFINIBAND:
1938                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
1939                         break;
1940                 case IB_LINK_LAYER_ETHERNET:
1941                         ret = cma_resolve_iboe_route(id_priv);
1942                         break;
1943                 default:
1944                         ret = -ENOSYS;
1945                 }
1946                 break;
1947         case RDMA_TRANSPORT_IWARP:
1948                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1949                 break;
1950         default:
1951                 ret = -ENOSYS;
1952                 break;
1953         }
1954         if (ret)
1955                 goto err;
1956
1957         return 0;
1958 err:
1959         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
1960         cma_deref_id(id_priv);
1961         return ret;
1962 }
1963 EXPORT_SYMBOL(rdma_resolve_route);
1964
1965 static void cma_set_loopback(struct sockaddr *addr)
1966 {
1967         switch (addr->sa_family) {
1968         case AF_INET:
1969                 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
1970                 break;
1971         case AF_INET6:
1972                 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
1973                               0, 0, 0, htonl(1));
1974                 break;
1975         default:
1976                 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
1977                             0, 0, 0, htonl(1));
1978                 break;
1979         }
1980 }
1981
1982 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1983 {
1984         struct cma_device *cma_dev, *cur_dev;
1985         struct ib_port_attr port_attr;
1986         union ib_gid gid;
1987         u16 pkey;
1988         int ret;
1989         u8 p;
1990
1991         cma_dev = NULL;
1992         mutex_lock(&lock);
1993         list_for_each_entry(cur_dev, &dev_list, list) {
1994                 if (cma_family(id_priv) == AF_IB &&
1995                     rdma_node_get_transport(cur_dev->device->node_type) != RDMA_TRANSPORT_IB)
1996                         continue;
1997
1998                 if (!cma_dev)
1999                         cma_dev = cur_dev;
2000
2001                 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
2002                         if (!ib_query_port(cur_dev->device, p, &port_attr) &&
2003                             port_attr.state == IB_PORT_ACTIVE) {
2004                                 cma_dev = cur_dev;
2005                                 goto port_found;
2006                         }
2007                 }
2008         }
2009
2010         if (!cma_dev) {
2011                 ret = -ENODEV;
2012                 goto out;
2013         }
2014
2015         p = 1;
2016
2017 port_found:
2018         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
2019         if (ret)
2020                 goto out;
2021
2022         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
2023         if (ret)
2024                 goto out;
2025
2026         id_priv->id.route.addr.dev_addr.dev_type =
2027                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
2028                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
2029
2030         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2031         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
2032         id_priv->id.port_num = p;
2033         cma_attach_to_dev(id_priv, cma_dev);
2034         cma_set_loopback(cma_src_addr(id_priv));
2035 out:
2036         mutex_unlock(&lock);
2037         return ret;
2038 }
2039
2040 static void addr_handler(int status, struct sockaddr *src_addr,
2041                          struct rdma_dev_addr *dev_addr, void *context)
2042 {
2043         struct rdma_id_private *id_priv = context;
2044         struct rdma_cm_event event;
2045
2046         memset(&event, 0, sizeof event);
2047         mutex_lock(&id_priv->handler_mutex);
2048         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
2049                            RDMA_CM_ADDR_RESOLVED))
2050                 goto out;
2051
2052         if (!status && !id_priv->cma_dev)
2053                 status = cma_acquire_dev(id_priv);
2054
2055         if (status) {
2056                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2057                                    RDMA_CM_ADDR_BOUND))
2058                         goto out;
2059                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2060                 event.status = status;
2061         } else {
2062                 memcpy(cma_src_addr(id_priv), src_addr, rdma_addr_size(src_addr));
2063                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2064         }
2065
2066         if (id_priv->id.event_handler(&id_priv->id, &event)) {
2067                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2068                 mutex_unlock(&id_priv->handler_mutex);
2069                 cma_deref_id(id_priv);
2070                 rdma_destroy_id(&id_priv->id);
2071                 return;
2072         }
2073 out:
2074         mutex_unlock(&id_priv->handler_mutex);
2075         cma_deref_id(id_priv);
2076 }
2077
2078 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2079 {
2080         struct cma_work *work;
2081         union ib_gid gid;
2082         int ret;
2083
2084         work = kzalloc(sizeof *work, GFP_KERNEL);
2085         if (!work)
2086                 return -ENOMEM;
2087
2088         if (!id_priv->cma_dev) {
2089                 ret = cma_bind_loopback(id_priv);
2090                 if (ret)
2091                         goto err;
2092         }
2093
2094         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2095         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2096
2097         work->id = id_priv;
2098         INIT_WORK(&work->work, cma_work_handler);
2099         work->old_state = RDMA_CM_ADDR_QUERY;
2100         work->new_state = RDMA_CM_ADDR_RESOLVED;
2101         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2102         queue_work(cma_wq, &work->work);
2103         return 0;
2104 err:
2105         kfree(work);
2106         return ret;
2107 }
2108
2109 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2110                          struct sockaddr *dst_addr)
2111 {
2112         if (!src_addr || !src_addr->sa_family) {
2113                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2114                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2115                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2116                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2117                 }
2118         }
2119         return rdma_bind_addr(id, src_addr);
2120 }
2121
2122 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2123                       struct sockaddr *dst_addr, int timeout_ms)
2124 {
2125         struct rdma_id_private *id_priv;
2126         int ret;
2127
2128         id_priv = container_of(id, struct rdma_id_private, id);
2129         if (id_priv->state == RDMA_CM_IDLE) {
2130                 ret = cma_bind_addr(id, src_addr, dst_addr);
2131                 if (ret)
2132                         return ret;
2133         }
2134
2135         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2136                 return -EINVAL;
2137
2138         atomic_inc(&id_priv->refcount);
2139         memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
2140         if (cma_any_addr(dst_addr))
2141                 ret = cma_resolve_loopback(id_priv);
2142         else
2143                 ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv),
2144                                       dst_addr, &id->route.addr.dev_addr,
2145                                       timeout_ms, addr_handler, id_priv);
2146         if (ret)
2147                 goto err;
2148
2149         return 0;
2150 err:
2151         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2152         cma_deref_id(id_priv);
2153         return ret;
2154 }
2155 EXPORT_SYMBOL(rdma_resolve_addr);
2156
2157 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2158 {
2159         struct rdma_id_private *id_priv;
2160         unsigned long flags;
2161         int ret;
2162
2163         id_priv = container_of(id, struct rdma_id_private, id);
2164         spin_lock_irqsave(&id_priv->lock, flags);
2165         if (reuse || id_priv->state == RDMA_CM_IDLE) {
2166                 id_priv->reuseaddr = reuse;
2167                 ret = 0;
2168         } else {
2169                 ret = -EINVAL;
2170         }
2171         spin_unlock_irqrestore(&id_priv->lock, flags);
2172         return ret;
2173 }
2174 EXPORT_SYMBOL(rdma_set_reuseaddr);
2175
2176 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2177 {
2178         struct rdma_id_private *id_priv;
2179         unsigned long flags;
2180         int ret;
2181
2182         id_priv = container_of(id, struct rdma_id_private, id);
2183         spin_lock_irqsave(&id_priv->lock, flags);
2184         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2185                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2186                 id_priv->afonly = afonly;
2187                 ret = 0;
2188         } else {
2189                 ret = -EINVAL;
2190         }
2191         spin_unlock_irqrestore(&id_priv->lock, flags);
2192         return ret;
2193 }
2194 EXPORT_SYMBOL(rdma_set_afonly);
2195
2196 static void cma_bind_port(struct rdma_bind_list *bind_list,
2197                           struct rdma_id_private *id_priv)
2198 {
2199         struct sockaddr *addr;
2200         struct sockaddr_ib *sib;
2201         u64 sid, mask;
2202         __be16 port;
2203
2204         addr = cma_src_addr(id_priv);
2205         port = htons(bind_list->port);
2206
2207         switch (addr->sa_family) {
2208         case AF_INET:
2209                 ((struct sockaddr_in *) addr)->sin_port = port;
2210                 break;
2211         case AF_INET6:
2212                 ((struct sockaddr_in6 *) addr)->sin6_port = port;
2213                 break;
2214         case AF_IB:
2215                 sib = (struct sockaddr_ib *) addr;
2216                 sid = be64_to_cpu(sib->sib_sid);
2217                 mask = be64_to_cpu(sib->sib_sid_mask);
2218                 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
2219                 sib->sib_sid_mask = cpu_to_be64(~0ULL);
2220                 break;
2221         }
2222         id_priv->bind_list = bind_list;
2223         hlist_add_head(&id_priv->node, &bind_list->owners);
2224 }
2225
2226 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2227                           unsigned short snum)
2228 {
2229         struct rdma_bind_list *bind_list;
2230         int ret;
2231
2232         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2233         if (!bind_list)
2234                 return -ENOMEM;
2235
2236         ret = idr_alloc(ps, bind_list, snum, snum + 1, GFP_KERNEL);
2237         if (ret < 0)
2238                 goto err;
2239
2240         bind_list->ps = ps;
2241         bind_list->port = (unsigned short)ret;
2242         cma_bind_port(bind_list, id_priv);
2243         return 0;
2244 err:
2245         kfree(bind_list);
2246         return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
2247 }
2248
2249 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2250 {
2251         static unsigned int last_used_port;
2252         int low, high, remaining;
2253         unsigned int rover;
2254
2255         inet_get_local_port_range(&low, &high);
2256         remaining = (high - low) + 1;
2257         rover = net_random() % remaining + low;
2258 retry:
2259         if (last_used_port != rover &&
2260             !idr_find(ps, (unsigned short) rover)) {
2261                 int ret = cma_alloc_port(ps, id_priv, rover);
2262                 /*
2263                  * Remember previously used port number in order to avoid
2264                  * re-using same port immediately after it is closed.
2265                  */
2266                 if (!ret)
2267                         last_used_port = rover;
2268                 if (ret != -EADDRNOTAVAIL)
2269                         return ret;
2270         }
2271         if (--remaining) {
2272                 rover++;
2273                 if ((rover < low) || (rover > high))
2274                         rover = low;
2275                 goto retry;
2276         }
2277         return -EADDRNOTAVAIL;
2278 }
2279
2280 /*
2281  * Check that the requested port is available.  This is called when trying to
2282  * bind to a specific port, or when trying to listen on a bound port.  In
2283  * the latter case, the provided id_priv may already be on the bind_list, but
2284  * we still need to check that it's okay to start listening.
2285  */
2286 static int cma_check_port(struct rdma_bind_list *bind_list,
2287                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
2288 {
2289         struct rdma_id_private *cur_id;
2290         struct sockaddr *addr, *cur_addr;
2291
2292         addr = cma_src_addr(id_priv);
2293         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
2294                 if (id_priv == cur_id)
2295                         continue;
2296
2297                 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2298                     cur_id->reuseaddr)
2299                         continue;
2300
2301                 cur_addr = cma_src_addr(cur_id);
2302                 if (id_priv->afonly && cur_id->afonly &&
2303                     (addr->sa_family != cur_addr->sa_family))
2304                         continue;
2305
2306                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2307                         return -EADDRNOTAVAIL;
2308
2309                 if (!cma_addr_cmp(addr, cur_addr))
2310                         return -EADDRINUSE;
2311         }
2312         return 0;
2313 }
2314
2315 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2316 {
2317         struct rdma_bind_list *bind_list;
2318         unsigned short snum;
2319         int ret;
2320
2321         snum = ntohs(cma_port(cma_src_addr(id_priv)));
2322         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2323                 return -EACCES;
2324
2325         bind_list = idr_find(ps, snum);
2326         if (!bind_list) {
2327                 ret = cma_alloc_port(ps, id_priv, snum);
2328         } else {
2329                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2330                 if (!ret)
2331                         cma_bind_port(bind_list, id_priv);
2332         }
2333         return ret;
2334 }
2335
2336 static int cma_bind_listen(struct rdma_id_private *id_priv)
2337 {
2338         struct rdma_bind_list *bind_list = id_priv->bind_list;
2339         int ret = 0;
2340
2341         mutex_lock(&lock);
2342         if (bind_list->owners.first->next)
2343                 ret = cma_check_port(bind_list, id_priv, 0);
2344         mutex_unlock(&lock);
2345         return ret;
2346 }
2347
2348 static struct idr *cma_select_inet_ps(struct rdma_id_private *id_priv)
2349 {
2350         switch (id_priv->id.ps) {
2351         case RDMA_PS_SDP:
2352                 return &sdp_ps;
2353         case RDMA_PS_TCP:
2354                 return &tcp_ps;
2355         case RDMA_PS_UDP:
2356                 return &udp_ps;
2357         case RDMA_PS_IPOIB:
2358                 return &ipoib_ps;
2359         case RDMA_PS_IB:
2360                 return &ib_ps;
2361         default:
2362                 return NULL;
2363         }
2364 }
2365
2366 static struct idr *cma_select_ib_ps(struct rdma_id_private *id_priv)
2367 {
2368         struct idr *ps = NULL;
2369         struct sockaddr_ib *sib;
2370         u64 sid_ps, mask, sid;
2371
2372         sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2373         mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
2374         sid = be64_to_cpu(sib->sib_sid) & mask;
2375
2376         if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
2377                 sid_ps = RDMA_IB_IP_PS_IB;
2378                 ps = &ib_ps;
2379         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
2380                    (sid == (RDMA_IB_IP_PS_TCP & mask))) {
2381                 sid_ps = RDMA_IB_IP_PS_TCP;
2382                 ps = &tcp_ps;
2383         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
2384                    (sid == (RDMA_IB_IP_PS_UDP & mask))) {
2385                 sid_ps = RDMA_IB_IP_PS_UDP;
2386                 ps = &udp_ps;
2387         }
2388
2389         if (ps) {
2390                 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
2391                 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
2392                                                 be64_to_cpu(sib->sib_sid_mask));
2393         }
2394         return ps;
2395 }
2396
2397 static int cma_get_port(struct rdma_id_private *id_priv)
2398 {
2399         struct idr *ps;
2400         int ret;
2401
2402         if (cma_family(id_priv) != AF_IB)
2403                 ps = cma_select_inet_ps(id_priv);
2404         else
2405                 ps = cma_select_ib_ps(id_priv);
2406         if (!ps)
2407                 return -EPROTONOSUPPORT;
2408
2409         mutex_lock(&lock);
2410         if (cma_any_port(cma_src_addr(id_priv)))
2411                 ret = cma_alloc_any_port(ps, id_priv);
2412         else
2413                 ret = cma_use_port(ps, id_priv);
2414         mutex_unlock(&lock);
2415
2416         return ret;
2417 }
2418
2419 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2420                                struct sockaddr *addr)
2421 {
2422 #if IS_ENABLED(CONFIG_IPV6)
2423         struct sockaddr_in6 *sin6;
2424
2425         if (addr->sa_family != AF_INET6)
2426                 return 0;
2427
2428         sin6 = (struct sockaddr_in6 *) addr;
2429         if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2430             !sin6->sin6_scope_id)
2431                         return -EINVAL;
2432
2433         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2434 #endif
2435         return 0;
2436 }
2437
2438 int rdma_listen(struct rdma_cm_id *id, int backlog)
2439 {
2440         struct rdma_id_private *id_priv;
2441         int ret;
2442
2443         id_priv = container_of(id, struct rdma_id_private, id);
2444         if (id_priv->state == RDMA_CM_IDLE) {
2445                 id->route.addr.src_addr.ss_family = AF_INET;
2446                 ret = rdma_bind_addr(id, cma_src_addr(id_priv));
2447                 if (ret)
2448                         return ret;
2449         }
2450
2451         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2452                 return -EINVAL;
2453
2454         if (id_priv->reuseaddr) {
2455                 ret = cma_bind_listen(id_priv);
2456                 if (ret)
2457                         goto err;
2458         }
2459
2460         id_priv->backlog = backlog;
2461         if (id->device) {
2462                 switch (rdma_node_get_transport(id->device->node_type)) {
2463                 case RDMA_TRANSPORT_IB:
2464                         ret = cma_ib_listen(id_priv);
2465                         if (ret)
2466                                 goto err;
2467                         break;
2468                 case RDMA_TRANSPORT_IWARP:
2469                         ret = cma_iw_listen(id_priv, backlog);
2470                         if (ret)
2471                                 goto err;
2472                         break;
2473                 default:
2474                         ret = -ENOSYS;
2475                         goto err;
2476                 }
2477         } else
2478                 cma_listen_on_all(id_priv);
2479
2480         return 0;
2481 err:
2482         id_priv->backlog = 0;
2483         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2484         return ret;
2485 }
2486 EXPORT_SYMBOL(rdma_listen);
2487
2488 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2489 {
2490         struct rdma_id_private *id_priv;
2491         int ret;
2492
2493         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
2494             addr->sa_family != AF_IB)
2495                 return -EAFNOSUPPORT;
2496
2497         id_priv = container_of(id, struct rdma_id_private, id);
2498         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2499                 return -EINVAL;
2500
2501         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2502         if (ret)
2503                 goto err1;
2504
2505         if (!cma_any_addr(addr)) {
2506                 ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
2507                 if (ret)
2508                         goto err1;
2509
2510                 ret = cma_acquire_dev(id_priv);
2511                 if (ret)
2512                         goto err1;
2513         }
2514
2515         memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
2516         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2517                 if (addr->sa_family == AF_INET)
2518                         id_priv->afonly = 1;
2519 #if IS_ENABLED(CONFIG_IPV6)
2520                 else if (addr->sa_family == AF_INET6)
2521                         id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
2522 #endif
2523         }
2524         ret = cma_get_port(id_priv);
2525         if (ret)
2526                 goto err2;
2527
2528         return 0;
2529 err2:
2530         if (id_priv->cma_dev)
2531                 cma_release_dev(id_priv);
2532 err1:
2533         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2534         return ret;
2535 }
2536 EXPORT_SYMBOL(rdma_bind_addr);
2537
2538 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
2539 {
2540         struct cma_hdr *cma_hdr;
2541         struct sdp_hh *sdp_hdr;
2542
2543         if (cma_family(id_priv) == AF_INET) {
2544                 struct sockaddr_in *src4, *dst4;
2545
2546                 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
2547                 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
2548
2549                 switch (id_priv->id.ps) {
2550                 case RDMA_PS_SDP:
2551                         sdp_hdr = hdr;
2552                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2553                                 return -EINVAL;
2554                         sdp_set_ip_ver(sdp_hdr, 4);
2555                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2556                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2557                         sdp_hdr->port = src4->sin_port;
2558                         break;
2559                 default:
2560                         cma_hdr = hdr;
2561                         cma_hdr->cma_version = CMA_VERSION;
2562                         cma_set_ip_ver(cma_hdr, 4);
2563                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2564                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2565                         cma_hdr->port = src4->sin_port;
2566                         break;
2567                 }
2568         } else {
2569                 struct sockaddr_in6 *src6, *dst6;
2570
2571                 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2572                 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
2573
2574                 switch (id_priv->id.ps) {
2575                 case RDMA_PS_SDP:
2576                         sdp_hdr = hdr;
2577                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2578                                 return -EINVAL;
2579                         sdp_set_ip_ver(sdp_hdr, 6);
2580                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2581                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2582                         sdp_hdr->port = src6->sin6_port;
2583                         break;
2584                 default:
2585                         cma_hdr = hdr;
2586                         cma_hdr->cma_version = CMA_VERSION;
2587                         cma_set_ip_ver(cma_hdr, 6);
2588                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2589                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2590                         cma_hdr->port = src6->sin6_port;
2591                         break;
2592                 }
2593         }
2594         return 0;
2595 }
2596
2597 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2598                                 struct ib_cm_event *ib_event)
2599 {
2600         struct rdma_id_private *id_priv = cm_id->context;
2601         struct rdma_cm_event event;
2602         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2603         int ret = 0;
2604
2605         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2606                 return 0;
2607
2608         memset(&event, 0, sizeof event);
2609         switch (ib_event->event) {
2610         case IB_CM_SIDR_REQ_ERROR:
2611                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2612                 event.status = -ETIMEDOUT;
2613                 break;
2614         case IB_CM_SIDR_REP_RECEIVED:
2615                 event.param.ud.private_data = ib_event->private_data;
2616                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2617                 if (rep->status != IB_SIDR_SUCCESS) {
2618                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2619                         event.status = ib_event->param.sidr_rep_rcvd.status;
2620                         break;
2621                 }
2622                 ret = cma_set_qkey(id_priv);
2623                 if (ret) {
2624                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2625                         event.status = -EINVAL;
2626                         break;
2627                 }
2628                 if (id_priv->qkey != rep->qkey) {
2629                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2630                         event.status = -EINVAL;
2631                         break;
2632                 }
2633                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2634                                      id_priv->id.route.path_rec,
2635                                      &event.param.ud.ah_attr);
2636                 event.param.ud.qp_num = rep->qpn;
2637                 event.param.ud.qkey = rep->qkey;
2638                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2639                 event.status = 0;
2640                 break;
2641         default:
2642                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2643                        ib_event->event);
2644                 goto out;
2645         }
2646
2647         ret = id_priv->id.event_handler(&id_priv->id, &event);
2648         if (ret) {
2649                 /* Destroy the CM ID by returning a non-zero value. */
2650                 id_priv->cm_id.ib = NULL;
2651                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2652                 mutex_unlock(&id_priv->handler_mutex);
2653                 rdma_destroy_id(&id_priv->id);
2654                 return ret;
2655         }
2656 out:
2657         mutex_unlock(&id_priv->handler_mutex);
2658         return ret;
2659 }
2660
2661 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2662                               struct rdma_conn_param *conn_param)
2663 {
2664         struct ib_cm_sidr_req_param req;
2665         struct ib_cm_id *id;
2666         int ret;
2667
2668         req.private_data_len = sizeof(struct cma_hdr) +
2669                                conn_param->private_data_len;
2670         if (req.private_data_len < conn_param->private_data_len)
2671                 return -EINVAL;
2672
2673         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2674         if (!req.private_data)
2675                 return -ENOMEM;
2676
2677         if (conn_param->private_data && conn_param->private_data_len)
2678                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2679                        conn_param->private_data, conn_param->private_data_len);
2680
2681         ret = cma_format_hdr((void *) req.private_data, id_priv);
2682         if (ret)
2683                 goto out;
2684
2685         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2686                              id_priv);
2687         if (IS_ERR(id)) {
2688                 ret = PTR_ERR(id);
2689                 goto out;
2690         }
2691         id_priv->cm_id.ib = id;
2692
2693         req.path = id_priv->id.route.path_rec;
2694         req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
2695         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2696         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2697
2698         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2699         if (ret) {
2700                 ib_destroy_cm_id(id_priv->cm_id.ib);
2701                 id_priv->cm_id.ib = NULL;
2702         }
2703 out:
2704         kfree(req.private_data);
2705         return ret;
2706 }
2707
2708 static int cma_connect_ib(struct rdma_id_private *id_priv,
2709                           struct rdma_conn_param *conn_param)
2710 {
2711         struct ib_cm_req_param req;
2712         struct rdma_route *route;
2713         void *private_data;
2714         struct ib_cm_id *id;
2715         int offset, ret;
2716
2717         memset(&req, 0, sizeof req);
2718         offset = cma_user_data_offset(id_priv->id.ps);
2719         req.private_data_len = offset + conn_param->private_data_len;
2720         if (req.private_data_len < conn_param->private_data_len)
2721                 return -EINVAL;
2722
2723         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2724         if (!private_data)
2725                 return -ENOMEM;
2726
2727         if (conn_param->private_data && conn_param->private_data_len)
2728                 memcpy(private_data + offset, conn_param->private_data,
2729                        conn_param->private_data_len);
2730
2731         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2732         if (IS_ERR(id)) {
2733                 ret = PTR_ERR(id);
2734                 goto out;
2735         }
2736         id_priv->cm_id.ib = id;
2737
2738         route = &id_priv->id.route;
2739         ret = cma_format_hdr(private_data, id_priv);
2740         if (ret)
2741                 goto out;
2742         req.private_data = private_data;
2743
2744         req.primary_path = &route->path_rec[0];
2745         if (route->num_paths == 2)
2746                 req.alternate_path = &route->path_rec[1];
2747
2748         req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
2749         req.qp_num = id_priv->qp_num;
2750         req.qp_type = id_priv->id.qp_type;
2751         req.starting_psn = id_priv->seq_num;
2752         req.responder_resources = conn_param->responder_resources;
2753         req.initiator_depth = conn_param->initiator_depth;
2754         req.flow_control = conn_param->flow_control;
2755         req.retry_count = min_t(u8, 7, conn_param->retry_count);
2756         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2757         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2758         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2759         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2760         req.srq = id_priv->srq ? 1 : 0;
2761
2762         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2763 out:
2764         if (ret && !IS_ERR(id)) {
2765                 ib_destroy_cm_id(id);
2766                 id_priv->cm_id.ib = NULL;
2767         }
2768
2769         kfree(private_data);
2770         return ret;
2771 }
2772
2773 static int cma_connect_iw(struct rdma_id_private *id_priv,
2774                           struct rdma_conn_param *conn_param)
2775 {
2776         struct iw_cm_id *cm_id;
2777         struct sockaddr_in* sin;
2778         int ret;
2779         struct iw_cm_conn_param iw_param;
2780
2781         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2782         if (IS_ERR(cm_id))
2783                 return PTR_ERR(cm_id);
2784
2785         id_priv->cm_id.iw = cm_id;
2786
2787         sin = (struct sockaddr_in *) cma_src_addr(id_priv);
2788         cm_id->local_addr = *sin;
2789
2790         sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
2791         cm_id->remote_addr = *sin;
2792
2793         ret = cma_modify_qp_rtr(id_priv, conn_param);
2794         if (ret)
2795                 goto out;
2796
2797         if (conn_param) {
2798                 iw_param.ord = conn_param->initiator_depth;
2799                 iw_param.ird = conn_param->responder_resources;
2800                 iw_param.private_data = conn_param->private_data;
2801                 iw_param.private_data_len = conn_param->private_data_len;
2802                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2803         } else {
2804                 memset(&iw_param, 0, sizeof iw_param);
2805                 iw_param.qpn = id_priv->qp_num;
2806         }
2807         ret = iw_cm_connect(cm_id, &iw_param);
2808 out:
2809         if (ret) {
2810                 iw_destroy_cm_id(cm_id);
2811                 id_priv->cm_id.iw = NULL;
2812         }
2813         return ret;
2814 }
2815
2816 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2817 {
2818         struct rdma_id_private *id_priv;
2819         int ret;
2820
2821         id_priv = container_of(id, struct rdma_id_private, id);
2822         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2823                 return -EINVAL;
2824
2825         if (!id->qp) {
2826                 id_priv->qp_num = conn_param->qp_num;
2827                 id_priv->srq = conn_param->srq;
2828         }
2829
2830         switch (rdma_node_get_transport(id->device->node_type)) {
2831         case RDMA_TRANSPORT_IB:
2832                 if (id->qp_type == IB_QPT_UD)
2833                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2834                 else
2835                         ret = cma_connect_ib(id_priv, conn_param);
2836                 break;
2837         case RDMA_TRANSPORT_IWARP:
2838                 ret = cma_connect_iw(id_priv, conn_param);
2839                 break;
2840         default:
2841                 ret = -ENOSYS;
2842                 break;
2843         }
2844         if (ret)
2845                 goto err;
2846
2847         return 0;
2848 err:
2849         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2850         return ret;
2851 }
2852 EXPORT_SYMBOL(rdma_connect);
2853
2854 static int cma_accept_ib(struct rdma_id_private *id_priv,
2855                          struct rdma_conn_param *conn_param)
2856 {
2857         struct ib_cm_rep_param rep;
2858         int ret;
2859
2860         ret = cma_modify_qp_rtr(id_priv, conn_param);
2861         if (ret)
2862                 goto out;
2863
2864         ret = cma_modify_qp_rts(id_priv, conn_param);
2865         if (ret)
2866                 goto out;
2867
2868         memset(&rep, 0, sizeof rep);
2869         rep.qp_num = id_priv->qp_num;
2870         rep.starting_psn = id_priv->seq_num;
2871         rep.private_data = conn_param->private_data;
2872         rep.private_data_len = conn_param->private_data_len;
2873         rep.responder_resources = conn_param->responder_resources;
2874         rep.initiator_depth = conn_param->initiator_depth;
2875         rep.failover_accepted = 0;
2876         rep.flow_control = conn_param->flow_control;
2877         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2878         rep.srq = id_priv->srq ? 1 : 0;
2879
2880         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2881 out:
2882         return ret;
2883 }
2884
2885 static int cma_accept_iw(struct rdma_id_private *id_priv,
2886                   struct rdma_conn_param *conn_param)
2887 {
2888         struct iw_cm_conn_param iw_param;
2889         int ret;
2890
2891         ret = cma_modify_qp_rtr(id_priv, conn_param);
2892         if (ret)
2893                 return ret;
2894
2895         iw_param.ord = conn_param->initiator_depth;
2896         iw_param.ird = conn_param->responder_resources;
2897         iw_param.private_data = conn_param->private_data;
2898         iw_param.private_data_len = conn_param->private_data_len;
2899         if (id_priv->id.qp) {
2900                 iw_param.qpn = id_priv->qp_num;
2901         } else
2902                 iw_param.qpn = conn_param->qp_num;
2903
2904         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2905 }
2906
2907 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2908                              enum ib_cm_sidr_status status,
2909                              const void *private_data, int private_data_len)
2910 {
2911         struct ib_cm_sidr_rep_param rep;
2912         int ret;
2913
2914         memset(&rep, 0, sizeof rep);
2915         rep.status = status;
2916         if (status == IB_SIDR_SUCCESS) {
2917                 ret = cma_set_qkey(id_priv);
2918                 if (ret)
2919                         return ret;
2920                 rep.qp_num = id_priv->qp_num;
2921                 rep.qkey = id_priv->qkey;
2922         }
2923         rep.private_data = private_data;
2924         rep.private_data_len = private_data_len;
2925
2926         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2927 }
2928
2929 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2930 {
2931         struct rdma_id_private *id_priv;
2932         int ret;
2933
2934         id_priv = container_of(id, struct rdma_id_private, id);
2935
2936         id_priv->owner = task_pid_nr(current);
2937
2938         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
2939                 return -EINVAL;
2940
2941         if (!id->qp && conn_param) {
2942                 id_priv->qp_num = conn_param->qp_num;
2943                 id_priv->srq = conn_param->srq;
2944         }
2945
2946         switch (rdma_node_get_transport(id->device->node_type)) {
2947         case RDMA_TRANSPORT_IB:
2948                 if (id->qp_type == IB_QPT_UD) {
2949                         if (conn_param)
2950                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2951                                                         conn_param->private_data,
2952                                                         conn_param->private_data_len);
2953                         else
2954                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2955                                                         NULL, 0);
2956                 } else {
2957                         if (conn_param)
2958                                 ret = cma_accept_ib(id_priv, conn_param);
2959                         else
2960                                 ret = cma_rep_recv(id_priv);
2961                 }
2962                 break;
2963         case RDMA_TRANSPORT_IWARP:
2964                 ret = cma_accept_iw(id_priv, conn_param);
2965                 break;
2966         default:
2967                 ret = -ENOSYS;
2968                 break;
2969         }
2970
2971         if (ret)
2972                 goto reject;
2973
2974         return 0;
2975 reject:
2976         cma_modify_qp_err(id_priv);
2977         rdma_reject(id, NULL, 0);
2978         return ret;
2979 }
2980 EXPORT_SYMBOL(rdma_accept);
2981
2982 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2983 {
2984         struct rdma_id_private *id_priv;
2985         int ret;
2986
2987         id_priv = container_of(id, struct rdma_id_private, id);
2988         if (!id_priv->cm_id.ib)
2989                 return -EINVAL;
2990
2991         switch (id->device->node_type) {
2992         case RDMA_NODE_IB_CA:
2993                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2994                 break;
2995         default:
2996                 ret = 0;
2997                 break;
2998         }
2999         return ret;
3000 }
3001 EXPORT_SYMBOL(rdma_notify);
3002
3003 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
3004                 u8 private_data_len)
3005 {
3006         struct rdma_id_private *id_priv;
3007         int ret;
3008
3009         id_priv = container_of(id, struct rdma_id_private, id);
3010         if (!id_priv->cm_id.ib)
3011                 return -EINVAL;
3012
3013         switch (rdma_node_get_transport(id->device->node_type)) {
3014         case RDMA_TRANSPORT_IB:
3015                 if (id->qp_type == IB_QPT_UD)
3016                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
3017                                                 private_data, private_data_len);
3018                 else
3019                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
3020                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
3021                                              0, private_data, private_data_len);
3022                 break;
3023         case RDMA_TRANSPORT_IWARP:
3024                 ret = iw_cm_reject(id_priv->cm_id.iw,
3025                                    private_data, private_data_len);
3026                 break;
3027         default:
3028                 ret = -ENOSYS;
3029                 break;
3030         }
3031         return ret;
3032 }
3033 EXPORT_SYMBOL(rdma_reject);
3034
3035 int rdma_disconnect(struct rdma_cm_id *id)
3036 {
3037         struct rdma_id_private *id_priv;
3038         int ret;
3039
3040         id_priv = container_of(id, struct rdma_id_private, id);
3041         if (!id_priv->cm_id.ib)
3042                 return -EINVAL;
3043
3044         switch (rdma_node_get_transport(id->device->node_type)) {
3045         case RDMA_TRANSPORT_IB:
3046                 ret = cma_modify_qp_err(id_priv);
3047                 if (ret)
3048                         goto out;
3049                 /* Initiate or respond to a disconnect. */
3050                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
3051                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
3052                 break;
3053         case RDMA_TRANSPORT_IWARP:
3054                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
3055                 break;
3056         default:
3057                 ret = -EINVAL;
3058                 break;
3059         }
3060 out:
3061         return ret;
3062 }
3063 EXPORT_SYMBOL(rdma_disconnect);
3064
3065 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
3066 {
3067         struct rdma_id_private *id_priv;
3068         struct cma_multicast *mc = multicast->context;
3069         struct rdma_cm_event event;
3070         int ret;
3071
3072         id_priv = mc->id_priv;
3073         if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
3074             cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
3075                 return 0;
3076
3077         mutex_lock(&id_priv->qp_mutex);
3078         if (!status && id_priv->id.qp)
3079                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
3080                                          be16_to_cpu(multicast->rec.mlid));
3081         mutex_unlock(&id_priv->qp_mutex);
3082
3083         memset(&event, 0, sizeof event);
3084         event.status = status;
3085         event.param.ud.private_data = mc->context;
3086         if (!status) {
3087                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
3088                 ib_init_ah_from_mcmember(id_priv->id.device,
3089                                          id_priv->id.port_num, &multicast->rec,
3090                                          &event.param.ud.ah_attr);
3091                 event.param.ud.qp_num = 0xFFFFFF;
3092                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
3093         } else
3094                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
3095
3096         ret = id_priv->id.event_handler(&id_priv->id, &event);
3097         if (ret) {
3098                 cma_exch(id_priv, RDMA_CM_DESTROYING);
3099                 mutex_unlock(&id_priv->handler_mutex);
3100                 rdma_destroy_id(&id_priv->id);
3101                 return 0;
3102         }
3103
3104         mutex_unlock(&id_priv->handler_mutex);
3105         return 0;
3106 }
3107
3108 static void cma_set_mgid(struct rdma_id_private *id_priv,
3109                          struct sockaddr *addr, union ib_gid *mgid)
3110 {
3111         unsigned char mc_map[MAX_ADDR_LEN];
3112         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3113         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3114         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3115
3116         if (cma_any_addr(addr)) {
3117                 memset(mgid, 0, sizeof *mgid);
3118         } else if ((addr->sa_family == AF_INET6) &&
3119                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3120                                                                  0xFF10A01B)) {
3121                 /* IPv6 address is an SA assigned MGID. */
3122                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3123         } else if ((addr->sa_family == AF_INET6)) {
3124                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3125                 if (id_priv->id.ps == RDMA_PS_UDP)
3126                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3127                 *mgid = *(union ib_gid *) (mc_map + 4);
3128         } else {
3129                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3130                 if (id_priv->id.ps == RDMA_PS_UDP)
3131                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3132                 *mgid = *(union ib_gid *) (mc_map + 4);
3133         }
3134 }
3135
3136 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3137                                  struct cma_multicast *mc)
3138 {
3139         struct ib_sa_mcmember_rec rec;
3140         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3141         ib_sa_comp_mask comp_mask;
3142         int ret;
3143
3144         ib_addr_get_mgid(dev_addr, &rec.mgid);
3145         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
3146                                      &rec.mgid, &rec);
3147         if (ret)
3148                 return ret;
3149
3150         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3151         if (id_priv->id.ps == RDMA_PS_UDP)
3152                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3153         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3154         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3155         rec.join_state = 1;
3156
3157         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3158                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3159                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3160                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
3161                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3162
3163         if (id_priv->id.ps == RDMA_PS_IPOIB)
3164                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3165                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3166                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3167                              IB_SA_MCMEMBER_REC_MTU |
3168                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
3169
3170         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3171                                                 id_priv->id.port_num, &rec,
3172                                                 comp_mask, GFP_KERNEL,
3173                                                 cma_ib_mc_handler, mc);
3174         return PTR_RET(mc->multicast.ib);
3175 }
3176
3177 static void iboe_mcast_work_handler(struct work_struct *work)
3178 {
3179         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3180         struct cma_multicast *mc = mw->mc;
3181         struct ib_sa_multicast *m = mc->multicast.ib;
3182
3183         mc->multicast.ib->context = mc;
3184         cma_ib_mc_handler(0, m);
3185         kref_put(&mc->mcref, release_mc);
3186         kfree(mw);
3187 }
3188
3189 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3190 {
3191         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3192         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3193
3194         if (cma_any_addr(addr)) {
3195                 memset(mgid, 0, sizeof *mgid);
3196         } else if (addr->sa_family == AF_INET6) {
3197                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3198         } else {
3199                 mgid->raw[0] = 0xff;
3200                 mgid->raw[1] = 0x0e;
3201                 mgid->raw[2] = 0;
3202                 mgid->raw[3] = 0;
3203                 mgid->raw[4] = 0;
3204                 mgid->raw[5] = 0;
3205                 mgid->raw[6] = 0;
3206                 mgid->raw[7] = 0;
3207                 mgid->raw[8] = 0;
3208                 mgid->raw[9] = 0;
3209                 mgid->raw[10] = 0xff;
3210                 mgid->raw[11] = 0xff;
3211                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3212         }
3213 }
3214
3215 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3216                                    struct cma_multicast *mc)
3217 {
3218         struct iboe_mcast_work *work;
3219         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3220         int err;
3221         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3222         struct net_device *ndev = NULL;
3223
3224         if (cma_zero_addr((struct sockaddr *)&mc->addr))
3225                 return -EINVAL;
3226
3227         work = kzalloc(sizeof *work, GFP_KERNEL);
3228         if (!work)
3229                 return -ENOMEM;
3230
3231         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3232         if (!mc->multicast.ib) {
3233                 err = -ENOMEM;
3234                 goto out1;
3235         }
3236
3237         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3238
3239         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3240         if (id_priv->id.ps == RDMA_PS_UDP)
3241                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3242
3243         if (dev_addr->bound_dev_if)
3244                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3245         if (!ndev) {
3246                 err = -ENODEV;
3247                 goto out2;
3248         }
3249         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3250         mc->multicast.ib->rec.hop_limit = 1;
3251         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3252         dev_put(ndev);
3253         if (!mc->multicast.ib->rec.mtu) {
3254                 err = -EINVAL;
3255                 goto out2;
3256         }
3257         iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3258         work->id = id_priv;
3259         work->mc = mc;
3260         INIT_WORK(&work->work, iboe_mcast_work_handler);
3261         kref_get(&mc->mcref);
3262         queue_work(cma_wq, &work->work);
3263
3264         return 0;
3265
3266 out2:
3267         kfree(mc->multicast.ib);
3268 out1:
3269         kfree(work);
3270         return err;
3271 }
3272
3273 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3274                         void *context)
3275 {
3276         struct rdma_id_private *id_priv;
3277         struct cma_multicast *mc;
3278         int ret;
3279
3280         id_priv = container_of(id, struct rdma_id_private, id);
3281         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3282             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3283                 return -EINVAL;
3284
3285         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3286         if (!mc)
3287                 return -ENOMEM;
3288
3289         memcpy(&mc->addr, addr, rdma_addr_size(addr));
3290         mc->context = context;
3291         mc->id_priv = id_priv;
3292
3293         spin_lock(&id_priv->lock);
3294         list_add(&mc->list, &id_priv->mc_list);
3295         spin_unlock(&id_priv->lock);
3296
3297         switch (rdma_node_get_transport(id->device->node_type)) {
3298         case RDMA_TRANSPORT_IB:
3299                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3300                 case IB_LINK_LAYER_INFINIBAND:
3301                         ret = cma_join_ib_multicast(id_priv, mc);
3302                         break;
3303                 case IB_LINK_LAYER_ETHERNET:
3304                         kref_init(&mc->mcref);
3305                         ret = cma_iboe_join_multicast(id_priv, mc);
3306                         break;
3307                 default:
3308                         ret = -EINVAL;
3309                 }
3310                 break;
3311         default:
3312                 ret = -ENOSYS;
3313                 break;
3314         }
3315
3316         if (ret) {
3317                 spin_lock_irq(&id_priv->lock);
3318                 list_del(&mc->list);
3319                 spin_unlock_irq(&id_priv->lock);
3320                 kfree(mc);
3321         }
3322         return ret;
3323 }
3324 EXPORT_SYMBOL(rdma_join_multicast);
3325
3326 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3327 {
3328         struct rdma_id_private *id_priv;
3329         struct cma_multicast *mc;
3330
3331         id_priv = container_of(id, struct rdma_id_private, id);
3332         spin_lock_irq(&id_priv->lock);
3333         list_for_each_entry(mc, &id_priv->mc_list, list) {
3334                 if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
3335                         list_del(&mc->list);
3336                         spin_unlock_irq(&id_priv->lock);
3337
3338                         if (id->qp)
3339                                 ib_detach_mcast(id->qp,
3340                                                 &mc->multicast.ib->rec.mgid,
3341                                                 be16_to_cpu(mc->multicast.ib->rec.mlid));
3342                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3343                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3344                                 case IB_LINK_LAYER_INFINIBAND:
3345                                         ib_sa_free_multicast(mc->multicast.ib);
3346                                         kfree(mc);
3347                                         break;
3348                                 case IB_LINK_LAYER_ETHERNET:
3349                                         kref_put(&mc->mcref, release_mc);
3350                                         break;
3351                                 default:
3352                                         break;
3353                                 }
3354                         }
3355                         return;
3356                 }
3357         }
3358         spin_unlock_irq(&id_priv->lock);
3359 }
3360 EXPORT_SYMBOL(rdma_leave_multicast);
3361
3362 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3363 {
3364         struct rdma_dev_addr *dev_addr;
3365         struct cma_ndev_work *work;
3366
3367         dev_addr = &id_priv->id.route.addr.dev_addr;
3368
3369         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3370             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3371                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3372                        ndev->name, &id_priv->id);
3373                 work = kzalloc(sizeof *work, GFP_KERNEL);
3374                 if (!work)
3375                         return -ENOMEM;
3376
3377                 INIT_WORK(&work->work, cma_ndev_work_handler);
3378                 work->id = id_priv;
3379                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3380                 atomic_inc(&id_priv->refcount);
3381                 queue_work(cma_wq, &work->work);
3382         }
3383
3384         return 0;
3385 }
3386
3387 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3388                                void *ptr)
3389 {
3390         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3391         struct cma_device *cma_dev;
3392         struct rdma_id_private *id_priv;
3393         int ret = NOTIFY_DONE;
3394
3395         if (dev_net(ndev) != &init_net)
3396                 return NOTIFY_DONE;
3397
3398         if (event != NETDEV_BONDING_FAILOVER)
3399                 return NOTIFY_DONE;
3400
3401         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3402                 return NOTIFY_DONE;
3403
3404         mutex_lock(&lock);
3405         list_for_each_entry(cma_dev, &dev_list, list)
3406                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3407                         ret = cma_netdev_change(ndev, id_priv);
3408                         if (ret)
3409                                 goto out;
3410                 }
3411
3412 out:
3413         mutex_unlock(&lock);
3414         return ret;
3415 }
3416
3417 static struct notifier_block cma_nb = {
3418         .notifier_call = cma_netdev_callback
3419 };
3420
3421 static void cma_add_one(struct ib_device *device)
3422 {
3423         struct cma_device *cma_dev;
3424         struct rdma_id_private *id_priv;
3425
3426         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3427         if (!cma_dev)
3428                 return;
3429
3430         cma_dev->device = device;
3431
3432         init_completion(&cma_dev->comp);
3433         atomic_set(&cma_dev->refcount, 1);
3434         INIT_LIST_HEAD(&cma_dev->id_list);
3435         ib_set_client_data(device, &cma_client, cma_dev);
3436
3437         mutex_lock(&lock);
3438         list_add_tail(&cma_dev->list, &dev_list);
3439         list_for_each_entry(id_priv, &listen_any_list, list)
3440                 cma_listen_on_dev(id_priv, cma_dev);
3441         mutex_unlock(&lock);
3442 }
3443
3444 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3445 {
3446         struct rdma_cm_event event;
3447         enum rdma_cm_state state;
3448         int ret = 0;
3449
3450         /* Record that we want to remove the device */
3451         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3452         if (state == RDMA_CM_DESTROYING)
3453                 return 0;
3454
3455         cma_cancel_operation(id_priv, state);
3456         mutex_lock(&id_priv->handler_mutex);
3457
3458         /* Check for destruction from another callback. */
3459         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3460                 goto out;
3461
3462         memset(&event, 0, sizeof event);
3463         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3464         ret = id_priv->id.event_handler(&id_priv->id, &event);
3465 out:
3466         mutex_unlock(&id_priv->handler_mutex);
3467         return ret;
3468 }
3469
3470 static void cma_process_remove(struct cma_device *cma_dev)
3471 {
3472         struct rdma_id_private *id_priv;
3473         int ret;
3474
3475         mutex_lock(&lock);
3476         while (!list_empty(&cma_dev->id_list)) {
3477                 id_priv = list_entry(cma_dev->id_list.next,
3478                                      struct rdma_id_private, list);
3479
3480                 list_del(&id_priv->listen_list);
3481                 list_del_init(&id_priv->list);
3482                 atomic_inc(&id_priv->refcount);
3483                 mutex_unlock(&lock);
3484
3485                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3486                 cma_deref_id(id_priv);
3487                 if (ret)
3488                         rdma_destroy_id(&id_priv->id);
3489
3490                 mutex_lock(&lock);
3491         }
3492         mutex_unlock(&lock);
3493
3494         cma_deref_dev(cma_dev);
3495         wait_for_completion(&cma_dev->comp);
3496 }
3497
3498 static void cma_remove_one(struct ib_device *device)
3499 {
3500         struct cma_device *cma_dev;
3501
3502         cma_dev = ib_get_client_data(device, &cma_client);
3503         if (!cma_dev)
3504                 return;
3505
3506         mutex_lock(&lock);
3507         list_del(&cma_dev->list);
3508         mutex_unlock(&lock);
3509
3510         cma_process_remove(cma_dev);
3511         kfree(cma_dev);
3512 }
3513
3514 static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
3515 {
3516         struct nlmsghdr *nlh;
3517         struct rdma_cm_id_stats *id_stats;
3518         struct rdma_id_private *id_priv;
3519         struct rdma_cm_id *id = NULL;
3520         struct cma_device *cma_dev;
3521         int i_dev = 0, i_id = 0;
3522
3523         /*
3524          * We export all of the IDs as a sequence of messages.  Each
3525          * ID gets its own netlink message.
3526          */
3527         mutex_lock(&lock);
3528
3529         list_for_each_entry(cma_dev, &dev_list, list) {
3530                 if (i_dev < cb->args[0]) {
3531                         i_dev++;
3532                         continue;
3533                 }
3534
3535                 i_id = 0;
3536                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3537                         if (i_id < cb->args[1]) {
3538                                 i_id++;
3539                                 continue;
3540                         }
3541
3542                         id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
3543                                                 sizeof *id_stats, RDMA_NL_RDMA_CM,
3544                                                 RDMA_NL_RDMA_CM_ID_STATS);
3545                         if (!id_stats)
3546                                 goto out;
3547
3548                         memset(id_stats, 0, sizeof *id_stats);
3549                         id = &id_priv->id;
3550                         id_stats->node_type = id->route.addr.dev_addr.dev_type;
3551                         id_stats->port_num = id->port_num;
3552                         id_stats->bound_dev_if =
3553                                 id->route.addr.dev_addr.bound_dev_if;
3554
3555                         if (cma_family(id_priv) == AF_INET) {
3556                                 if (ibnl_put_attr(skb, nlh,
3557                                                   sizeof(struct sockaddr_in),
3558                                                   cma_src_addr(id_priv),
3559                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3560                                         goto out;
3561                                 }
3562                                 if (ibnl_put_attr(skb, nlh,
3563                                                   sizeof(struct sockaddr_in),
3564                                                   cma_dst_addr(id_priv),
3565                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3566                                         goto out;
3567                                 }
3568                         } else if (cma_family(id_priv) == AF_INET6) {
3569                                 if (ibnl_put_attr(skb, nlh,
3570                                                   sizeof(struct sockaddr_in6),
3571                                                   cma_src_addr(id_priv),
3572                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3573                                         goto out;
3574                                 }
3575                                 if (ibnl_put_attr(skb, nlh,
3576                                                   sizeof(struct sockaddr_in6),
3577                                                   cma_dst_addr(id_priv),
3578                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3579                                         goto out;
3580                                 }
3581                         }
3582
3583                         id_stats->pid           = id_priv->owner;
3584                         id_stats->port_space    = id->ps;
3585                         id_stats->cm_state      = id_priv->state;
3586                         id_stats->qp_num        = id_priv->qp_num;
3587                         id_stats->qp_type       = id->qp_type;
3588
3589                         i_id++;
3590                 }
3591
3592                 cb->args[1] = 0;
3593                 i_dev++;
3594         }
3595
3596 out:
3597         mutex_unlock(&lock);
3598         cb->args[0] = i_dev;
3599         cb->args[1] = i_id;
3600
3601         return skb->len;
3602 }
3603
3604 static const struct ibnl_client_cbs cma_cb_table[] = {
3605         [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
3606                                        .module = THIS_MODULE },
3607 };
3608
3609 static int __init cma_init(void)
3610 {
3611         int ret;
3612
3613         cma_wq = create_singlethread_workqueue("rdma_cm");
3614         if (!cma_wq)
3615                 return -ENOMEM;
3616
3617         ib_sa_register_client(&sa_client);
3618         rdma_addr_register_client(&addr_client);
3619         register_netdevice_notifier(&cma_nb);
3620
3621         ret = ib_register_client(&cma_client);
3622         if (ret)
3623                 goto err;
3624
3625         if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
3626                 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
3627
3628         return 0;
3629
3630 err:
3631         unregister_netdevice_notifier(&cma_nb);
3632         rdma_addr_unregister_client(&addr_client);
3633         ib_sa_unregister_client(&sa_client);
3634         destroy_workqueue(cma_wq);
3635         return ret;
3636 }
3637
3638 static void __exit cma_cleanup(void)
3639 {
3640         ibnl_remove_client(RDMA_NL_RDMA_CM);
3641         ib_unregister_client(&cma_client);
3642         unregister_netdevice_notifier(&cma_nb);
3643         rdma_addr_unregister_client(&addr_client);
3644         ib_sa_unregister_client(&sa_client);
3645         destroy_workqueue(cma_wq);
3646         idr_destroy(&sdp_ps);
3647         idr_destroy(&tcp_ps);
3648         idr_destroy(&udp_ps);
3649         idr_destroy(&ipoib_ps);
3650         idr_destroy(&ib_ps);
3651 }
3652
3653 module_init(cma_init);
3654 module_exit(cma_cleanup);