drivers/infiniband/core/multicast.c

   1 /*
   2  * Copyright (c) 2006 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/completion.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/export.h>
  38 #include <linux/slab.h>
  39 #include <linux/bitops.h>
  40 #include <linux/random.h>
  41
  42 #include <rdma/ib_cache.h>
  43 #include "sa.h"
  44
  45 static void mcast_add_one(struct ib_device *device);
  46 static void mcast_remove_one(struct ib_device *device, void *client_data);
  47
  48 static struct ib_client mcast_client = {
  49         .name   = "ib_multicast",
  50         .add    = mcast_add_one,
  51         .remove = mcast_remove_one
  52 };
  53
  54 static struct ib_sa_client      sa_client;
  55 static struct workqueue_struct  *mcast_wq;
  56 static union ib_gid mgid0;
  57
  58 struct mcast_device;
  59
  60 struct mcast_port {
  61         struct mcast_device     *dev;
  62         spinlock_t              lock;
  63         struct rb_root          table;
  64         atomic_t                refcount;
  65         struct completion       comp;
  66         u8                      port_num;
  67 };
  68
  69 struct mcast_device {
  70         struct ib_device        *device;
  71         struct ib_event_handler event_handler;
  72         int                     start_port;
  73         int                     end_port;
  74         struct mcast_port       port[0];
  75 };
  76
  77 enum mcast_state {
  78         MCAST_JOINING,
  79         MCAST_MEMBER,
  80         MCAST_ERROR,
  81 };
  82
  83 enum mcast_group_state {
  84         MCAST_IDLE,
  85         MCAST_BUSY,
  86         MCAST_GROUP_ERROR,
  87         MCAST_PKEY_EVENT
  88 };
  89
  90 enum {
  91         MCAST_INVALID_PKEY_INDEX = 0xFFFF
  92 };
  93
  94 struct mcast_member;
  95
  96 /*
  97 * There are 4 types of join states:
  98 * FullMember, NonMember, SendOnlyNonMember, SendOnlyFullMember.
  99 */
 100 enum {
 101         FULLMEMBER_JOIN,
 102         NONMEMBER_JOIN,
 103         SENDONLY_NONMEBER_JOIN,
 104         SENDONLY_FULLMEMBER_JOIN,
 105         NUM_JOIN_MEMBERSHIP_TYPES,
 106 };
 107
 108 struct mcast_group {
 109         struct ib_sa_mcmember_rec rec;
 110         struct rb_node          node;
 111         struct mcast_port       *port;
 112         spinlock_t              lock;
 113         struct work_struct      work;
 114         struct list_head        pending_list;
 115         struct list_head        active_list;
 116         struct mcast_member     *last_join;
 117         int                     members[NUM_JOIN_MEMBERSHIP_TYPES];
 118         atomic_t                refcount;
 119         enum mcast_group_state  state;
 120         struct ib_sa_query      *query;
 121         int                     query_id;
 122         u16                     pkey_index;
 123         u8                      leave_state;
 124         int                     retries;
 125 };
 126
 127 struct mcast_member {
 128         struct ib_sa_multicast  multicast;
 129         struct ib_sa_client     *client;
 130         struct mcast_group      *group;
 131         struct list_head        list;
 132         enum mcast_state        state;
 133         atomic_t                refcount;
 134         struct completion       comp;
 135 };
 136
 137 static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 138                          void *context);
 139 static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
 140                           void *context);
 141
 142 static struct mcast_group *mcast_find(struct mcast_port *port,
 143                                       union ib_gid *mgid)
 144 {
 145         struct rb_node *node = port->table.rb_node;
 146         struct mcast_group *group;
 147         int ret;
 148
 149         while (node) {
 150                 group = rb_entry(node, struct mcast_group, node);
 151                 ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid);
 152                 if (!ret)
 153                         return group;
 154
 155                 if (ret < 0)
 156                         node = node->rb_left;
 157                 else
 158                         node = node->rb_right;
 159         }
 160         return NULL;
 161 }
 162
 163 static struct mcast_group *mcast_insert(struct mcast_port *port,
 164                                         struct mcast_group *group,
 165                                         int allow_duplicates)
 166 {
 167         struct rb_node **link = &port->table.rb_node;
 168         struct rb_node *parent = NULL;
 169         struct mcast_group *cur_group;
 170         int ret;
 171
 172         while (*link) {
 173                 parent = *link;
 174                 cur_group = rb_entry(parent, struct mcast_group, node);
 175
 176                 ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw,
 177                              sizeof group->rec.mgid);
 178                 if (ret < 0)
 179                         link = &(*link)->rb_left;
 180                 else if (ret > 0)
 181                         link = &(*link)->rb_right;
 182                 else if (allow_duplicates)
 183                         link = &(*link)->rb_left;
 184                 else
 185                         return cur_group;
 186         }
 187         rb_link_node(&group->node, parent, link);
 188         rb_insert_color(&group->node, &port->table);
 189         return NULL;
 190 }
 191
 192 static void deref_port(struct mcast_port *port)
 193 {
 194         if (atomic_dec_and_test(&port->refcount))
 195                 complete(&port->comp);
 196 }
 197
 198 static void release_group(struct mcast_group *group)
 199 {
 200         struct mcast_port *port = group->port;
 201         unsigned long flags;
 202
 203         spin_lock_irqsave(&port->lock, flags);
 204         if (atomic_dec_and_test(&group->refcount)) {
 205                 rb_erase(&group->node, &port->table);
 206                 spin_unlock_irqrestore(&port->lock, flags);
 207                 kfree(group);
 208                 deref_port(port);
 209         } else
 210                 spin_unlock_irqrestore(&port->lock, flags);
 211 }
 212
 213 static void deref_member(struct mcast_member *member)
 214 {
 215         if (atomic_dec_and_test(&member->refcount))
 216                 complete(&member->comp);
 217 }
 218
 219 static void queue_join(struct mcast_member *member)
 220 {
 221         struct mcast_group *group = member->group;
 222         unsigned long flags;
 223
 224         spin_lock_irqsave(&group->lock, flags);
 225         list_add_tail(&member->list, &group->pending_list);
 226         if (group->state == MCAST_IDLE) {
 227                 group->state = MCAST_BUSY;
 228                 atomic_inc(&group->refcount);
 229                 queue_work(mcast_wq, &group->work);
 230         }
 231         spin_unlock_irqrestore(&group->lock, flags);
 232 }
 233
 234 /*
 235  * A multicast group has four types of members: full member, non member,
 236  * sendonly non member and sendonly full member.
 237  * We need to keep track of the number of members of each
 238  * type based on their join state.  Adjust the number of members the belong to
 239  * the specified join states.
 240  */
 241 static void adjust_membership(struct mcast_group *group, u8 join_state, int inc)
 242 {
 243         int i;
 244
 245         for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++, join_state >>= 1)
 246                 if (join_state & 0x1)
 247                         group->members[i] += inc;
 248 }
 249
 250 /*
 251  * If a multicast group has zero members left for a particular join state, but
 252  * the group is still a member with the SA, we need to leave that join state.
 253  * Determine which join states we still belong to, but that do not have any
 254  * active members.
 255  */
 256 static u8 get_leave_state(struct mcast_group *group)
 257 {
 258         u8 leave_state = 0;
 259         int i;
 260
 261         for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++)
 262                 if (!group->members[i])
 263                         leave_state |= (0x1 << i);
 264
 265         return leave_state & group->rec.join_state;
 266 }
 267
 268 static int check_selector(ib_sa_comp_mask comp_mask,
 269                           ib_sa_comp_mask selector_mask,
 270                           ib_sa_comp_mask value_mask,
 271                           u8 selector, u8 src_value, u8 dst_value)
 272 {
 273         int err;
 274
 275         if (!(comp_mask & selector_mask) || !(comp_mask & value_mask))
 276                 return 0;
 277
 278         switch (selector) {
 279         case IB_SA_GT:
 280                 err = (src_value <= dst_value);
 281                 break;
 282         case IB_SA_LT:
 283                 err = (src_value >= dst_value);
 284                 break;
 285         case IB_SA_EQ:
 286                 err = (src_value != dst_value);
 287                 break;
 288         default:
 289                 err = 0;
 290                 break;
 291         }
 292
 293         return err;
 294 }
 295
 296 static int cmp_rec(struct ib_sa_mcmember_rec *src,
 297                    struct ib_sa_mcmember_rec *dst, ib_sa_comp_mask comp_mask)
 298 {
 299         /* MGID must already match */
 300
 301         if (comp_mask & IB_SA_MCMEMBER_REC_PORT_GID &&
 302             memcmp(&src->port_gid, &dst->port_gid, sizeof src->port_gid))
 303                 return -EINVAL;
 304         if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey)
 305                 return -EINVAL;
 306         if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid)
 307                 return -EINVAL;
 308         if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR,
 309                            IB_SA_MCMEMBER_REC_MTU, dst->mtu_selector,
 310                            src->mtu, dst->mtu))
 311                 return -EINVAL;
 312         if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS &&
 313             src->traffic_class != dst->traffic_class)
 314                 return -EINVAL;
 315         if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey)
 316                 return -EINVAL;
 317         if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR,
 318                            IB_SA_MCMEMBER_REC_RATE, dst->rate_selector,
 319                            src->rate, dst->rate))
 320                 return -EINVAL;
 321         if (check_selector(comp_mask,
 322                            IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR,
 323                            IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME,
 324                            dst->packet_life_time_selector,
 325                            src->packet_life_time, dst->packet_life_time))
 326                 return -EINVAL;
 327         if (comp_mask & IB_SA_MCMEMBER_REC_SL && src->sl != dst->sl)
 328                 return -EINVAL;
 329         if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL &&
 330             src->flow_label != dst->flow_label)
 331                 return -EINVAL;
 332         if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT &&
 333             src->hop_limit != dst->hop_limit)
 334                 return -EINVAL;
 335         if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && src->scope != dst->scope)
 336                 return -EINVAL;
 337
 338         /* join_state checked separately, proxy_join ignored */
 339
 340         return 0;
 341 }
 342
 343 static int send_join(struct mcast_group *group, struct mcast_member *member)
 344 {
 345         struct mcast_port *port = group->port;
 346         int ret;
 347
 348         group->last_join = member;
 349         ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
 350                                        port->port_num, IB_MGMT_METHOD_SET,
 351                                        &member->multicast.rec,
 352                                        member->multicast.comp_mask,
 353                                        3000, GFP_KERNEL, join_handler, group,
 354                                        &group->query);
 355         if (ret >= 0) {
 356                 group->query_id = ret;
 357                 ret = 0;
 358         }
 359         return ret;
 360 }
 361
 362 static int send_leave(struct mcast_group *group, u8 leave_state)
 363 {
 364         struct mcast_port *port = group->port;
 365         struct ib_sa_mcmember_rec rec;
 366         int ret;
 367
 368         rec = group->rec;
 369         rec.join_state = leave_state;
 370         group->leave_state = leave_state;
 371
 372         ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
 373                                        port->port_num, IB_SA_METHOD_DELETE, &rec,
 374                                        IB_SA_MCMEMBER_REC_MGID     |
 375                                        IB_SA_MCMEMBER_REC_PORT_GID |
 376                                        IB_SA_MCMEMBER_REC_JOIN_STATE,
 377                                        3000, GFP_KERNEL, leave_handler,
 378                                        group, &group->query);
 379         if (ret >= 0) {
 380                 group->query_id = ret;
 381                 ret = 0;
 382         }
 383         return ret;
 384 }
 385
 386 static void join_group(struct mcast_group *group, struct mcast_member *member,
 387                        u8 join_state)
 388 {
 389         member->state = MCAST_MEMBER;
 390         adjust_membership(group, join_state, 1);
 391         group->rec.join_state |= join_state;
 392         member->multicast.rec = group->rec;
 393         member->multicast.rec.join_state = join_state;
 394         list_move(&member->list, &group->active_list);
 395 }
 396
 397 static int fail_join(struct mcast_group *group, struct mcast_member *member,
 398                      int status)
 399 {
 400         spin_lock_irq(&group->lock);
 401         list_del_init(&member->list);
 402         spin_unlock_irq(&group->lock);
 403         return member->multicast.callback(status, &member->multicast);
 404 }
 405
 406 static void process_group_error(struct mcast_group *group)
 407 {
 408         struct mcast_member *member;
 409         int ret = 0;
 410         u16 pkey_index;
 411
 412         if (group->state == MCAST_PKEY_EVENT)
 413                 ret = ib_find_pkey(group->port->dev->device,
 414                                    group->port->port_num,
 415                                    be16_to_cpu(group->rec.pkey), &pkey_index);
 416
 417         spin_lock_irq(&group->lock);
 418         if (group->state == MCAST_PKEY_EVENT && !ret &&
 419             group->pkey_index == pkey_index)
 420                 goto out;
 421
 422         while (!list_empty(&group->active_list)) {
 423                 member = list_entry(group->active_list.next,
 424                                     struct mcast_member, list);
 425                 atomic_inc(&member->refcount);
 426                 list_del_init(&member->list);
 427                 adjust_membership(group, member->multicast.rec.join_state, -1);
 428                 member->state = MCAST_ERROR;
 429                 spin_unlock_irq(&group->lock);
 430
 431                 ret = member->multicast.callback(-ENETRESET,
 432                                                  &member->multicast);
 433                 deref_member(member);
 434                 if (ret)
 435                         ib_sa_free_multicast(&member->multicast);
 436                 spin_lock_irq(&group->lock);
 437         }
 438
 439         group->rec.join_state = 0;
 440 out:
 441         group->state = MCAST_BUSY;
 442         spin_unlock_irq(&group->lock);
 443 }
 444
 445 static void mcast_work_handler(struct work_struct *work)
 446 {
 447         struct mcast_group *group;
 448         struct mcast_member *member;
 449         struct ib_sa_multicast *multicast;
 450         int status, ret;
 451         u8 join_state;
 452
 453         group = container_of(work, typeof(*group), work);
 454 retest:
 455         spin_lock_irq(&group->lock);
 456         while (!list_empty(&group->pending_list) ||
 457                (group->state != MCAST_BUSY)) {
 458
 459                 if (group->state != MCAST_BUSY) {
 460                         spin_unlock_irq(&group->lock);
 461                         process_group_error(group);
 462                         goto retest;
 463                 }
 464
 465                 member = list_entry(group->pending_list.next,
 466                                     struct mcast_member, list);
 467                 multicast = &member->multicast;
 468                 join_state = multicast->rec.join_state;
 469                 atomic_inc(&member->refcount);
 470
 471                 if (join_state == (group->rec.join_state & join_state)) {
 472                         status = cmp_rec(&group->rec, &multicast->rec,
 473                                          multicast->comp_mask);
 474                         if (!status)
 475                                 join_group(group, member, join_state);
 476                         else
 477                                 list_del_init(&member->list);
 478                         spin_unlock_irq(&group->lock);
 479                         ret = multicast->callback(status, multicast);
 480                 } else {
 481                         spin_unlock_irq(&group->lock);
 482                         status = send_join(group, member);
 483                         if (!status) {
 484                                 deref_member(member);
 485                                 return;
 486                         }
 487                         ret = fail_join(group, member, status);
 488                 }
 489
 490                 deref_member(member);
 491                 if (ret)
 492                         ib_sa_free_multicast(&member->multicast);
 493                 spin_lock_irq(&group->lock);
 494         }
 495
 496         join_state = get_leave_state(group);
 497         if (join_state) {
 498                 group->rec.join_state &= ~join_state;
 499                 spin_unlock_irq(&group->lock);
 500                 if (send_leave(group, join_state))
 501                         goto retest;
 502         } else {
 503                 group->state = MCAST_IDLE;
 504                 spin_unlock_irq(&group->lock);
 505                 release_group(group);
 506         }
 507 }
 508
 509 /*
 510  * Fail a join request if it is still active - at the head of the pending queue.
 511  */
 512 static void process_join_error(struct mcast_group *group, int status)
 513 {
 514         struct mcast_member *member;
 515         int ret;
 516
 517         spin_lock_irq(&group->lock);
 518         member = list_entry(group->pending_list.next,
 519                             struct mcast_member, list);
 520         if (group->last_join == member) {
 521                 atomic_inc(&member->refcount);
 522                 list_del_init(&member->list);
 523                 spin_unlock_irq(&group->lock);
 524                 ret = member->multicast.callback(status, &member->multicast);
 525                 deref_member(member);
 526                 if (ret)
 527                         ib_sa_free_multicast(&member->multicast);
 528         } else
 529                 spin_unlock_irq(&group->lock);
 530 }
 531
 532 static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 533                          void *context)
 534 {
 535         struct mcast_group *group = context;
 536         u16 pkey_index = MCAST_INVALID_PKEY_INDEX;
 537
 538         if (status)
 539                 process_join_error(group, status);
 540         else {
 541                 int mgids_changed, is_mgid0;
 542                 ib_find_pkey(group->port->dev->device, group->port->port_num,
 543                              be16_to_cpu(rec->pkey), &pkey_index);
 544
 545                 spin_lock_irq(&group->port->lock);
 546                 if (group->state == MCAST_BUSY &&
 547                     group->pkey_index == MCAST_INVALID_PKEY_INDEX)
 548                         group->pkey_index = pkey_index;
 549                 mgids_changed = memcmp(&rec->mgid, &group->rec.mgid,
 550                                        sizeof(group->rec.mgid));
 551                 group->rec = *rec;
 552                 if (mgids_changed) {
 553                         rb_erase(&group->node, &group->port->table);
 554                         is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
 555                                            sizeof(mgid0));
 556                         mcast_insert(group->port, group, is_mgid0);
 557                 }
 558                 spin_unlock_irq(&group->port->lock);
 559         }
 560         mcast_work_handler(&group->work);
 561 }
 562
 563 static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
 564                           void *context)
 565 {
 566         struct mcast_group *group = context;
 567
 568         if (status && group->retries > 0 &&
 569             !send_leave(group, group->leave_state))
 570                 group->retries--;
 571         else
 572                 mcast_work_handler(&group->work);
 573 }
 574
 575 static struct mcast_group *acquire_group(struct mcast_port *port,
 576                                          union ib_gid *mgid, gfp_t gfp_mask)
 577 {
 578         struct mcast_group *group, *cur_group;
 579         unsigned long flags;
 580         int is_mgid0;
 581
 582         is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0);
 583         if (!is_mgid0) {
 584                 spin_lock_irqsave(&port->lock, flags);
 585                 group = mcast_find(port, mgid);
 586                 if (group)
 587                         goto found;
 588                 spin_unlock_irqrestore(&port->lock, flags);
 589         }
 590
 591         group = kzalloc(sizeof *group, gfp_mask);
 592         if (!group)
 593                 return NULL;
 594
 595         group->retries = 3;
 596         group->port = port;
 597         group->rec.mgid = *mgid;
 598         group->pkey_index = MCAST_INVALID_PKEY_INDEX;
 599         INIT_LIST_HEAD(&group->pending_list);
 600         INIT_LIST_HEAD(&group->active_list);
 601         INIT_WORK(&group->work, mcast_work_handler);
 602         spin_lock_init(&group->lock);
 603
 604         spin_lock_irqsave(&port->lock, flags);
 605         cur_group = mcast_insert(port, group, is_mgid0);
 606         if (cur_group) {
 607                 kfree(group);
 608                 group = cur_group;
 609         } else
 610                 atomic_inc(&port->refcount);
 611 found:
 612         atomic_inc(&group->refcount);
 613         spin_unlock_irqrestore(&port->lock, flags);
 614         return group;
 615 }
 616
 617 /*
 618  * We serialize all join requests to a single group to make our lives much
 619  * easier.  Otherwise, two users could try to join the same group
 620  * simultaneously, with different configurations, one could leave while the
 621  * join is in progress, etc., which makes locking around error recovery
 622  * difficult.
 623  */
 624 struct ib_sa_multicast *
 625 ib_sa_join_multicast(struct ib_sa_client *client,
 626                      struct ib_device *device, u8 port_num,
 627                      struct ib_sa_mcmember_rec *rec,
 628                      ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
 629                      int (*callback)(int status,
 630                                      struct ib_sa_multicast *multicast),
 631                      void *context)
 632 {
 633         struct mcast_device *dev;
 634         struct mcast_member *member;
 635         struct ib_sa_multicast *multicast;
 636         int ret;
 637
 638         dev = ib_get_client_data(device, &mcast_client);
 639         if (!dev)
 640                 return ERR_PTR(-ENODEV);
 641
 642         member = kmalloc(sizeof *member, gfp_mask);
 643         if (!member)
 644                 return ERR_PTR(-ENOMEM);
 645
 646         ib_sa_client_get(client);
 647         member->client = client;
 648         member->multicast.rec = *rec;
 649         member->multicast.comp_mask = comp_mask;
 650         member->multicast.callback = callback;
 651         member->multicast.context = context;
 652         init_completion(&member->comp);
 653         atomic_set(&member->refcount, 1);
 654         member->state = MCAST_JOINING;
 655
 656         member->group = acquire_group(&dev->port[port_num - dev->start_port],
 657                                       &rec->mgid, gfp_mask);
 658         if (!member->group) {
 659                 ret = -ENOMEM;
 660                 goto err;
 661         }
 662
 663         /*
 664          * The user will get the multicast structure in their callback.  They
 665          * could then free the multicast structure before we can return from
 666          * this routine.  So we save the pointer to return before queuing
 667          * any callback.
 668          */
 669         multicast = &member->multicast;
 670         queue_join(member);
 671         return multicast;
 672
 673 err:
 674         ib_sa_client_put(client);
 675         kfree(member);
 676         return ERR_PTR(ret);
 677 }
 678 EXPORT_SYMBOL(ib_sa_join_multicast);
 679
 680 void ib_sa_free_multicast(struct ib_sa_multicast *multicast)
 681 {
 682         struct mcast_member *member;
 683         struct mcast_group *group;
 684
 685         member = container_of(multicast, struct mcast_member, multicast);
 686         group = member->group;
 687
 688         spin_lock_irq(&group->lock);
 689         if (member->state == MCAST_MEMBER)
 690                 adjust_membership(group, multicast->rec.join_state, -1);
 691
 692         list_del_init(&member->list);
 693
 694         if (group->state == MCAST_IDLE) {
 695                 group->state = MCAST_BUSY;
 696                 spin_unlock_irq(&group->lock);
 697                 /* Continue to hold reference on group until callback */
 698                 queue_work(mcast_wq, &group->work);
 699         } else {
 700                 spin_unlock_irq(&group->lock);
 701                 release_group(group);
 702         }
 703
 704         deref_member(member);
 705         wait_for_completion(&member->comp);
 706         ib_sa_client_put(member->client);
 707         kfree(member);
 708 }
 709 EXPORT_SYMBOL(ib_sa_free_multicast);
 710
 711 int ib_sa_get_mcmember_rec(struct ib_device *device, u8 port_num,
 712                            union ib_gid *mgid, struct ib_sa_mcmember_rec *rec)
 713 {
 714         struct mcast_device *dev;
 715         struct mcast_port *port;
 716         struct mcast_group *group;
 717         unsigned long flags;
 718         int ret = 0;
 719
 720         dev = ib_get_client_data(device, &mcast_client);
 721         if (!dev)
 722                 return -ENODEV;
 723
 724         port = &dev->port[port_num - dev->start_port];
 725         spin_lock_irqsave(&port->lock, flags);
 726         group = mcast_find(port, mgid);
 727         if (group)
 728                 *rec = group->rec;
 729         else
 730                 ret = -EADDRNOTAVAIL;
 731         spin_unlock_irqrestore(&port->lock, flags);
 732
 733         return ret;
 734 }
 735 EXPORT_SYMBOL(ib_sa_get_mcmember_rec);
 736
 737 int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
 738                              struct ib_sa_mcmember_rec *rec,
 739                              struct net_device *ndev,
 740                              enum ib_gid_type gid_type,
 741                              struct ib_ah_attr *ah_attr)
 742 {
 743         int ret;
 744         u16 gid_index;
 745         u8 p;
 746
 747         if (rdma_protocol_roce(device, port_num)) {
 748                 ret = ib_find_cached_gid_by_port(device, &rec->port_gid,
 749                                                  gid_type, port_num,
 750                                                  ndev,
 751                                                  &gid_index);
 752         } else if (rdma_protocol_ib(device, port_num)) {
 753                 ret = ib_find_cached_gid(device, &rec->port_gid,
 754                                          IB_GID_TYPE_IB, NULL, &p,
 755                                          &gid_index);
 756         } else {
 757                 ret = -EINVAL;
 758         }
 759
 760         if (ret)
 761                 return ret;
 762
 763         memset(ah_attr, 0, sizeof *ah_attr);
 764         ah_attr->dlid = be16_to_cpu(rec->mlid);
 765         ah_attr->sl = rec->sl;
 766         ah_attr->port_num = port_num;
 767         ah_attr->static_rate = rec->rate;
 768
 769         ah_attr->ah_flags = IB_AH_GRH;
 770         ah_attr->grh.dgid = rec->mgid;
 771
 772         ah_attr->grh.sgid_index = (u8) gid_index;
 773         ah_attr->grh.flow_label = be32_to_cpu(rec->flow_label);
 774         ah_attr->grh.hop_limit = rec->hop_limit;
 775         ah_attr->grh.traffic_class = rec->traffic_class;
 776
 777         return 0;
 778 }
 779 EXPORT_SYMBOL(ib_init_ah_from_mcmember);
 780
 781 static void mcast_groups_event(struct mcast_port *port,
 782                                enum mcast_group_state state)
 783 {
 784         struct mcast_group *group;
 785         struct rb_node *node;
 786         unsigned long flags;
 787
 788         spin_lock_irqsave(&port->lock, flags);
 789         for (node = rb_first(&port->table); node; node = rb_next(node)) {
 790                 group = rb_entry(node, struct mcast_group, node);
 791                 spin_lock(&group->lock);
 792                 if (group->state == MCAST_IDLE) {
 793                         atomic_inc(&group->refcount);
 794                         queue_work(mcast_wq, &group->work);
 795                 }
 796                 if (group->state != MCAST_GROUP_ERROR)
 797                         group->state = state;
 798                 spin_unlock(&group->lock);
 799         }
 800         spin_unlock_irqrestore(&port->lock, flags);
 801 }
 802
 803 static void mcast_event_handler(struct ib_event_handler *handler,
 804                                 struct ib_event *event)
 805 {
 806         struct mcast_device *dev;
 807         int index;
 808
 809         dev = container_of(handler, struct mcast_device, event_handler);
 810         if (!rdma_cap_ib_mcast(dev->device, event->element.port_num))
 811                 return;
 812
 813         index = event->element.port_num - dev->start_port;
 814
 815         switch (event->event) {
 816         case IB_EVENT_PORT_ERR:
 817         case IB_EVENT_LID_CHANGE:
 818         case IB_EVENT_SM_CHANGE:
 819         case IB_EVENT_CLIENT_REREGISTER:
 820                 mcast_groups_event(&dev->port[index], MCAST_GROUP_ERROR);
 821                 break;
 822         case IB_EVENT_PKEY_CHANGE:
 823                 mcast_groups_event(&dev->port[index], MCAST_PKEY_EVENT);
 824                 break;
 825         default:
 826                 break;
 827         }
 828 }
 829
 830 static void mcast_add_one(struct ib_device *device)
 831 {
 832         struct mcast_device *dev;
 833         struct mcast_port *port;
 834         int i;
 835         int count = 0;
 836
 837         dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 838                       GFP_KERNEL);
 839         if (!dev)
 840                 return;
 841
 842         dev->start_port = rdma_start_port(device);
 843         dev->end_port = rdma_end_port(device);
 844
 845         for (i = 0; i <= dev->end_port - dev->start_port; i++) {
 846                 if (!rdma_cap_ib_mcast(device, dev->start_port + i))
 847                         continue;
 848                 port = &dev->port[i];
 849                 port->dev = dev;
 850                 port->port_num = dev->start_port + i;
 851                 spin_lock_init(&port->lock);
 852                 port->table = RB_ROOT;
 853                 init_completion(&port->comp);
 854                 atomic_set(&port->refcount, 1);
 855                 ++count;
 856         }
 857
 858         if (!count) {
 859                 kfree(dev);
 860                 return;
 861         }
 862
 863         dev->device = device;
 864         ib_set_client_data(device, &mcast_client, dev);
 865
 866         INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler);
 867         ib_register_event_handler(&dev->event_handler);
 868 }
 869
 870 static void mcast_remove_one(struct ib_device *device, void *client_data)
 871 {
 872         struct mcast_device *dev = client_data;
 873         struct mcast_port *port;
 874         int i;
 875
 876         if (!dev)
 877                 return;
 878
 879         ib_unregister_event_handler(&dev->event_handler);
 880         flush_workqueue(mcast_wq);
 881
 882         for (i = 0; i <= dev->end_port - dev->start_port; i++) {
 883                 if (rdma_cap_ib_mcast(device, dev->start_port + i)) {
 884                         port = &dev->port[i];
 885                         deref_port(port);
 886                         wait_for_completion(&port->comp);
 887                 }
 888         }
 889
 890         kfree(dev);
 891 }
 892
 893 int mcast_init(void)
 894 {
 895         int ret;
 896
 897         mcast_wq = create_singlethread_workqueue("ib_mcast");
 898         if (!mcast_wq)
 899                 return -ENOMEM;
 900
 901         ib_sa_register_client(&sa_client);
 902
 903         ret = ib_register_client(&mcast_client);
 904         if (ret)
 905                 goto err;
 906         return 0;
 907
 908 err:
 909         ib_sa_unregister_client(&sa_client);
 910         destroy_workqueue(mcast_wq);
 911         return ret;
 912 }
 913
 914 void mcast_cleanup(void)
 915 {
 916         ib_unregister_client(&mcast_client);
 917         ib_sa_unregister_client(&sa_client);
 918         destroy_workqueue(mcast_wq);
 919 }