drivers/infiniband/hw/mlx4/main.c

   1 /*
   2  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/init.h>
  36 #include <linux/slab.h>
  37 #include <linux/errno.h>
  38 #include <linux/netdevice.h>
  39 #include <linux/inetdevice.h>
  40 #include <linux/rtnetlink.h>
  41 #include <linux/if_vlan.h>
  42
  43 #include <rdma/ib_smi.h>
  44 #include <rdma/ib_user_verbs.h>
  45 #include <rdma/ib_addr.h>
  46
  47 #include <linux/mlx4/driver.h>
  48 #include <linux/mlx4/cmd.h>
  49
  50 #include "mlx4_ib.h"
  51 #include "user.h"
  52
  53 #define DRV_NAME        MLX4_IB_DRV_NAME
  54 #define DRV_VERSION     "1.0"
  55 #define DRV_RELDATE     "April 4, 2008"
  56
  57 #define MLX4_IB_FLOW_MAX_PRIO 0xFFF
  58
  59 MODULE_AUTHOR("Roland Dreier");
  60 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
  61 MODULE_LICENSE("Dual BSD/GPL");
  62 MODULE_VERSION(DRV_VERSION);
  63
  64 int mlx4_ib_sm_guid_assign = 1;
  65 module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444);
  66 MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 1)");
  67
  68 static const char mlx4_ib_version[] =
  69         DRV_NAME ": Mellanox ConnectX InfiniBand driver v"
  70         DRV_VERSION " (" DRV_RELDATE ")\n";
  71
  72 struct update_gid_work {
  73         struct work_struct      work;
  74         union ib_gid            gids[128];
  75         struct mlx4_ib_dev     *dev;
  76         int                     port;
  77 };
  78
  79 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);
  80
  81 static struct workqueue_struct *wq;
  82
  83 static void init_query_mad(struct ib_smp *mad)
  84 {
  85         mad->base_version  = 1;
  86         mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
  87         mad->class_version = 1;
  88         mad->method        = IB_MGMT_METHOD_GET;
  89 }
  90
  91 static union ib_gid zgid;
  92
  93 static int check_flow_steering_support(struct mlx4_dev *dev)
  94 {
  95         int ib_num_ports = 0;
  96         int i;
  97
  98         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
  99                 ib_num_ports++;
 100
 101         if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
 102                 if (ib_num_ports || mlx4_is_mfunc(dev)) {
 103                         pr_warn("Device managed flow steering is unavailable "
 104                                 "for IB ports or in multifunction env.\n");
 105                         return 0;
 106                 }
 107                 return 1;
 108         }
 109         return 0;
 110 }
 111
 112 static int mlx4_ib_query_device(struct ib_device *ibdev,
 113                                 struct ib_device_attr *props)
 114 {
 115         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 116         struct ib_smp *in_mad  = NULL;
 117         struct ib_smp *out_mad = NULL;
 118         int err = -ENOMEM;
 119
 120         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 121         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 122         if (!in_mad || !out_mad)
 123                 goto out;
 124
 125         init_query_mad(in_mad);
 126         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
 127
 128         err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,
 129                            1, NULL, NULL, in_mad, out_mad);
 130         if (err)
 131                 goto out;
 132
 133         memset(props, 0, sizeof *props);
 134
 135         props->fw_ver = dev->dev->caps.fw_ver;
 136         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
 137                 IB_DEVICE_PORT_ACTIVE_EVENT             |
 138                 IB_DEVICE_SYS_IMAGE_GUID                |
 139                 IB_DEVICE_RC_RNR_NAK_GEN                |
 140                 IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 141         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR)
 142                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 143         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR)
 144                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 145         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM)
 146                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 147         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT)
 148                 props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
 149         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM)
 150                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
 151         if (dev->dev->caps.max_gso_sz && dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH)
 152                 props->device_cap_flags |= IB_DEVICE_UD_TSO;
 153         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY)
 154                 props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
 155         if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) &&
 156             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) &&
 157             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR))
 158                 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
 159         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)
 160                 props->device_cap_flags |= IB_DEVICE_XRC;
 161         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)
 162                 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW;
 163         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
 164                 if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_WIN_TYPE_2B)
 165                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;
 166                 else
 167                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2A;
 168         if (check_flow_steering_support(dev->dev))
 169                 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
 170         }
 171
 172         props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
 173                 0xffffff;
 174         props->vendor_part_id      = dev->dev->pdev->device;
 175         props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
 176         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
 177
 178         props->max_mr_size         = ~0ull;
 179         props->page_size_cap       = dev->dev->caps.page_size_cap;
 180         props->max_qp              = dev->dev->quotas.qp;
 181         props->max_qp_wr           = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE;
 182         props->max_sge             = min(dev->dev->caps.max_sq_sg,
 183                                          dev->dev->caps.max_rq_sg);
 184         props->max_cq              = dev->dev->quotas.cq;
 185         props->max_cqe             = dev->dev->caps.max_cqes;
 186         props->max_mr              = dev->dev->quotas.mpt;
 187         props->max_pd              = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds;
 188         props->max_qp_rd_atom      = dev->dev->caps.max_qp_dest_rdma;
 189         props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma;
 190         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
 191         props->max_srq             = dev->dev->quotas.srq;
 192         props->max_srq_wr          = dev->dev->caps.max_srq_wqes - 1;
 193         props->max_srq_sge         = dev->dev->caps.max_srq_sge;
 194         props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES;
 195         props->local_ca_ack_delay  = dev->dev->caps.local_ca_ack_delay;
 196         props->atomic_cap          = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ?
 197                 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
 198         props->masked_atomic_cap   = props->atomic_cap;
 199         props->max_pkeys           = dev->dev->caps.pkey_table_len[1];
 200         props->max_mcast_grp       = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms;
 201         props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm;
 202         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 203                                            props->max_mcast_grp;
 204         props->max_map_per_fmr = dev->dev->caps.max_fmr_maps;
 205
 206 out:
 207         kfree(in_mad);
 208         kfree(out_mad);
 209
 210         return err;
 211 }
 212
 213 static enum rdma_link_layer
 214 mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num)
 215 {
 216         struct mlx4_dev *dev = to_mdev(device)->dev;
 217
 218         return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ?
 219                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 220 }
 221
 222 static int ib_link_query_port(struct ib_device *ibdev, u8 port,
 223                               struct ib_port_attr *props, int netw_view)
 224 {
 225         struct ib_smp *in_mad  = NULL;
 226         struct ib_smp *out_mad = NULL;
 227         int ext_active_speed;
 228         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 229         int err = -ENOMEM;
 230
 231         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 232         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 233         if (!in_mad || !out_mad)
 234                 goto out;
 235
 236         init_query_mad(in_mad);
 237         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 238         in_mad->attr_mod = cpu_to_be32(port);
 239
 240         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
 241                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 242
 243         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
 244                                 in_mad, out_mad);
 245         if (err)
 246                 goto out;
 247
 248
 249         props->lid              = be16_to_cpup((__be16 *) (out_mad->data + 16));
 250         props->lmc              = out_mad->data[34] & 0x7;
 251         props->sm_lid           = be16_to_cpup((__be16 *) (out_mad->data + 18));
 252         props->sm_sl            = out_mad->data[36] & 0xf;
 253         props->state            = out_mad->data[32] & 0xf;
 254         props->phys_state       = out_mad->data[33] >> 4;
 255         props->port_cap_flags   = be32_to_cpup((__be32 *) (out_mad->data + 20));
 256         if (netw_view)
 257                 props->gid_tbl_len = out_mad->data[50];
 258         else
 259                 props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port];
 260         props->max_msg_sz       = to_mdev(ibdev)->dev->caps.max_msg_sz;
 261         props->pkey_tbl_len     = to_mdev(ibdev)->dev->caps.pkey_table_len[port];
 262         props->bad_pkey_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 46));
 263         props->qkey_viol_cntr   = be16_to_cpup((__be16 *) (out_mad->data + 48));
 264         props->active_width     = out_mad->data[31] & 0xf;
 265         props->active_speed     = out_mad->data[35] >> 4;
 266         props->max_mtu          = out_mad->data[41] & 0xf;
 267         props->active_mtu       = out_mad->data[36] >> 4;
 268         props->subnet_timeout   = out_mad->data[51] & 0x1f;
 269         props->max_vl_num       = out_mad->data[37] >> 4;
 270         props->init_type_reply  = out_mad->data[41] >> 4;
 271
 272         /* Check if extended speeds (EDR/FDR/...) are supported */
 273         if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) {
 274                 ext_active_speed = out_mad->data[62] >> 4;
 275
 276                 switch (ext_active_speed) {
 277                 case 1:
 278                         props->active_speed = IB_SPEED_FDR;
 279                         break;
 280                 case 2:
 281                         props->active_speed = IB_SPEED_EDR;
 282                         break;
 283                 }
 284         }
 285
 286         /* If reported active speed is QDR, check if is FDR-10 */
 287         if (props->active_speed == IB_SPEED_QDR) {
 288                 init_query_mad(in_mad);
 289                 in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;
 290                 in_mad->attr_mod = cpu_to_be32(port);
 291
 292                 err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port,
 293                                    NULL, NULL, in_mad, out_mad);
 294                 if (err)
 295                         goto out;
 296
 297                 /* Checking LinkSpeedActive for FDR-10 */
 298                 if (out_mad->data[15] & 0x1)
 299                         props->active_speed = IB_SPEED_FDR10;
 300         }
 301
 302         /* Avoid wrong speed value returned by FW if the IB link is down. */
 303         if (props->state == IB_PORT_DOWN)
 304                  props->active_speed = IB_SPEED_SDR;
 305
 306 out:
 307         kfree(in_mad);
 308         kfree(out_mad);
 309         return err;
 310 }
 311
 312 static u8 state_to_phys_state(enum ib_port_state state)
 313 {
 314         return state == IB_PORT_ACTIVE ? 5 : 3;
 315 }
 316
 317 static int eth_link_query_port(struct ib_device *ibdev, u8 port,
 318                                struct ib_port_attr *props, int netw_view)
 319 {
 320
 321         struct mlx4_ib_dev *mdev = to_mdev(ibdev);
 322         struct mlx4_ib_iboe *iboe = &mdev->iboe;
 323         struct net_device *ndev;
 324         enum ib_mtu tmp;
 325         struct mlx4_cmd_mailbox *mailbox;
 326         int err = 0;
 327
 328         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
 329         if (IS_ERR(mailbox))
 330                 return PTR_ERR(mailbox);
 331
 332         err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0,
 333                            MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B,
 334                            MLX4_CMD_WRAPPED);
 335         if (err)
 336                 goto out;
 337
 338         props->active_width     =  (((u8 *)mailbox->buf)[5] == 0x40) ?
 339                                                 IB_WIDTH_4X : IB_WIDTH_1X;
 340         props->active_speed     = IB_SPEED_QDR;
 341         props->port_cap_flags   = IB_PORT_CM_SUP;
 342         props->gid_tbl_len      = mdev->dev->caps.gid_table_len[port];
 343         props->max_msg_sz       = mdev->dev->caps.max_msg_sz;
 344         props->pkey_tbl_len     = 1;
 345         props->max_mtu          = IB_MTU_4096;
 346         props->max_vl_num       = 2;
 347         props->state            = IB_PORT_DOWN;
 348         props->phys_state       = state_to_phys_state(props->state);
 349         props->active_mtu       = IB_MTU_256;
 350         spin_lock(&iboe->lock);
 351         ndev = iboe->netdevs[port - 1];
 352         if (!ndev)
 353                 goto out_unlock;
 354
 355         tmp = iboe_get_mtu(ndev->mtu);
 356         props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256;
 357
 358         props->state            = (netif_running(ndev) && netif_carrier_ok(ndev)) ?
 359                                         IB_PORT_ACTIVE : IB_PORT_DOWN;
 360         props->phys_state       = state_to_phys_state(props->state);
 361 out_unlock:
 362         spin_unlock(&iboe->lock);
 363 out:
 364         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 365         return err;
 366 }
 367
 368 int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
 369                          struct ib_port_attr *props, int netw_view)
 370 {
 371         int err;
 372
 373         memset(props, 0, sizeof *props);
 374
 375         err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ?
 376                 ib_link_query_port(ibdev, port, props, netw_view) :
 377                                 eth_link_query_port(ibdev, port, props, netw_view);
 378
 379         return err;
 380 }
 381
 382 static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
 383                               struct ib_port_attr *props)
 384 {
 385         /* returns host view */
 386         return __mlx4_ib_query_port(ibdev, port, props, 0);
 387 }
 388
 389 int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 390                         union ib_gid *gid, int netw_view)
 391 {
 392         struct ib_smp *in_mad  = NULL;
 393         struct ib_smp *out_mad = NULL;
 394         int err = -ENOMEM;
 395         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 396         int clear = 0;
 397         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 398
 399         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 400         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 401         if (!in_mad || !out_mad)
 402                 goto out;
 403
 404         init_query_mad(in_mad);
 405         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 406         in_mad->attr_mod = cpu_to_be32(port);
 407
 408         if (mlx4_is_mfunc(dev->dev) && netw_view)
 409                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 410
 411         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad);
 412         if (err)
 413                 goto out;
 414
 415         memcpy(gid->raw, out_mad->data + 8, 8);
 416
 417         if (mlx4_is_mfunc(dev->dev) && !netw_view) {
 418                 if (index) {
 419                         /* For any index > 0, return the null guid */
 420                         err = 0;
 421                         clear = 1;
 422                         goto out;
 423                 }
 424         }
 425
 426         init_query_mad(in_mad);
 427         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
 428         in_mad->attr_mod = cpu_to_be32(index / 8);
 429
 430         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port,
 431                            NULL, NULL, in_mad, out_mad);
 432         if (err)
 433                 goto out;
 434
 435         memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
 436
 437 out:
 438         if (clear)
 439                 memset(gid->raw + 8, 0, 8);
 440         kfree(in_mad);
 441         kfree(out_mad);
 442         return err;
 443 }
 444
 445 static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index,
 446                           union ib_gid *gid)
 447 {
 448         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 449
 450         *gid = dev->iboe.gid_table[port - 1][index];
 451
 452         return 0;
 453 }
 454
 455 static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 456                              union ib_gid *gid)
 457 {
 458         if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND)
 459                 return __mlx4_ib_query_gid(ibdev, port, index, gid, 0);
 460         else
 461                 return iboe_query_gid(ibdev, port, index, gid);
 462 }
 463
 464 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 465                          u16 *pkey, int netw_view)
 466 {
 467         struct ib_smp *in_mad  = NULL;
 468         struct ib_smp *out_mad = NULL;
 469         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 470         int err = -ENOMEM;
 471
 472         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 473         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 474         if (!in_mad || !out_mad)
 475                 goto out;
 476
 477         init_query_mad(in_mad);
 478         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
 479         in_mad->attr_mod = cpu_to_be32(index / 32);
 480
 481         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
 482                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 483
 484         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
 485                            in_mad, out_mad);
 486         if (err)
 487                 goto out;
 488
 489         *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);
 490
 491 out:
 492         kfree(in_mad);
 493         kfree(out_mad);
 494         return err;
 495 }
 496
 497 static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
 498 {
 499         return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0);
 500 }
 501
 502 static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
 503                                  struct ib_device_modify *props)
 504 {
 505         struct mlx4_cmd_mailbox *mailbox;
 506         unsigned long flags;
 507
 508         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 509                 return -EOPNOTSUPP;
 510
 511         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
 512                 return 0;
 513
 514         if (mlx4_is_slave(to_mdev(ibdev)->dev))
 515                 return -EOPNOTSUPP;
 516
 517         spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags);
 518         memcpy(ibdev->node_desc, props->node_desc, 64);
 519         spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags);
 520
 521         /*
 522          * If possible, pass node desc to FW, so it can generate
 523          * a 144 trap.  If cmd fails, just ignore.
 524          */
 525         mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev);
 526         if (IS_ERR(mailbox))
 527                 return 0;
 528
 529         memset(mailbox->buf, 0, 256);
 530         memcpy(mailbox->buf, props->node_desc, 64);
 531         mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0,
 532                  MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
 533
 534         mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox);
 535
 536         return 0;
 537 }
 538
 539 static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols,
 540                          u32 cap_mask)
 541 {
 542         struct mlx4_cmd_mailbox *mailbox;
 543         int err;
 544         u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH;
 545
 546         mailbox = mlx4_alloc_cmd_mailbox(dev->dev);
 547         if (IS_ERR(mailbox))
 548                 return PTR_ERR(mailbox);
 549
 550         memset(mailbox->buf, 0, 256);
 551
 552         if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
 553                 *(u8 *) mailbox->buf         = !!reset_qkey_viols << 6;
 554                 ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
 555         } else {
 556                 ((u8 *) mailbox->buf)[3]     = !!reset_qkey_viols;
 557                 ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask);
 558         }
 559
 560         err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT,
 561                        MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);
 562
 563         mlx4_free_cmd_mailbox(dev->dev, mailbox);
 564         return err;
 565 }
 566
 567 static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
 568                                struct ib_port_modify *props)
 569 {
 570         struct ib_port_attr attr;
 571         u32 cap_mask;
 572         int err;
 573
 574         mutex_lock(&to_mdev(ibdev)->cap_mask_mutex);
 575
 576         err = mlx4_ib_query_port(ibdev, port, &attr);
 577         if (err)
 578                 goto out;
 579
 580         cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
 581                 ~props->clr_port_cap_mask;
 582
 583         err = mlx4_SET_PORT(to_mdev(ibdev), port,
 584                             !!(mask & IB_PORT_RESET_QKEY_CNTR),
 585                             cap_mask);
 586
 587 out:
 588         mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
 589         return err;
 590 }
 591
 592 static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
 593                                                   struct ib_udata *udata)
 594 {
 595         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 596         struct mlx4_ib_ucontext *context;
 597         struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
 598         struct mlx4_ib_alloc_ucontext_resp resp;
 599         int err;
 600
 601         if (!dev->ib_active)
 602                 return ERR_PTR(-EAGAIN);
 603
 604         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
 605                 resp_v3.qp_tab_size      = dev->dev->caps.num_qps;
 606                 resp_v3.bf_reg_size      = dev->dev->caps.bf_reg_size;
 607                 resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
 608         } else {
 609                 resp.dev_caps         = dev->dev->caps.userspace_caps;
 610                 resp.qp_tab_size      = dev->dev->caps.num_qps;
 611                 resp.bf_reg_size      = dev->dev->caps.bf_reg_size;
 612                 resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
 613                 resp.cqe_size         = dev->dev->caps.cqe_size;
 614         }
 615
 616         context = kmalloc(sizeof *context, GFP_KERNEL);
 617         if (!context)
 618                 return ERR_PTR(-ENOMEM);
 619
 620         err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);
 621         if (err) {
 622                 kfree(context);
 623                 return ERR_PTR(err);
 624         }
 625
 626         INIT_LIST_HEAD(&context->db_page_list);
 627         mutex_init(&context->db_page_mutex);
 628
 629         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
 630                 err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
 631         else
 632                 err = ib_copy_to_udata(udata, &resp, sizeof(resp));
 633
 634         if (err) {
 635                 mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
 636                 kfree(context);
 637                 return ERR_PTR(-EFAULT);
 638         }
 639
 640         return &context->ibucontext;
 641 }
 642
 643 static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 644 {
 645         struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);
 646
 647         mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);
 648         kfree(context);
 649
 650         return 0;
 651 }
 652
 653 static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 654 {
 655         struct mlx4_ib_dev *dev = to_mdev(context->device);
 656
 657         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 658                 return -EINVAL;
 659
 660         if (vma->vm_pgoff == 0) {
 661                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 662
 663                 if (io_remap_pfn_range(vma, vma->vm_start,
 664                                        to_mucontext(context)->uar.pfn,
 665                                        PAGE_SIZE, vma->vm_page_prot))
 666                         return -EAGAIN;
 667         } else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) {
 668                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 669
 670                 if (io_remap_pfn_range(vma, vma->vm_start,
 671                                        to_mucontext(context)->uar.pfn +
 672                                        dev->dev->caps.num_uars,
 673                                        PAGE_SIZE, vma->vm_page_prot))
 674                         return -EAGAIN;
 675         } else
 676                 return -EINVAL;
 677
 678         return 0;
 679 }
 680
 681 static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev,
 682                                       struct ib_ucontext *context,
 683                                       struct ib_udata *udata)
 684 {
 685         struct mlx4_ib_pd *pd;
 686         int err;
 687
 688         pd = kmalloc(sizeof *pd, GFP_KERNEL);
 689         if (!pd)
 690                 return ERR_PTR(-ENOMEM);
 691
 692         err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);
 693         if (err) {
 694                 kfree(pd);
 695                 return ERR_PTR(err);
 696         }
 697
 698         if (context)
 699                 if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) {
 700                         mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
 701                         kfree(pd);
 702                         return ERR_PTR(-EFAULT);
 703                 }
 704
 705         return &pd->ibpd;
 706 }
 707
 708 static int mlx4_ib_dealloc_pd(struct ib_pd *pd)
 709 {
 710         mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
 711         kfree(pd);
 712
 713         return 0;
 714 }
 715
 716 static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
 717                                           struct ib_ucontext *context,
 718                                           struct ib_udata *udata)
 719 {
 720         struct mlx4_ib_xrcd *xrcd;
 721         int err;
 722
 723         if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
 724                 return ERR_PTR(-ENOSYS);
 725
 726         xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);
 727         if (!xrcd)
 728                 return ERR_PTR(-ENOMEM);
 729
 730         err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);
 731         if (err)
 732                 goto err1;
 733
 734         xrcd->pd = ib_alloc_pd(ibdev);
 735         if (IS_ERR(xrcd->pd)) {
 736                 err = PTR_ERR(xrcd->pd);
 737                 goto err2;
 738         }
 739
 740         xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0);
 741         if (IS_ERR(xrcd->cq)) {
 742                 err = PTR_ERR(xrcd->cq);
 743                 goto err3;
 744         }
 745
 746         return &xrcd->ibxrcd;
 747
 748 err3:
 749         ib_dealloc_pd(xrcd->pd);
 750 err2:
 751         mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
 752 err1:
 753         kfree(xrcd);
 754         return ERR_PTR(err);
 755 }
 756
 757 static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 758 {
 759         ib_destroy_cq(to_mxrcd(xrcd)->cq);
 760         ib_dealloc_pd(to_mxrcd(xrcd)->pd);
 761         mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
 762         kfree(xrcd);
 763
 764         return 0;
 765 }
 766
 767 static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
 768 {
 769         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
 770         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
 771         struct mlx4_ib_gid_entry *ge;
 772
 773         ge = kzalloc(sizeof *ge, GFP_KERNEL);
 774         if (!ge)
 775                 return -ENOMEM;
 776
 777         ge->gid = *gid;
 778         if (mlx4_ib_add_mc(mdev, mqp, gid)) {
 779                 ge->port = mqp->port;
 780                 ge->added = 1;
 781         }
 782
 783         mutex_lock(&mqp->mutex);
 784         list_add_tail(&ge->list, &mqp->gid_list);
 785         mutex_unlock(&mqp->mutex);
 786
 787         return 0;
 788 }
 789
 790 int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,
 791                    union ib_gid *gid)
 792 {
 793         u8 mac[6];
 794         struct net_device *ndev;
 795         int ret = 0;
 796
 797         if (!mqp->port)
 798                 return 0;
 799
 800         spin_lock(&mdev->iboe.lock);
 801         ndev = mdev->iboe.netdevs[mqp->port - 1];
 802         if (ndev)
 803                 dev_hold(ndev);
 804         spin_unlock(&mdev->iboe.lock);
 805
 806         if (ndev) {
 807                 rdma_get_mcast_mac((struct in6_addr *)gid, mac);
 808                 rtnl_lock();
 809                 dev_mc_add(mdev->iboe.netdevs[mqp->port - 1], mac);
 810                 ret = 1;
 811                 rtnl_unlock();
 812                 dev_put(ndev);
 813         }
 814
 815         return ret;
 816 }
 817
 818 struct mlx4_ib_steering {
 819         struct list_head list;
 820         u64 reg_id;
 821         union ib_gid gid;
 822 };
 823
 824 static int parse_flow_attr(struct mlx4_dev *dev,
 825                            union ib_flow_spec *ib_spec,
 826                            struct _rule_hw *mlx4_spec)
 827 {
 828         enum mlx4_net_trans_rule_id type;
 829
 830         switch (ib_spec->type) {
 831         case IB_FLOW_SPEC_ETH:
 832                 type = MLX4_NET_TRANS_RULE_ID_ETH;
 833                 memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac,
 834                        ETH_ALEN);
 835                 memcpy(mlx4_spec->eth.dst_mac_msk, ib_spec->eth.mask.dst_mac,
 836                        ETH_ALEN);
 837                 mlx4_spec->eth.vlan_tag = ib_spec->eth.val.vlan_tag;
 838                 mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag;
 839                 break;
 840
 841         case IB_FLOW_SPEC_IPV4:
 842                 type = MLX4_NET_TRANS_RULE_ID_IPV4;
 843                 mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip;
 844                 mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip;
 845                 mlx4_spec->ipv4.dst_ip = ib_spec->ipv4.val.dst_ip;
 846                 mlx4_spec->ipv4.dst_ip_msk = ib_spec->ipv4.mask.dst_ip;
 847                 break;
 848
 849         case IB_FLOW_SPEC_TCP:
 850         case IB_FLOW_SPEC_UDP:
 851                 type = ib_spec->type == IB_FLOW_SPEC_TCP ?
 852                                         MLX4_NET_TRANS_RULE_ID_TCP :
 853                                         MLX4_NET_TRANS_RULE_ID_UDP;
 854                 mlx4_spec->tcp_udp.dst_port = ib_spec->tcp_udp.val.dst_port;
 855                 mlx4_spec->tcp_udp.dst_port_msk = ib_spec->tcp_udp.mask.dst_port;
 856                 mlx4_spec->tcp_udp.src_port = ib_spec->tcp_udp.val.src_port;
 857                 mlx4_spec->tcp_udp.src_port_msk = ib_spec->tcp_udp.mask.src_port;
 858                 break;
 859
 860         default:
 861                 return -EINVAL;
 862         }
 863         if (mlx4_map_sw_to_hw_steering_id(dev, type) < 0 ||
 864             mlx4_hw_rule_sz(dev, type) < 0)
 865                 return -EINVAL;
 866         mlx4_spec->id = cpu_to_be16(mlx4_map_sw_to_hw_steering_id(dev, type));
 867         mlx4_spec->size = mlx4_hw_rule_sz(dev, type) >> 2;
 868         return mlx4_hw_rule_sz(dev, type);
 869 }
 870
 871 static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr,
 872                           int domain,
 873                           enum mlx4_net_trans_promisc_mode flow_type,
 874                           u64 *reg_id)
 875 {
 876         int ret, i;
 877         int size = 0;
 878         void *ib_flow;
 879         struct mlx4_ib_dev *mdev = to_mdev(qp->device);
 880         struct mlx4_cmd_mailbox *mailbox;
 881         struct mlx4_net_trans_rule_hw_ctrl *ctrl;
 882         size_t rule_size = sizeof(struct mlx4_net_trans_rule_hw_ctrl) +
 883                            (sizeof(struct _rule_hw) * flow_attr->num_of_specs);
 884
 885         static const u16 __mlx4_domain[] = {
 886                 [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS,
 887                 [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL,
 888                 [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS,
 889                 [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC,
 890         };
 891
 892         if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) {
 893                 pr_err("Invalid priority value %d\n", flow_attr->priority);
 894                 return -EINVAL;
 895         }
 896
 897         if (domain >= IB_FLOW_DOMAIN_NUM) {
 898                 pr_err("Invalid domain value %d\n", domain);
 899                 return -EINVAL;
 900         }
 901
 902         if (mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0)
 903                 return -EINVAL;
 904
 905         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
 906         if (IS_ERR(mailbox))
 907                 return PTR_ERR(mailbox);
 908         memset(mailbox->buf, 0, rule_size);
 909         ctrl = mailbox->buf;
 910
 911         ctrl->prio = cpu_to_be16(__mlx4_domain[domain] |
 912                                  flow_attr->priority);
 913         ctrl->type = mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type);
 914         ctrl->port = flow_attr->port;
 915         ctrl->qpn = cpu_to_be32(qp->qp_num);
 916
 917         ib_flow = flow_attr + 1;
 918         size += sizeof(struct mlx4_net_trans_rule_hw_ctrl);
 919         for (i = 0; i < flow_attr->num_of_specs; i++) {
 920                 ret = parse_flow_attr(mdev->dev, ib_flow, mailbox->buf + size);
 921                 if (ret < 0) {
 922                         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 923                         return -EINVAL;
 924                 }
 925                 ib_flow += ((union ib_flow_spec *) ib_flow)->size;
 926                 size += ret;
 927         }
 928
 929         ret = mlx4_cmd_imm(mdev->dev, mailbox->dma, reg_id, size >> 2, 0,
 930                            MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
 931                            MLX4_CMD_NATIVE);
 932         if (ret == -ENOMEM)
 933                 pr_err("mcg table is full. Fail to register network rule.\n");
 934         else if (ret == -ENXIO)
 935                 pr_err("Device managed flow steering is disabled. Fail to register network rule.\n");
 936         else if (ret)
 937                 pr_err("Invalid argumant. Fail to register network rule.\n");
 938
 939         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 940         return ret;
 941 }
 942
 943 static int __mlx4_ib_destroy_flow(struct mlx4_dev *dev, u64 reg_id)
 944 {
 945         int err;
 946         err = mlx4_cmd(dev, reg_id, 0, 0,
 947                        MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
 948                        MLX4_CMD_NATIVE);
 949         if (err)
 950                 pr_err("Fail to detach network rule. registration id = 0x%llx\n",
 951                        reg_id);
 952         return err;
 953 }
 954
 955 static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
 956                                     struct ib_flow_attr *flow_attr,
 957                                     int domain)
 958 {
 959         int err = 0, i = 0;
 960         struct mlx4_ib_flow *mflow;
 961         enum mlx4_net_trans_promisc_mode type[2];
 962
 963         memset(type, 0, sizeof(type));
 964
 965         mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
 966         if (!mflow) {
 967                 err = -ENOMEM;
 968                 goto err_free;
 969         }
 970
 971         switch (flow_attr->type) {
 972         case IB_FLOW_ATTR_NORMAL:
 973                 type[0] = MLX4_FS_REGULAR;
 974                 break;
 975
 976         case IB_FLOW_ATTR_ALL_DEFAULT:
 977                 type[0] = MLX4_FS_ALL_DEFAULT;
 978                 break;
 979
 980         case IB_FLOW_ATTR_MC_DEFAULT:
 981                 type[0] = MLX4_FS_MC_DEFAULT;
 982                 break;
 983
 984         case IB_FLOW_ATTR_SNIFFER:
 985                 type[0] = MLX4_FS_UC_SNIFFER;
 986                 type[1] = MLX4_FS_MC_SNIFFER;
 987                 break;
 988
 989         default:
 990                 err = -EINVAL;
 991                 goto err_free;
 992         }
 993
 994         while (i < ARRAY_SIZE(type) && type[i]) {
 995                 err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i],
 996                                             &mflow->reg_id[i]);
 997                 if (err)
 998                         goto err_free;
 999                 i++;
1000         }
1001
1002         return &mflow->ibflow;
1003
1004 err_free:
1005         kfree(mflow);
1006         return ERR_PTR(err);
1007 }
1008
1009 static int mlx4_ib_destroy_flow(struct ib_flow *flow_id)
1010 {
1011         int err, ret = 0;
1012         int i = 0;
1013         struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device);
1014         struct mlx4_ib_flow *mflow = to_mflow(flow_id);
1015
1016         while (i < ARRAY_SIZE(mflow->reg_id) && mflow->reg_id[i]) {
1017                 err = __mlx4_ib_destroy_flow(mdev->dev, mflow->reg_id[i]);
1018                 if (err)
1019                         ret = err;
1020                 i++;
1021         }
1022
1023         kfree(mflow);
1024         return ret;
1025 }
1026
1027 static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1028 {
1029         int err;
1030         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1031         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1032         u64 reg_id;
1033         struct mlx4_ib_steering *ib_steering = NULL;
1034
1035         if (mdev->dev->caps.steering_mode ==
1036             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1037                 ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL);
1038                 if (!ib_steering)
1039                         return -ENOMEM;
1040         }
1041
1042         err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw, mqp->port,
1043                                     !!(mqp->flags &
1044                                        MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK),
1045                                     MLX4_PROT_IB_IPV6, &reg_id);
1046         if (err)
1047                 goto err_malloc;
1048
1049         err = add_gid_entry(ibqp, gid);
1050         if (err)
1051                 goto err_add;
1052
1053         if (ib_steering) {
1054                 memcpy(ib_steering->gid.raw, gid->raw, 16);
1055                 ib_steering->reg_id = reg_id;
1056                 mutex_lock(&mqp->mutex);
1057                 list_add(&ib_steering->list, &mqp->steering_rules);
1058                 mutex_unlock(&mqp->mutex);
1059         }
1060         return 0;
1061
1062 err_add:
1063         mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1064                               MLX4_PROT_IB_IPV6, reg_id);
1065 err_malloc:
1066         kfree(ib_steering);
1067
1068         return err;
1069 }
1070
1071 static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw)
1072 {
1073         struct mlx4_ib_gid_entry *ge;
1074         struct mlx4_ib_gid_entry *tmp;
1075         struct mlx4_ib_gid_entry *ret = NULL;
1076
1077         list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1078                 if (!memcmp(raw, ge->gid.raw, 16)) {
1079                         ret = ge;
1080                         break;
1081                 }
1082         }
1083
1084         return ret;
1085 }
1086
1087 static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1088 {
1089         int err;
1090         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1091         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1092         u8 mac[6];
1093         struct net_device *ndev;
1094         struct mlx4_ib_gid_entry *ge;
1095         u64 reg_id = 0;
1096
1097         if (mdev->dev->caps.steering_mode ==
1098             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1099                 struct mlx4_ib_steering *ib_steering;
1100
1101                 mutex_lock(&mqp->mutex);
1102                 list_for_each_entry(ib_steering, &mqp->steering_rules, list) {
1103                         if (!memcmp(ib_steering->gid.raw, gid->raw, 16)) {
1104                                 list_del(&ib_steering->list);
1105                                 break;
1106                         }
1107                 }
1108                 mutex_unlock(&mqp->mutex);
1109                 if (&ib_steering->list == &mqp->steering_rules) {
1110                         pr_err("Couldn't find reg_id for mgid. Steering rule is left attached\n");
1111                         return -EINVAL;
1112                 }
1113                 reg_id = ib_steering->reg_id;
1114                 kfree(ib_steering);
1115         }
1116
1117         err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1118                                     MLX4_PROT_IB_IPV6, reg_id);
1119         if (err)
1120                 return err;
1121
1122         mutex_lock(&mqp->mutex);
1123         ge = find_gid_entry(mqp, gid->raw);
1124         if (ge) {
1125                 spin_lock(&mdev->iboe.lock);
1126                 ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL;
1127                 if (ndev)
1128                         dev_hold(ndev);
1129                 spin_unlock(&mdev->iboe.lock);
1130                 rdma_get_mcast_mac((struct in6_addr *)gid, mac);
1131                 if (ndev) {
1132                         rtnl_lock();
1133                         dev_mc_del(mdev->iboe.netdevs[ge->port - 1], mac);
1134                         rtnl_unlock();
1135                         dev_put(ndev);
1136                 }
1137                 list_del(&ge->list);
1138                 kfree(ge);
1139         } else
1140                 pr_warn("could not find mgid entry\n");
1141
1142         mutex_unlock(&mqp->mutex);
1143
1144         return 0;
1145 }
1146
1147 static int init_node_data(struct mlx4_ib_dev *dev)
1148 {
1149         struct ib_smp *in_mad  = NULL;
1150         struct ib_smp *out_mad = NULL;
1151         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
1152         int err = -ENOMEM;
1153
1154         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1155         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1156         if (!in_mad || !out_mad)
1157                 goto out;
1158
1159         init_query_mad(in_mad);
1160         in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1161         if (mlx4_is_master(dev->dev))
1162                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
1163
1164         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1165         if (err)
1166                 goto out;
1167
1168         memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
1169
1170         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1171
1172         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1173         if (err)
1174                 goto out;
1175
1176         dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32));
1177         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1178
1179 out:
1180         kfree(in_mad);
1181         kfree(out_mad);
1182         return err;
1183 }
1184
1185 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
1186                         char *buf)
1187 {
1188         struct mlx4_ib_dev *dev =
1189                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1190         return sprintf(buf, "MT%d\n", dev->dev->pdev->device);
1191 }
1192
1193 static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
1194                            char *buf)
1195 {
1196         struct mlx4_ib_dev *dev =
1197                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1198         return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32),
1199                        (int) (dev->dev->caps.fw_ver >> 16) & 0xffff,
1200                        (int) dev->dev->caps.fw_ver & 0xffff);
1201 }
1202
1203 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
1204                         char *buf)
1205 {
1206         struct mlx4_ib_dev *dev =
1207                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1208         return sprintf(buf, "%x\n", dev->dev->rev_id);
1209 }
1210
1211 static ssize_t show_board(struct device *device, struct device_attribute *attr,
1212                           char *buf)
1213 {
1214         struct mlx4_ib_dev *dev =
1215                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1216         return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN,
1217                        dev->dev->board_id);
1218 }
1219
1220 static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
1221 static DEVICE_ATTR(fw_ver,   S_IRUGO, show_fw_ver, NULL);
1222 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
1223 static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
1224
1225 static struct device_attribute *mlx4_class_attributes[] = {
1226         &dev_attr_hw_rev,
1227         &dev_attr_fw_ver,
1228         &dev_attr_hca_type,
1229         &dev_attr_board_id
1230 };
1231
1232 static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev)
1233 {
1234         memcpy(eui, dev->dev_addr, 3);
1235         memcpy(eui + 5, dev->dev_addr + 3, 3);
1236         if (vlan_id < 0x1000) {
1237                 eui[3] = vlan_id >> 8;
1238                 eui[4] = vlan_id & 0xff;
1239         } else {
1240                 eui[3] = 0xff;
1241                 eui[4] = 0xfe;
1242         }
1243         eui[0] ^= 2;
1244 }
1245
1246 static void update_gids_task(struct work_struct *work)
1247 {
1248         struct update_gid_work *gw = container_of(work, struct update_gid_work, work);
1249         struct mlx4_cmd_mailbox *mailbox;
1250         union ib_gid *gids;
1251         int err;
1252         struct mlx4_dev *dev = gw->dev->dev;
1253
1254         mailbox = mlx4_alloc_cmd_mailbox(dev);
1255         if (IS_ERR(mailbox)) {
1256                 pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox));
1257                 return;
1258         }
1259
1260         gids = mailbox->buf;
1261         memcpy(gids, gw->gids, sizeof gw->gids);
1262
1263         err = mlx4_cmd(dev, mailbox->dma, MLX4_SET_PORT_GID_TABLE << 8 | gw->port,
1264                        1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
1265                        MLX4_CMD_WRAPPED);
1266         if (err)
1267                 pr_warn("set port command failed\n");
1268         else {
1269                 memcpy(gw->dev->iboe.gid_table[gw->port - 1], gw->gids, sizeof gw->gids);
1270                 mlx4_ib_dispatch_event(gw->dev, gw->port, IB_EVENT_GID_CHANGE);
1271         }
1272
1273         mlx4_free_cmd_mailbox(dev, mailbox);
1274         kfree(gw);
1275 }
1276
1277 static int update_ipv6_gids(struct mlx4_ib_dev *dev, int port, int clear)
1278 {
1279         struct net_device *ndev = dev->iboe.netdevs[port - 1];
1280         struct update_gid_work *work;
1281         struct net_device *tmp;
1282         int i;
1283         u8 *hits;
1284         int ret;
1285         union ib_gid gid;
1286         int free;
1287         int found;
1288         int need_update = 0;
1289         u16 vid;
1290
1291         work = kzalloc(sizeof *work, GFP_ATOMIC);
1292         if (!work)
1293                 return -ENOMEM;
1294
1295         hits = kzalloc(128, GFP_ATOMIC);
1296         if (!hits) {
1297                 ret = -ENOMEM;
1298                 goto out;
1299         }
1300
1301         rcu_read_lock();
1302         for_each_netdev_rcu(&init_net, tmp) {
1303                 if (ndev && (tmp == ndev || rdma_vlan_dev_real_dev(tmp) == ndev)) {
1304                         gid.global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
1305                         vid = rdma_vlan_dev_vlan_id(tmp);
1306                         mlx4_addrconf_ifid_eui48(&gid.raw[8], vid, ndev);
1307                         found = 0;
1308                         free = -1;
1309                         for (i = 0; i < 128; ++i) {
1310                                 if (free < 0 &&
1311                                     !memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid))
1312                                         free = i;
1313                                 if (!memcmp(&dev->iboe.gid_table[port - 1][i], &gid, sizeof gid)) {
1314                                         hits[i] = 1;
1315                                         found = 1;
1316                                         break;
1317                                 }
1318                         }
1319
1320                         if (!found) {
1321                                 if (tmp == ndev &&
1322                                     (memcmp(&dev->iboe.gid_table[port - 1][0],
1323                                             &gid, sizeof gid) ||
1324                                      !memcmp(&dev->iboe.gid_table[port - 1][0],
1325                                              &zgid, sizeof gid))) {
1326                                         dev->iboe.gid_table[port - 1][0] = gid;
1327                                         ++need_update;
1328                                         hits[0] = 1;
1329                                 } else if (free >= 0) {
1330                                         dev->iboe.gid_table[port - 1][free] = gid;
1331                                         hits[free] = 1;
1332                                         ++need_update;
1333                                 }
1334                         }
1335                 }
1336         }
1337         rcu_read_unlock();
1338
1339         for (i = 0; i < 128; ++i)
1340                 if (!hits[i]) {
1341                         if (memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid))
1342                                 ++need_update;
1343                         dev->iboe.gid_table[port - 1][i] = zgid;
1344                 }
1345
1346         if (need_update) {
1347                 memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof work->gids);
1348                 INIT_WORK(&work->work, update_gids_task);
1349                 work->port = port;
1350                 work->dev = dev;
1351                 queue_work(wq, &work->work);
1352         } else
1353                 kfree(work);
1354
1355         kfree(hits);
1356         return 0;
1357
1358 out:
1359         kfree(work);
1360         return ret;
1361 }
1362
1363 static void handle_en_event(struct mlx4_ib_dev *dev, int port, unsigned long event)
1364 {
1365         switch (event) {
1366         case NETDEV_UP:
1367         case NETDEV_CHANGEADDR:
1368                 update_ipv6_gids(dev, port, 0);
1369                 break;
1370
1371         case NETDEV_DOWN:
1372                 update_ipv6_gids(dev, port, 1);
1373                 dev->iboe.netdevs[port - 1] = NULL;
1374         }
1375 }
1376
1377 static void netdev_added(struct mlx4_ib_dev *dev, int port)
1378 {
1379         update_ipv6_gids(dev, port, 0);
1380 }
1381
1382 static void netdev_removed(struct mlx4_ib_dev *dev, int port)
1383 {
1384         update_ipv6_gids(dev, port, 1);
1385 }
1386
1387 static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event,
1388                                 void *ptr)
1389 {
1390         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1391         struct mlx4_ib_dev *ibdev;
1392         struct net_device *oldnd;
1393         struct mlx4_ib_iboe *iboe;
1394         int port;
1395
1396         if (!net_eq(dev_net(dev), &init_net))
1397                 return NOTIFY_DONE;
1398
1399         ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb);
1400         iboe = &ibdev->iboe;
1401
1402         spin_lock(&iboe->lock);
1403         mlx4_foreach_ib_transport_port(port, ibdev->dev) {
1404                 oldnd = iboe->netdevs[port - 1];
1405                 iboe->netdevs[port - 1] =
1406                         mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port);
1407                 if (oldnd != iboe->netdevs[port - 1]) {
1408                         if (iboe->netdevs[port - 1])
1409                                 netdev_added(ibdev, port);
1410                         else
1411                                 netdev_removed(ibdev, port);
1412                 }
1413         }
1414
1415         if (dev == iboe->netdevs[0] ||
1416             (iboe->netdevs[0] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[0]))
1417                 handle_en_event(ibdev, 1, event);
1418         else if (dev == iboe->netdevs[1]
1419                  || (iboe->netdevs[1] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[1]))
1420                 handle_en_event(ibdev, 2, event);
1421
1422         spin_unlock(&iboe->lock);
1423
1424         return NOTIFY_DONE;
1425 }
1426
1427 static void init_pkeys(struct mlx4_ib_dev *ibdev)
1428 {
1429         int port;
1430         int slave;
1431         int i;
1432
1433         if (mlx4_is_master(ibdev->dev)) {
1434                 for (slave = 0; slave <= ibdev->dev->num_vfs; ++slave) {
1435                         for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
1436                                 for (i = 0;
1437                                      i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
1438                                      ++i) {
1439                                         ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] =
1440                                         /* master has the identity virt2phys pkey mapping */
1441                                                 (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i :
1442                                                         ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1;
1443                                         mlx4_sync_pkey_table(ibdev->dev, slave, port, i,
1444                                                              ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]);
1445                                 }
1446                         }
1447                 }
1448                 /* initialize pkey cache */
1449                 for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
1450                         for (i = 0;
1451                              i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
1452                              ++i)
1453                                 ibdev->pkeys.phys_pkey_cache[port-1][i] =
1454                                         (i) ? 0 : 0xFFFF;
1455                 }
1456         }
1457 }
1458
1459 static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
1460 {
1461         char name[32];
1462         int eq_per_port = 0;
1463         int added_eqs = 0;
1464         int total_eqs = 0;
1465         int i, j, eq;
1466
1467         /* Legacy mode or comp_pool is not large enough */
1468         if (dev->caps.comp_pool == 0 ||
1469             dev->caps.num_ports > dev->caps.comp_pool)
1470                 return;
1471
1472         eq_per_port = rounddown_pow_of_two(dev->caps.comp_pool/
1473                                         dev->caps.num_ports);
1474
1475         /* Init eq table */
1476         added_eqs = 0;
1477         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
1478                 added_eqs += eq_per_port;
1479
1480         total_eqs = dev->caps.num_comp_vectors + added_eqs;
1481
1482         ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL);
1483         if (!ibdev->eq_table)
1484                 return;
1485
1486         ibdev->eq_added = added_eqs;
1487
1488         eq = 0;
1489         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) {
1490                 for (j = 0; j < eq_per_port; j++) {
1491                         sprintf(name, "mlx4-ib-%d-%d@%s",
1492                                 i, j, dev->pdev->bus->name);
1493                         /* Set IRQ for specific name (per ring) */
1494                         if (mlx4_assign_eq(dev, name, NULL,
1495                                            &ibdev->eq_table[eq])) {
1496                                 /* Use legacy (same as mlx4_en driver) */
1497                                 pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq);
1498                                 ibdev->eq_table[eq] =
1499                                         (eq % dev->caps.num_comp_vectors);
1500                         }
1501                         eq++;
1502                 }
1503         }
1504
1505         /* Fill the reset of the vector with legacy EQ */
1506         for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++)
1507                 ibdev->eq_table[eq++] = i;
1508
1509         /* Advertise the new number of EQs to clients */
1510         ibdev->ib_dev.num_comp_vectors = total_eqs;
1511 }
1512
1513 static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
1514 {
1515         int i;
1516
1517         /* no additional eqs were added */
1518         if (!ibdev->eq_table)
1519                 return;
1520
1521         /* Reset the advertised EQ number */
1522         ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
1523
1524         /* Free only the added eqs */
1525         for (i = 0; i < ibdev->eq_added; i++) {
1526                 /* Don't free legacy eqs if used */
1527                 if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors)
1528                         continue;
1529                 mlx4_release_eq(dev, ibdev->eq_table[i]);
1530         }
1531
1532         kfree(ibdev->eq_table);
1533 }
1534
1535 static void *mlx4_ib_add(struct mlx4_dev *dev)
1536 {
1537         struct mlx4_ib_dev *ibdev;
1538         int num_ports = 0;
1539         int i, j;
1540         int err;
1541         struct mlx4_ib_iboe *iboe;
1542
1543         pr_info_once("%s", mlx4_ib_version);
1544
1545         mlx4_foreach_non_ib_transport_port(i, dev)
1546                 num_ports++;
1547
1548         if (mlx4_is_mfunc(dev) && num_ports) {
1549                 dev_err(&dev->pdev->dev, "RoCE is not supported over SRIOV as yet\n");
1550                 return NULL;
1551         }
1552
1553         num_ports = 0;
1554         mlx4_foreach_ib_transport_port(i, dev)
1555                 num_ports++;
1556
1557         /* No point in registering a device with no ports... */
1558         if (num_ports == 0)
1559                 return NULL;
1560
1561         ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev);
1562         if (!ibdev) {
1563                 dev_err(&dev->pdev->dev, "Device struct alloc failed\n");
1564                 return NULL;
1565         }
1566
1567         iboe = &ibdev->iboe;
1568
1569         if (mlx4_pd_alloc(dev, &ibdev->priv_pdn))
1570                 goto err_dealloc;
1571
1572         if (mlx4_uar_alloc(dev, &ibdev->priv_uar))
1573                 goto err_pd;
1574
1575         ibdev->uar_map = ioremap((phys_addr_t) ibdev->priv_uar.pfn << PAGE_SHIFT,
1576                                  PAGE_SIZE);
1577         if (!ibdev->uar_map)
1578                 goto err_uar;
1579         MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);
1580
1581         ibdev->dev = dev;
1582
1583         strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
1584         ibdev->ib_dev.owner             = THIS_MODULE;
1585         ibdev->ib_dev.node_type         = RDMA_NODE_IB_CA;
1586         ibdev->ib_dev.local_dma_lkey    = dev->caps.reserved_lkey;
1587         ibdev->num_ports                = num_ports;
1588         ibdev->ib_dev.phys_port_cnt     = ibdev->num_ports;
1589         ibdev->ib_dev.num_comp_vectors  = dev->caps.num_comp_vectors;
1590         ibdev->ib_dev.dma_device        = &dev->pdev->dev;
1591
1592         if (dev->caps.userspace_caps)
1593                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;
1594         else
1595                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;
1596
1597         ibdev->ib_dev.uverbs_cmd_mask   =
1598                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
1599                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
1600                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
1601                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
1602                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
1603                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
1604                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
1605                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1606                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
1607                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
1608                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
1609                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
1610                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
1611                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
1612                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
1613                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
1614                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
1615                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
1616                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
1617                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
1618                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
1619                 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)         |
1620                 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
1621
1622         ibdev->ib_dev.query_device      = mlx4_ib_query_device;
1623         ibdev->ib_dev.query_port        = mlx4_ib_query_port;
1624         ibdev->ib_dev.get_link_layer    = mlx4_ib_port_link_layer;
1625         ibdev->ib_dev.query_gid         = mlx4_ib_query_gid;
1626         ibdev->ib_dev.query_pkey        = mlx4_ib_query_pkey;
1627         ibdev->ib_dev.modify_device     = mlx4_ib_modify_device;
1628         ibdev->ib_dev.modify_port       = mlx4_ib_modify_port;
1629         ibdev->ib_dev.alloc_ucontext    = mlx4_ib_alloc_ucontext;
1630         ibdev->ib_dev.dealloc_ucontext  = mlx4_ib_dealloc_ucontext;
1631         ibdev->ib_dev.mmap              = mlx4_ib_mmap;
1632         ibdev->ib_dev.alloc_pd          = mlx4_ib_alloc_pd;
1633         ibdev->ib_dev.dealloc_pd        = mlx4_ib_dealloc_pd;
1634         ibdev->ib_dev.create_ah         = mlx4_ib_create_ah;
1635         ibdev->ib_dev.query_ah          = mlx4_ib_query_ah;
1636         ibdev->ib_dev.destroy_ah        = mlx4_ib_destroy_ah;
1637         ibdev->ib_dev.create_srq        = mlx4_ib_create_srq;
1638         ibdev->ib_dev.modify_srq        = mlx4_ib_modify_srq;
1639         ibdev->ib_dev.query_srq         = mlx4_ib_query_srq;
1640         ibdev->ib_dev.destroy_srq       = mlx4_ib_destroy_srq;
1641         ibdev->ib_dev.post_srq_recv     = mlx4_ib_post_srq_recv;
1642         ibdev->ib_dev.create_qp         = mlx4_ib_create_qp;
1643         ibdev->ib_dev.modify_qp         = mlx4_ib_modify_qp;
1644         ibdev->ib_dev.query_qp          = mlx4_ib_query_qp;
1645         ibdev->ib_dev.destroy_qp        = mlx4_ib_destroy_qp;
1646         ibdev->ib_dev.post_send         = mlx4_ib_post_send;
1647         ibdev->ib_dev.post_recv         = mlx4_ib_post_recv;
1648         ibdev->ib_dev.create_cq         = mlx4_ib_create_cq;
1649         ibdev->ib_dev.modify_cq         = mlx4_ib_modify_cq;
1650         ibdev->ib_dev.resize_cq         = mlx4_ib_resize_cq;
1651         ibdev->ib_dev.destroy_cq        = mlx4_ib_destroy_cq;
1652         ibdev->ib_dev.poll_cq           = mlx4_ib_poll_cq;
1653         ibdev->ib_dev.req_notify_cq     = mlx4_ib_arm_cq;
1654         ibdev->ib_dev.get_dma_mr        = mlx4_ib_get_dma_mr;
1655         ibdev->ib_dev.reg_user_mr       = mlx4_ib_reg_user_mr;
1656         ibdev->ib_dev.dereg_mr          = mlx4_ib_dereg_mr;
1657         ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr;
1658         ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list;
1659         ibdev->ib_dev.free_fast_reg_page_list  = mlx4_ib_free_fast_reg_page_list;
1660         ibdev->ib_dev.attach_mcast      = mlx4_ib_mcg_attach;
1661         ibdev->ib_dev.detach_mcast      = mlx4_ib_mcg_detach;
1662         ibdev->ib_dev.process_mad       = mlx4_ib_process_mad;
1663
1664         if (!mlx4_is_slave(ibdev->dev)) {
1665                 ibdev->ib_dev.alloc_fmr         = mlx4_ib_fmr_alloc;
1666                 ibdev->ib_dev.map_phys_fmr      = mlx4_ib_map_phys_fmr;
1667                 ibdev->ib_dev.unmap_fmr         = mlx4_ib_unmap_fmr;
1668                 ibdev->ib_dev.dealloc_fmr       = mlx4_ib_fmr_dealloc;
1669         }
1670
1671         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
1672             dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
1673                 ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw;
1674                 ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw;
1675                 ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw;
1676
1677                 ibdev->ib_dev.uverbs_cmd_mask |=
1678                         (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
1679                         (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
1680         }
1681
1682         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) {
1683                 ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd;
1684                 ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd;
1685                 ibdev->ib_dev.uverbs_cmd_mask |=
1686                         (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
1687                         (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
1688         }
1689
1690         if (check_flow_steering_support(dev)) {
1691                 ibdev->ib_dev.create_flow       = mlx4_ib_create_flow;
1692                 ibdev->ib_dev.destroy_flow      = mlx4_ib_destroy_flow;
1693
1694 #ifdef CONFIG_INFINIBAND_EXPERIMENTAL_UVERBS_FLOW_STEERING
1695                 ibdev->ib_dev.uverbs_cmd_mask   |=
1696                         (1ull << IB_USER_VERBS_CMD_CREATE_FLOW) |
1697                         (1ull << IB_USER_VERBS_CMD_DESTROY_FLOW);
1698 #endif /* CONFIG_INFINIBAND_EXPERIMENTAL_UVERBS_FLOW_STEERING */
1699         }
1700
1701         mlx4_ib_alloc_eqs(dev, ibdev);
1702
1703         spin_lock_init(&iboe->lock);
1704
1705         if (init_node_data(ibdev))
1706                 goto err_map;
1707
1708         for (i = 0; i < ibdev->num_ports; ++i) {
1709                 if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
1710                                                 IB_LINK_LAYER_ETHERNET) {
1711                         err = mlx4_counter_alloc(ibdev->dev, &ibdev->counters[i]);
1712                         if (err)
1713                                 ibdev->counters[i] = -1;
1714                 } else
1715                                 ibdev->counters[i] = -1;
1716         }
1717
1718         spin_lock_init(&ibdev->sm_lock);
1719         mutex_init(&ibdev->cap_mask_mutex);
1720
1721         if (ib_register_device(&ibdev->ib_dev, NULL))
1722                 goto err_counter;
1723
1724         if (mlx4_ib_mad_init(ibdev))
1725                 goto err_reg;
1726
1727         if (mlx4_ib_init_sriov(ibdev))
1728                 goto err_mad;
1729
1730         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE && !iboe->nb.notifier_call) {
1731                 iboe->nb.notifier_call = mlx4_ib_netdev_event;
1732                 err = register_netdevice_notifier(&iboe->nb);
1733                 if (err)
1734                         goto err_sriov;
1735         }
1736
1737         for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) {
1738                 if (device_create_file(&ibdev->ib_dev.dev,
1739                                        mlx4_class_attributes[j]))
1740                         goto err_notif;
1741         }
1742
1743         ibdev->ib_active = true;
1744
1745         if (mlx4_is_mfunc(ibdev->dev))
1746                 init_pkeys(ibdev);
1747
1748         /* create paravirt contexts for any VFs which are active */
1749         if (mlx4_is_master(ibdev->dev)) {
1750                 for (j = 0; j < MLX4_MFUNC_MAX; j++) {
1751                         if (j == mlx4_master_func_num(ibdev->dev))
1752                                 continue;
1753                         if (mlx4_is_slave_active(ibdev->dev, j))
1754                                 do_slave_init(ibdev, j, 1);
1755                 }
1756         }
1757         return ibdev;
1758
1759 err_notif:
1760         if (unregister_netdevice_notifier(&ibdev->iboe.nb))
1761                 pr_warn("failure unregistering notifier\n");
1762         flush_workqueue(wq);
1763
1764 err_sriov:
1765         mlx4_ib_close_sriov(ibdev);
1766
1767 err_mad:
1768         mlx4_ib_mad_cleanup(ibdev);
1769
1770 err_reg:
1771         ib_unregister_device(&ibdev->ib_dev);
1772
1773 err_counter:
1774         for (; i; --i)
1775                 if (ibdev->counters[i - 1] != -1)
1776                         mlx4_counter_free(ibdev->dev, ibdev->counters[i - 1]);
1777
1778 err_map:
1779         iounmap(ibdev->uar_map);
1780
1781 err_uar:
1782         mlx4_uar_free(dev, &ibdev->priv_uar);
1783
1784 err_pd:
1785         mlx4_pd_free(dev, ibdev->priv_pdn);
1786
1787 err_dealloc:
1788         ib_dealloc_device(&ibdev->ib_dev);
1789
1790         return NULL;
1791 }
1792
1793 static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
1794 {
1795         struct mlx4_ib_dev *ibdev = ibdev_ptr;
1796         int p;
1797
1798         mlx4_ib_close_sriov(ibdev);
1799         mlx4_ib_mad_cleanup(ibdev);
1800         ib_unregister_device(&ibdev->ib_dev);
1801         if (ibdev->iboe.nb.notifier_call) {
1802                 if (unregister_netdevice_notifier(&ibdev->iboe.nb))
1803                         pr_warn("failure unregistering notifier\n");
1804                 ibdev->iboe.nb.notifier_call = NULL;
1805         }
1806         iounmap(ibdev->uar_map);
1807         for (p = 0; p < ibdev->num_ports; ++p)
1808                 if (ibdev->counters[p] != -1)
1809                         mlx4_counter_free(ibdev->dev, ibdev->counters[p]);
1810         mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB)
1811                 mlx4_CLOSE_PORT(dev, p);
1812
1813         mlx4_ib_free_eqs(dev, ibdev);
1814
1815         mlx4_uar_free(dev, &ibdev->priv_uar);
1816         mlx4_pd_free(dev, ibdev->priv_pdn);
1817         ib_dealloc_device(&ibdev->ib_dev);
1818 }
1819
1820 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)
1821 {
1822         struct mlx4_ib_demux_work **dm = NULL;
1823         struct mlx4_dev *dev = ibdev->dev;
1824         int i;
1825         unsigned long flags;
1826
1827         if (!mlx4_is_master(dev))
1828                 return;
1829
1830         dm = kcalloc(dev->caps.num_ports, sizeof *dm, GFP_ATOMIC);
1831         if (!dm) {
1832                 pr_err("failed to allocate memory for tunneling qp update\n");
1833                 goto out;
1834         }
1835
1836         for (i = 0; i < dev->caps.num_ports; i++) {
1837                 dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC);
1838                 if (!dm[i]) {
1839                         pr_err("failed to allocate memory for tunneling qp update work struct\n");
1840                         for (i = 0; i < dev->caps.num_ports; i++) {
1841                                 if (dm[i])
1842                                         kfree(dm[i]);
1843                         }
1844                         goto out;
1845                 }
1846         }
1847         /* initialize or tear down tunnel QPs for the slave */
1848         for (i = 0; i < dev->caps.num_ports; i++) {
1849                 INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work);
1850                 dm[i]->port = i + 1;
1851                 dm[i]->slave = slave;
1852                 dm[i]->do_init = do_init;
1853                 dm[i]->dev = ibdev;
1854                 spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags);
1855                 if (!ibdev->sriov.is_going_down)
1856                         queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work);
1857                 spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);
1858         }
1859 out:
1860         kfree(dm);
1861         return;
1862 }
1863
1864 static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
1865                           enum mlx4_dev_event event, unsigned long param)
1866 {
1867         struct ib_event ibev;
1868         struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);
1869         struct mlx4_eqe *eqe = NULL;
1870         struct ib_event_work *ew;
1871         int p = 0;
1872
1873         if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE)
1874                 eqe = (struct mlx4_eqe *)param;
1875         else
1876                 p = (int) param;
1877
1878         switch (event) {
1879         case MLX4_DEV_EVENT_PORT_UP:
1880                 if (p > ibdev->num_ports)
1881                         return;
1882                 if (mlx4_is_master(dev) &&
1883                     rdma_port_get_link_layer(&ibdev->ib_dev, p) ==
1884                         IB_LINK_LAYER_INFINIBAND) {
1885                         mlx4_ib_invalidate_all_guid_record(ibdev, p);
1886                 }
1887                 ibev.event = IB_EVENT_PORT_ACTIVE;
1888                 break;
1889
1890         case MLX4_DEV_EVENT_PORT_DOWN:
1891                 if (p > ibdev->num_ports)
1892                         return;
1893                 ibev.event = IB_EVENT_PORT_ERR;
1894                 break;
1895
1896         case MLX4_DEV_EVENT_CATASTROPHIC_ERROR:
1897                 ibdev->ib_active = false;
1898                 ibev.event = IB_EVENT_DEVICE_FATAL;
1899                 break;
1900
1901         case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:
1902                 ew = kmalloc(sizeof *ew, GFP_ATOMIC);
1903                 if (!ew) {
1904                         pr_err("failed to allocate memory for events work\n");
1905                         break;
1906                 }
1907
1908                 INIT_WORK(&ew->work, handle_port_mgmt_change_event);
1909                 memcpy(&ew->ib_eqe, eqe, sizeof *eqe);
1910                 ew->ib_dev = ibdev;
1911                 /* need to queue only for port owner, which uses GEN_EQE */
1912                 if (mlx4_is_master(dev))
1913                         queue_work(wq, &ew->work);
1914                 else
1915                         handle_port_mgmt_change_event(&ew->work);
1916                 return;
1917
1918         case MLX4_DEV_EVENT_SLAVE_INIT:
1919                 /* here, p is the slave id */
1920                 do_slave_init(ibdev, p, 1);
1921                 return;
1922
1923         case MLX4_DEV_EVENT_SLAVE_SHUTDOWN:
1924                 /* here, p is the slave id */
1925                 do_slave_init(ibdev, p, 0);
1926                 return;
1927
1928         default:
1929                 return;
1930         }
1931
1932         ibev.device           = ibdev_ptr;
1933         ibev.element.port_num = (u8) p;
1934
1935         ib_dispatch_event(&ibev);
1936 }
1937
1938 static struct mlx4_interface mlx4_ib_interface = {
1939         .add            = mlx4_ib_add,
1940         .remove         = mlx4_ib_remove,
1941         .event          = mlx4_ib_event,
1942         .protocol       = MLX4_PROT_IB_IPV6
1943 };
1944
1945 static int __init mlx4_ib_init(void)
1946 {
1947         int err;
1948
1949         wq = create_singlethread_workqueue("mlx4_ib");
1950         if (!wq)
1951                 return -ENOMEM;
1952
1953         err = mlx4_ib_mcg_init();
1954         if (err)
1955                 goto clean_wq;
1956
1957         err = mlx4_register_interface(&mlx4_ib_interface);
1958         if (err)
1959                 goto clean_mcg;
1960
1961         return 0;
1962
1963 clean_mcg:
1964         mlx4_ib_mcg_destroy();
1965
1966 clean_wq:
1967         destroy_workqueue(wq);
1968         return err;
1969 }
1970
1971 static void __exit mlx4_ib_cleanup(void)
1972 {
1973         mlx4_unregister_interface(&mlx4_ib_interface);
1974         mlx4_ib_mcg_destroy();
1975         destroy_workqueue(wq);
1976 }
1977
1978 module_init(mlx4_ib_init);
1979 module_exit(mlx4_ib_cleanup);