drivers/infiniband/hw/mlx5/qp.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/module.h>
  34 #include <rdma/ib_umem.h>
  35 #include <rdma/ib_cache.h>
  36 #include <rdma/ib_user_verbs.h>
  37 #include "mlx5_ib.h"
  38 #include "user.h"
  39
  40 /* not supported currently */
  41 static int wq_signature;
  42
  43 enum {
  44         MLX5_IB_ACK_REQ_FREQ    = 8,
  45 };
  46
  47 enum {
  48         MLX5_IB_DEFAULT_SCHED_QUEUE     = 0x83,
  49         MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
  50         MLX5_IB_LINK_TYPE_IB            = 0,
  51         MLX5_IB_LINK_TYPE_ETH           = 1
  52 };
  53
  54 enum {
  55         MLX5_IB_SQ_STRIDE       = 6,
  56         MLX5_IB_CACHE_LINE_SIZE = 64,
  57 };
  58
  59 static const u32 mlx5_ib_opcode[] = {
  60         [IB_WR_SEND]                            = MLX5_OPCODE_SEND,
  61         [IB_WR_LSO]                             = MLX5_OPCODE_LSO,
  62         [IB_WR_SEND_WITH_IMM]                   = MLX5_OPCODE_SEND_IMM,
  63         [IB_WR_RDMA_WRITE]                      = MLX5_OPCODE_RDMA_WRITE,
  64         [IB_WR_RDMA_WRITE_WITH_IMM]             = MLX5_OPCODE_RDMA_WRITE_IMM,
  65         [IB_WR_RDMA_READ]                       = MLX5_OPCODE_RDMA_READ,
  66         [IB_WR_ATOMIC_CMP_AND_SWP]              = MLX5_OPCODE_ATOMIC_CS,
  67         [IB_WR_ATOMIC_FETCH_AND_ADD]            = MLX5_OPCODE_ATOMIC_FA,
  68         [IB_WR_SEND_WITH_INV]                   = MLX5_OPCODE_SEND_INVAL,
  69         [IB_WR_LOCAL_INV]                       = MLX5_OPCODE_UMR,
  70         [IB_WR_REG_MR]                          = MLX5_OPCODE_UMR,
  71         [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]       = MLX5_OPCODE_ATOMIC_MASKED_CS,
  72         [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]     = MLX5_OPCODE_ATOMIC_MASKED_FA,
  73         [MLX5_IB_WR_UMR]                        = MLX5_OPCODE_UMR,
  74 };
  75
  76 struct mlx5_wqe_eth_pad {
  77         u8 rsvd0[16];
  78 };
  79
  80 enum raw_qp_set_mask_map {
  81         MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID         = 1UL << 0,
  82 };
  83
  84 struct mlx5_modify_raw_qp_param {
  85         u16 operation;
  86
  87         u32 set_mask; /* raw_qp_set_mask_map */
  88         u8 rq_q_ctr_id;
  89 };
  90
  91 static void get_cqs(enum ib_qp_type qp_type,
  92                     struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
  93                     struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
  94
  95 static int is_qp0(enum ib_qp_type qp_type)
  96 {
  97         return qp_type == IB_QPT_SMI;
  98 }
  99
 100 static int is_sqp(enum ib_qp_type qp_type)
 101 {
 102         return is_qp0(qp_type) || is_qp1(qp_type);
 103 }
 104
 105 static void *get_wqe(struct mlx5_ib_qp *qp, int offset)
 106 {
 107         return mlx5_buf_offset(&qp->buf, offset);
 108 }
 109
 110 static void *get_recv_wqe(struct mlx5_ib_qp *qp, int n)
 111 {
 112         return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
 113 }
 114
 115 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n)
 116 {
 117         return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE));
 118 }
 119
 120 /**
 121  * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space.
 122  *
 123  * @qp: QP to copy from.
 124  * @send: copy from the send queue when non-zero, use the receive queue
 125  *        otherwise.
 126  * @wqe_index:  index to start copying from. For send work queues, the
 127  *              wqe_index is in units of MLX5_SEND_WQE_BB.
 128  *              For receive work queue, it is the number of work queue
 129  *              element in the queue.
 130  * @buffer: destination buffer.
 131  * @length: maximum number of bytes to copy.
 132  *
 133  * Copies at least a single WQE, but may copy more data.
 134  *
 135  * Return: the number of bytes copied, or an error code.
 136  */
 137 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
 138                           void *buffer, u32 length,
 139                           struct mlx5_ib_qp_base *base)
 140 {
 141         struct ib_device *ibdev = qp->ibqp.device;
 142         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 143         struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq;
 144         size_t offset;
 145         size_t wq_end;
 146         struct ib_umem *umem = base->ubuffer.umem;
 147         u32 first_copy_length;
 148         int wqe_length;
 149         int ret;
 150
 151         if (wq->wqe_cnt == 0) {
 152                 mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n",
 153                             qp->ibqp.qp_type);
 154                 return -EINVAL;
 155         }
 156
 157         offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift);
 158         wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift);
 159
 160         if (send && length < sizeof(struct mlx5_wqe_ctrl_seg))
 161                 return -EINVAL;
 162
 163         if (offset > umem->length ||
 164             (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length))
 165                 return -EINVAL;
 166
 167         first_copy_length = min_t(u32, offset + length, wq_end) - offset;
 168         ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length);
 169         if (ret)
 170                 return ret;
 171
 172         if (send) {
 173                 struct mlx5_wqe_ctrl_seg *ctrl = buffer;
 174                 int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 175
 176                 wqe_length = ds * MLX5_WQE_DS_UNITS;
 177         } else {
 178                 wqe_length = 1 << wq->wqe_shift;
 179         }
 180
 181         if (wqe_length <= first_copy_length)
 182                 return first_copy_length;
 183
 184         ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset,
 185                                 wqe_length - first_copy_length);
 186         if (ret)
 187                 return ret;
 188
 189         return wqe_length;
 190 }
 191
 192 static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
 193 {
 194         struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 195         struct ib_event event;
 196
 197         if (type == MLX5_EVENT_TYPE_PATH_MIG) {
 198                 /* This event is only valid for trans_qps */
 199                 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
 200         }
 201
 202         if (ibqp->event_handler) {
 203                 event.device     = ibqp->device;
 204                 event.element.qp = ibqp;
 205                 switch (type) {
 206                 case MLX5_EVENT_TYPE_PATH_MIG:
 207                         event.event = IB_EVENT_PATH_MIG;
 208                         break;
 209                 case MLX5_EVENT_TYPE_COMM_EST:
 210                         event.event = IB_EVENT_COMM_EST;
 211                         break;
 212                 case MLX5_EVENT_TYPE_SQ_DRAINED:
 213                         event.event = IB_EVENT_SQ_DRAINED;
 214                         break;
 215                 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 216                         event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 217                         break;
 218                 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 219                         event.event = IB_EVENT_QP_FATAL;
 220                         break;
 221                 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 222                         event.event = IB_EVENT_PATH_MIG_ERR;
 223                         break;
 224                 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 225                         event.event = IB_EVENT_QP_REQ_ERR;
 226                         break;
 227                 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 228                         event.event = IB_EVENT_QP_ACCESS_ERR;
 229                         break;
 230                 default:
 231                         pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
 232                         return;
 233                 }
 234
 235                 ibqp->event_handler(&event, ibqp->qp_context);
 236         }
 237 }
 238
 239 static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
 240                        int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
 241 {
 242         int wqe_size;
 243         int wq_size;
 244
 245         /* Sanity check RQ size before proceeding */
 246         if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
 247                 return -EINVAL;
 248
 249         if (!has_rq) {
 250                 qp->rq.max_gs = 0;
 251                 qp->rq.wqe_cnt = 0;
 252                 qp->rq.wqe_shift = 0;
 253                 cap->max_recv_wr = 0;
 254                 cap->max_recv_sge = 0;
 255         } else {
 256                 if (ucmd) {
 257                         qp->rq.wqe_cnt = ucmd->rq_wqe_count;
 258                         qp->rq.wqe_shift = ucmd->rq_wqe_shift;
 259                         qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
 260                         qp->rq.max_post = qp->rq.wqe_cnt;
 261                 } else {
 262                         wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;
 263                         wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
 264                         wqe_size = roundup_pow_of_two(wqe_size);
 265                         wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
 266                         wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
 267                         qp->rq.wqe_cnt = wq_size / wqe_size;
 268                         if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
 269                                 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
 270                                             wqe_size,
 271                                             MLX5_CAP_GEN(dev->mdev,
 272                                                          max_wqe_sz_rq));
 273                                 return -EINVAL;
 274                         }
 275                         qp->rq.wqe_shift = ilog2(wqe_size);
 276                         qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
 277                         qp->rq.max_post = qp->rq.wqe_cnt;
 278                 }
 279         }
 280
 281         return 0;
 282 }
 283
 284 static int sq_overhead(struct ib_qp_init_attr *attr)
 285 {
 286         int size = 0;
 287
 288         switch (attr->qp_type) {
 289         case IB_QPT_XRC_INI:
 290                 size += sizeof(struct mlx5_wqe_xrc_seg);
 291                 /* fall through */
 292         case IB_QPT_RC:
 293                 size += sizeof(struct mlx5_wqe_ctrl_seg) +
 294                         max(sizeof(struct mlx5_wqe_atomic_seg) +
 295                             sizeof(struct mlx5_wqe_raddr_seg),
 296                             sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 297                             sizeof(struct mlx5_mkey_seg));
 298                 break;
 299
 300         case IB_QPT_XRC_TGT:
 301                 return 0;
 302
 303         case IB_QPT_UC:
 304                 size += sizeof(struct mlx5_wqe_ctrl_seg) +
 305                         max(sizeof(struct mlx5_wqe_raddr_seg),
 306                             sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 307                             sizeof(struct mlx5_mkey_seg));
 308                 break;
 309
 310         case IB_QPT_UD:
 311                 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
 312                         size += sizeof(struct mlx5_wqe_eth_pad) +
 313                                 sizeof(struct mlx5_wqe_eth_seg);
 314                 /* fall through */
 315         case IB_QPT_SMI:
 316         case MLX5_IB_QPT_HW_GSI:
 317                 size += sizeof(struct mlx5_wqe_ctrl_seg) +
 318                         sizeof(struct mlx5_wqe_datagram_seg);
 319                 break;
 320
 321         case MLX5_IB_QPT_REG_UMR:
 322                 size += sizeof(struct mlx5_wqe_ctrl_seg) +
 323                         sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 324                         sizeof(struct mlx5_mkey_seg);
 325                 break;
 326
 327         default:
 328                 return -EINVAL;
 329         }
 330
 331         return size;
 332 }
 333
 334 static int calc_send_wqe(struct ib_qp_init_attr *attr)
 335 {
 336         int inl_size = 0;
 337         int size;
 338
 339         size = sq_overhead(attr);
 340         if (size < 0)
 341                 return size;
 342
 343         if (attr->cap.max_inline_data) {
 344                 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
 345                         attr->cap.max_inline_data;
 346         }
 347
 348         size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
 349         if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
 350             ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
 351                         return MLX5_SIG_WQE_SIZE;
 352         else
 353                 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
 354 }
 355
 356 static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 357                         struct mlx5_ib_qp *qp)
 358 {
 359         int wqe_size;
 360         int wq_size;
 361
 362         if (!attr->cap.max_send_wr)
 363                 return 0;
 364
 365         wqe_size = calc_send_wqe(attr);
 366         mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
 367         if (wqe_size < 0)
 368                 return wqe_size;
 369
 370         if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 371                 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
 372                             wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 373                 return -EINVAL;
 374         }
 375
 376         qp->max_inline_data = wqe_size - sq_overhead(attr) -
 377                               sizeof(struct mlx5_wqe_inline_seg);
 378         attr->cap.max_inline_data = qp->max_inline_data;
 379
 380         if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
 381                 qp->signature_en = true;
 382
 383         wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
 384         qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
 385         if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 386                 mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n",
 387                             qp->sq.wqe_cnt,
 388                             1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 389                 return -ENOMEM;
 390         }
 391         qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 392         qp->sq.max_gs = attr->cap.max_send_sge;
 393         qp->sq.max_post = wq_size / wqe_size;
 394         attr->cap.max_send_wr = qp->sq.max_post;
 395
 396         return wq_size;
 397 }
 398
 399 static int set_user_buf_size(struct mlx5_ib_dev *dev,
 400                             struct mlx5_ib_qp *qp,
 401                             struct mlx5_ib_create_qp *ucmd,
 402                             struct mlx5_ib_qp_base *base,
 403                             struct ib_qp_init_attr *attr)
 404 {
 405         int desc_sz = 1 << qp->sq.wqe_shift;
 406
 407         if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 408                 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
 409                              desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 410                 return -EINVAL;
 411         }
 412
 413         if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) {
 414                 mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n",
 415                              ucmd->sq_wqe_count, ucmd->sq_wqe_count);
 416                 return -EINVAL;
 417         }
 418
 419         qp->sq.wqe_cnt = ucmd->sq_wqe_count;
 420
 421         if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 422                 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
 423                              qp->sq.wqe_cnt,
 424                              1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 425                 return -EINVAL;
 426         }
 427
 428         if (attr->qp_type == IB_QPT_RAW_PACKET) {
 429                 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 430                 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
 431         } else {
 432                 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 433                                          (qp->sq.wqe_cnt << 6);
 434         }
 435
 436         return 0;
 437 }
 438
 439 static int qp_has_rq(struct ib_qp_init_attr *attr)
 440 {
 441         if (attr->qp_type == IB_QPT_XRC_INI ||
 442             attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
 443             attr->qp_type == MLX5_IB_QPT_REG_UMR ||
 444             !attr->cap.max_recv_wr)
 445                 return 0;
 446
 447         return 1;
 448 }
 449
 450 static int first_med_uuar(void)
 451 {
 452         return 1;
 453 }
 454
 455 static int next_uuar(int n)
 456 {
 457         n++;
 458
 459         while (((n % 4) & 2))
 460                 n++;
 461
 462         return n;
 463 }
 464
 465 static int num_med_uuar(struct mlx5_uuar_info *uuari)
 466 {
 467         int n;
 468
 469         n = uuari->num_uars * MLX5_NON_FP_BF_REGS_PER_PAGE -
 470                 uuari->num_low_latency_uuars - 1;
 471
 472         return n >= 0 ? n : 0;
 473 }
 474
 475 static int max_uuari(struct mlx5_uuar_info *uuari)
 476 {
 477         return uuari->num_uars * 4;
 478 }
 479
 480 static int first_hi_uuar(struct mlx5_uuar_info *uuari)
 481 {
 482         int med;
 483         int i;
 484         int t;
 485
 486         med = num_med_uuar(uuari);
 487         for (t = 0, i = first_med_uuar();; i = next_uuar(i)) {
 488                 t++;
 489                 if (t == med)
 490                         return next_uuar(i);
 491         }
 492
 493         return 0;
 494 }
 495
 496 static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari)
 497 {
 498         int i;
 499
 500         for (i = first_hi_uuar(uuari); i < max_uuari(uuari); i = next_uuar(i)) {
 501                 if (!test_bit(i, uuari->bitmap)) {
 502                         set_bit(i, uuari->bitmap);
 503                         uuari->count[i]++;
 504                         return i;
 505                 }
 506         }
 507
 508         return -ENOMEM;
 509 }
 510
 511 static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari)
 512 {
 513         int minidx = first_med_uuar();
 514         int i;
 515
 516         for (i = first_med_uuar(); i < first_hi_uuar(uuari); i = next_uuar(i)) {
 517                 if (uuari->count[i] < uuari->count[minidx])
 518                         minidx = i;
 519         }
 520
 521         uuari->count[minidx]++;
 522         return minidx;
 523 }
 524
 525 static int alloc_uuar(struct mlx5_uuar_info *uuari,
 526                       enum mlx5_ib_latency_class lat)
 527 {
 528         int uuarn = -EINVAL;
 529
 530         mutex_lock(&uuari->lock);
 531         switch (lat) {
 532         case MLX5_IB_LATENCY_CLASS_LOW:
 533                 uuarn = 0;
 534                 uuari->count[uuarn]++;
 535                 break;
 536
 537         case MLX5_IB_LATENCY_CLASS_MEDIUM:
 538                 if (uuari->ver < 2)
 539                         uuarn = -ENOMEM;
 540                 else
 541                         uuarn = alloc_med_class_uuar(uuari);
 542                 break;
 543
 544         case MLX5_IB_LATENCY_CLASS_HIGH:
 545                 if (uuari->ver < 2)
 546                         uuarn = -ENOMEM;
 547                 else
 548                         uuarn = alloc_high_class_uuar(uuari);
 549                 break;
 550
 551         case MLX5_IB_LATENCY_CLASS_FAST_PATH:
 552                 uuarn = 2;
 553                 break;
 554         }
 555         mutex_unlock(&uuari->lock);
 556
 557         return uuarn;
 558 }
 559
 560 static void free_med_class_uuar(struct mlx5_uuar_info *uuari, int uuarn)
 561 {
 562         clear_bit(uuarn, uuari->bitmap);
 563         --uuari->count[uuarn];
 564 }
 565
 566 static void free_high_class_uuar(struct mlx5_uuar_info *uuari, int uuarn)
 567 {
 568         clear_bit(uuarn, uuari->bitmap);
 569         --uuari->count[uuarn];
 570 }
 571
 572 static void free_uuar(struct mlx5_uuar_info *uuari, int uuarn)
 573 {
 574         int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE;
 575         int high_uuar = nuuars - uuari->num_low_latency_uuars;
 576
 577         mutex_lock(&uuari->lock);
 578         if (uuarn == 0) {
 579                 --uuari->count[uuarn];
 580                 goto out;
 581         }
 582
 583         if (uuarn < high_uuar) {
 584                 free_med_class_uuar(uuari, uuarn);
 585                 goto out;
 586         }
 587
 588         free_high_class_uuar(uuari, uuarn);
 589
 590 out:
 591         mutex_unlock(&uuari->lock);
 592 }
 593
 594 static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
 595 {
 596         switch (state) {
 597         case IB_QPS_RESET:      return MLX5_QP_STATE_RST;
 598         case IB_QPS_INIT:       return MLX5_QP_STATE_INIT;
 599         case IB_QPS_RTR:        return MLX5_QP_STATE_RTR;
 600         case IB_QPS_RTS:        return MLX5_QP_STATE_RTS;
 601         case IB_QPS_SQD:        return MLX5_QP_STATE_SQD;
 602         case IB_QPS_SQE:        return MLX5_QP_STATE_SQER;
 603         case IB_QPS_ERR:        return MLX5_QP_STATE_ERR;
 604         default:                return -1;
 605         }
 606 }
 607
 608 static int to_mlx5_st(enum ib_qp_type type)
 609 {
 610         switch (type) {
 611         case IB_QPT_RC:                 return MLX5_QP_ST_RC;
 612         case IB_QPT_UC:                 return MLX5_QP_ST_UC;
 613         case IB_QPT_UD:                 return MLX5_QP_ST_UD;
 614         case MLX5_IB_QPT_REG_UMR:       return MLX5_QP_ST_REG_UMR;
 615         case IB_QPT_XRC_INI:
 616         case IB_QPT_XRC_TGT:            return MLX5_QP_ST_XRC;
 617         case IB_QPT_SMI:                return MLX5_QP_ST_QP0;
 618         case MLX5_IB_QPT_HW_GSI:        return MLX5_QP_ST_QP1;
 619         case IB_QPT_RAW_IPV6:           return MLX5_QP_ST_RAW_IPV6;
 620         case IB_QPT_RAW_PACKET:
 621         case IB_QPT_RAW_ETHERTYPE:      return MLX5_QP_ST_RAW_ETHERTYPE;
 622         case IB_QPT_MAX:
 623         default:                return -EINVAL;
 624         }
 625 }
 626
 627 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
 628                              struct mlx5_ib_cq *recv_cq);
 629 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
 630                                struct mlx5_ib_cq *recv_cq);
 631
 632 static int uuarn_to_uar_index(struct mlx5_uuar_info *uuari, int uuarn)
 633 {
 634         return uuari->uars[uuarn / MLX5_BF_REGS_PER_PAGE].index;
 635 }
 636
 637 static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
 638                             struct ib_pd *pd,
 639                             unsigned long addr, size_t size,
 640                             struct ib_umem **umem,
 641                             int *npages, int *page_shift, int *ncont,
 642                             u32 *offset)
 643 {
 644         int err;
 645
 646         *umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0);
 647         if (IS_ERR(*umem)) {
 648                 mlx5_ib_dbg(dev, "umem_get failed\n");
 649                 return PTR_ERR(*umem);
 650         }
 651
 652         mlx5_ib_cont_pages(*umem, addr, npages, page_shift, ncont, NULL);
 653
 654         err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
 655         if (err) {
 656                 mlx5_ib_warn(dev, "bad offset\n");
 657                 goto err_umem;
 658         }
 659
 660         mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
 661                     addr, size, *npages, *page_shift, *ncont, *offset);
 662
 663         return 0;
 664
 665 err_umem:
 666         ib_umem_release(*umem);
 667         *umem = NULL;
 668
 669         return err;
 670 }
 671
 672 static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq)
 673 {
 674         struct mlx5_ib_ucontext *context;
 675
 676         context = to_mucontext(pd->uobject->context);
 677         mlx5_ib_db_unmap_user(context, &rwq->db);
 678         if (rwq->umem)
 679                 ib_umem_release(rwq->umem);
 680 }
 681
 682 static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 683                           struct mlx5_ib_rwq *rwq,
 684                           struct mlx5_ib_create_wq *ucmd)
 685 {
 686         struct mlx5_ib_ucontext *context;
 687         int page_shift = 0;
 688         int npages;
 689         u32 offset = 0;
 690         int ncont = 0;
 691         int err;
 692
 693         if (!ucmd->buf_addr)
 694                 return -EINVAL;
 695
 696         context = to_mucontext(pd->uobject->context);
 697         rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,
 698                                rwq->buf_size, 0, 0);
 699         if (IS_ERR(rwq->umem)) {
 700                 mlx5_ib_dbg(dev, "umem_get failed\n");
 701                 err = PTR_ERR(rwq->umem);
 702                 return err;
 703         }
 704
 705         mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, &npages, &page_shift,
 706                            &ncont, NULL);
 707         err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
 708                                      &rwq->rq_page_offset);
 709         if (err) {
 710                 mlx5_ib_warn(dev, "bad offset\n");
 711                 goto err_umem;
 712         }
 713
 714         rwq->rq_num_pas = ncont;
 715         rwq->page_shift = page_shift;
 716         rwq->log_page_size =  page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 717         rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
 718
 719         mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
 720                     (unsigned long long)ucmd->buf_addr, rwq->buf_size,
 721                     npages, page_shift, ncont, offset);
 722
 723         err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);
 724         if (err) {
 725                 mlx5_ib_dbg(dev, "map failed\n");
 726                 goto err_umem;
 727         }
 728
 729         rwq->create_type = MLX5_WQ_USER;
 730         return 0;
 731
 732 err_umem:
 733         ib_umem_release(rwq->umem);
 734         return err;
 735 }
 736
 737 static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 738                           struct mlx5_ib_qp *qp, struct ib_udata *udata,
 739                           struct ib_qp_init_attr *attr,
 740                           u32 **in,
 741                           struct mlx5_ib_create_qp_resp *resp, int *inlen,
 742                           struct mlx5_ib_qp_base *base)
 743 {
 744         struct mlx5_ib_ucontext *context;
 745         struct mlx5_ib_create_qp ucmd;
 746         struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 747         int page_shift = 0;
 748         int uar_index;
 749         int npages;
 750         u32 offset = 0;
 751         int uuarn;
 752         int ncont = 0;
 753         __be64 *pas;
 754         void *qpc;
 755         int err;
 756
 757         err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
 758         if (err) {
 759                 mlx5_ib_dbg(dev, "copy failed\n");
 760                 return err;
 761         }
 762
 763         context = to_mucontext(pd->uobject->context);
 764         /*
 765          * TBD: should come from the verbs when we have the API
 766          */
 767         if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
 768                 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
 769                 uuarn = MLX5_CROSS_CHANNEL_UUAR;
 770         else {
 771                 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);
 772                 if (uuarn < 0) {
 773                         mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
 774                         mlx5_ib_dbg(dev, "reverting to medium latency\n");
 775                         uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM);
 776                         if (uuarn < 0) {
 777                                 mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n");
 778                                 mlx5_ib_dbg(dev, "reverting to high latency\n");
 779                                 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW);
 780                                 if (uuarn < 0) {
 781                                         mlx5_ib_warn(dev, "uuar allocation failed\n");
 782                                         return uuarn;
 783                                 }
 784                         }
 785                 }
 786         }
 787
 788         uar_index = uuarn_to_uar_index(&context->uuari, uuarn);
 789         mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index);
 790
 791         qp->rq.offset = 0;
 792         qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 793         qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 794
 795         err = set_user_buf_size(dev, qp, &ucmd, base, attr);
 796         if (err)
 797                 goto err_uuar;
 798
 799         if (ucmd.buf_addr && ubuffer->buf_size) {
 800                 ubuffer->buf_addr = ucmd.buf_addr;
 801                 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr,
 802                                        ubuffer->buf_size,
 803                                        &ubuffer->umem, &npages, &page_shift,
 804                                        &ncont, &offset);
 805                 if (err)
 806                         goto err_uuar;
 807         } else {
 808                 ubuffer->umem = NULL;
 809         }
 810
 811         *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
 812                  MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
 813         *in = mlx5_vzalloc(*inlen);
 814         if (!*in) {
 815                 err = -ENOMEM;
 816                 goto err_umem;
 817         }
 818
 819         pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
 820         if (ubuffer->umem)
 821                 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
 822
 823         qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
 824
 825         MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 826         MLX5_SET(qpc, qpc, page_offset, offset);
 827
 828         MLX5_SET(qpc, qpc, uar_page, uar_index);
 829         resp->uuar_index = uuarn;
 830         qp->uuarn = uuarn;
 831
 832         err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
 833         if (err) {
 834                 mlx5_ib_dbg(dev, "map failed\n");
 835                 goto err_free;
 836         }
 837
 838         err = ib_copy_to_udata(udata, resp, sizeof(*resp));
 839         if (err) {
 840                 mlx5_ib_dbg(dev, "copy failed\n");
 841                 goto err_unmap;
 842         }
 843         qp->create_type = MLX5_QP_USER;
 844
 845         return 0;
 846
 847 err_unmap:
 848         mlx5_ib_db_unmap_user(context, &qp->db);
 849
 850 err_free:
 851         kvfree(*in);
 852
 853 err_umem:
 854         if (ubuffer->umem)
 855                 ib_umem_release(ubuffer->umem);
 856
 857 err_uuar:
 858         free_uuar(&context->uuari, uuarn);
 859         return err;
 860 }
 861
 862 static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp,
 863                             struct mlx5_ib_qp_base *base)
 864 {
 865         struct mlx5_ib_ucontext *context;
 866
 867         context = to_mucontext(pd->uobject->context);
 868         mlx5_ib_db_unmap_user(context, &qp->db);
 869         if (base->ubuffer.umem)
 870                 ib_umem_release(base->ubuffer.umem);
 871         free_uuar(&context->uuari, qp->uuarn);
 872 }
 873
 874 static int create_kernel_qp(struct mlx5_ib_dev *dev,
 875                             struct ib_qp_init_attr *init_attr,
 876                             struct mlx5_ib_qp *qp,
 877                             u32 **in, int *inlen,
 878                             struct mlx5_ib_qp_base *base)
 879 {
 880         enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
 881         struct mlx5_uuar_info *uuari;
 882         int uar_index;
 883         void *qpc;
 884         int uuarn;
 885         int err;
 886
 887         uuari = &dev->mdev->priv.uuari;
 888         if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
 889                                         IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
 890                                         IB_QP_CREATE_IPOIB_UD_LSO |
 891                                         mlx5_ib_create_qp_sqpn_qp1()))
 892                 return -EINVAL;
 893
 894         if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
 895                 lc = MLX5_IB_LATENCY_CLASS_FAST_PATH;
 896
 897         uuarn = alloc_uuar(uuari, lc);
 898         if (uuarn < 0) {
 899                 mlx5_ib_dbg(dev, "\n");
 900                 return -ENOMEM;
 901         }
 902
 903         qp->bf = &uuari->bfs[uuarn];
 904         uar_index = qp->bf->uar->index;
 905
 906         err = calc_sq_size(dev, init_attr, qp);
 907         if (err < 0) {
 908                 mlx5_ib_dbg(dev, "err %d\n", err);
 909                 goto err_uuar;
 910         }
 911
 912         qp->rq.offset = 0;
 913         qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 914         base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
 915
 916         err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
 917         if (err) {
 918                 mlx5_ib_dbg(dev, "err %d\n", err);
 919                 goto err_uuar;
 920         }
 921
 922         qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
 923         *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
 924                  MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
 925         *in = mlx5_vzalloc(*inlen);
 926         if (!*in) {
 927                 err = -ENOMEM;
 928                 goto err_buf;
 929         }
 930
 931         qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
 932         MLX5_SET(qpc, qpc, uar_page, uar_index);
 933         MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 934
 935         /* Set "fast registration enabled" for all kernel QPs */
 936         MLX5_SET(qpc, qpc, fre, 1);
 937         MLX5_SET(qpc, qpc, rlky, 1);
 938
 939         if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
 940                 MLX5_SET(qpc, qpc, deth_sqpn, 1);
 941                 qp->flags |= MLX5_IB_QP_SQPN_QP1;
 942         }
 943
 944         mlx5_fill_page_array(&qp->buf,
 945                              (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas));
 946
 947         err = mlx5_db_alloc(dev->mdev, &qp->db);
 948         if (err) {
 949                 mlx5_ib_dbg(dev, "err %d\n", err);
 950                 goto err_free;
 951         }
 952
 953         qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid), GFP_KERNEL);
 954         qp->sq.wr_data = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data), GFP_KERNEL);
 955         qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(*qp->rq.wrid), GFP_KERNEL);
 956         qp->sq.w_list = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.w_list), GFP_KERNEL);
 957         qp->sq.wqe_head = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head), GFP_KERNEL);
 958
 959         if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
 960             !qp->sq.w_list || !qp->sq.wqe_head) {
 961                 err = -ENOMEM;
 962                 goto err_wrid;
 963         }
 964         qp->create_type = MLX5_QP_KERNEL;
 965
 966         return 0;
 967
 968 err_wrid:
 969         mlx5_db_free(dev->mdev, &qp->db);
 970         kfree(qp->sq.wqe_head);
 971         kfree(qp->sq.w_list);
 972         kfree(qp->sq.wrid);
 973         kfree(qp->sq.wr_data);
 974         kfree(qp->rq.wrid);
 975
 976 err_free:
 977         kvfree(*in);
 978
 979 err_buf:
 980         mlx5_buf_free(dev->mdev, &qp->buf);
 981
 982 err_uuar:
 983         free_uuar(&dev->mdev->priv.uuari, uuarn);
 984         return err;
 985 }
 986
 987 static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 988 {
 989         mlx5_db_free(dev->mdev, &qp->db);
 990         kfree(qp->sq.wqe_head);
 991         kfree(qp->sq.w_list);
 992         kfree(qp->sq.wrid);
 993         kfree(qp->sq.wr_data);
 994         kfree(qp->rq.wrid);
 995         mlx5_buf_free(dev->mdev, &qp->buf);
 996         free_uuar(&dev->mdev->priv.uuari, qp->bf->uuarn);
 997 }
 998
 999 static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
1000 {
1001         if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
1002             (attr->qp_type == IB_QPT_XRC_INI))
1003                 return MLX5_SRQ_RQ;
1004         else if (!qp->has_rq)
1005                 return MLX5_ZERO_LEN_RQ;
1006         else
1007                 return MLX5_NON_ZERO_RQ;
1008 }
1009
1010 static int is_connected(enum ib_qp_type qp_type)
1011 {
1012         if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC)
1013                 return 1;
1014
1015         return 0;
1016 }
1017
1018 static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1019                                     struct mlx5_ib_sq *sq, u32 tdn)
1020 {
1021         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
1022         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1023
1024         MLX5_SET(tisc, tisc, transport_domain, tdn);
1025         return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
1026 }
1027
1028 static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1029                                       struct mlx5_ib_sq *sq)
1030 {
1031         mlx5_core_destroy_tis(dev->mdev, sq->tisn);
1032 }
1033
1034 static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1035                                    struct mlx5_ib_sq *sq, void *qpin,
1036                                    struct ib_pd *pd)
1037 {
1038         struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1039         __be64 *pas;
1040         void *in;
1041         void *sqc;
1042         void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1043         void *wq;
1044         int inlen;
1045         int err;
1046         int page_shift = 0;
1047         int npages;
1048         int ncont = 0;
1049         u32 offset = 0;
1050
1051         err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size,
1052                                &sq->ubuffer.umem, &npages, &page_shift,
1053                                &ncont, &offset);
1054         if (err)
1055                 return err;
1056
1057         inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
1058         in = mlx5_vzalloc(inlen);
1059         if (!in) {
1060                 err = -ENOMEM;
1061                 goto err_umem;
1062         }
1063
1064         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1065         MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1066         MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1067         MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1068         MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1069         MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1070         MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
1071
1072         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1073         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1074         MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1075         MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1076         MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1077         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1078         MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1079         MLX5_SET(wq, wq, log_wq_pg_sz,  page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1080         MLX5_SET(wq, wq, page_offset, offset);
1081
1082         pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1083         mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1084
1085         err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp);
1086
1087         kvfree(in);
1088
1089         if (err)
1090                 goto err_umem;
1091
1092         return 0;
1093
1094 err_umem:
1095         ib_umem_release(sq->ubuffer.umem);
1096         sq->ubuffer.umem = NULL;
1097
1098         return err;
1099 }
1100
1101 static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1102                                      struct mlx5_ib_sq *sq)
1103 {
1104         mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1105         ib_umem_release(sq->ubuffer.umem);
1106 }
1107
1108 static int get_rq_pas_size(void *qpc)
1109 {
1110         u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
1111         u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
1112         u32 log_rq_size   = MLX5_GET(qpc, qpc, log_rq_size);
1113         u32 page_offset   = MLX5_GET(qpc, qpc, page_offset);
1114         u32 po_quanta     = 1 << (log_page_size - 6);
1115         u32 rq_sz         = 1 << (log_rq_size + 4 + log_rq_stride);
1116         u32 page_size     = 1 << log_page_size;
1117         u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
1118         u32 rq_num_pas    = (rq_sz_po + page_size - 1) / page_size;
1119
1120         return rq_num_pas * sizeof(u64);
1121 }
1122
1123 static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1124                                    struct mlx5_ib_rq *rq, void *qpin)
1125 {
1126         struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
1127         __be64 *pas;
1128         __be64 *qp_pas;
1129         void *in;
1130         void *rqc;
1131         void *wq;
1132         void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1133         int inlen;
1134         int err;
1135         u32 rq_pas_size = get_rq_pas_size(qpc);
1136
1137         inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
1138         in = mlx5_vzalloc(inlen);
1139         if (!in)
1140                 return -ENOMEM;
1141
1142         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1143         MLX5_SET(rqc, rqc, vsd, 1);
1144         MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1145         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1146         MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1147         MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1148         MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1149
1150         if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS)
1151                 MLX5_SET(rqc, rqc, scatter_fcs, 1);
1152
1153         wq = MLX5_ADDR_OF(rqc, rqc, wq);
1154         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1155         MLX5_SET(wq, wq, end_padding_mode,
1156                  MLX5_GET(qpc, qpc, end_padding_mode));
1157         MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
1158         MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1159         MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1160         MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1161         MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
1162         MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1163
1164         pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1165         qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1166         memcpy(pas, qp_pas, rq_pas_size);
1167
1168         err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp);
1169
1170         kvfree(in);
1171
1172         return err;
1173 }
1174
1175 static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1176                                      struct mlx5_ib_rq *rq)
1177 {
1178         mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp);
1179 }
1180
1181 static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1182                                     struct mlx5_ib_rq *rq, u32 tdn)
1183 {
1184         u32 *in;
1185         void *tirc;
1186         int inlen;
1187         int err;
1188
1189         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1190         in = mlx5_vzalloc(inlen);
1191         if (!in)
1192                 return -ENOMEM;
1193
1194         tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1195         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1196         MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1197         MLX5_SET(tirc, tirc, transport_domain, tdn);
1198
1199         err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
1200
1201         kvfree(in);
1202
1203         return err;
1204 }
1205
1206 static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1207                                       struct mlx5_ib_rq *rq)
1208 {
1209         mlx5_core_destroy_tir(dev->mdev, rq->tirn);
1210 }
1211
1212 static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1213                                 u32 *in,
1214                                 struct ib_pd *pd)
1215 {
1216         struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1217         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1218         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1219         struct ib_uobject *uobj = pd->uobject;
1220         struct ib_ucontext *ucontext = uobj->context;
1221         struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1222         int err;
1223         u32 tdn = mucontext->tdn;
1224
1225         if (qp->sq.wqe_cnt) {
1226                 err = create_raw_packet_qp_tis(dev, sq, tdn);
1227                 if (err)
1228                         return err;
1229
1230                 err = create_raw_packet_qp_sq(dev, sq, in, pd);
1231                 if (err)
1232                         goto err_destroy_tis;
1233
1234                 sq->base.container_mibqp = qp;
1235         }
1236
1237         if (qp->rq.wqe_cnt) {
1238                 rq->base.container_mibqp = qp;
1239
1240                 err = create_raw_packet_qp_rq(dev, rq, in);
1241                 if (err)
1242                         goto err_destroy_sq;
1243
1244
1245                 err = create_raw_packet_qp_tir(dev, rq, tdn);
1246                 if (err)
1247                         goto err_destroy_rq;
1248         }
1249
1250         qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1251                                                      rq->base.mqp.qpn;
1252
1253         return 0;
1254
1255 err_destroy_rq:
1256         destroy_raw_packet_qp_rq(dev, rq);
1257 err_destroy_sq:
1258         if (!qp->sq.wqe_cnt)
1259                 return err;
1260         destroy_raw_packet_qp_sq(dev, sq);
1261 err_destroy_tis:
1262         destroy_raw_packet_qp_tis(dev, sq);
1263
1264         return err;
1265 }
1266
1267 static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1268                                   struct mlx5_ib_qp *qp)
1269 {
1270         struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1271         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1272         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1273
1274         if (qp->rq.wqe_cnt) {
1275                 destroy_raw_packet_qp_tir(dev, rq);
1276                 destroy_raw_packet_qp_rq(dev, rq);
1277         }
1278
1279         if (qp->sq.wqe_cnt) {
1280                 destroy_raw_packet_qp_sq(dev, sq);
1281                 destroy_raw_packet_qp_tis(dev, sq);
1282         }
1283 }
1284
1285 static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1286                                     struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1287 {
1288         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1289         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1290
1291         sq->sq = &qp->sq;
1292         rq->rq = &qp->rq;
1293         sq->doorbell = &qp->db;
1294         rq->doorbell = &qp->db;
1295 }
1296
1297 static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1298 {
1299         mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn);
1300 }
1301
1302 static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1303                                  struct ib_pd *pd,
1304                                  struct ib_qp_init_attr *init_attr,
1305                                  struct ib_udata *udata)
1306 {
1307         struct ib_uobject *uobj = pd->uobject;
1308         struct ib_ucontext *ucontext = uobj->context;
1309         struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1310         struct mlx5_ib_create_qp_resp resp = {};
1311         int inlen;
1312         int err;
1313         u32 *in;
1314         void *tirc;
1315         void *hfso;
1316         u32 selected_fields = 0;
1317         size_t min_resp_len;
1318         u32 tdn = mucontext->tdn;
1319         struct mlx5_ib_create_qp_rss ucmd = {};
1320         size_t required_cmd_sz;
1321
1322         if (init_attr->qp_type != IB_QPT_RAW_PACKET)
1323                 return -EOPNOTSUPP;
1324
1325         if (init_attr->create_flags || init_attr->send_cq)
1326                 return -EINVAL;
1327
1328         min_resp_len = offsetof(typeof(resp), uuar_index) + sizeof(resp.uuar_index);
1329         if (udata->outlen < min_resp_len)
1330                 return -EINVAL;
1331
1332         required_cmd_sz = offsetof(typeof(ucmd), reserved1) + sizeof(ucmd.reserved1);
1333         if (udata->inlen < required_cmd_sz) {
1334                 mlx5_ib_dbg(dev, "invalid inlen\n");
1335                 return -EINVAL;
1336         }
1337
1338         if (udata->inlen > sizeof(ucmd) &&
1339             !ib_is_udata_cleared(udata, sizeof(ucmd),
1340                                  udata->inlen - sizeof(ucmd))) {
1341                 mlx5_ib_dbg(dev, "inlen is not supported\n");
1342                 return -EOPNOTSUPP;
1343         }
1344
1345         if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
1346                 mlx5_ib_dbg(dev, "copy failed\n");
1347                 return -EFAULT;
1348         }
1349
1350         if (ucmd.comp_mask) {
1351                 mlx5_ib_dbg(dev, "invalid comp mask\n");
1352                 return -EOPNOTSUPP;
1353         }
1354
1355         if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)) || ucmd.reserved1) {
1356                 mlx5_ib_dbg(dev, "invalid reserved\n");
1357                 return -EOPNOTSUPP;
1358         }
1359
1360         err = ib_copy_to_udata(udata, &resp, min_resp_len);
1361         if (err) {
1362                 mlx5_ib_dbg(dev, "copy failed\n");
1363                 return -EINVAL;
1364         }
1365
1366         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1367         in = mlx5_vzalloc(inlen);
1368         if (!in)
1369                 return -ENOMEM;
1370
1371         tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1372         MLX5_SET(tirc, tirc, disp_type,
1373                  MLX5_TIRC_DISP_TYPE_INDIRECT);
1374         MLX5_SET(tirc, tirc, indirect_table,
1375                  init_attr->rwq_ind_tbl->ind_tbl_num);
1376         MLX5_SET(tirc, tirc, transport_domain, tdn);
1377
1378         hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1379         switch (ucmd.rx_hash_function) {
1380         case MLX5_RX_HASH_FUNC_TOEPLITZ:
1381         {
1382                 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1383                 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1384
1385                 if (len != ucmd.rx_key_len) {
1386                         err = -EINVAL;
1387                         goto err;
1388                 }
1389
1390                 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1391                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1392                 memcpy(rss_key, ucmd.rx_hash_key, len);
1393                 break;
1394         }
1395         default:
1396                 err = -EOPNOTSUPP;
1397                 goto err;
1398         }
1399
1400         if (!ucmd.rx_hash_fields_mask) {
1401                 /* special case when this TIR serves as steering entry without hashing */
1402                 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1403                         goto create_tir;
1404                 err = -EINVAL;
1405                 goto err;
1406         }
1407
1408         if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1409              (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1410              ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1411              (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1412                 err = -EINVAL;
1413                 goto err;
1414         }
1415
1416         /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1417         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1418             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1419                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1420                          MLX5_L3_PROT_TYPE_IPV4);
1421         else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1422                  (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1423                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1424                          MLX5_L3_PROT_TYPE_IPV6);
1425
1426         if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1427              (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) &&
1428              ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1429              (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))) {
1430                 err = -EINVAL;
1431                 goto err;
1432         }
1433
1434         /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1435         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1436             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1437                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1438                          MLX5_L4_PROT_TYPE_TCP);
1439         else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1440                  (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1441                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1442                          MLX5_L4_PROT_TYPE_UDP);
1443
1444         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1445             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1446                 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1447
1448         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1449             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1450                 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1451
1452         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1453             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1454                 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1455
1456         if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1457             (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1458                 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1459
1460         MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1461
1462 create_tir:
1463         err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn);
1464
1465         if (err)
1466                 goto err;
1467
1468         kvfree(in);
1469         /* qpn is reserved for that QP */
1470         qp->trans_qp.base.mqp.qpn = 0;
1471         return 0;
1472
1473 err:
1474         kvfree(in);
1475         return err;
1476 }
1477
1478 static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1479                             struct ib_qp_init_attr *init_attr,
1480                             struct ib_udata *udata, struct mlx5_ib_qp *qp)
1481 {
1482         struct mlx5_ib_resources *devr = &dev->devr;
1483         int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
1484         struct mlx5_core_dev *mdev = dev->mdev;
1485         struct mlx5_ib_create_qp_resp resp;
1486         struct mlx5_ib_cq *send_cq;
1487         struct mlx5_ib_cq *recv_cq;
1488         unsigned long flags;
1489         u32 uidx = MLX5_IB_DEFAULT_UIDX;
1490         struct mlx5_ib_create_qp ucmd;
1491         struct mlx5_ib_qp_base *base;
1492         void *qpc;
1493         u32 *in;
1494         int err;
1495
1496         base = init_attr->qp_type == IB_QPT_RAW_PACKET ?
1497                &qp->raw_packet_qp.rq.base :
1498                &qp->trans_qp.base;
1499
1500         if (init_attr->qp_type != IB_QPT_RAW_PACKET)
1501                 mlx5_ib_odp_create_qp(qp);
1502
1503         mutex_init(&qp->mutex);
1504         spin_lock_init(&qp->sq.lock);
1505         spin_lock_init(&qp->rq.lock);
1506
1507         if (init_attr->rwq_ind_tbl) {
1508                 if (!udata)
1509                         return -ENOSYS;
1510
1511                 err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);
1512                 return err;
1513         }
1514
1515         if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
1516                 if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {
1517                         mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
1518                         return -EINVAL;
1519                 } else {
1520                         qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1521                 }
1522         }
1523
1524         if (init_attr->create_flags &
1525                         (IB_QP_CREATE_CROSS_CHANNEL |
1526                          IB_QP_CREATE_MANAGED_SEND |
1527                          IB_QP_CREATE_MANAGED_RECV)) {
1528                 if (!MLX5_CAP_GEN(mdev, cd)) {
1529                         mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
1530                         return -EINVAL;
1531                 }
1532                 if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
1533                         qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;
1534                 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
1535                         qp->flags |= MLX5_IB_QP_MANAGED_SEND;
1536                 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
1537                         qp->flags |= MLX5_IB_QP_MANAGED_RECV;
1538         }
1539
1540         if (init_attr->qp_type == IB_QPT_UD &&
1541             (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
1542                 if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1543                         mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
1544                         return -EOPNOTSUPP;
1545                 }
1546
1547         if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1548                 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1549                         mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");
1550                         return -EOPNOTSUPP;
1551                 }
1552                 if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) ||
1553                     !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
1554                         mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");
1555                         return -EOPNOTSUPP;
1556                 }
1557                 qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS;
1558         }
1559
1560         if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1561                 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1562
1563         if (pd && pd->uobject) {
1564                 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
1565                         mlx5_ib_dbg(dev, "copy failed\n");
1566                         return -EFAULT;
1567                 }
1568
1569                 err = get_qp_user_index(to_mucontext(pd->uobject->context),
1570                                         &ucmd, udata->inlen, &uidx);
1571                 if (err)
1572                         return err;
1573
1574                 qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
1575                 qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
1576         } else {
1577                 qp->wq_sig = !!wq_signature;
1578         }
1579
1580         qp->has_rq = qp_has_rq(init_attr);
1581         err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
1582                           qp, (pd && pd->uobject) ? &ucmd : NULL);
1583         if (err) {
1584                 mlx5_ib_dbg(dev, "err %d\n", err);
1585                 return err;
1586         }
1587
1588         if (pd) {
1589                 if (pd->uobject) {
1590                         __u32 max_wqes =
1591                                 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1592                         mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);
1593                         if (ucmd.rq_wqe_shift != qp->rq.wqe_shift ||
1594                             ucmd.rq_wqe_count != qp->rq.wqe_cnt) {
1595                                 mlx5_ib_dbg(dev, "invalid rq params\n");
1596                                 return -EINVAL;
1597                         }
1598                         if (ucmd.sq_wqe_count > max_wqes) {
1599                                 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n",
1600                                             ucmd.sq_wqe_count, max_wqes);
1601                                 return -EINVAL;
1602                         }
1603                         if (init_attr->create_flags &
1604                             mlx5_ib_create_qp_sqpn_qp1()) {
1605                                 mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");
1606                                 return -EINVAL;
1607                         }
1608                         err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
1609                                              &resp, &inlen, base);
1610                         if (err)
1611                                 mlx5_ib_dbg(dev, "err %d\n", err);
1612                 } else {
1613                         err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
1614                                                base);
1615                         if (err)
1616                                 mlx5_ib_dbg(dev, "err %d\n", err);
1617                 }
1618
1619                 if (err)
1620                         return err;
1621         } else {
1622                 in = mlx5_vzalloc(inlen);
1623                 if (!in)
1624                         return -ENOMEM;
1625
1626                 qp->create_type = MLX5_QP_EMPTY;
1627         }
1628
1629         if (is_sqp(init_attr->qp_type))
1630                 qp->port = init_attr->port_num;
1631
1632         qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1633
1634         MLX5_SET(qpc, qpc, st, to_mlx5_st(init_attr->qp_type));
1635         MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
1636
1637         if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
1638                 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
1639         else
1640                 MLX5_SET(qpc, qpc, latency_sensitive, 1);
1641
1642
1643         if (qp->wq_sig)
1644                 MLX5_SET(qpc, qpc, wq_signature, 1);
1645
1646         if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
1647                 MLX5_SET(qpc, qpc, block_lb_mc, 1);
1648
1649         if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
1650                 MLX5_SET(qpc, qpc, cd_master, 1);
1651         if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
1652                 MLX5_SET(qpc, qpc, cd_slave_send, 1);
1653         if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
1654                 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
1655
1656         if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
1657                 int rcqe_sz;
1658                 int scqe_sz;
1659
1660                 rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq);
1661                 scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq);
1662
1663                 if (rcqe_sz == 128)
1664                         MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
1665                 else
1666                         MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE);
1667
1668                 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) {
1669                         if (scqe_sz == 128)
1670                                 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1671                         else
1672                                 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1673                 }
1674         }
1675
1676         if (qp->rq.wqe_cnt) {
1677                 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
1678                 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
1679         }
1680
1681         MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
1682
1683         if (qp->sq.wqe_cnt)
1684                 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
1685         else
1686                 MLX5_SET(qpc, qpc, no_sq, 1);
1687
1688         /* Set default resources */
1689         switch (init_attr->qp_type) {
1690         case IB_QPT_XRC_TGT:
1691                 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1692                 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
1693                 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1694                 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
1695                 break;
1696         case IB_QPT_XRC_INI:
1697                 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1698                 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1699                 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1700                 break;
1701         default:
1702                 if (init_attr->srq) {
1703                         MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
1704                         MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
1705                 } else {
1706                         MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1707                         MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
1708                 }
1709         }
1710
1711         if (init_attr->send_cq)
1712                 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
1713
1714         if (init_attr->recv_cq)
1715                 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
1716
1717         MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
1718
1719         /* 0xffffff means we ask to work with cqe version 0 */
1720         if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
1721                 MLX5_SET(qpc, qpc, user_index, uidx);
1722
1723         /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
1724         if (init_attr->qp_type == IB_QPT_UD &&
1725             (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
1726                 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
1727                 qp->flags |= MLX5_IB_QP_LSO;
1728         }
1729
1730         if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
1731                 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
1732                 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
1733                 err = create_raw_packet_qp(dev, qp, in, pd);
1734         } else {
1735                 err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
1736         }
1737
1738         if (err) {
1739                 mlx5_ib_dbg(dev, "create qp failed\n");
1740                 goto err_create;
1741         }
1742
1743         kvfree(in);
1744
1745         base->container_mibqp = qp;
1746         base->mqp.event = mlx5_ib_qp_event;
1747
1748         get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq,
1749                 &send_cq, &recv_cq);
1750         spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1751         mlx5_ib_lock_cqs(send_cq, recv_cq);
1752         /* Maintain device to QPs access, needed for further handling via reset
1753          * flow
1754          */
1755         list_add_tail(&qp->qps_list, &dev->qp_list);
1756         /* Maintain CQ to QPs access, needed for further handling via reset flow
1757          */
1758         if (send_cq)
1759                 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
1760         if (recv_cq)
1761                 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
1762         mlx5_ib_unlock_cqs(send_cq, recv_cq);
1763         spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1764
1765         return 0;
1766
1767 err_create:
1768         if (qp->create_type == MLX5_QP_USER)
1769                 destroy_qp_user(pd, qp, base);
1770         else if (qp->create_type == MLX5_QP_KERNEL)
1771                 destroy_qp_kernel(dev, qp);
1772
1773         kvfree(in);
1774         return err;
1775 }
1776
1777 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
1778         __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1779 {
1780         if (send_cq) {
1781                 if (recv_cq) {
1782                         if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
1783                                 spin_lock(&send_cq->lock);
1784                                 spin_lock_nested(&recv_cq->lock,
1785                                                  SINGLE_DEPTH_NESTING);
1786                         } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
1787                                 spin_lock(&send_cq->lock);
1788                                 __acquire(&recv_cq->lock);
1789                         } else {
1790                                 spin_lock(&recv_cq->lock);
1791                                 spin_lock_nested(&send_cq->lock,
1792                                                  SINGLE_DEPTH_NESTING);
1793                         }
1794                 } else {
1795                         spin_lock(&send_cq->lock);
1796                         __acquire(&recv_cq->lock);
1797                 }
1798         } else if (recv_cq) {
1799                 spin_lock(&recv_cq->lock);
1800                 __acquire(&send_cq->lock);
1801         } else {
1802                 __acquire(&send_cq->lock);
1803                 __acquire(&recv_cq->lock);
1804         }
1805 }
1806
1807 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
1808         __releases(&send_cq->lock) __releases(&recv_cq->lock)
1809 {
1810         if (send_cq) {
1811                 if (recv_cq) {
1812                         if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
1813                                 spin_unlock(&recv_cq->lock);
1814                                 spin_unlock(&send_cq->lock);
1815                         } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
1816                                 __release(&recv_cq->lock);
1817                                 spin_unlock(&send_cq->lock);
1818                         } else {
1819                                 spin_unlock(&send_cq->lock);
1820                                 spin_unlock(&recv_cq->lock);
1821                         }
1822                 } else {
1823                         __release(&recv_cq->lock);
1824                         spin_unlock(&send_cq->lock);
1825                 }
1826         } else if (recv_cq) {
1827                 __release(&send_cq->lock);
1828                 spin_unlock(&recv_cq->lock);
1829         } else {
1830                 __release(&recv_cq->lock);
1831                 __release(&send_cq->lock);
1832         }
1833 }
1834
1835 static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp)
1836 {
1837         return to_mpd(qp->ibqp.pd);
1838 }
1839
1840 static void get_cqs(enum ib_qp_type qp_type,
1841                     struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
1842                     struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
1843 {
1844         switch (qp_type) {
1845         case IB_QPT_XRC_TGT:
1846                 *send_cq = NULL;
1847                 *recv_cq = NULL;
1848                 break;
1849         case MLX5_IB_QPT_REG_UMR:
1850         case IB_QPT_XRC_INI:
1851                 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
1852                 *recv_cq = NULL;
1853                 break;
1854
1855         case IB_QPT_SMI:
1856         case MLX5_IB_QPT_HW_GSI:
1857         case IB_QPT_RC:
1858         case IB_QPT_UC:
1859         case IB_QPT_UD:
1860         case IB_QPT_RAW_IPV6:
1861         case IB_QPT_RAW_ETHERTYPE:
1862         case IB_QPT_RAW_PACKET:
1863                 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
1864                 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
1865                 break;
1866
1867         case IB_QPT_MAX:
1868         default:
1869                 *send_cq = NULL;
1870                 *recv_cq = NULL;
1871                 break;
1872         }
1873 }
1874
1875 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1876                                 const struct mlx5_modify_raw_qp_param *raw_qp_param);
1877
1878 static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1879 {
1880         struct mlx5_ib_cq *send_cq, *recv_cq;
1881         struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
1882         unsigned long flags;
1883         int err;
1884
1885         if (qp->ibqp.rwq_ind_tbl) {
1886                 destroy_rss_raw_qp_tir(dev, qp);
1887                 return;
1888         }
1889
1890         base = qp->ibqp.qp_type == IB_QPT_RAW_PACKET ?
1891                &qp->raw_packet_qp.rq.base :
1892                &qp->trans_qp.base;
1893
1894         if (qp->state != IB_QPS_RESET) {
1895                 if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) {
1896                         mlx5_ib_qp_disable_pagefaults(qp);
1897                         err = mlx5_core_qp_modify(dev->mdev,
1898                                                   MLX5_CMD_OP_2RST_QP, 0,
1899                                                   NULL, &base->mqp);
1900                 } else {
1901                         struct mlx5_modify_raw_qp_param raw_qp_param = {
1902                                 .operation = MLX5_CMD_OP_2RST_QP
1903                         };
1904
1905                         err = modify_raw_packet_qp(dev, qp, &raw_qp_param);
1906                 }
1907                 if (err)
1908                         mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
1909                                      base->mqp.qpn);
1910         }
1911
1912         get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
1913                 &send_cq, &recv_cq);
1914
1915         spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1916         mlx5_ib_lock_cqs(send_cq, recv_cq);
1917         /* del from lists under both locks above to protect reset flow paths */
1918         list_del(&qp->qps_list);
1919         if (send_cq)
1920                 list_del(&qp->cq_send_list);
1921
1922         if (recv_cq)
1923                 list_del(&qp->cq_recv_list);
1924
1925         if (qp->create_type == MLX5_QP_KERNEL) {
1926                 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
1927                                    qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
1928                 if (send_cq != recv_cq)
1929                         __mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
1930                                            NULL);
1931         }
1932         mlx5_ib_unlock_cqs(send_cq, recv_cq);
1933         spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1934
1935         if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
1936                 destroy_raw_packet_qp(dev, qp);
1937         } else {
1938                 err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
1939                 if (err)
1940                         mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
1941                                      base->mqp.qpn);
1942         }
1943
1944         if (qp->create_type == MLX5_QP_KERNEL)
1945                 destroy_qp_kernel(dev, qp);
1946         else if (qp->create_type == MLX5_QP_USER)
1947                 destroy_qp_user(&get_pd(qp)->ibpd, qp, base);
1948 }
1949
1950 static const char *ib_qp_type_str(enum ib_qp_type type)
1951 {
1952         switch (type) {
1953         case IB_QPT_SMI:
1954                 return "IB_QPT_SMI";
1955         case IB_QPT_GSI:
1956                 return "IB_QPT_GSI";
1957         case IB_QPT_RC:
1958                 return "IB_QPT_RC";
1959         case IB_QPT_UC:
1960                 return "IB_QPT_UC";
1961         case IB_QPT_UD:
1962                 return "IB_QPT_UD";
1963         case IB_QPT_RAW_IPV6:
1964                 return "IB_QPT_RAW_IPV6";
1965         case IB_QPT_RAW_ETHERTYPE:
1966                 return "IB_QPT_RAW_ETHERTYPE";
1967         case IB_QPT_XRC_INI:
1968                 return "IB_QPT_XRC_INI";
1969         case IB_QPT_XRC_TGT:
1970                 return "IB_QPT_XRC_TGT";
1971         case IB_QPT_RAW_PACKET:
1972                 return "IB_QPT_RAW_PACKET";
1973         case MLX5_IB_QPT_REG_UMR:
1974                 return "MLX5_IB_QPT_REG_UMR";
1975         case IB_QPT_MAX:
1976         default:
1977                 return "Invalid QP type";
1978         }
1979 }
1980
1981 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
1982                                 struct ib_qp_init_attr *init_attr,
1983                                 struct ib_udata *udata)
1984 {
1985         struct mlx5_ib_dev *dev;
1986         struct mlx5_ib_qp *qp;
1987         u16 xrcdn = 0;
1988         int err;
1989
1990         if (pd) {
1991                 dev = to_mdev(pd->device);
1992
1993                 if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
1994                         if (!pd->uobject) {
1995                                 mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
1996                                 return ERR_PTR(-EINVAL);
1997                         } else if (!to_mucontext(pd->uobject->context)->cqe_version) {
1998                                 mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
1999                                 return ERR_PTR(-EINVAL);
2000                         }
2001                 }
2002         } else {
2003                 /* being cautious here */
2004                 if (init_attr->qp_type != IB_QPT_XRC_TGT &&
2005                     init_attr->qp_type != MLX5_IB_QPT_REG_UMR) {
2006                         pr_warn("%s: no PD for transport %s\n", __func__,
2007                                 ib_qp_type_str(init_attr->qp_type));
2008                         return ERR_PTR(-EINVAL);
2009                 }
2010                 dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
2011         }
2012
2013         switch (init_attr->qp_type) {
2014         case IB_QPT_XRC_TGT:
2015         case IB_QPT_XRC_INI:
2016                 if (!MLX5_CAP_GEN(dev->mdev, xrc)) {
2017                         mlx5_ib_dbg(dev, "XRC not supported\n");
2018                         return ERR_PTR(-ENOSYS);
2019                 }
2020                 init_attr->recv_cq = NULL;
2021                 if (init_attr->qp_type == IB_QPT_XRC_TGT) {
2022                         xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
2023                         init_attr->send_cq = NULL;
2024                 }
2025
2026                 /* fall through */
2027         case IB_QPT_RAW_PACKET:
2028         case IB_QPT_RC:
2029         case IB_QPT_UC:
2030         case IB_QPT_UD:
2031         case IB_QPT_SMI:
2032         case MLX5_IB_QPT_HW_GSI:
2033         case MLX5_IB_QPT_REG_UMR:
2034                 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2035                 if (!qp)
2036                         return ERR_PTR(-ENOMEM);
2037
2038                 err = create_qp_common(dev, pd, init_attr, udata, qp);
2039                 if (err) {
2040                         mlx5_ib_dbg(dev, "create_qp_common failed\n");
2041                         kfree(qp);
2042                         return ERR_PTR(err);
2043                 }
2044
2045                 if (is_qp0(init_attr->qp_type))
2046                         qp->ibqp.qp_num = 0;
2047                 else if (is_qp1(init_attr->qp_type))
2048                         qp->ibqp.qp_num = 1;
2049                 else
2050                         qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
2051
2052                 mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
2053                             qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
2054                             to_mcq(init_attr->recv_cq)->mcq.cqn,
2055                             to_mcq(init_attr->send_cq)->mcq.cqn);
2056
2057                 qp->trans_qp.xrcdn = xrcdn;
2058
2059                 break;
2060
2061         case IB_QPT_GSI:
2062                 return mlx5_ib_gsi_create_qp(pd, init_attr);
2063
2064         case IB_QPT_RAW_IPV6:
2065         case IB_QPT_RAW_ETHERTYPE:
2066         case IB_QPT_MAX:
2067         default:
2068                 mlx5_ib_dbg(dev, "unsupported qp type %d\n",
2069                             init_attr->qp_type);
2070                 /* Don't support raw QPs */
2071                 return ERR_PTR(-EINVAL);
2072         }
2073
2074         return &qp->ibqp;
2075 }
2076
2077 int mlx5_ib_destroy_qp(struct ib_qp *qp)
2078 {
2079         struct mlx5_ib_dev *dev = to_mdev(qp->device);
2080         struct mlx5_ib_qp *mqp = to_mqp(qp);
2081
2082         if (unlikely(qp->qp_type == IB_QPT_GSI))
2083                 return mlx5_ib_gsi_destroy_qp(qp);
2084
2085         destroy_qp_common(dev, mqp);
2086
2087         kfree(mqp);
2088
2089         return 0;
2090 }
2091
2092 static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr,
2093                                    int attr_mask)
2094 {
2095         u32 hw_access_flags = 0;
2096         u8 dest_rd_atomic;
2097         u32 access_flags;
2098
2099         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2100                 dest_rd_atomic = attr->max_dest_rd_atomic;
2101         else
2102                 dest_rd_atomic = qp->trans_qp.resp_depth;
2103
2104         if (attr_mask & IB_QP_ACCESS_FLAGS)
2105                 access_flags = attr->qp_access_flags;
2106         else
2107                 access_flags = qp->trans_qp.atomic_rd_en;
2108
2109         if (!dest_rd_atomic)
2110                 access_flags &= IB_ACCESS_REMOTE_WRITE;
2111
2112         if (access_flags & IB_ACCESS_REMOTE_READ)
2113                 hw_access_flags |= MLX5_QP_BIT_RRE;
2114         if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
2115                 hw_access_flags |= (MLX5_QP_BIT_RAE | MLX5_ATOMIC_MODE_CX);
2116         if (access_flags & IB_ACCESS_REMOTE_WRITE)
2117                 hw_access_flags |= MLX5_QP_BIT_RWE;
2118
2119         return cpu_to_be32(hw_access_flags);
2120 }
2121
2122 enum {
2123         MLX5_PATH_FLAG_FL       = 1 << 0,
2124         MLX5_PATH_FLAG_FREE_AR  = 1 << 1,
2125         MLX5_PATH_FLAG_COUNTER  = 1 << 2,
2126 };
2127
2128 static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
2129 {
2130         if (rate == IB_RATE_PORT_CURRENT) {
2131                 return 0;
2132         } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) {
2133                 return -EINVAL;
2134         } else {
2135                 while (rate != IB_RATE_2_5_GBPS &&
2136                        !(1 << (rate + MLX5_STAT_RATE_OFFSET) &
2137                          MLX5_CAP_GEN(dev->mdev, stat_rate_support)))
2138                         --rate;
2139         }
2140
2141         return rate + MLX5_STAT_RATE_OFFSET;
2142 }
2143
2144 static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
2145                                       struct mlx5_ib_sq *sq, u8 sl)
2146 {
2147         void *in;
2148         void *tisc;
2149         int inlen;
2150         int err;
2151
2152         inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
2153         in = mlx5_vzalloc(inlen);
2154         if (!in)
2155                 return -ENOMEM;
2156
2157         MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
2158
2159         tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2160         MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
2161
2162         err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
2163
2164         kvfree(in);
2165
2166         return err;
2167 }
2168
2169 static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2170                          const struct ib_ah_attr *ah,
2171                          struct mlx5_qp_path *path, u8 port, int attr_mask,
2172                          u32 path_flags, const struct ib_qp_attr *attr,
2173                          bool alt)
2174 {
2175         enum rdma_link_layer ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2176         int err;
2177
2178         if (attr_mask & IB_QP_PKEY_INDEX)
2179                 path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
2180                                                      attr->pkey_index);
2181
2182         if (ah->ah_flags & IB_AH_GRH) {
2183                 if (ah->grh.sgid_index >=
2184                     dev->mdev->port_caps[port - 1].gid_table_len) {
2185                         pr_err("sgid_index (%u) too large. max is %d\n",
2186                                ah->grh.sgid_index,
2187                                dev->mdev->port_caps[port - 1].gid_table_len);
2188                         return -EINVAL;
2189                 }
2190         }
2191
2192         if (ll == IB_LINK_LAYER_ETHERNET) {
2193                 if (!(ah->ah_flags & IB_AH_GRH))
2194                         return -EINVAL;
2195                 memcpy(path->rmac, ah->dmac, sizeof(ah->dmac));
2196                 path->udp_sport = mlx5_get_roce_udp_sport(dev, port,
2197                                                           ah->grh.sgid_index);
2198                 path->dci_cfi_prio_sl = (ah->sl & 0x7) << 4;
2199         } else {
2200                 path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
2201                 path->fl_free_ar |=
2202                         (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
2203                 path->rlid = cpu_to_be16(ah->dlid);
2204                 path->grh_mlid = ah->src_path_bits & 0x7f;
2205                 if (ah->ah_flags & IB_AH_GRH)
2206                         path->grh_mlid  |= 1 << 7;
2207                 path->dci_cfi_prio_sl = ah->sl & 0xf;
2208         }
2209
2210         if (ah->ah_flags & IB_AH_GRH) {
2211                 path->mgid_index = ah->grh.sgid_index;
2212                 path->hop_limit  = ah->grh.hop_limit;
2213                 path->tclass_flowlabel =
2214                         cpu_to_be32((ah->grh.traffic_class << 20) |
2215                                     (ah->grh.flow_label));
2216                 memcpy(path->rgid, ah->grh.dgid.raw, 16);
2217         }
2218
2219         err = ib_rate_to_mlx5(dev, ah->static_rate);
2220         if (err < 0)
2221                 return err;
2222         path->static_rate = err;
2223         path->port = port;
2224
2225         if (attr_mask & IB_QP_TIMEOUT)
2226                 path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
2227
2228         if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
2229                 return modify_raw_packet_eth_prio(dev->mdev,
2230                                                   &qp->raw_packet_qp.sq,
2231                                                   ah->sl & 0xf);
2232
2233         return 0;
2234 }
2235
2236 static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
2237         [MLX5_QP_STATE_INIT] = {
2238                 [MLX5_QP_STATE_INIT] = {
2239                         [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE            |
2240                                           MLX5_QP_OPTPAR_RAE            |
2241                                           MLX5_QP_OPTPAR_RWE            |
2242                                           MLX5_QP_OPTPAR_PKEY_INDEX     |
2243                                           MLX5_QP_OPTPAR_PRI_PORT,
2244                         [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE            |
2245                                           MLX5_QP_OPTPAR_PKEY_INDEX     |
2246                                           MLX5_QP_OPTPAR_PRI_PORT,
2247                         [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |
2248                                           MLX5_QP_OPTPAR_Q_KEY          |
2249                                           MLX5_QP_OPTPAR_PRI_PORT,
2250                 },
2251                 [MLX5_QP_STATE_RTR] = {
2252                         [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
2253                                           MLX5_QP_OPTPAR_RRE            |
2254                                           MLX5_QP_OPTPAR_RAE            |
2255                                           MLX5_QP_OPTPAR_RWE            |
2256                                           MLX5_QP_OPTPAR_PKEY_INDEX,
2257                         [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
2258                                           MLX5_QP_OPTPAR_RWE            |
2259                                           MLX5_QP_OPTPAR_PKEY_INDEX,
2260                         [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |
2261                                           MLX5_QP_OPTPAR_Q_KEY,
2262                         [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX    |
2263                                            MLX5_QP_OPTPAR_Q_KEY,
2264                         [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2265                                           MLX5_QP_OPTPAR_RRE            |
2266                                           MLX5_QP_OPTPAR_RAE            |
2267                                           MLX5_QP_OPTPAR_RWE            |
2268                                           MLX5_QP_OPTPAR_PKEY_INDEX,
2269                 },
2270         },
2271         [MLX5_QP_STATE_RTR] = {
2272                 [MLX5_QP_STATE_RTS] = {
2273                         [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
2274                                           MLX5_QP_OPTPAR_RRE            |
2275                                           MLX5_QP_OPTPAR_RAE            |
2276                                           MLX5_QP_OPTPAR_RWE            |
2277                                           MLX5_QP_OPTPAR_PM_STATE       |
2278                                           MLX5_QP_OPTPAR_RNR_TIMEOUT,
2279                         [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
2280                                           MLX5_QP_OPTPAR_RWE            |
2281                                           MLX5_QP_OPTPAR_PM_STATE,
2282                         [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
2283                 },
2284         },
2285         [MLX5_QP_STATE_RTS] = {
2286                 [MLX5_QP_STATE_RTS] = {
2287                         [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE            |
2288                                           MLX5_QP_OPTPAR_RAE            |
2289                                           MLX5_QP_OPTPAR_RWE            |
2290                                           MLX5_QP_OPTPAR_RNR_TIMEOUT    |
2291                                           MLX5_QP_OPTPAR_PM_STATE       |
2292                                           MLX5_QP_OPTPAR_ALT_ADDR_PATH,
2293                         [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE            |
2294                                           MLX5_QP_OPTPAR_PM_STATE       |
2295                                           MLX5_QP_OPTPAR_ALT_ADDR_PATH,
2296                         [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY          |
2297                                           MLX5_QP_OPTPAR_SRQN           |
2298                                           MLX5_QP_OPTPAR_CQN_RCV,
2299                 },
2300         },
2301         [MLX5_QP_STATE_SQER] = {
2302                 [MLX5_QP_STATE_RTS] = {
2303                         [MLX5_QP_ST_UD]  = MLX5_QP_OPTPAR_Q_KEY,
2304                         [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
2305                         [MLX5_QP_ST_UC]  = MLX5_QP_OPTPAR_RWE,
2306                         [MLX5_QP_ST_RC]  = MLX5_QP_OPTPAR_RNR_TIMEOUT   |
2307                                            MLX5_QP_OPTPAR_RWE           |
2308                                            MLX5_QP_OPTPAR_RAE           |
2309                                            MLX5_QP_OPTPAR_RRE,
2310                 },
2311         },
2312 };
2313
2314 static int ib_nr_to_mlx5_nr(int ib_mask)
2315 {
2316         switch (ib_mask) {
2317         case IB_QP_STATE:
2318                 return 0;
2319         case IB_QP_CUR_STATE:
2320                 return 0;
2321         case IB_QP_EN_SQD_ASYNC_NOTIFY:
2322                 return 0;
2323         case IB_QP_ACCESS_FLAGS:
2324                 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
2325                         MLX5_QP_OPTPAR_RAE;
2326         case IB_QP_PKEY_INDEX:
2327                 return MLX5_QP_OPTPAR_PKEY_INDEX;
2328         case IB_QP_PORT:
2329                 return MLX5_QP_OPTPAR_PRI_PORT;
2330         case IB_QP_QKEY:
2331                 return MLX5_QP_OPTPAR_Q_KEY;
2332         case IB_QP_AV:
2333                 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
2334                         MLX5_QP_OPTPAR_PRI_PORT;
2335         case IB_QP_PATH_MTU:
2336                 return 0;
2337         case IB_QP_TIMEOUT:
2338                 return MLX5_QP_OPTPAR_ACK_TIMEOUT;
2339         case IB_QP_RETRY_CNT:
2340                 return MLX5_QP_OPTPAR_RETRY_COUNT;
2341         case IB_QP_RNR_RETRY:
2342                 return MLX5_QP_OPTPAR_RNR_RETRY;
2343         case IB_QP_RQ_PSN:
2344                 return 0;
2345         case IB_QP_MAX_QP_RD_ATOMIC:
2346                 return MLX5_QP_OPTPAR_SRA_MAX;
2347         case IB_QP_ALT_PATH:
2348                 return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
2349         case IB_QP_MIN_RNR_TIMER:
2350                 return MLX5_QP_OPTPAR_RNR_TIMEOUT;
2351         case IB_QP_SQ_PSN:
2352                 return 0;
2353         case IB_QP_MAX_DEST_RD_ATOMIC:
2354                 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
2355                         MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
2356         case IB_QP_PATH_MIG_STATE:
2357                 return MLX5_QP_OPTPAR_PM_STATE;
2358         case IB_QP_CAP:
2359                 return 0;
2360         case IB_QP_DEST_QPN:
2361                 return 0;
2362         }
2363         return 0;
2364 }
2365
2366 static int ib_mask_to_mlx5_opt(int ib_mask)
2367 {
2368         int result = 0;
2369         int i;
2370
2371         for (i = 0; i < 8 * sizeof(int); i++) {
2372                 if ((1 << i) & ib_mask)
2373                         result |= ib_nr_to_mlx5_nr(1 << i);
2374         }
2375
2376         return result;
2377 }
2378
2379 static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2380                                    struct mlx5_ib_rq *rq, int new_state,
2381                                    const struct mlx5_modify_raw_qp_param *raw_qp_param)
2382 {
2383         void *in;
2384         void *rqc;
2385         int inlen;
2386         int err;
2387
2388         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
2389         in = mlx5_vzalloc(inlen);
2390         if (!in)
2391                 return -ENOMEM;
2392
2393         MLX5_SET(modify_rq_in, in, rq_state, rq->state);
2394
2395         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
2396         MLX5_SET(rqc, rqc, state, new_state);
2397
2398         if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
2399                 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
2400                         MLX5_SET64(modify_rq_in, in, modify_bitmask,
2401                                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_MODIFY_RQ_COUNTER_SET_ID);
2402                         MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
2403                 } else
2404                         pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
2405                                      dev->ib_dev.name);
2406         }
2407
2408         err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen);
2409         if (err)
2410                 goto out;
2411
2412         rq->state = new_state;
2413
2414 out:
2415         kvfree(in);
2416         return err;
2417 }
2418
2419 static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
2420                                    struct mlx5_ib_sq *sq, int new_state)
2421 {
2422         void *in;
2423         void *sqc;
2424         int inlen;
2425         int err;
2426
2427         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
2428         in = mlx5_vzalloc(inlen);
2429         if (!in)
2430                 return -ENOMEM;
2431
2432         MLX5_SET(modify_sq_in, in, sq_state, sq->state);
2433
2434         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
2435         MLX5_SET(sqc, sqc, state, new_state);
2436
2437         err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
2438         if (err)
2439                 goto out;
2440
2441         sq->state = new_state;
2442
2443 out:
2444         kvfree(in);
2445         return err;
2446 }
2447
2448 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2449                                 const struct mlx5_modify_raw_qp_param *raw_qp_param)
2450 {
2451         struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
2452         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
2453         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
2454         int rq_state;
2455         int sq_state;
2456         int err;
2457
2458         switch (raw_qp_param->operation) {
2459         case MLX5_CMD_OP_RST2INIT_QP:
2460                 rq_state = MLX5_RQC_STATE_RDY;
2461                 sq_state = MLX5_SQC_STATE_RDY;
2462                 break;
2463         case MLX5_CMD_OP_2ERR_QP:
2464                 rq_state = MLX5_RQC_STATE_ERR;
2465                 sq_state = MLX5_SQC_STATE_ERR;
2466                 break;
2467         case MLX5_CMD_OP_2RST_QP:
2468                 rq_state = MLX5_RQC_STATE_RST;
2469                 sq_state = MLX5_SQC_STATE_RST;
2470                 break;
2471         case MLX5_CMD_OP_INIT2INIT_QP:
2472         case MLX5_CMD_OP_INIT2RTR_QP:
2473         case MLX5_CMD_OP_RTR2RTS_QP:
2474         case MLX5_CMD_OP_RTS2RTS_QP:
2475                 if (raw_qp_param->set_mask)
2476                         return -EINVAL;
2477                 else
2478                         return 0;
2479         default:
2480                 WARN_ON(1);
2481                 return -EINVAL;
2482         }
2483
2484         if (qp->rq.wqe_cnt) {
2485                 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
2486                 if (err)
2487                         return err;
2488         }
2489
2490         if (qp->sq.wqe_cnt)
2491                 return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state);
2492
2493         return 0;
2494 }
2495
2496 static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
2497                                const struct ib_qp_attr *attr, int attr_mask,
2498                                enum ib_qp_state cur_state, enum ib_qp_state new_state)
2499 {
2500         static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
2501                 [MLX5_QP_STATE_RST] = {
2502                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2503                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2504                         [MLX5_QP_STATE_INIT]    = MLX5_CMD_OP_RST2INIT_QP,
2505                 },
2506                 [MLX5_QP_STATE_INIT]  = {
2507                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2508                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2509                         [MLX5_QP_STATE_INIT]    = MLX5_CMD_OP_INIT2INIT_QP,
2510                         [MLX5_QP_STATE_RTR]     = MLX5_CMD_OP_INIT2RTR_QP,
2511                 },
2512                 [MLX5_QP_STATE_RTR]   = {
2513                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2514                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2515                         [MLX5_QP_STATE_RTS]     = MLX5_CMD_OP_RTR2RTS_QP,
2516                 },
2517                 [MLX5_QP_STATE_RTS]   = {
2518                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2519                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2520                         [MLX5_QP_STATE_RTS]     = MLX5_CMD_OP_RTS2RTS_QP,
2521                 },
2522                 [MLX5_QP_STATE_SQD] = {
2523                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2524                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2525                 },
2526                 [MLX5_QP_STATE_SQER] = {
2527                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2528                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2529                         [MLX5_QP_STATE_RTS]     = MLX5_CMD_OP_SQERR2RTS_QP,
2530                 },
2531                 [MLX5_QP_STATE_ERR] = {
2532                         [MLX5_QP_STATE_RST]     = MLX5_CMD_OP_2RST_QP,
2533                         [MLX5_QP_STATE_ERR]     = MLX5_CMD_OP_2ERR_QP,
2534                 }
2535         };
2536
2537         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2538         struct mlx5_ib_qp *qp = to_mqp(ibqp);
2539         struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
2540         struct mlx5_ib_cq *send_cq, *recv_cq;
2541         struct mlx5_qp_context *context;
2542         struct mlx5_ib_pd *pd;
2543         struct mlx5_ib_port *mibport = NULL;
2544         enum mlx5_qp_state mlx5_cur, mlx5_new;
2545         enum mlx5_qp_optpar optpar;
2546         int sqd_event;
2547         int mlx5_st;
2548         int err;
2549         u16 op;
2550
2551         context = kzalloc(sizeof(*context), GFP_KERNEL);
2552         if (!context)
2553                 return -ENOMEM;
2554
2555         err = to_mlx5_st(ibqp->qp_type);
2556         if (err < 0) {
2557                 mlx5_ib_dbg(dev, "unsupported qp type %d\n", ibqp->qp_type);
2558                 goto out;
2559         }
2560
2561         context->flags = cpu_to_be32(err << 16);
2562
2563         if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
2564                 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2565         } else {
2566                 switch (attr->path_mig_state) {
2567                 case IB_MIG_MIGRATED:
2568                         context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2569                         break;
2570                 case IB_MIG_REARM:
2571                         context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11);
2572                         break;
2573                 case IB_MIG_ARMED:
2574                         context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
2575                         break;
2576                 }
2577         }
2578
2579         if (is_sqp(ibqp->qp_type)) {
2580                 context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
2581         } else if (ibqp->qp_type == IB_QPT_UD ||
2582                    ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
2583                 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
2584         } else if (attr_mask & IB_QP_PATH_MTU) {
2585                 if (attr->path_mtu < IB_MTU_256 ||
2586                     attr->path_mtu > IB_MTU_4096) {
2587                         mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
2588                         err = -EINVAL;
2589                         goto out;
2590                 }
2591                 context->mtu_msgmax = (attr->path_mtu << 5) |
2592                                       (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
2593         }
2594
2595         if (attr_mask & IB_QP_DEST_QPN)
2596                 context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
2597
2598         if (attr_mask & IB_QP_PKEY_INDEX)
2599                 context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
2600
2601         /* todo implement counter_index functionality */
2602
2603         if (is_sqp(ibqp->qp_type))
2604                 context->pri_path.port = qp->port;
2605
2606         if (attr_mask & IB_QP_PORT)
2607                 context->pri_path.port = attr->port_num;
2608
2609         if (attr_mask & IB_QP_AV) {
2610                 err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
2611                                     attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
2612                                     attr_mask, 0, attr, false);
2613                 if (err)
2614                         goto out;
2615         }
2616
2617         if (attr_mask & IB_QP_TIMEOUT)
2618                 context->pri_path.ackto_lt |= attr->timeout << 3;
2619
2620         if (attr_mask & IB_QP_ALT_PATH) {
2621                 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
2622                                     &context->alt_path,
2623                                     attr->alt_port_num,
2624                                     attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,
2625                                     0, attr, true);
2626                 if (err)
2627                         goto out;
2628         }
2629
2630         pd = get_pd(qp);
2631         get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
2632                 &send_cq, &recv_cq);
2633
2634         context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
2635         context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
2636         context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
2637         context->params1  = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
2638
2639         if (attr_mask & IB_QP_RNR_RETRY)
2640                 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
2641
2642         if (attr_mask & IB_QP_RETRY_CNT)
2643                 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
2644
2645         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2646                 if (attr->max_rd_atomic)
2647                         context->params1 |=
2648                                 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
2649         }
2650
2651         if (attr_mask & IB_QP_SQ_PSN)
2652                 context->next_send_psn = cpu_to_be32(attr->sq_psn);
2653
2654         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2655                 if (attr->max_dest_rd_atomic)
2656                         context->params2 |=
2657                                 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
2658         }
2659
2660         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
2661                 context->params2 |= to_mlx5_access_flags(qp, attr, attr_mask);
2662
2663         if (attr_mask & IB_QP_MIN_RNR_TIMER)
2664                 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
2665
2666         if (attr_mask & IB_QP_RQ_PSN)
2667                 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
2668
2669         if (attr_mask & IB_QP_QKEY)
2670                 context->qkey = cpu_to_be32(attr->qkey);
2671
2672         if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
2673                 context->db_rec_addr = cpu_to_be64(qp->db.dma);
2674
2675         if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD  &&
2676             attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
2677                 sqd_event = 1;
2678         else
2679                 sqd_event = 0;
2680
2681         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2682                 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
2683                                qp->port) - 1;
2684                 mibport = &dev->port[port_num];
2685                 context->qp_counter_set_usr_page |=
2686                         cpu_to_be32((u32)(mibport->q_cnt_id) << 24);
2687         }
2688
2689         if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
2690                 context->sq_crq_size |= cpu_to_be16(1 << 4);
2691
2692         if (qp->flags & MLX5_IB_QP_SQPN_QP1)
2693                 context->deth_sqpn = cpu_to_be32(1);
2694
2695         mlx5_cur = to_mlx5_state(cur_state);
2696         mlx5_new = to_mlx5_state(new_state);
2697         mlx5_st = to_mlx5_st(ibqp->qp_type);
2698         if (mlx5_st < 0)
2699                 goto out;
2700
2701         /* If moving to a reset or error state, we must disable page faults on
2702          * this QP and flush all current page faults. Otherwise a stale page
2703          * fault may attempt to work on this QP after it is reset and moved
2704          * again to RTS, and may cause the driver and the device to get out of
2705          * sync. */
2706         if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2707             (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) &&
2708             (qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
2709                 mlx5_ib_qp_disable_pagefaults(qp);
2710
2711         if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
2712             !optab[mlx5_cur][mlx5_new])
2713                 goto out;
2714
2715         op = optab[mlx5_cur][mlx5_new];
2716         optpar = ib_mask_to_mlx5_opt(attr_mask);
2717         optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
2718
2719         if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
2720                 struct mlx5_modify_raw_qp_param raw_qp_param = {};
2721
2722                 raw_qp_param.operation = op;
2723                 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2724                         raw_qp_param.rq_q_ctr_id = mibport->q_cnt_id;
2725                         raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
2726                 }
2727                 err = modify_raw_packet_qp(dev, qp, &raw_qp_param);
2728         } else {
2729                 err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
2730                                           &base->mqp);
2731         }
2732
2733         if (err)
2734                 goto out;
2735
2736         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT &&
2737             (qp->ibqp.qp_type != IB_QPT_RAW_PACKET))
2738                 mlx5_ib_qp_enable_pagefaults(qp);
2739
2740         qp->state = new_state;
2741
2742         if (attr_mask & IB_QP_ACCESS_FLAGS)
2743                 qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
2744         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2745                 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
2746         if (attr_mask & IB_QP_PORT)
2747                 qp->port = attr->port_num;
2748         if (attr_mask & IB_QP_ALT_PATH)
2749                 qp->trans_qp.alt_port = attr->alt_port_num;
2750
2751         /*
2752          * If we moved a kernel QP to RESET, clean up all old CQ
2753          * entries and reinitialize the QP.
2754          */
2755         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
2756                 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
2757                                  ibqp->srq ? to_msrq(ibqp->srq) : NULL);
2758                 if (send_cq != recv_cq)
2759                         mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
2760
2761                 qp->rq.head = 0;
2762                 qp->rq.tail = 0;
2763                 qp->sq.head = 0;
2764                 qp->sq.tail = 0;
2765                 qp->sq.cur_post = 0;
2766                 qp->sq.last_poll = 0;
2767                 qp->db.db[MLX5_RCV_DBR] = 0;
2768                 qp->db.db[MLX5_SND_DBR] = 0;
2769         }
2770
2771 out:
2772         kfree(context);
2773         return err;
2774 }
2775
2776 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2777                       int attr_mask, struct ib_udata *udata)
2778 {
2779         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2780         struct mlx5_ib_qp *qp = to_mqp(ibqp);
2781         enum ib_qp_type qp_type;
2782         enum ib_qp_state cur_state, new_state;
2783         int err = -EINVAL;
2784         int port;
2785         enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2786
2787         if (ibqp->rwq_ind_tbl)
2788                 return -ENOSYS;
2789
2790         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
2791                 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
2792
2793         qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
2794                 IB_QPT_GSI : ibqp->qp_type;
2795
2796         mutex_lock(&qp->mutex);
2797
2798         cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2799         new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2800
2801         if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
2802                 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2803                 ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
2804         }
2805
2806         if (qp_type != MLX5_IB_QPT_REG_UMR &&
2807             !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
2808                 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
2809                             cur_state, new_state, ibqp->qp_type, attr_mask);
2810                 goto out;
2811         }
2812
2813         if ((attr_mask & IB_QP_PORT) &&
2814             (attr->port_num == 0 ||
2815              attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports))) {
2816                 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
2817                             attr->port_num, dev->num_ports);
2818                 goto out;
2819         }
2820
2821         if (attr_mask & IB_QP_PKEY_INDEX) {
2822                 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2823                 if (attr->pkey_index >=
2824                     dev->mdev->port_caps[port - 1].pkey_table_len) {
2825                         mlx5_ib_dbg(dev, "invalid pkey index %d\n",
2826                                     attr->pkey_index);
2827                         goto out;
2828                 }
2829         }
2830
2831         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2832             attr->max_rd_atomic >
2833             (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) {
2834                 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
2835                             attr->max_rd_atomic);
2836                 goto out;
2837         }
2838
2839         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2840             attr->max_dest_rd_atomic >
2841             (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) {
2842                 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
2843                             attr->max_dest_rd_atomic);
2844                 goto out;
2845         }
2846
2847         if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2848                 err = 0;
2849                 goto out;
2850         }
2851
2852         err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
2853
2854 out:
2855         mutex_unlock(&qp->mutex);
2856         return err;
2857 }
2858
2859 static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
2860 {
2861         struct mlx5_ib_cq *cq;
2862         unsigned cur;
2863
2864         cur = wq->head - wq->tail;
2865         if (likely(cur + nreq < wq->max_post))
2866                 return 0;
2867
2868         cq = to_mcq(ib_cq);
2869         spin_lock(&cq->lock);
2870         cur = wq->head - wq->tail;
2871         spin_unlock(&cq->lock);
2872
2873         return cur + nreq >= wq->max_post;
2874 }
2875
2876 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
2877                                           u64 remote_addr, u32 rkey)
2878 {
2879         rseg->raddr    = cpu_to_be64(remote_addr);
2880         rseg->rkey     = cpu_to_be32(rkey);
2881         rseg->reserved = 0;
2882 }
2883
2884 static void *set_eth_seg(struct mlx5_wqe_eth_seg *eseg,
2885                          struct ib_send_wr *wr, void *qend,
2886                          struct mlx5_ib_qp *qp, int *size)
2887 {
2888         void *seg = eseg;
2889
2890         memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
2891
2892         if (wr->send_flags & IB_SEND_IP_CSUM)
2893                 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
2894                                  MLX5_ETH_WQE_L4_CSUM;
2895
2896         seg += sizeof(struct mlx5_wqe_eth_seg);
2897         *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
2898
2899         if (wr->opcode == IB_WR_LSO) {
2900                 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
2901                 int size_of_inl_hdr_start = sizeof(eseg->inline_hdr_start);
2902                 u64 left, leftlen, copysz;
2903                 void *pdata = ud_wr->header;
2904
2905                 left = ud_wr->hlen;
2906                 eseg->mss = cpu_to_be16(ud_wr->mss);
2907                 eseg->inline_hdr_sz = cpu_to_be16(left);
2908
2909                 /*
2910                  * check if there is space till the end of queue, if yes,
2911                  * copy all in one shot, otherwise copy till the end of queue,
2912                  * rollback and than the copy the left
2913                  */
2914                 leftlen = qend - (void *)eseg->inline_hdr_start;
2915                 copysz = min_t(u64, leftlen, left);
2916
2917                 memcpy(seg - size_of_inl_hdr_start, pdata, copysz);
2918
2919                 if (likely(copysz > size_of_inl_hdr_start)) {
2920                         seg += ALIGN(copysz - size_of_inl_hdr_start, 16);
2921                         *size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16;
2922                 }
2923
2924                 if (unlikely(copysz < left)) { /* the last wqe in the queue */
2925                         seg = mlx5_get_send_wqe(qp, 0);
2926                         left -= copysz;
2927                         pdata += copysz;
2928                         memcpy(seg, pdata, left);
2929                         seg += ALIGN(left, 16);
2930                         *size += ALIGN(left, 16) / 16;
2931                 }
2932         }
2933
2934         return seg;
2935 }
2936
2937 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
2938                              struct ib_send_wr *wr)
2939 {
2940         memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
2941         dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
2942         dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
2943 }
2944
2945 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
2946 {
2947         dseg->byte_count = cpu_to_be32(sg->length);
2948         dseg->lkey       = cpu_to_be32(sg->lkey);
2949         dseg->addr       = cpu_to_be64(sg->addr);
2950 }
2951
2952 static __be16 get_klm_octo(int npages)
2953 {
2954         return cpu_to_be16(ALIGN(npages, 8) / 2);
2955 }
2956
2957 static __be64 frwr_mkey_mask(void)
2958 {
2959         u64 result;
2960
2961         result = MLX5_MKEY_MASK_LEN             |
2962                 MLX5_MKEY_MASK_PAGE_SIZE        |
2963                 MLX5_MKEY_MASK_START_ADDR       |
2964                 MLX5_MKEY_MASK_EN_RINVAL        |
2965                 MLX5_MKEY_MASK_KEY              |
2966                 MLX5_MKEY_MASK_LR               |
2967                 MLX5_MKEY_MASK_LW               |
2968                 MLX5_MKEY_MASK_RR               |
2969                 MLX5_MKEY_MASK_RW               |
2970                 MLX5_MKEY_MASK_A                |
2971                 MLX5_MKEY_MASK_SMALL_FENCE      |
2972                 MLX5_MKEY_MASK_FREE;
2973
2974         return cpu_to_be64(result);
2975 }
2976
2977 static __be64 sig_mkey_mask(void)
2978 {
2979         u64 result;
2980
2981         result = MLX5_MKEY_MASK_LEN             |
2982                 MLX5_MKEY_MASK_PAGE_SIZE        |
2983                 MLX5_MKEY_MASK_START_ADDR       |
2984                 MLX5_MKEY_MASK_EN_SIGERR        |
2985                 MLX5_MKEY_MASK_EN_RINVAL        |
2986                 MLX5_MKEY_MASK_KEY              |
2987                 MLX5_MKEY_MASK_LR               |
2988                 MLX5_MKEY_MASK_LW               |
2989                 MLX5_MKEY_MASK_RR               |
2990                 MLX5_MKEY_MASK_RW               |
2991                 MLX5_MKEY_MASK_SMALL_FENCE      |
2992                 MLX5_MKEY_MASK_FREE             |
2993                 MLX5_MKEY_MASK_BSF_EN;
2994
2995         return cpu_to_be64(result);
2996 }
2997
2998 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
2999                                 struct mlx5_ib_mr *mr)
3000 {
3001         int ndescs = mr->ndescs;
3002
3003         memset(umr, 0, sizeof(*umr));
3004
3005         if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
3006                 /* KLMs take twice the size of MTTs */
3007                 ndescs *= 2;
3008
3009         umr->flags = MLX5_UMR_CHECK_NOT_FREE;
3010         umr->klm_octowords = get_klm_octo(ndescs);
3011         umr->mkey_mask = frwr_mkey_mask();
3012 }
3013
3014 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
3015 {
3016         memset(umr, 0, sizeof(*umr));
3017         umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
3018         umr->flags = 1 << 7;
3019 }
3020
3021 static __be64 get_umr_reg_mr_mask(void)
3022 {
3023         u64 result;
3024
3025         result = MLX5_MKEY_MASK_LEN             |
3026                  MLX5_MKEY_MASK_PAGE_SIZE       |
3027                  MLX5_MKEY_MASK_START_ADDR      |
3028                  MLX5_MKEY_MASK_PD              |
3029                  MLX5_MKEY_MASK_LR              |
3030                  MLX5_MKEY_MASK_LW              |
3031                  MLX5_MKEY_MASK_KEY             |
3032                  MLX5_MKEY_MASK_RR              |
3033                  MLX5_MKEY_MASK_RW              |
3034                  MLX5_MKEY_MASK_A               |
3035                  MLX5_MKEY_MASK_FREE;
3036
3037         return cpu_to_be64(result);
3038 }
3039
3040 static __be64 get_umr_unreg_mr_mask(void)
3041 {
3042         u64 result;
3043
3044         result = MLX5_MKEY_MASK_FREE;
3045
3046         return cpu_to_be64(result);
3047 }
3048
3049 static __be64 get_umr_update_mtt_mask(void)
3050 {
3051         u64 result;
3052
3053         result = MLX5_MKEY_MASK_FREE;
3054
3055         return cpu_to_be64(result);
3056 }
3057
3058 static __be64 get_umr_update_translation_mask(void)
3059 {
3060         u64 result;
3061
3062         result = MLX5_MKEY_MASK_LEN |
3063                  MLX5_MKEY_MASK_PAGE_SIZE |
3064                  MLX5_MKEY_MASK_START_ADDR |
3065                  MLX5_MKEY_MASK_KEY |
3066                  MLX5_MKEY_MASK_FREE;
3067
3068         return cpu_to_be64(result);
3069 }
3070
3071 static __be64 get_umr_update_access_mask(void)
3072 {
3073         u64 result;
3074
3075         result = MLX5_MKEY_MASK_LW |
3076                  MLX5_MKEY_MASK_RR |
3077                  MLX5_MKEY_MASK_RW |
3078                  MLX5_MKEY_MASK_A |
3079                  MLX5_MKEY_MASK_KEY |
3080                  MLX5_MKEY_MASK_FREE;
3081
3082         return cpu_to_be64(result);
3083 }
3084
3085 static __be64 get_umr_update_pd_mask(void)
3086 {
3087         u64 result;
3088
3089         result = MLX5_MKEY_MASK_PD |
3090                  MLX5_MKEY_MASK_KEY |
3091                  MLX5_MKEY_MASK_FREE;
3092
3093         return cpu_to_be64(result);
3094 }
3095
3096 static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
3097                                 struct ib_send_wr *wr)
3098 {
3099         struct mlx5_umr_wr *umrwr = umr_wr(wr);
3100
3101         memset(umr, 0, sizeof(*umr));
3102
3103         if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
3104                 umr->flags = MLX5_UMR_CHECK_FREE; /* fail if free */
3105         else
3106                 umr->flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */
3107
3108         if (!(wr->send_flags & MLX5_IB_SEND_UMR_UNREG)) {
3109                 umr->klm_octowords = get_klm_octo(umrwr->npages);
3110                 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT) {
3111                         umr->mkey_mask = get_umr_update_mtt_mask();
3112                         umr->bsf_octowords = get_klm_octo(umrwr->target.offset);
3113                         umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
3114                 }
3115                 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
3116                         umr->mkey_mask |= get_umr_update_translation_mask();
3117                 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_ACCESS)
3118                         umr->mkey_mask |= get_umr_update_access_mask();
3119                 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD)
3120                         umr->mkey_mask |= get_umr_update_pd_mask();
3121                 if (!umr->mkey_mask)
3122                         umr->mkey_mask = get_umr_reg_mr_mask();
3123         } else {
3124                 umr->mkey_mask = get_umr_unreg_mr_mask();
3125         }
3126
3127         if (!wr->num_sge)
3128                 umr->flags |= MLX5_UMR_INLINE;
3129 }
3130
3131 static u8 get_umr_flags(int acc)
3132 {
3133         return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
3134                (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
3135                (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
3136                (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
3137                 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
3138 }
3139
3140 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
3141                              struct mlx5_ib_mr *mr,
3142                              u32 key, int access)
3143 {
3144         int ndescs = ALIGN(mr->ndescs, 8) >> 1;
3145
3146         memset(seg, 0, sizeof(*seg));
3147
3148         if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
3149                 seg->log2_page_size = ilog2(mr->ibmr.page_size);
3150         else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
3151                 /* KLMs take twice the size of MTTs */
3152                 ndescs *= 2;
3153
3154         seg->flags = get_umr_flags(access) | mr->access_mode;
3155         seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
3156         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
3157         seg->start_addr = cpu_to_be64(mr->ibmr.iova);
3158         seg->len = cpu_to_be64(mr->ibmr.length);
3159         seg->xlt_oct_size = cpu_to_be32(ndescs);
3160 }
3161
3162 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
3163 {
3164         memset(seg, 0, sizeof(*seg));
3165         seg->status = MLX5_MKEY_STATUS_FREE;
3166 }
3167
3168 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr)
3169 {
3170         struct mlx5_umr_wr *umrwr = umr_wr(wr);
3171
3172         memset(seg, 0, sizeof(*seg));
3173         if (wr->send_flags & MLX5_IB_SEND_UMR_UNREG) {
3174                 seg->status = MLX5_MKEY_STATUS_FREE;
3175                 return;
3176         }
3177
3178         seg->flags = convert_access(umrwr->access_flags);
3179         if (!(wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT)) {
3180                 if (umrwr->pd)
3181                         seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
3182                 seg->start_addr = cpu_to_be64(umrwr->target.virt_addr);
3183         }
3184         seg->len = cpu_to_be64(umrwr->length);
3185         seg->log2_page_size = umrwr->page_shift;
3186         seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
3187                                        mlx5_mkey_variant(umrwr->mkey));
3188 }
3189
3190 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
3191                              struct mlx5_ib_mr *mr,
3192                              struct mlx5_ib_pd *pd)
3193 {
3194         int bcount = mr->desc_size * mr->ndescs;
3195
3196         dseg->addr = cpu_to_be64(mr->desc_map);
3197         dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
3198         dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
3199 }
3200
3201 static __be32 send_ieth(struct ib_send_wr *wr)
3202 {
3203         switch (wr->opcode) {
3204         case IB_WR_SEND_WITH_IMM:
3205         case IB_WR_RDMA_WRITE_WITH_IMM:
3206                 return wr->ex.imm_data;
3207
3208         case IB_WR_SEND_WITH_INV:
3209                 return cpu_to_be32(wr->ex.invalidate_rkey);
3210
3211         default:
3212                 return 0;
3213         }
3214 }
3215
3216 static u8 calc_sig(void *wqe, int size)
3217 {
3218         u8 *p = wqe;
3219         u8 res = 0;
3220         int i;
3221
3222         for (i = 0; i < size; i++)
3223                 res ^= p[i];
3224
3225         return ~res;
3226 }
3227
3228 static u8 wq_sig(void *wqe)
3229 {
3230         return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
3231 }
3232
3233 static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,
3234                             void *wqe, int *sz)
3235 {
3236         struct mlx5_wqe_inline_seg *seg;
3237         void *qend = qp->sq.qend;
3238         void *addr;
3239         int inl = 0;
3240         int copy;
3241         int len;
3242         int i;
3243
3244         seg = wqe;
3245         wqe += sizeof(*seg);
3246         for (i = 0; i < wr->num_sge; i++) {
3247                 addr = (void *)(unsigned long)(wr->sg_list[i].addr);
3248                 len  = wr->sg_list[i].length;
3249                 inl += len;
3250
3251                 if (unlikely(inl > qp->max_inline_data))
3252                         return -ENOMEM;
3253
3254                 if (unlikely(wqe + len > qend)) {
3255                         copy = qend - wqe;
3256                         memcpy(wqe, addr, copy);
3257                         addr += copy;
3258                         len -= copy;
3259                         wqe = mlx5_get_send_wqe(qp, 0);
3260                 }
3261                 memcpy(wqe, addr, len);
3262                 wqe += len;
3263         }
3264
3265         seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
3266
3267         *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
3268
3269         return 0;
3270 }
3271
3272 static u16 prot_field_size(enum ib_signature_type type)
3273 {
3274         switch (type) {
3275         case IB_SIG_TYPE_T10_DIF:
3276                 return MLX5_DIF_SIZE;
3277         default:
3278                 return 0;
3279         }
3280 }
3281
3282 static u8 bs_selector(int block_size)
3283 {
3284         switch (block_size) {
3285         case 512:           return 0x1;
3286         case 520:           return 0x2;
3287         case 4096:          return 0x3;
3288         case 4160:          return 0x4;
3289         case 1073741824:    return 0x5;
3290         default:            return 0;
3291         }
3292 }
3293
3294 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
3295                               struct mlx5_bsf_inl *inl)
3296 {
3297         /* Valid inline section and allow BSF refresh */
3298         inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
3299                                        MLX5_BSF_REFRESH_DIF);
3300         inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
3301         inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
3302         /* repeating block */
3303         inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
3304         inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
3305                         MLX5_DIF_CRC : MLX5_DIF_IPCS;
3306
3307         if (domain->sig.dif.ref_remap)
3308                 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
3309
3310         if (domain->sig.dif.app_escape) {
3311                 if (domain->sig.dif.ref_escape)
3312                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
3313                 else
3314                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
3315         }
3316
3317         inl->dif_app_bitmask_check =
3318                 cpu_to_be16(domain->sig.dif.apptag_check_mask);
3319 }
3320
3321 static int mlx5_set_bsf(struct ib_mr *sig_mr,
3322                         struct ib_sig_attrs *sig_attrs,
3323                         struct mlx5_bsf *bsf, u32 data_size)
3324 {
3325         struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
3326         struct mlx5_bsf_basic *basic = &bsf->basic;
3327         struct ib_sig_domain *mem = &sig_attrs->mem;
3328         struct ib_sig_domain *wire = &sig_attrs->wire;
3329
3330         memset(bsf, 0, sizeof(*bsf));
3331
3332         /* Basic + Extended + Inline */
3333         basic->bsf_size_sbs = 1 << 7;
3334         /* Input domain check byte mask */
3335         basic->check_byte_mask = sig_attrs->check_mask;
3336         basic->raw_data_size = cpu_to_be32(data_size);
3337
3338         /* Memory domain */
3339         switch (sig_attrs->mem.sig_type) {
3340         case IB_SIG_TYPE_NONE:
3341                 break;
3342         case IB_SIG_TYPE_T10_DIF:
3343                 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
3344                 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
3345                 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
3346                 break;
3347         default:
3348                 return -EINVAL;
3349         }
3350
3351         /* Wire domain */
3352         switch (sig_attrs->wire.sig_type) {
3353         case IB_SIG_TYPE_NONE:
3354                 break;
3355         case IB_SIG_TYPE_T10_DIF:
3356                 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
3357                     mem->sig_type == wire->sig_type) {
3358                         /* Same block structure */
3359                         basic->bsf_size_sbs |= 1 << 4;
3360                         if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
3361                                 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
3362                         if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
3363                                 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
3364                         if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
3365                                 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
3366                 } else
3367                         basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
3368
3369                 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
3370                 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
3371                 break;
3372         default:
3373                 return -EINVAL;
3374         }
3375
3376         return 0;
3377 }
3378
3379 static int set_sig_data_segment(struct ib_sig_handover_wr *wr,
3380                                 struct mlx5_ib_qp *qp, void **seg, int *size)
3381 {
3382         struct ib_sig_attrs *sig_attrs = wr->sig_attrs;
3383         struct ib_mr *sig_mr = wr->sig_mr;
3384         struct mlx5_bsf *bsf;
3385         u32 data_len = wr->wr.sg_list->length;
3386         u32 data_key = wr->wr.sg_list->lkey;
3387         u64 data_va = wr->wr.sg_list->addr;
3388         int ret;
3389         int wqe_size;
3390
3391         if (!wr->prot ||
3392             (data_key == wr->prot->lkey &&
3393              data_va == wr->prot->addr &&
3394              data_len == wr->prot->length)) {
3395                 /**
3396                  * Source domain doesn't contain signature information
3397                  * or data and protection are interleaved in memory.
3398                  * So need construct:
3399                  *                  ------------------
3400                  *                 |     data_klm     |
3401                  *                  ------------------
3402                  *                 |       BSF        |
3403                  *                  ------------------
3404                  **/
3405                 struct mlx5_klm *data_klm = *seg;
3406
3407                 data_klm->bcount = cpu_to_be32(data_len);
3408                 data_klm->key = cpu_to_be32(data_key);
3409                 data_klm->va = cpu_to_be64(data_va);
3410                 wqe_size = ALIGN(sizeof(*data_klm), 64);
3411         } else {
3412                 /**
3413                  * Source domain contains signature information
3414                  * So need construct a strided block format:
3415                  *               ---------------------------
3416                  *              |     stride_block_ctrl     |
3417                  *               ---------------------------
3418                  *              |          data_klm         |
3419                  *               ---------------------------
3420                  *              |          prot_klm         |
3421                  *               ---------------------------
3422                  *              |             BSF           |
3423                  *               ---------------------------
3424                  **/
3425                 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
3426                 struct mlx5_stride_block_entry *data_sentry;
3427                 struct mlx5_stride_block_entry *prot_sentry;
3428                 u32 prot_key = wr->prot->lkey;
3429                 u64 prot_va = wr->prot->addr;
3430                 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
3431                 int prot_size;
3432
3433                 sblock_ctrl = *seg;
3434                 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
3435                 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
3436
3437                 prot_size = prot_field_size(sig_attrs->mem.sig_type);
3438                 if (!prot_size) {
3439                         pr_err("Bad block size given: %u\n", block_size);
3440                         return -EINVAL;
3441                 }
3442                 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
3443                                                             prot_size);
3444                 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
3445                 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
3446                 sblock_ctrl->num_entries = cpu_to_be16(2);
3447
3448                 data_sentry->bcount = cpu_to_be16(block_size);
3449                 data_sentry->key = cpu_to_be32(data_key);
3450                 data_sentry->va = cpu_to_be64(data_va);
3451                 data_sentry->stride = cpu_to_be16(block_size);
3452
3453                 prot_sentry->bcount = cpu_to_be16(prot_size);
3454                 prot_sentry->key = cpu_to_be32(prot_key);
3455                 prot_sentry->va = cpu_to_be64(prot_va);
3456                 prot_sentry->stride = cpu_to_be16(prot_size);
3457
3458                 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
3459                                  sizeof(*prot_sentry), 64);
3460         }
3461
3462         *seg += wqe_size;
3463         *size += wqe_size / 16;
3464         if (unlikely((*seg == qp->sq.qend)))
3465                 *seg = mlx5_get_send_wqe(qp, 0);
3466
3467         bsf = *seg;
3468         ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
3469         if (ret)
3470                 return -EINVAL;
3471
3472         *seg += sizeof(*bsf);
3473         *size += sizeof(*bsf) / 16;
3474         if (unlikely((*seg == qp->sq.qend)))
3475                 *seg = mlx5_get_send_wqe(qp, 0);
3476
3477         return 0;
3478 }
3479
3480 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
3481                                  struct ib_sig_handover_wr *wr, u32 nelements,
3482                                  u32 length, u32 pdn)
3483 {
3484         struct ib_mr *sig_mr = wr->sig_mr;
3485         u32 sig_key = sig_mr->rkey;
3486         u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
3487
3488         memset(seg, 0, sizeof(*seg));
3489
3490         seg->flags = get_umr_flags(wr->access_flags) |
3491                                    MLX5_MKC_ACCESS_MODE_KLMS;
3492         seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
3493         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
3494                                     MLX5_MKEY_BSF_EN | pdn);
3495         seg->len = cpu_to_be64(length);
3496         seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements)));
3497         seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
3498 }
3499
3500 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
3501                                 u32 nelements)
3502 {
3503         memset(umr, 0, sizeof(*umr));
3504
3505         umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
3506         umr->klm_octowords = get_klm_octo(nelements);
3507         umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
3508         umr->mkey_mask = sig_mkey_mask();
3509 }
3510
3511
3512 static int set_sig_umr_wr(struct ib_send_wr *send_wr, struct mlx5_ib_qp *qp,
3513                           void **seg, int *size)
3514 {
3515         struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);
3516         struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);
3517         u32 pdn = get_pd(qp)->pdn;
3518         u32 klm_oct_size;
3519         int region_len, ret;
3520
3521         if (unlikely(wr->wr.num_sge != 1) ||
3522             unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) ||
3523             unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) ||
3524             unlikely(!sig_mr->sig->sig_status_checked))
3525                 return -EINVAL;
3526
3527         /* length of the protected region, data + protection */
3528         region_len = wr->wr.sg_list->length;
3529         if (wr->prot &&
3530             (wr->prot->lkey != wr->wr.sg_list->lkey  ||
3531              wr->prot->addr != wr->wr.sg_list->addr  ||
3532              wr->prot->length != wr->wr.sg_list->length))
3533                 region_len += wr->prot->length;
3534
3535         /**
3536          * KLM octoword size - if protection was provided
3537          * then we use strided block format (3 octowords),
3538          * else we use single KLM (1 octoword)
3539          **/
3540         klm_oct_size = wr->prot ? 3 : 1;
3541
3542         set_sig_umr_segment(*seg, klm_oct_size);
3543         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
3544         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
3545         if (unlikely((*seg == qp->sq.qend)))
3546                 *seg = mlx5_get_send_wqe(qp, 0);
3547
3548         set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);
3549         *seg += sizeof(struct mlx5_mkey_seg);
3550         *size += sizeof(struct mlx5_mkey_seg) / 16;
3551         if (unlikely((*seg == qp->sq.qend)))
3552                 *seg = mlx5_get_send_wqe(qp, 0);
3553
3554         ret = set_sig_data_segment(wr, qp, seg, size);
3555         if (ret)
3556                 return ret;
3557
3558         sig_mr->sig->sig_status_checked = false;
3559         return 0;
3560 }
3561
3562 static int set_psv_wr(struct ib_sig_domain *domain,
3563                       u32 psv_idx, void **seg, int *size)
3564 {
3565         struct mlx5_seg_set_psv *psv_seg = *seg;
3566
3567         memset(psv_seg, 0, sizeof(*psv_seg));
3568         psv_seg->psv_num = cpu_to_be32(psv_idx);
3569         switch (domain->sig_type) {
3570         case IB_SIG_TYPE_NONE:
3571                 break;
3572         case IB_SIG_TYPE_T10_DIF:
3573                 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
3574                                                      domain->sig.dif.app_tag);
3575                 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
3576                 break;
3577         default:
3578                 pr_err("Bad signature type given.\n");
3579                 return 1;
3580         }
3581
3582         *seg += sizeof(*psv_seg);
3583         *size += sizeof(*psv_seg) / 16;
3584
3585         return 0;
3586 }
3587
3588 static int set_reg_wr(struct mlx5_ib_qp *qp,
3589                       struct ib_reg_wr *wr,
3590                       void **seg, int *size)
3591 {
3592         struct mlx5_ib_mr *mr = to_mmr(wr->mr);
3593         struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
3594
3595         if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
3596                 mlx5_ib_warn(to_mdev(qp->ibqp.device),
3597                              "Invalid IB_SEND_INLINE send flag\n");
3598                 return -EINVAL;
3599         }
3600
3601         set_reg_umr_seg(*seg, mr);
3602         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
3603         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
3604         if (unlikely((*seg == qp->sq.qend)))
3605                 *seg = mlx5_get_send_wqe(qp, 0);
3606
3607         set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
3608         *seg += sizeof(struct mlx5_mkey_seg);
3609         *size += sizeof(struct mlx5_mkey_seg) / 16;
3610         if (unlikely((*seg == qp->sq.qend)))
3611                 *seg = mlx5_get_send_wqe(qp, 0);
3612
3613         set_reg_data_seg(*seg, mr, pd);
3614         *seg += sizeof(struct mlx5_wqe_data_seg);
3615         *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
3616
3617         return 0;
3618 }
3619
3620 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size)
3621 {
3622         set_linv_umr_seg(*seg);
3623         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
3624         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
3625         if (unlikely((*seg == qp->sq.qend)))
3626                 *seg = mlx5_get_send_wqe(qp, 0);
3627         set_linv_mkey_seg(*seg);
3628         *seg += sizeof(struct mlx5_mkey_seg);
3629         *size += sizeof(struct mlx5_mkey_seg) / 16;
3630         if (unlikely((*seg == qp->sq.qend)))
3631                 *seg = mlx5_get_send_wqe(qp, 0);
3632 }
3633
3634 static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
3635 {
3636         __be32 *p = NULL;
3637         int tidx = idx;
3638         int i, j;
3639
3640         pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx));
3641         for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
3642                 if ((i & 0xf) == 0) {
3643                         void *buf = mlx5_get_send_wqe(qp, tidx);
3644                         tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1);
3645                         p = buf;
3646                         j = 0;
3647                 }
3648                 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
3649                          be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
3650                          be32_to_cpu(p[j + 3]));
3651         }
3652 }
3653
3654 static void mlx5_bf_copy(u64 __iomem *dst, u64 *src,
3655                          unsigned bytecnt, struct mlx5_ib_qp *qp)
3656 {
3657         while (bytecnt > 0) {
3658                 __iowrite64_copy(dst++, src++, 8);
3659                 __iowrite64_copy(dst++, src++, 8);
3660                 __iowrite64_copy(dst++, src++, 8);
3661                 __iowrite64_copy(dst++, src++, 8);
3662                 __iowrite64_copy(dst++, src++, 8);
3663                 __iowrite64_copy(dst++, src++, 8);
3664                 __iowrite64_copy(dst++, src++, 8);
3665                 __iowrite64_copy(dst++, src++, 8);
3666                 bytecnt -= 64;
3667                 if (unlikely(src == qp->sq.qend))
3668                         src = mlx5_get_send_wqe(qp, 0);
3669         }
3670 }
3671
3672 static u8 get_fence(u8 fence, struct ib_send_wr *wr)
3673 {
3674         if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
3675                      wr->send_flags & IB_SEND_FENCE))
3676                 return MLX5_FENCE_MODE_STRONG_ORDERING;
3677
3678         if (unlikely(fence)) {
3679                 if (wr->send_flags & IB_SEND_FENCE)
3680                         return MLX5_FENCE_MODE_SMALL_AND_FENCE;
3681                 else
3682                         return fence;
3683         } else if (unlikely(wr->send_flags & IB_SEND_FENCE)) {
3684                 return MLX5_FENCE_MODE_FENCE;
3685         }
3686
3687         return 0;
3688 }
3689
3690 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
3691                      struct mlx5_wqe_ctrl_seg **ctrl,
3692                      struct ib_send_wr *wr, unsigned *idx,
3693                      int *size, int nreq)
3694 {
3695         int err = 0;
3696
3697         if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) {
3698                 err = -ENOMEM;
3699                 return err;
3700         }
3701
3702         *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
3703         *seg = mlx5_get_send_wqe(qp, *idx);
3704         *ctrl = *seg;
3705         *(uint32_t *)(*seg + 8) = 0;
3706         (*ctrl)->imm = send_ieth(wr);
3707         (*ctrl)->fm_ce_se = qp->sq_signal_bits |
3708                 (wr->send_flags & IB_SEND_SIGNALED ?
3709                  MLX5_WQE_CTRL_CQ_UPDATE : 0) |
3710                 (wr->send_flags & IB_SEND_SOLICITED ?
3711                  MLX5_WQE_CTRL_SOLICITED : 0);
3712
3713         *seg += sizeof(**ctrl);
3714         *size = sizeof(**ctrl) / 16;
3715
3716         return err;
3717 }
3718
3719 static void finish_wqe(struct mlx5_ib_qp *qp,
3720                        struct mlx5_wqe_ctrl_seg *ctrl,
3721                        u8 size, unsigned idx, u64 wr_id,
3722                        int nreq, u8 fence, u8 next_fence,
3723                        u32 mlx5_opcode)
3724 {
3725         u8 opmod = 0;
3726
3727         ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
3728                                              mlx5_opcode | ((u32)opmod << 24));
3729         ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
3730         ctrl->fm_ce_se |= fence;
3731         qp->fm_cache = next_fence;
3732         if (unlikely(qp->wq_sig))
3733                 ctrl->signature = wq_sig(ctrl);
3734
3735         qp->sq.wrid[idx] = wr_id;
3736         qp->sq.w_list[idx].opcode = mlx5_opcode;
3737         qp->sq.wqe_head[idx] = qp->sq.head + nreq;
3738         qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
3739         qp->sq.w_list[idx].next = qp->sq.cur_post;
3740 }
3741
3742
3743 int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3744                       struct ib_send_wr **bad_wr)
3745 {
3746         struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
3747         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3748         struct mlx5_core_dev *mdev = dev->mdev;
3749         struct mlx5_ib_qp *qp;
3750         struct mlx5_ib_mr *mr;
3751         struct mlx5_wqe_data_seg *dpseg;
3752         struct mlx5_wqe_xrc_seg *xrc;
3753         struct mlx5_bf *bf;
3754         int uninitialized_var(size);
3755         void *qend;
3756         unsigned long flags;
3757         unsigned idx;
3758         int err = 0;
3759         int inl = 0;
3760         int num_sge;
3761         void *seg;
3762         int nreq;
3763         int i;
3764         u8 next_fence = 0;
3765         u8 fence;
3766
3767         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
3768                 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
3769
3770         qp = to_mqp(ibqp);
3771         bf = qp->bf;
3772         qend = qp->sq.qend;
3773
3774         spin_lock_irqsave(&qp->sq.lock, flags);
3775
3776         if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
3777                 err = -EIO;
3778                 *bad_wr = wr;
3779                 nreq = 0;
3780                 goto out;
3781         }
3782
3783         for (nreq = 0; wr; nreq++, wr = wr->next) {
3784                 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
3785                         mlx5_ib_warn(dev, "\n");
3786                         err = -EINVAL;
3787                         *bad_wr = wr;
3788                         goto out;
3789                 }
3790
3791                 fence = qp->fm_cache;
3792                 num_sge = wr->num_sge;
3793                 if (unlikely(num_sge > qp->sq.max_gs)) {
3794                         mlx5_ib_warn(dev, "\n");
3795                         err = -ENOMEM;
3796                         *bad_wr = wr;
3797                         goto out;
3798                 }
3799
3800                 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
3801                 if (err) {
3802                         mlx5_ib_warn(dev, "\n");
3803                         err = -ENOMEM;
3804                         *bad_wr = wr;
3805                         goto out;
3806                 }
3807
3808                 switch (ibqp->qp_type) {
3809                 case IB_QPT_XRC_INI:
3810                         xrc = seg;
3811                         seg += sizeof(*xrc);
3812                         size += sizeof(*xrc) / 16;
3813                         /* fall through */
3814                 case IB_QPT_RC:
3815                         switch (wr->opcode) {
3816                         case IB_WR_RDMA_READ:
3817                         case IB_WR_RDMA_WRITE:
3818                         case IB_WR_RDMA_WRITE_WITH_IMM:
3819                                 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
3820                                               rdma_wr(wr)->rkey);
3821                                 seg += sizeof(struct mlx5_wqe_raddr_seg);
3822                                 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
3823                                 break;
3824
3825                         case IB_WR_ATOMIC_CMP_AND_SWP:
3826                         case IB_WR_ATOMIC_FETCH_AND_ADD:
3827                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
3828                                 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
3829                                 err = -ENOSYS;
3830                                 *bad_wr = wr;
3831                                 goto out;
3832
3833                         case IB_WR_LOCAL_INV:
3834                                 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
3835                                 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
3836                                 ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
3837                                 set_linv_wr(qp, &seg, &size);
3838                                 num_sge = 0;
3839                                 break;
3840
3841                         case IB_WR_REG_MR:
3842                                 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
3843                                 qp->sq.wr_data[idx] = IB_WR_REG_MR;
3844                                 ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
3845                                 err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
3846                                 if (err) {
3847                                         *bad_wr = wr;
3848                                         goto out;
3849                                 }
3850                                 num_sge = 0;
3851                                 break;
3852
3853                         case IB_WR_REG_SIG_MR:
3854                                 qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
3855                                 mr = to_mmr(sig_handover_wr(wr)->sig_mr);
3856
3857                                 ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
3858                                 err = set_sig_umr_wr(wr, qp, &seg, &size);
3859                                 if (err) {
3860                                         mlx5_ib_warn(dev, "\n");
3861                                         *bad_wr = wr;
3862                                         goto out;
3863                                 }
3864
3865                                 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
3866                                            nreq, get_fence(fence, wr),
3867                                            next_fence, MLX5_OPCODE_UMR);
3868                                 /*
3869                                  * SET_PSV WQEs are not signaled and solicited
3870                                  * on error
3871                                  */
3872                                 wr->send_flags &= ~IB_SEND_SIGNALED;
3873                                 wr->send_flags |= IB_SEND_SOLICITED;
3874                                 err = begin_wqe(qp, &seg, &ctrl, wr,
3875                                                 &idx, &size, nreq);
3876                                 if (err) {
3877                                         mlx5_ib_warn(dev, "\n");
3878                                         err = -ENOMEM;
3879                                         *bad_wr = wr;
3880                                         goto out;
3881                                 }
3882
3883                                 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,
3884                                                  mr->sig->psv_memory.psv_idx, &seg,
3885                                                  &size);
3886                                 if (err) {
3887                                         mlx5_ib_warn(dev, "\n");
3888                                         *bad_wr = wr;
3889                                         goto out;
3890                                 }
3891
3892                                 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
3893                                            nreq, get_fence(fence, wr),
3894                                            next_fence, MLX5_OPCODE_SET_PSV);
3895                                 err = begin_wqe(qp, &seg, &ctrl, wr,
3896                                                 &idx, &size, nreq);
3897                                 if (err) {
3898                                         mlx5_ib_warn(dev, "\n");
3899                                         err = -ENOMEM;
3900                                         *bad_wr = wr;
3901                                         goto out;
3902                                 }
3903
3904                                 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
3905                                 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
3906                                                  mr->sig->psv_wire.psv_idx, &seg,
3907                                                  &size);
3908                                 if (err) {
3909                                         mlx5_ib_warn(dev, "\n");
3910                                         *bad_wr = wr;
3911                                         goto out;
3912                                 }
3913
3914                                 finish_wqe(qp, ctrl, size, idx, wr->wr_id,
3915                                            nreq, get_fence(fence, wr),
3916                                            next_fence, MLX5_OPCODE_SET_PSV);
3917                                 num_sge = 0;
3918                                 goto skip_psv;
3919
3920                         default:
3921                                 break;
3922                         }
3923                         break;
3924
3925                 case IB_QPT_UC:
3926                         switch (wr->opcode) {
3927                         case IB_WR_RDMA_WRITE:
3928                         case IB_WR_RDMA_WRITE_WITH_IMM:
3929                                 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
3930                                               rdma_wr(wr)->rkey);
3931                                 seg  += sizeof(struct mlx5_wqe_raddr_seg);
3932                                 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
3933                                 break;
3934
3935                         default:
3936                                 break;
3937                         }
3938                         break;
3939
3940                 case IB_QPT_SMI:
3941                 case MLX5_IB_QPT_HW_GSI:
3942                         set_datagram_seg(seg, wr);
3943                         seg += sizeof(struct mlx5_wqe_datagram_seg);
3944                         size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
3945                         if (unlikely((seg == qend)))
3946                                 seg = mlx5_get_send_wqe(qp, 0);
3947                         break;
3948                 case IB_QPT_UD:
3949                         set_datagram_seg(seg, wr);
3950                         seg += sizeof(struct mlx5_wqe_datagram_seg);
3951                         size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
3952
3953                         if (unlikely((seg == qend)))
3954                                 seg = mlx5_get_send_wqe(qp, 0);
3955
3956                         /* handle qp that supports ud offload */
3957                         if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
3958                                 struct mlx5_wqe_eth_pad *pad;
3959
3960                                 pad = seg;
3961                                 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
3962                                 seg += sizeof(struct mlx5_wqe_eth_pad);
3963                                 size += sizeof(struct mlx5_wqe_eth_pad) / 16;
3964
3965                                 seg = set_eth_seg(seg, wr, qend, qp, &size);
3966
3967                                 if (unlikely((seg == qend)))
3968                                         seg = mlx5_get_send_wqe(qp, 0);
3969                         }
3970                         break;
3971                 case MLX5_IB_QPT_REG_UMR:
3972                         if (wr->opcode != MLX5_IB_WR_UMR) {
3973                                 err = -EINVAL;
3974                                 mlx5_ib_warn(dev, "bad opcode\n");
3975                                 goto out;
3976                         }
3977                         qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
3978                         ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
3979                         set_reg_umr_segment(seg, wr);
3980                         seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
3981                         size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
3982                         if (unlikely((seg == qend)))
3983                                 seg = mlx5_get_send_wqe(qp, 0);
3984                         set_reg_mkey_segment(seg, wr);
3985                         seg += sizeof(struct mlx5_mkey_seg);
3986                         size += sizeof(struct mlx5_mkey_seg) / 16;
3987                         if (unlikely((seg == qend)))
3988                                 seg = mlx5_get_send_wqe(qp, 0);
3989                         break;
3990
3991                 default:
3992                         break;
3993                 }
3994
3995                 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
3996                         int uninitialized_var(sz);
3997
3998                         err = set_data_inl_seg(qp, wr, seg, &sz);
3999                         if (unlikely(err)) {
4000                                 mlx5_ib_warn(dev, "\n");
4001                                 *bad_wr = wr;
4002                                 goto out;
4003                         }
4004                         inl = 1;
4005                         size += sz;
4006                 } else {
4007                         dpseg = seg;
4008                         for (i = 0; i < num_sge; i++) {
4009                                 if (unlikely(dpseg == qend)) {
4010                                         seg = mlx5_get_send_wqe(qp, 0);
4011                                         dpseg = seg;
4012                                 }
4013                                 if (likely(wr->sg_list[i].length)) {
4014                                         set_data_ptr_seg(dpseg, wr->sg_list + i);
4015                                         size += sizeof(struct mlx5_wqe_data_seg) / 16;
4016                                         dpseg++;
4017                                 }
4018                         }
4019                 }
4020
4021                 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4022                            get_fence(fence, wr), next_fence,
4023                            mlx5_ib_opcode[wr->opcode]);
4024 skip_psv:
4025                 if (0)
4026                         dump_wqe(qp, idx, size);
4027         }
4028
4029 out:
4030         if (likely(nreq)) {
4031                 qp->sq.head += nreq;
4032
4033                 /* Make sure that descriptors are written before
4034                  * updating doorbell record and ringing the doorbell
4035                  */
4036                 wmb();
4037
4038                 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
4039
4040                 /* Make sure doorbell record is visible to the HCA before
4041                  * we hit doorbell */
4042                 wmb();
4043
4044                 if (bf->need_lock)
4045                         spin_lock(&bf->lock);
4046                 else
4047                         __acquire(&bf->lock);
4048
4049                 /* TBD enable WC */
4050                 if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) {
4051                         mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp);
4052                         /* wc_wmb(); */
4053                 } else {
4054                         mlx5_write64((__be32 *)ctrl, bf->regreg + bf->offset,
4055                                      MLX5_GET_DOORBELL_LOCK(&bf->lock32));
4056                         /* Make sure doorbells don't leak out of SQ spinlock
4057                          * and reach the HCA out of order.
4058                          */
4059                         mmiowb();
4060                 }
4061                 bf->offset ^= bf->buf_size;
4062                 if (bf->need_lock)
4063                         spin_unlock(&bf->lock);
4064                 else
4065                         __release(&bf->lock);
4066         }
4067
4068         spin_unlock_irqrestore(&qp->sq.lock, flags);
4069
4070         return err;
4071 }
4072
4073 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)
4074 {
4075         sig->signature = calc_sig(sig, size);
4076 }
4077
4078 int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
4079                       struct ib_recv_wr **bad_wr)
4080 {
4081         struct mlx5_ib_qp *qp = to_mqp(ibqp);
4082         struct mlx5_wqe_data_seg *scat;
4083         struct mlx5_rwqe_sig *sig;
4084         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4085         struct mlx5_core_dev *mdev = dev->mdev;
4086         unsigned long flags;
4087         int err = 0;
4088         int nreq;
4089         int ind;
4090         int i;
4091
4092         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4093                 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
4094
4095         spin_lock_irqsave(&qp->rq.lock, flags);
4096
4097         if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
4098                 err = -EIO;
4099                 *bad_wr = wr;
4100                 nreq = 0;
4101                 goto out;
4102         }
4103
4104         ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
4105
4106         for (nreq = 0; wr; nreq++, wr = wr->next) {
4107                 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
4108                         err = -ENOMEM;
4109                         *bad_wr = wr;
4110                         goto out;
4111                 }
4112
4113                 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
4114                         err = -EINVAL;
4115                         *bad_wr = wr;
4116                         goto out;
4117                 }
4118
4119                 scat = get_recv_wqe(qp, ind);
4120                 if (qp->wq_sig)
4121                         scat++;
4122
4123                 for (i = 0; i < wr->num_sge; i++)
4124                         set_data_ptr_seg(scat + i, wr->sg_list + i);
4125
4126                 if (i < qp->rq.max_gs) {
4127                         scat[i].byte_count = 0;
4128                         scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
4129                         scat[i].addr       = 0;
4130                 }
4131
4132                 if (qp->wq_sig) {
4133                         sig = (struct mlx5_rwqe_sig *)scat;
4134                         set_sig_seg(sig, (qp->rq.max_gs + 1) << 2);
4135                 }
4136
4137                 qp->rq.wrid[ind] = wr->wr_id;
4138
4139                 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
4140         }
4141
4142 out:
4143         if (likely(nreq)) {
4144                 qp->rq.head += nreq;
4145
4146                 /* Make sure that descriptors are written before
4147                  * doorbell record.
4148                  */
4149                 wmb();
4150
4151                 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
4152         }
4153
4154         spin_unlock_irqrestore(&qp->rq.lock, flags);
4155
4156         return err;
4157 }
4158
4159 static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4160 {
4161         switch (mlx5_state) {
4162         case MLX5_QP_STATE_RST:      return IB_QPS_RESET;
4163         case MLX5_QP_STATE_INIT:     return IB_QPS_INIT;
4164         case MLX5_QP_STATE_RTR:      return IB_QPS_RTR;
4165         case MLX5_QP_STATE_RTS:      return IB_QPS_RTS;
4166         case MLX5_QP_STATE_SQ_DRAINING:
4167         case MLX5_QP_STATE_SQD:      return IB_QPS_SQD;
4168         case MLX5_QP_STATE_SQER:     return IB_QPS_SQE;
4169         case MLX5_QP_STATE_ERR:      return IB_QPS_ERR;
4170         default:                     return -1;
4171         }
4172 }
4173
4174 static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4175 {
4176         switch (mlx5_mig_state) {
4177         case MLX5_QP_PM_ARMED:          return IB_MIG_ARMED;
4178         case MLX5_QP_PM_REARM:          return IB_MIG_REARM;
4179         case MLX5_QP_PM_MIGRATED:       return IB_MIG_MIGRATED;
4180         default: return -1;
4181         }
4182 }
4183
4184 static int to_ib_qp_access_flags(int mlx5_flags)
4185 {
4186         int ib_flags = 0;
4187
4188         if (mlx5_flags & MLX5_QP_BIT_RRE)
4189                 ib_flags |= IB_ACCESS_REMOTE_READ;
4190         if (mlx5_flags & MLX5_QP_BIT_RWE)
4191                 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4192         if (mlx5_flags & MLX5_QP_BIT_RAE)
4193                 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4194
4195         return ib_flags;
4196 }
4197
4198 static void to_ib_ah_attr(struct mlx5_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr,
4199                                 struct mlx5_qp_path *path)
4200 {
4201         struct mlx5_core_dev *dev = ibdev->mdev;
4202
4203         memset(ib_ah_attr, 0, sizeof(*ib_ah_attr));
4204         ib_ah_attr->port_num      = path->port;
4205
4206         if (ib_ah_attr->port_num == 0 ||
4207             ib_ah_attr->port_num > MLX5_CAP_GEN(dev, num_ports))
4208                 return;
4209
4210         ib_ah_attr->sl = path->dci_cfi_prio_sl & 0xf;
4211
4212         ib_ah_attr->dlid          = be16_to_cpu(path->rlid);
4213         ib_ah_attr->src_path_bits = path->grh_mlid & 0x7f;
4214         ib_ah_attr->static_rate   = path->static_rate ? path->static_rate - 5 : 0;
4215         ib_ah_attr->ah_flags      = (path->grh_mlid & (1 << 7)) ? IB_AH_GRH : 0;
4216         if (ib_ah_attr->ah_flags) {
4217                 ib_ah_attr->grh.sgid_index = path->mgid_index;
4218                 ib_ah_attr->grh.hop_limit  = path->hop_limit;
4219                 ib_ah_attr->grh.traffic_class =
4220                         (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
4221                 ib_ah_attr->grh.flow_label =
4222                         be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
4223                 memcpy(ib_ah_attr->grh.dgid.raw,
4224                        path->rgid, sizeof(ib_ah_attr->grh.dgid.raw));
4225         }
4226 }
4227
4228 static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4229                                         struct mlx5_ib_sq *sq,
4230                                         u8 *sq_state)
4231 {
4232         void *out;
4233         void *sqc;
4234         int inlen;
4235         int err;
4236
4237         inlen = MLX5_ST_SZ_BYTES(query_sq_out);
4238         out = mlx5_vzalloc(inlen);
4239         if (!out)
4240                 return -ENOMEM;
4241
4242         err = mlx5_core_query_sq(dev->mdev, sq->base.mqp.qpn, out);
4243         if (err)
4244                 goto out;
4245
4246         sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
4247         *sq_state = MLX5_GET(sqc, sqc, state);
4248         sq->state = *sq_state;
4249
4250 out:
4251         kvfree(out);
4252         return err;
4253 }
4254
4255 static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4256                                         struct mlx5_ib_rq *rq,
4257                                         u8 *rq_state)
4258 {
4259         void *out;
4260         void *rqc;
4261         int inlen;
4262         int err;
4263
4264         inlen = MLX5_ST_SZ_BYTES(query_rq_out);
4265         out = mlx5_vzalloc(inlen);
4266         if (!out)
4267                 return -ENOMEM;
4268
4269         err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4270         if (err)
4271                 goto out;
4272
4273         rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4274         *rq_state = MLX5_GET(rqc, rqc, state);
4275         rq->state = *rq_state;
4276
4277 out:
4278         kvfree(out);
4279         return err;
4280 }
4281
4282 static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4283                                   struct mlx5_ib_qp *qp, u8 *qp_state)
4284 {
4285         static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4286                 [MLX5_RQC_STATE_RST] = {
4287                         [MLX5_SQC_STATE_RST]    = IB_QPS_RESET,
4288                         [MLX5_SQC_STATE_RDY]    = MLX5_QP_STATE_BAD,
4289                         [MLX5_SQC_STATE_ERR]    = MLX5_QP_STATE_BAD,
4290                         [MLX5_SQ_STATE_NA]      = IB_QPS_RESET,
4291                 },
4292                 [MLX5_RQC_STATE_RDY] = {
4293                         [MLX5_SQC_STATE_RST]    = MLX5_QP_STATE_BAD,
4294                         [MLX5_SQC_STATE_RDY]    = MLX5_QP_STATE,
4295                         [MLX5_SQC_STATE_ERR]    = IB_QPS_SQE,
4296                         [MLX5_SQ_STATE_NA]      = MLX5_QP_STATE,
4297                 },
4298                 [MLX5_RQC_STATE_ERR] = {
4299                         [MLX5_SQC_STATE_RST]    = MLX5_QP_STATE_BAD,
4300                         [MLX5_SQC_STATE_RDY]    = MLX5_QP_STATE_BAD,
4301                         [MLX5_SQC_STATE_ERR]    = IB_QPS_ERR,
4302                         [MLX5_SQ_STATE_NA]      = IB_QPS_ERR,
4303                 },
4304                 [MLX5_RQ_STATE_NA] = {
4305                         [MLX5_SQC_STATE_RST]    = IB_QPS_RESET,
4306                         [MLX5_SQC_STATE_RDY]    = MLX5_QP_STATE,
4307                         [MLX5_SQC_STATE_ERR]    = MLX5_QP_STATE,
4308                         [MLX5_SQ_STATE_NA]      = MLX5_QP_STATE_BAD,
4309                 },
4310         };
4311
4312         *qp_state = sqrq_trans[rq_state][sq_state];
4313
4314         if (*qp_state == MLX5_QP_STATE_BAD) {
4315                 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4316                      qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4317                      qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4318                 return -EINVAL;
4319         }
4320
4321         if (*qp_state == MLX5_QP_STATE)
4322                 *qp_state = qp->state;
4323
4324         return 0;
4325 }
4326
4327 static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4328                                      struct mlx5_ib_qp *qp,
4329                                      u8 *raw_packet_qp_state)
4330 {
4331         struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4332         struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4333         struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4334         int err;
4335         u8 sq_state = MLX5_SQ_STATE_NA;
4336         u8 rq_state = MLX5_RQ_STATE_NA;
4337
4338         if (qp->sq.wqe_cnt) {
4339                 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4340                 if (err)
4341                         return err;
4342         }
4343
4344         if (qp->rq.wqe_cnt) {
4345                 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4346                 if (err)
4347                         return err;
4348         }
4349
4350         return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4351                                       raw_packet_qp_state);
4352 }
4353
4354 static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4355                          struct ib_qp_attr *qp_attr)
4356 {
4357         int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
4358         struct mlx5_qp_context *context;
4359         int mlx5_state;
4360         u32 *outb;
4361         int err = 0;
4362
4363         outb = kzalloc(outlen, GFP_KERNEL);
4364         if (!outb)
4365                 return -ENOMEM;
4366
4367         err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
4368                                  outlen);
4369         if (err)
4370                 goto out;
4371
4372         /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
4373         context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
4374
4375         mlx5_state = be32_to_cpu(context->flags) >> 28;
4376
4377         qp->state                    = to_ib_qp_state(mlx5_state);
4378         qp_attr->path_mtu            = context->mtu_msgmax >> 5;
4379         qp_attr->path_mig_state      =
4380                 to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
4381         qp_attr->qkey                = be32_to_cpu(context->qkey);
4382         qp_attr->rq_psn              = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
4383         qp_attr->sq_psn              = be32_to_cpu(context->next_send_psn) & 0xffffff;
4384         qp_attr->dest_qp_num         = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
4385         qp_attr->qp_access_flags     =
4386                 to_ib_qp_access_flags(be32_to_cpu(context->params2));
4387
4388         if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4389                 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
4390                 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
4391                 qp_attr->alt_pkey_index =
4392                         be16_to_cpu(context->alt_path.pkey_index);
4393                 qp_attr->alt_port_num   = qp_attr->alt_ah_attr.port_num;
4394         }
4395
4396         qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
4397         qp_attr->port_num = context->pri_path.port;
4398
4399         /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4400         qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
4401
4402         qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
4403
4404         qp_attr->max_dest_rd_atomic =
4405                 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
4406         qp_attr->min_rnr_timer      =
4407                 (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
4408         qp_attr->timeout            = context->pri_path.ackto_lt >> 3;
4409         qp_attr->retry_cnt          = (be32_to_cpu(context->params1) >> 16) & 0x7;
4410         qp_attr->rnr_retry          = (be32_to_cpu(context->params1) >> 13) & 0x7;
4411         qp_attr->alt_timeout        = context->alt_path.ackto_lt >> 3;
4412
4413 out:
4414         kfree(outb);
4415         return err;
4416 }
4417
4418 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4419                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
4420 {
4421         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4422         struct mlx5_ib_qp *qp = to_mqp(ibqp);
4423         int err = 0;
4424         u8 raw_packet_qp_state;
4425
4426         if (ibqp->rwq_ind_tbl)
4427                 return -ENOSYS;
4428
4429         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4430                 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
4431                                             qp_init_attr);
4432
4433 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4434         /*
4435          * Wait for any outstanding page faults, in case the user frees memory
4436          * based upon this query's result.
4437          */
4438         flush_workqueue(mlx5_ib_page_fault_wq);
4439 #endif
4440
4441         mutex_lock(&qp->mutex);
4442
4443         if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
4444                 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
4445                 if (err)
4446                         goto out;
4447                 qp->state = raw_packet_qp_state;
4448                 qp_attr->port_num = 1;
4449         } else {
4450                 err = query_qp_attr(dev, qp, qp_attr);
4451                 if (err)
4452                         goto out;
4453         }
4454
4455         qp_attr->qp_state            = qp->state;
4456         qp_attr->cur_qp_state        = qp_attr->qp_state;
4457         qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
4458         qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
4459
4460         if (!ibqp->uobject) {
4461                 qp_attr->cap.max_send_wr  = qp->sq.max_post;
4462                 qp_attr->cap.max_send_sge = qp->sq.max_gs;
4463                 qp_init_attr->qp_context = ibqp->qp_context;
4464         } else {
4465                 qp_attr->cap.max_send_wr  = 0;
4466                 qp_attr->cap.max_send_sge = 0;
4467         }
4468
4469         qp_init_attr->qp_type = ibqp->qp_type;
4470         qp_init_attr->recv_cq = ibqp->recv_cq;
4471         qp_init_attr->send_cq = ibqp->send_cq;
4472         qp_init_attr->srq = ibqp->srq;
4473         qp_attr->cap.max_inline_data = qp->max_inline_data;
4474
4475         qp_init_attr->cap            = qp_attr->cap;
4476
4477         qp_init_attr->create_flags = 0;
4478         if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
4479                 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
4480
4481         if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
4482                 qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;
4483         if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
4484                 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;
4485         if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
4486                 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;
4487         if (qp->flags & MLX5_IB_QP_SQPN_QP1)
4488                 qp_init_attr->create_flags |= mlx5_ib_create_qp_sqpn_qp1();
4489
4490         qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
4491                 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4492
4493 out:
4494         mutex_unlock(&qp->mutex);
4495         return err;
4496 }
4497
4498 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
4499                                           struct ib_ucontext *context,
4500                                           struct ib_udata *udata)
4501 {
4502         struct mlx5_ib_dev *dev = to_mdev(ibdev);
4503         struct mlx5_ib_xrcd *xrcd;
4504         int err;
4505
4506         if (!MLX5_CAP_GEN(dev->mdev, xrc))
4507                 return ERR_PTR(-ENOSYS);
4508
4509         xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
4510         if (!xrcd)
4511                 return ERR_PTR(-ENOMEM);
4512
4513         err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn);
4514         if (err) {
4515                 kfree(xrcd);
4516                 return ERR_PTR(-ENOMEM);
4517         }
4518
4519         return &xrcd->ibxrcd;
4520 }
4521
4522 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
4523 {
4524         struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
4525         u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
4526         int err;
4527
4528         err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
4529         if (err) {
4530                 mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
4531                 return err;
4532         }
4533
4534         kfree(xrcd);
4535
4536         return 0;
4537 }
4538
4539 static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
4540 {
4541         struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
4542         struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
4543         struct ib_event event;
4544
4545         if (rwq->ibwq.event_handler) {
4546                 event.device     = rwq->ibwq.device;
4547                 event.element.wq = &rwq->ibwq;
4548                 switch (type) {
4549                 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
4550                         event.event = IB_EVENT_WQ_FATAL;
4551                         break;
4552                 default:
4553                         mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
4554                         return;
4555                 }
4556
4557                 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
4558         }
4559 }
4560
4561 static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
4562                       struct ib_wq_init_attr *init_attr)
4563 {
4564         struct mlx5_ib_dev *dev;
4565         __be64 *rq_pas0;
4566         void *in;
4567         void *rqc;
4568         void *wq;
4569         int inlen;
4570         int err;
4571
4572         dev = to_mdev(pd->device);
4573
4574         inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
4575         in = mlx5_vzalloc(inlen);
4576         if (!in)
4577                 return -ENOMEM;
4578
4579         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
4580         MLX5_SET(rqc,  rqc, mem_rq_type,
4581                  MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
4582         MLX5_SET(rqc, rqc, user_index, rwq->user_index);
4583         MLX5_SET(rqc,  rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
4584         MLX5_SET(rqc,  rqc, state, MLX5_RQC_STATE_RST);
4585         MLX5_SET(rqc,  rqc, flush_in_error_en, 1);
4586         wq = MLX5_ADDR_OF(rqc, rqc, wq);
4587         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
4588         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
4589         MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
4590         MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
4591         MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
4592         MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
4593         MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
4594         MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
4595         MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
4596         rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
4597         mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
4598         err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
4599         kvfree(in);
4600         return err;
4601 }
4602
4603 static int set_user_rq_size(struct mlx5_ib_dev *dev,
4604                             struct ib_wq_init_attr *wq_init_attr,
4605                             struct mlx5_ib_create_wq *ucmd,
4606                             struct mlx5_ib_rwq *rwq)
4607 {
4608         /* Sanity check RQ size before proceeding */
4609         if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
4610                 return -EINVAL;
4611
4612         if (!ucmd->rq_wqe_count)
4613                 return -EINVAL;
4614
4615         rwq->wqe_count = ucmd->rq_wqe_count;
4616         rwq->wqe_shift = ucmd->rq_wqe_shift;
4617         rwq->buf_size = (rwq->wqe_count << rwq->wqe_shift);
4618         rwq->log_rq_stride = rwq->wqe_shift;
4619         rwq->log_rq_size = ilog2(rwq->wqe_count);
4620         return 0;
4621 }
4622
4623 static int prepare_user_rq(struct ib_pd *pd,
4624                            struct ib_wq_init_attr *init_attr,
4625                            struct ib_udata *udata,
4626                            struct mlx5_ib_rwq *rwq)
4627 {
4628         struct mlx5_ib_dev *dev = to_mdev(pd->device);
4629         struct mlx5_ib_create_wq ucmd = {};
4630         int err;
4631         size_t required_cmd_sz;
4632
4633         required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
4634         if (udata->inlen < required_cmd_sz) {
4635                 mlx5_ib_dbg(dev, "invalid inlen\n");
4636                 return -EINVAL;
4637         }
4638
4639         if (udata->inlen > sizeof(ucmd) &&
4640             !ib_is_udata_cleared(udata, sizeof(ucmd),
4641                                  udata->inlen - sizeof(ucmd))) {
4642                 mlx5_ib_dbg(dev, "inlen is not supported\n");
4643                 return -EOPNOTSUPP;
4644         }
4645
4646         if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
4647                 mlx5_ib_dbg(dev, "copy failed\n");
4648                 return -EFAULT;
4649         }
4650
4651         if (ucmd.comp_mask) {
4652                 mlx5_ib_dbg(dev, "invalid comp mask\n");
4653                 return -EOPNOTSUPP;
4654         }
4655
4656         if (ucmd.reserved) {
4657                 mlx5_ib_dbg(dev, "invalid reserved\n");
4658                 return -EOPNOTSUPP;
4659         }
4660
4661         err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
4662         if (err) {
4663                 mlx5_ib_dbg(dev, "err %d\n", err);
4664                 return err;
4665         }
4666
4667         err = create_user_rq(dev, pd, rwq, &ucmd);
4668         if (err) {
4669                 mlx5_ib_dbg(dev, "err %d\n", err);
4670                 if (err)
4671                         return err;
4672         }
4673
4674         rwq->user_index = ucmd.user_index;
4675         return 0;
4676 }
4677
4678 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
4679                                 struct ib_wq_init_attr *init_attr,
4680                                 struct ib_udata *udata)
4681 {
4682         struct mlx5_ib_dev *dev;
4683         struct mlx5_ib_rwq *rwq;
4684         struct mlx5_ib_create_wq_resp resp = {};
4685         size_t min_resp_len;
4686         int err;
4687
4688         if (!udata)
4689                 return ERR_PTR(-ENOSYS);
4690
4691         min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4692         if (udata->outlen && udata->outlen < min_resp_len)
4693                 return ERR_PTR(-EINVAL);
4694
4695         dev = to_mdev(pd->device);
4696         switch (init_attr->wq_type) {
4697         case IB_WQT_RQ:
4698                 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
4699                 if (!rwq)
4700                         return ERR_PTR(-ENOMEM);
4701                 err = prepare_user_rq(pd, init_attr, udata, rwq);
4702                 if (err)
4703                         goto err;
4704                 err = create_rq(rwq, pd, init_attr);
4705                 if (err)
4706                         goto err_user_rq;
4707                 break;
4708         default:
4709                 mlx5_ib_dbg(dev, "unsupported wq type %d\n",
4710                             init_attr->wq_type);
4711                 return ERR_PTR(-EINVAL);
4712         }
4713
4714         rwq->ibwq.wq_num = rwq->core_qp.qpn;
4715         rwq->ibwq.state = IB_WQS_RESET;
4716         if (udata->outlen) {
4717                 resp.response_length = offsetof(typeof(resp), response_length) +
4718                                 sizeof(resp.response_length);
4719                 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4720                 if (err)
4721                         goto err_copy;
4722         }
4723
4724         rwq->core_qp.event = mlx5_ib_wq_event;
4725         rwq->ibwq.event_handler = init_attr->event_handler;
4726         return &rwq->ibwq;
4727
4728 err_copy:
4729         mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
4730 err_user_rq:
4731         destroy_user_rq(pd, rwq);
4732 err:
4733         kfree(rwq);
4734         return ERR_PTR(err);
4735 }
4736
4737 int mlx5_ib_destroy_wq(struct ib_wq *wq)
4738 {
4739         struct mlx5_ib_dev *dev = to_mdev(wq->device);
4740         struct mlx5_ib_rwq *rwq = to_mrwq(wq);
4741
4742         mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
4743         destroy_user_rq(wq->pd, rwq);
4744         kfree(rwq);
4745
4746         return 0;
4747 }
4748
4749 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
4750                                                       struct ib_rwq_ind_table_init_attr *init_attr,
4751                                                       struct ib_udata *udata)
4752 {
4753         struct mlx5_ib_dev *dev = to_mdev(device);
4754         struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;
4755         int sz = 1 << init_attr->log_ind_tbl_size;
4756         struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
4757         size_t min_resp_len;
4758         int inlen;
4759         int err;
4760         int i;
4761         u32 *in;
4762         void *rqtc;
4763
4764         if (udata->inlen > 0 &&
4765             !ib_is_udata_cleared(udata, 0,
4766                                  udata->inlen))
4767                 return ERR_PTR(-EOPNOTSUPP);
4768
4769         min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4770         if (udata->outlen && udata->outlen < min_resp_len)
4771                 return ERR_PTR(-EINVAL);
4772
4773         rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);
4774         if (!rwq_ind_tbl)
4775                 return ERR_PTR(-ENOMEM);
4776
4777         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
4778         in = mlx5_vzalloc(inlen);
4779         if (!in) {
4780                 err = -ENOMEM;
4781                 goto err;
4782         }
4783
4784         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
4785
4786         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
4787         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
4788
4789         for (i = 0; i < sz; i++)
4790                 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
4791
4792         err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
4793         kvfree(in);
4794
4795         if (err)
4796                 goto err;
4797
4798         rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
4799         if (udata->outlen) {
4800                 resp.response_length = offsetof(typeof(resp), response_length) +
4801                                         sizeof(resp.response_length);
4802                 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4803                 if (err)
4804                         goto err_copy;
4805         }
4806
4807         return &rwq_ind_tbl->ib_rwq_ind_tbl;
4808
4809 err_copy:
4810         mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
4811 err:
4812         kfree(rwq_ind_tbl);
4813         return ERR_PTR(err);
4814 }
4815
4816 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4817 {
4818         struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
4819         struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
4820
4821         mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
4822
4823         kfree(rwq_ind_tbl);
4824         return 0;
4825 }
4826
4827 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
4828                       u32 wq_attr_mask, struct ib_udata *udata)
4829 {
4830         struct mlx5_ib_dev *dev = to_mdev(wq->device);
4831         struct mlx5_ib_rwq *rwq = to_mrwq(wq);
4832         struct mlx5_ib_modify_wq ucmd = {};
4833         size_t required_cmd_sz;
4834         int curr_wq_state;
4835         int wq_state;
4836         int inlen;
4837         int err;
4838         void *rqc;
4839         void *in;
4840
4841         required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
4842         if (udata->inlen < required_cmd_sz)
4843                 return -EINVAL;
4844
4845         if (udata->inlen > sizeof(ucmd) &&
4846             !ib_is_udata_cleared(udata, sizeof(ucmd),
4847                                  udata->inlen - sizeof(ucmd)))
4848                 return -EOPNOTSUPP;
4849
4850         if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
4851                 return -EFAULT;
4852
4853         if (ucmd.comp_mask || ucmd.reserved)
4854                 return -EOPNOTSUPP;
4855
4856         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
4857         in = mlx5_vzalloc(inlen);
4858         if (!in)
4859                 return -ENOMEM;
4860
4861         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
4862
4863         curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ?
4864                 wq_attr->curr_wq_state : wq->state;
4865         wq_state = (wq_attr_mask & IB_WQ_STATE) ?
4866                 wq_attr->wq_state : curr_wq_state;
4867         if (curr_wq_state == IB_WQS_ERR)
4868                 curr_wq_state = MLX5_RQC_STATE_ERR;
4869         if (wq_state == IB_WQS_ERR)
4870                 wq_state = MLX5_RQC_STATE_ERR;
4871         MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
4872         MLX5_SET(rqc, rqc, state, wq_state);
4873
4874         err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen);
4875         kvfree(in);
4876         if (!err)
4877                 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
4878
4879         return err;
4880 }