drivers/infiniband/hw/hns/hns_roce_hw_v1.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/platform_device.h>
  34 #include <linux/acpi.h>
  35 #include <rdma/ib_umem.h>
  36 #include "hns_roce_common.h"
  37 #include "hns_roce_device.h"
  38 #include "hns_roce_cmd.h"
  39 #include "hns_roce_hem.h"
  40 #include "hns_roce_hw_v1.h"
  41
  42 static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
  43 {
  44         dseg->lkey = cpu_to_le32(sg->lkey);
  45         dseg->addr = cpu_to_le64(sg->addr);
  46         dseg->len  = cpu_to_le32(sg->length);
  47 }
  48
  49 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
  50                           u32 rkey)
  51 {
  52         rseg->raddr = cpu_to_le64(remote_addr);
  53         rseg->rkey  = cpu_to_le32(rkey);
  54         rseg->len   = 0;
  55 }
  56
  57 int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
  58                           struct ib_send_wr **bad_wr)
  59 {
  60         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
  61         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
  62         struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
  63         struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
  64         struct hns_roce_wqe_data_seg *dseg = NULL;
  65         struct hns_roce_qp *qp = to_hr_qp(ibqp);
  66         struct device *dev = &hr_dev->pdev->dev;
  67         struct hns_roce_sq_db sq_db;
  68         int ps_opcode = 0, i = 0;
  69         unsigned long flags = 0;
  70         void *wqe = NULL;
  71         u32 doorbell[2];
  72         int nreq = 0;
  73         u32 ind = 0;
  74         int ret = 0;
  75
  76         spin_lock_irqsave(&qp->sq.lock, flags);
  77
  78         ind = qp->sq_next_wqe;
  79         for (nreq = 0; wr; ++nreq, wr = wr->next) {
  80                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
  81                         ret = -ENOMEM;
  82                         *bad_wr = wr;
  83                         goto out;
  84                 }
  85
  86                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
  87                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
  88                                 wr->num_sge, qp->sq.max_gs);
  89                         ret = -EINVAL;
  90                         *bad_wr = wr;
  91                         goto out;
  92                 }
  93
  94                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
  95                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
  96                                                                       wr->wr_id;
  97
  98                 /* Corresponding to the RC and RD type wqe process separately */
  99                 if (ibqp->qp_type == IB_QPT_GSI) {
 100                         ud_sq_wqe = wqe;
 101                         roce_set_field(ud_sq_wqe->dmac_h,
 102                                        UD_SEND_WQE_U32_4_DMAC_0_M,
 103                                        UD_SEND_WQE_U32_4_DMAC_0_S,
 104                                        ah->av.mac[0]);
 105                         roce_set_field(ud_sq_wqe->dmac_h,
 106                                        UD_SEND_WQE_U32_4_DMAC_1_M,
 107                                        UD_SEND_WQE_U32_4_DMAC_1_S,
 108                                        ah->av.mac[1]);
 109                         roce_set_field(ud_sq_wqe->dmac_h,
 110                                        UD_SEND_WQE_U32_4_DMAC_2_M,
 111                                        UD_SEND_WQE_U32_4_DMAC_2_S,
 112                                        ah->av.mac[2]);
 113                         roce_set_field(ud_sq_wqe->dmac_h,
 114                                        UD_SEND_WQE_U32_4_DMAC_3_M,
 115                                        UD_SEND_WQE_U32_4_DMAC_3_S,
 116                                        ah->av.mac[3]);
 117
 118                         roce_set_field(ud_sq_wqe->u32_8,
 119                                        UD_SEND_WQE_U32_8_DMAC_4_M,
 120                                        UD_SEND_WQE_U32_8_DMAC_4_S,
 121                                        ah->av.mac[4]);
 122                         roce_set_field(ud_sq_wqe->u32_8,
 123                                        UD_SEND_WQE_U32_8_DMAC_5_M,
 124                                        UD_SEND_WQE_U32_8_DMAC_5_S,
 125                                        ah->av.mac[5]);
 126                         roce_set_field(ud_sq_wqe->u32_8,
 127                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
 128                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
 129                                        HNS_ROCE_WQE_OPCODE_SEND);
 130                         roce_set_field(ud_sq_wqe->u32_8,
 131                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
 132                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
 133                                        2);
 134                         roce_set_bit(ud_sq_wqe->u32_8,
 135                                 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
 136                                 1);
 137
 138                         ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
 139                                 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 140                                 (wr->send_flags & IB_SEND_SOLICITED ?
 141                                 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 142                                 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
 143                                 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
 144
 145                         roce_set_field(ud_sq_wqe->u32_16,
 146                                        UD_SEND_WQE_U32_16_DEST_QP_M,
 147                                        UD_SEND_WQE_U32_16_DEST_QP_S,
 148                                        ud_wr(wr)->remote_qpn);
 149                         roce_set_field(ud_sq_wqe->u32_16,
 150                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
 151                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
 152                                        ah->av.stat_rate);
 153
 154                         roce_set_field(ud_sq_wqe->u32_36,
 155                                        UD_SEND_WQE_U32_36_FLOW_LABEL_M,
 156                                        UD_SEND_WQE_U32_36_FLOW_LABEL_S, 0);
 157                         roce_set_field(ud_sq_wqe->u32_36,
 158                                        UD_SEND_WQE_U32_36_PRIORITY_M,
 159                                        UD_SEND_WQE_U32_36_PRIORITY_S,
 160                                        ah->av.sl_tclass_flowlabel >>
 161                                        HNS_ROCE_SL_SHIFT);
 162                         roce_set_field(ud_sq_wqe->u32_36,
 163                                        UD_SEND_WQE_U32_36_SGID_INDEX_M,
 164                                        UD_SEND_WQE_U32_36_SGID_INDEX_S,
 165                                        hns_get_gid_index(hr_dev, qp->port,
 166                                                          ah->av.gid_index));
 167
 168                         roce_set_field(ud_sq_wqe->u32_40,
 169                                        UD_SEND_WQE_U32_40_HOP_LIMIT_M,
 170                                        UD_SEND_WQE_U32_40_HOP_LIMIT_S,
 171                                        ah->av.hop_limit);
 172                         roce_set_field(ud_sq_wqe->u32_40,
 173                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
 174                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, 0);
 175
 176                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
 177
 178                         ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
 179                         ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
 180                         ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
 181
 182                         ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
 183                         ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
 184                         ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
 185                         ind++;
 186                 } else if (ibqp->qp_type == IB_QPT_RC) {
 187                         ctrl = wqe;
 188                         memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
 189                         for (i = 0; i < wr->num_sge; i++)
 190                                 ctrl->msg_length += wr->sg_list[i].length;
 191
 192                         ctrl->sgl_pa_h = 0;
 193                         ctrl->flag = 0;
 194                         ctrl->imm_data = send_ieth(wr);
 195
 196                         /*Ctrl field, ctrl set type: sig, solic, imm, fence */
 197                         /* SO wait for conforming application scenarios */
 198                         ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
 199                                       cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 200                                       (wr->send_flags & IB_SEND_SOLICITED ?
 201                                       cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 202                                       ((wr->opcode == IB_WR_SEND_WITH_IMM ||
 203                                       wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
 204                                       cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
 205                                       (wr->send_flags & IB_SEND_FENCE ?
 206                                       (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
 207
 208                         wqe = (struct hns_roce_wqe_ctrl_seg *)wqe +
 209                                sizeof(struct hns_roce_wqe_ctrl_seg);
 210
 211                         switch (wr->opcode) {
 212                         case IB_WR_RDMA_READ:
 213                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
 214                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 215                                               atomic_wr(wr)->rkey);
 216                                 break;
 217                         case IB_WR_RDMA_WRITE:
 218                         case IB_WR_RDMA_WRITE_WITH_IMM:
 219                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
 220                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 221                                               atomic_wr(wr)->rkey);
 222                                 break;
 223                         case IB_WR_SEND:
 224                         case IB_WR_SEND_WITH_INV:
 225                         case IB_WR_SEND_WITH_IMM:
 226                                 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
 227                                 break;
 228                         case IB_WR_LOCAL_INV:
 229                                 break;
 230                         case IB_WR_ATOMIC_CMP_AND_SWP:
 231                         case IB_WR_ATOMIC_FETCH_AND_ADD:
 232                         case IB_WR_LSO:
 233                         default:
 234                                 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
 235                                 break;
 236                         }
 237                         ctrl->flag |= cpu_to_le32(ps_opcode);
 238                         wqe = (struct hns_roce_wqe_raddr_seg *)wqe +
 239                                sizeof(struct hns_roce_wqe_raddr_seg);
 240
 241                         dseg = wqe;
 242                         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
 243                                 if (ctrl->msg_length >
 244                                         hr_dev->caps.max_sq_inline) {
 245                                         ret = -EINVAL;
 246                                         *bad_wr = wr;
 247                                         dev_err(dev, "inline len(1-%d)=%d, illegal",
 248                                                 ctrl->msg_length,
 249                                                 hr_dev->caps.max_sq_inline);
 250                                         goto out;
 251                                 }
 252                                 for (i = 0; i < wr->num_sge; i++) {
 253                                         memcpy(wqe, ((void *) (uintptr_t)
 254                                                wr->sg_list[i].addr),
 255                                                wr->sg_list[i].length);
 256                                         wqe = (struct hns_roce_wqe_raddr_seg *)
 257                                                wqe + wr->sg_list[i].length;
 258                                 }
 259                                 ctrl->flag |= HNS_ROCE_WQE_INLINE;
 260                         } else {
 261                                 /*sqe num is two */
 262                                 for (i = 0; i < wr->num_sge; i++)
 263                                         set_data_seg(dseg + i, wr->sg_list + i);
 264
 265                                 ctrl->flag |= cpu_to_le32(wr->num_sge <<
 266                                               HNS_ROCE_WQE_SGE_NUM_BIT);
 267                         }
 268                         ind++;
 269                 } else {
 270                         dev_dbg(dev, "unSupported QP type\n");
 271                         break;
 272                 }
 273         }
 274
 275 out:
 276         /* Set DB return */
 277         if (likely(nreq)) {
 278                 qp->sq.head += nreq;
 279                 /* Memory barrier */
 280                 wmb();
 281
 282                 sq_db.u32_4 = 0;
 283                 sq_db.u32_8 = 0;
 284                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
 285                                SQ_DOORBELL_U32_4_SQ_HEAD_S,
 286                               (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
 287                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
 288                                SQ_DOORBELL_U32_4_PORT_S, qp->port);
 289                 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
 290                                SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
 291                 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
 292
 293                 doorbell[0] = sq_db.u32_4;
 294                 doorbell[1] = sq_db.u32_8;
 295
 296                 hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
 297                 qp->sq_next_wqe = ind;
 298         }
 299
 300         spin_unlock_irqrestore(&qp->sq.lock, flags);
 301
 302         return ret;
 303 }
 304
 305 int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 306                           struct ib_recv_wr **bad_wr)
 307 {
 308         int ret = 0;
 309         int nreq = 0;
 310         int ind = 0;
 311         int i = 0;
 312         u32 reg_val = 0;
 313         unsigned long flags = 0;
 314         struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
 315         struct hns_roce_wqe_data_seg *scat = NULL;
 316         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 317         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 318         struct device *dev = &hr_dev->pdev->dev;
 319         struct hns_roce_rq_db rq_db;
 320         uint32_t doorbell[2] = {0};
 321
 322         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 323         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
 324
 325         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 326                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
 327                         hr_qp->ibqp.recv_cq)) {
 328                         ret = -ENOMEM;
 329                         *bad_wr = wr;
 330                         goto out;
 331                 }
 332
 333                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
 334                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
 335                                 wr->num_sge, hr_qp->rq.max_gs);
 336                         ret = -EINVAL;
 337                         *bad_wr = wr;
 338                         goto out;
 339                 }
 340
 341                 ctrl = get_recv_wqe(hr_qp, ind);
 342
 343                 roce_set_field(ctrl->rwqe_byte_12,
 344                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
 345                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
 346                                wr->num_sge);
 347
 348                 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
 349
 350                 for (i = 0; i < wr->num_sge; i++)
 351                         set_data_seg(scat + i, wr->sg_list + i);
 352
 353                 hr_qp->rq.wrid[ind] = wr->wr_id;
 354
 355                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
 356         }
 357
 358 out:
 359         if (likely(nreq)) {
 360                 hr_qp->rq.head += nreq;
 361                 /* Memory barrier */
 362                 wmb();
 363
 364                 if (ibqp->qp_type == IB_QPT_GSI) {
 365                         /* SW update GSI rq header */
 366                         reg_val = roce_read(to_hr_dev(ibqp->device),
 367                                             ROCEE_QP1C_CFG3_0_REG +
 368                                             QP1C_CFGN_OFFSET * hr_qp->port);
 369                         roce_set_field(reg_val,
 370                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
 371                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
 372                                        hr_qp->rq.head);
 373                         roce_write(to_hr_dev(ibqp->device),
 374                                    ROCEE_QP1C_CFG3_0_REG +
 375                                    QP1C_CFGN_OFFSET * hr_qp->port, reg_val);
 376                 } else {
 377                         rq_db.u32_4 = 0;
 378                         rq_db.u32_8 = 0;
 379
 380                         roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
 381                                        RQ_DOORBELL_U32_4_RQ_HEAD_S,
 382                                        hr_qp->rq.head);
 383                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
 384                                        RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
 385                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
 386                                        RQ_DOORBELL_U32_8_CMD_S, 1);
 387                         roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
 388                                      1);
 389
 390                         doorbell[0] = rq_db.u32_4;
 391                         doorbell[1] = rq_db.u32_8;
 392
 393                         hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
 394                 }
 395         }
 396         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 397
 398         return ret;
 399 }
 400
 401 static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
 402                                        int sdb_mode, int odb_mode)
 403 {
 404         u32 val;
 405
 406         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 407         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
 408         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
 409         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 410 }
 411
 412 static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
 413                                      u32 odb_mode)
 414 {
 415         u32 val;
 416
 417         /* Configure SDB/ODB extend mode */
 418         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 419         roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
 420         roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
 421         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 422 }
 423
 424 static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
 425                              u32 sdb_alful)
 426 {
 427         u32 val;
 428
 429         /* Configure SDB */
 430         val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
 431         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
 432                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
 433         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
 434                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
 435         roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
 436 }
 437
 438 static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
 439                              u32 odb_alful)
 440 {
 441         u32 val;
 442
 443         /* Configure ODB */
 444         val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
 445         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
 446                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
 447         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
 448                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
 449         roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
 450 }
 451
 452 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
 453                                  u32 ext_sdb_alful)
 454 {
 455         struct device *dev = &hr_dev->pdev->dev;
 456         struct hns_roce_v1_priv *priv;
 457         struct hns_roce_db_table *db;
 458         dma_addr_t sdb_dma_addr;
 459         u32 val;
 460
 461         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 462         db = &priv->db_table;
 463
 464         /* Configure extend SDB threshold */
 465         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
 466         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
 467
 468         /* Configure extend SDB base addr */
 469         sdb_dma_addr = db->ext_db->sdb_buf_list->map;
 470         roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
 471
 472         /* Configure extend SDB depth */
 473         val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
 474         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
 475                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
 476                        db->ext_db->esdb_dep);
 477         /*
 478          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 479          * using 4K page, and shift more 32 because of
 480          * caculating the high 32 bit value evaluated to hardware.
 481          */
 482         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
 483                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
 484         roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
 485
 486         dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
 487         dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
 488                 ext_sdb_alept, ext_sdb_alful);
 489 }
 490
 491 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
 492                                  u32 ext_odb_alful)
 493 {
 494         struct device *dev = &hr_dev->pdev->dev;
 495         struct hns_roce_v1_priv *priv;
 496         struct hns_roce_db_table *db;
 497         dma_addr_t odb_dma_addr;
 498         u32 val;
 499
 500         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 501         db = &priv->db_table;
 502
 503         /* Configure extend ODB threshold */
 504         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
 505         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
 506
 507         /* Configure extend ODB base addr */
 508         odb_dma_addr = db->ext_db->odb_buf_list->map;
 509         roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
 510
 511         /* Configure extend ODB depth */
 512         val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
 513         roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
 514                        ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
 515                        db->ext_db->eodb_dep);
 516         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
 517                        ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
 518                        db->ext_db->eodb_dep);
 519         roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
 520
 521         dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
 522         dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
 523                 ext_odb_alept, ext_odb_alful);
 524 }
 525
 526 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
 527                                 u32 odb_ext_mod)
 528 {
 529         struct device *dev = &hr_dev->pdev->dev;
 530         struct hns_roce_v1_priv *priv;
 531         struct hns_roce_db_table *db;
 532         dma_addr_t sdb_dma_addr;
 533         dma_addr_t odb_dma_addr;
 534         int ret = 0;
 535
 536         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 537         db = &priv->db_table;
 538
 539         db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
 540         if (!db->ext_db)
 541                 return -ENOMEM;
 542
 543         if (sdb_ext_mod) {
 544                 db->ext_db->sdb_buf_list = kmalloc(
 545                                 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
 546                 if (!db->ext_db->sdb_buf_list) {
 547                         ret = -ENOMEM;
 548                         goto ext_sdb_buf_fail_out;
 549                 }
 550
 551                 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
 552                                                      HNS_ROCE_V1_EXT_SDB_SIZE,
 553                                                      &sdb_dma_addr, GFP_KERNEL);
 554                 if (!db->ext_db->sdb_buf_list->buf) {
 555                         ret = -ENOMEM;
 556                         goto alloc_sq_db_buf_fail;
 557                 }
 558                 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
 559
 560                 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
 561                 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
 562                                      HNS_ROCE_V1_EXT_SDB_ALFUL);
 563         } else
 564                 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
 565                                  HNS_ROCE_V1_SDB_ALFUL);
 566
 567         if (odb_ext_mod) {
 568                 db->ext_db->odb_buf_list = kmalloc(
 569                                 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
 570                 if (!db->ext_db->odb_buf_list) {
 571                         ret = -ENOMEM;
 572                         goto ext_odb_buf_fail_out;
 573                 }
 574
 575                 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
 576                                                      HNS_ROCE_V1_EXT_ODB_SIZE,
 577                                                      &odb_dma_addr, GFP_KERNEL);
 578                 if (!db->ext_db->odb_buf_list->buf) {
 579                         ret = -ENOMEM;
 580                         goto alloc_otr_db_buf_fail;
 581                 }
 582                 db->ext_db->odb_buf_list->map = odb_dma_addr;
 583
 584                 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
 585                 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
 586                                      HNS_ROCE_V1_EXT_ODB_ALFUL);
 587         } else
 588                 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
 589                                  HNS_ROCE_V1_ODB_ALFUL);
 590
 591         hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
 592
 593         return 0;
 594
 595 alloc_otr_db_buf_fail:
 596         kfree(db->ext_db->odb_buf_list);
 597
 598 ext_odb_buf_fail_out:
 599         if (sdb_ext_mod) {
 600                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 601                                   db->ext_db->sdb_buf_list->buf,
 602                                   db->ext_db->sdb_buf_list->map);
 603         }
 604
 605 alloc_sq_db_buf_fail:
 606         if (sdb_ext_mod)
 607                 kfree(db->ext_db->sdb_buf_list);
 608
 609 ext_sdb_buf_fail_out:
 610         kfree(db->ext_db);
 611         return ret;
 612 }
 613
 614 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 615 {
 616         struct device *dev = &hr_dev->pdev->dev;
 617         struct hns_roce_v1_priv *priv;
 618         struct hns_roce_db_table *db;
 619         u32 sdb_ext_mod;
 620         u32 odb_ext_mod;
 621         u32 sdb_evt_mod;
 622         u32 odb_evt_mod;
 623         int ret = 0;
 624
 625         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 626         db = &priv->db_table;
 627
 628         memset(db, 0, sizeof(*db));
 629
 630         /* Default DB mode */
 631         sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
 632         odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
 633         sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
 634         odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
 635
 636         db->sdb_ext_mod = sdb_ext_mod;
 637         db->odb_ext_mod = odb_ext_mod;
 638
 639         /* Init extend DB */
 640         ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
 641         if (ret) {
 642                 dev_err(dev, "Failed in extend DB configuration.\n");
 643                 return ret;
 644         }
 645
 646         hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
 647
 648         return 0;
 649 }
 650
 651 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 652 {
 653         struct device *dev = &hr_dev->pdev->dev;
 654         struct hns_roce_v1_priv *priv;
 655         struct hns_roce_db_table *db;
 656
 657         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 658         db = &priv->db_table;
 659
 660         if (db->sdb_ext_mod) {
 661                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 662                                   db->ext_db->sdb_buf_list->buf,
 663                                   db->ext_db->sdb_buf_list->map);
 664                 kfree(db->ext_db->sdb_buf_list);
 665         }
 666
 667         if (db->odb_ext_mod) {
 668                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
 669                                   db->ext_db->odb_buf_list->buf,
 670                                   db->ext_db->odb_buf_list->map);
 671                 kfree(db->ext_db->odb_buf_list);
 672         }
 673
 674         kfree(db->ext_db);
 675 }
 676
 677 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
 678 {
 679         int ret;
 680         int raq_shift = 0;
 681         dma_addr_t addr;
 682         u32 val;
 683         struct hns_roce_v1_priv *priv;
 684         struct hns_roce_raq_table *raq;
 685         struct device *dev = &hr_dev->pdev->dev;
 686
 687         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 688         raq = &priv->raq_table;
 689
 690         raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
 691         if (!raq->e_raq_buf)
 692                 return -ENOMEM;
 693
 694         raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
 695                                                  &addr, GFP_KERNEL);
 696         if (!raq->e_raq_buf->buf) {
 697                 ret = -ENOMEM;
 698                 goto err_dma_alloc_raq;
 699         }
 700         raq->e_raq_buf->map = addr;
 701
 702         /* Configure raq extended address. 48bit 4K align*/
 703         roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
 704
 705         /* Configure raq_shift */
 706         raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
 707         val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
 708         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
 709                        ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
 710         /*
 711          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 712          * using 4K page, and shift more 32 because of
 713          * caculating the high 32 bit value evaluated to hardware.
 714          */
 715         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
 716                        ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
 717                        raq->e_raq_buf->map >> 44);
 718         roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
 719         dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
 720
 721         /* Configure raq threshold */
 722         val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
 723         roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
 724                        ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
 725                        HNS_ROCE_V1_EXT_RAQ_WF);
 726         roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
 727         dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
 728
 729         /* Enable extend raq */
 730         val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
 731         roce_set_field(val,
 732                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
 733                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
 734                        POL_TIME_INTERVAL_VAL);
 735         roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
 736         roce_set_field(val,
 737                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
 738                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
 739                        2);
 740         roce_set_bit(val,
 741                      ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
 742         roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
 743         dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
 744
 745         /* Enable raq drop */
 746         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 747         roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
 748         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 749         dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
 750
 751         return 0;
 752
 753 err_dma_alloc_raq:
 754         kfree(raq->e_raq_buf);
 755         return ret;
 756 }
 757
 758 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
 759 {
 760         struct device *dev = &hr_dev->pdev->dev;
 761         struct hns_roce_v1_priv *priv;
 762         struct hns_roce_raq_table *raq;
 763
 764         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 765         raq = &priv->raq_table;
 766
 767         dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
 768                           raq->e_raq_buf->map);
 769         kfree(raq->e_raq_buf);
 770 }
 771
 772 static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
 773 {
 774         u32 val;
 775
 776         if (enable_flag) {
 777                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 778                  /* Open all ports */
 779                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 780                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
 781                                ALL_PORT_VAL_OPEN);
 782                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 783         } else {
 784                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 785                 /* Close all ports */
 786                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 787                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
 788                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 789         }
 790 }
 791
 792 /**
 793  * hns_roce_v1_reset - reset RoCE
 794  * @hr_dev: RoCE device struct pointer
 795  * @enable: true -- drop reset, false -- reset
 796  * return 0 - success , negative --fail
 797  */
 798 int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
 799 {
 800         struct device_node *dsaf_node;
 801         struct device *dev = &hr_dev->pdev->dev;
 802         struct device_node *np = dev->of_node;
 803         struct fwnode_handle *fwnode;
 804         int ret;
 805
 806         /* check if this is DT/ACPI case */
 807         if (dev_of_node(dev)) {
 808                 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
 809                 if (!dsaf_node) {
 810                         dev_err(dev, "could not find dsaf-handle\n");
 811                         return -EINVAL;
 812                 }
 813                 fwnode = &dsaf_node->fwnode;
 814         } else if (is_acpi_device_node(dev->fwnode)) {
 815                 struct acpi_reference_args args;
 816
 817                 ret = acpi_node_get_property_reference(dev->fwnode,
 818                                                        "dsaf-handle", 0, &args);
 819                 if (ret) {
 820                         dev_err(dev, "could not find dsaf-handle\n");
 821                         return ret;
 822                 }
 823                 fwnode = acpi_fwnode_handle(args.adev);
 824         } else {
 825                 dev_err(dev, "cannot read data from DT or ACPI\n");
 826                 return -ENXIO;
 827         }
 828
 829         ret = hns_dsaf_roce_reset(fwnode, false);
 830         if (ret)
 831                 return ret;
 832
 833         if (dereset) {
 834                 msleep(SLEEP_TIME_INTERVAL);
 835                 ret = hns_dsaf_roce_reset(fwnode, true);
 836         }
 837
 838         return ret;
 839 }
 840
 841 void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 842 {
 843         int i = 0;
 844         struct hns_roce_caps *caps = &hr_dev->caps;
 845
 846         hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
 847         hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
 848                                              ROCEE_VENDOR_PART_ID_REG));
 849         hr_dev->hw_rev = le32_to_cpu(roce_read(hr_dev, ROCEE_HW_VERSION_REG));
 850
 851         hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
 852                                              ROCEE_SYS_IMAGE_GUID_L_REG)) |
 853                                 ((u64)le32_to_cpu(roce_read(hr_dev,
 854                                             ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
 855
 856         caps->num_qps           = HNS_ROCE_V1_MAX_QP_NUM;
 857         caps->max_wqes          = HNS_ROCE_V1_MAX_WQE_NUM;
 858         caps->num_cqs           = HNS_ROCE_V1_MAX_CQ_NUM;
 859         caps->max_cqes          = HNS_ROCE_V1_MAX_CQE_NUM;
 860         caps->max_sq_sg         = HNS_ROCE_V1_SG_NUM;
 861         caps->max_rq_sg         = HNS_ROCE_V1_SG_NUM;
 862         caps->max_sq_inline     = HNS_ROCE_V1_INLINE_SIZE;
 863         caps->num_uars          = HNS_ROCE_V1_UAR_NUM;
 864         caps->phy_num_uars      = HNS_ROCE_V1_PHY_UAR_NUM;
 865         caps->num_aeq_vectors   = HNS_ROCE_AEQE_VEC_NUM;
 866         caps->num_comp_vectors  = HNS_ROCE_COMP_VEC_NUM;
 867         caps->num_other_vectors = HNS_ROCE_AEQE_OF_VEC_NUM;
 868         caps->num_mtpts         = HNS_ROCE_V1_MAX_MTPT_NUM;
 869         caps->num_mtt_segs      = HNS_ROCE_V1_MAX_MTT_SEGS;
 870         caps->num_pds           = HNS_ROCE_V1_MAX_PD_NUM;
 871         caps->max_qp_init_rdma  = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
 872         caps->max_qp_dest_rdma  = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
 873         caps->max_sq_desc_sz    = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
 874         caps->max_rq_desc_sz    = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
 875         caps->qpc_entry_sz      = HNS_ROCE_V1_QPC_ENTRY_SIZE;
 876         caps->irrl_entry_sz     = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
 877         caps->cqc_entry_sz      = HNS_ROCE_V1_CQC_ENTRY_SIZE;
 878         caps->mtpt_entry_sz     = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
 879         caps->mtt_entry_sz      = HNS_ROCE_V1_MTT_ENTRY_SIZE;
 880         caps->cq_entry_sz       = HNS_ROCE_V1_CQE_ENTRY_SIZE;
 881         caps->page_size_cap     = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
 882         caps->sqp_start         = 0;
 883         caps->reserved_lkey     = 0;
 884         caps->reserved_pds      = 0;
 885         caps->reserved_mrws     = 1;
 886         caps->reserved_uars     = 0;
 887         caps->reserved_cqs      = 0;
 888
 889         for (i = 0; i < caps->num_ports; i++)
 890                 caps->pkey_table_len[i] = 1;
 891
 892         for (i = 0; i < caps->num_ports; i++) {
 893                 /* Six ports shared 16 GID in v1 engine */
 894                 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
 895                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 896                                                  caps->num_ports;
 897                 else
 898                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 899                                                  caps->num_ports + 1;
 900         }
 901
 902         for (i = 0; i < caps->num_comp_vectors; i++)
 903                 caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
 904
 905         caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
 906         caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
 907                                                          ROCEE_ACK_DELAY_REG));
 908         caps->max_mtu = IB_MTU_2048;
 909 }
 910
 911 int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 912 {
 913         int ret;
 914         u32 val;
 915         struct device *dev = &hr_dev->pdev->dev;
 916
 917         /* DMAE user config */
 918         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
 919         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
 920                        ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
 921         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
 922                        ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
 923                        1 << PAGES_SHIFT_16);
 924         roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
 925
 926         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
 927         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
 928                        ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
 929         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
 930                        ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
 931                        1 << PAGES_SHIFT_16);
 932
 933         ret = hns_roce_db_init(hr_dev);
 934         if (ret) {
 935                 dev_err(dev, "doorbell init failed!\n");
 936                 return ret;
 937         }
 938
 939         ret = hns_roce_raq_init(hr_dev);
 940         if (ret) {
 941                 dev_err(dev, "raq init failed!\n");
 942                 goto error_failed_raq_init;
 943         }
 944
 945         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
 946
 947         return 0;
 948
 949 error_failed_raq_init:
 950         hns_roce_db_free(hr_dev);
 951         return ret;
 952 }
 953
 954 void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
 955 {
 956         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
 957         hns_roce_raq_free(hr_dev);
 958         hns_roce_db_free(hr_dev);
 959 }
 960
 961 void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
 962                          union ib_gid *gid)
 963 {
 964         u32 *p = NULL;
 965         u8 gid_idx = 0;
 966
 967         gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
 968
 969         p = (u32 *)&gid->raw[0];
 970         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
 971                        (HNS_ROCE_V1_GID_NUM * gid_idx));
 972
 973         p = (u32 *)&gid->raw[4];
 974         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
 975                        (HNS_ROCE_V1_GID_NUM * gid_idx));
 976
 977         p = (u32 *)&gid->raw[8];
 978         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
 979                        (HNS_ROCE_V1_GID_NUM * gid_idx));
 980
 981         p = (u32 *)&gid->raw[0xc];
 982         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
 983                        (HNS_ROCE_V1_GID_NUM * gid_idx));
 984 }
 985
 986 void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
 987 {
 988         u32 reg_smac_l;
 989         u16 reg_smac_h;
 990         u16 *p_h;
 991         u32 *p;
 992         u32 val;
 993
 994         p = (u32 *)(&addr[0]);
 995         reg_smac_l = *p;
 996         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
 997                        PHY_PORT_OFFSET * phy_port);
 998
 999         val = roce_read(hr_dev,
1000                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1001         p_h = (u16 *)(&addr[4]);
1002         reg_smac_h  = *p_h;
1003         roce_set_field(val, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1004                        ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1005         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1006                    val);
1007 }
1008
1009 void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1010                          enum ib_mtu mtu)
1011 {
1012         u32 val;
1013
1014         val = roce_read(hr_dev,
1015                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1016         roce_set_field(val, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1017                        ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1018         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1019                    val);
1020 }
1021
1022 int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1023                            unsigned long mtpt_idx)
1024 {
1025         struct hns_roce_v1_mpt_entry *mpt_entry;
1026         struct scatterlist *sg;
1027         u64 *pages;
1028         int entry;
1029         int i;
1030
1031         /* MPT filled into mailbox buf */
1032         mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1033         memset(mpt_entry, 0, sizeof(*mpt_entry));
1034
1035         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1036                        MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1037         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1038                        MPT_BYTE_4_KEY_S, mr->key);
1039         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1040                        MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1041         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1042         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1043                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1044         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1045         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1046                        MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1047         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1048         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1049                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1050         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1051                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1052         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1053                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1054         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1055                      0);
1056         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1057
1058         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1059                        MPT_BYTE_12_PBL_ADDR_H_S, 0);
1060         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1061                        MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1062
1063         mpt_entry->virt_addr_l = (u32)mr->iova;
1064         mpt_entry->virt_addr_h = (u32)(mr->iova >> 32);
1065         mpt_entry->length = (u32)mr->size;
1066
1067         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1068                        MPT_BYTE_28_PD_S, mr->pd);
1069         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1070                        MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1071         roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1072                        MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1073
1074         /* DMA momery regsiter */
1075         if (mr->type == MR_TYPE_DMA)
1076                 return 0;
1077
1078         pages = (u64 *) __get_free_page(GFP_KERNEL);
1079         if (!pages)
1080                 return -ENOMEM;
1081
1082         i = 0;
1083         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1084                 pages[i] = ((u64)sg_dma_address(sg)) >> 12;
1085
1086                 /* Directly record to MTPT table firstly 7 entry */
1087                 if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
1088                         break;
1089                 i++;
1090         }
1091
1092         /* Register user mr */
1093         for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
1094                 switch (i) {
1095                 case 0:
1096                         mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1097                         roce_set_field(mpt_entry->mpt_byte_36,
1098                                 MPT_BYTE_36_PA0_H_M,
1099                                 MPT_BYTE_36_PA0_H_S,
1100                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1101                         break;
1102                 case 1:
1103                         roce_set_field(mpt_entry->mpt_byte_36,
1104                                        MPT_BYTE_36_PA1_L_M,
1105                                        MPT_BYTE_36_PA1_L_S,
1106                                        cpu_to_le32((u32)(pages[i])));
1107                         roce_set_field(mpt_entry->mpt_byte_40,
1108                                 MPT_BYTE_40_PA1_H_M,
1109                                 MPT_BYTE_40_PA1_H_S,
1110                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1111                         break;
1112                 case 2:
1113                         roce_set_field(mpt_entry->mpt_byte_40,
1114                                        MPT_BYTE_40_PA2_L_M,
1115                                        MPT_BYTE_40_PA2_L_S,
1116                                        cpu_to_le32((u32)(pages[i])));
1117                         roce_set_field(mpt_entry->mpt_byte_44,
1118                                 MPT_BYTE_44_PA2_H_M,
1119                                 MPT_BYTE_44_PA2_H_S,
1120                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1121                         break;
1122                 case 3:
1123                         roce_set_field(mpt_entry->mpt_byte_44,
1124                                        MPT_BYTE_44_PA3_L_M,
1125                                        MPT_BYTE_44_PA3_L_S,
1126                                        cpu_to_le32((u32)(pages[i])));
1127                         roce_set_field(mpt_entry->mpt_byte_48,
1128                                 MPT_BYTE_48_PA3_H_M,
1129                                 MPT_BYTE_48_PA3_H_S,
1130                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_8)));
1131                         break;
1132                 case 4:
1133                         mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1134                         roce_set_field(mpt_entry->mpt_byte_56,
1135                                 MPT_BYTE_56_PA4_H_M,
1136                                 MPT_BYTE_56_PA4_H_S,
1137                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1138                         break;
1139                 case 5:
1140                         roce_set_field(mpt_entry->mpt_byte_56,
1141                                        MPT_BYTE_56_PA5_L_M,
1142                                        MPT_BYTE_56_PA5_L_S,
1143                                        cpu_to_le32((u32)(pages[i])));
1144                         roce_set_field(mpt_entry->mpt_byte_60,
1145                                 MPT_BYTE_60_PA5_H_M,
1146                                 MPT_BYTE_60_PA5_H_S,
1147                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1148                         break;
1149                 case 6:
1150                         roce_set_field(mpt_entry->mpt_byte_60,
1151                                        MPT_BYTE_60_PA6_L_M,
1152                                        MPT_BYTE_60_PA6_L_S,
1153                                        cpu_to_le32((u32)(pages[i])));
1154                         roce_set_field(mpt_entry->mpt_byte_64,
1155                                 MPT_BYTE_64_PA6_H_M,
1156                                 MPT_BYTE_64_PA6_H_S,
1157                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1158                         break;
1159                 default:
1160                         break;
1161                 }
1162         }
1163
1164         free_page((unsigned long) pages);
1165
1166         mpt_entry->pbl_addr_l = (u32)(mr->pbl_dma_addr);
1167
1168         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1169                        MPT_BYTE_12_PBL_ADDR_H_S,
1170                        ((u32)(mr->pbl_dma_addr >> 32)));
1171
1172         return 0;
1173 }
1174
1175 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1176 {
1177         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1178                                    n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
1179 }
1180
1181 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1182 {
1183         struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1184
1185         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1186         return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1187                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
1188 }
1189
1190 static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1191 {
1192         return get_sw_cqe(hr_cq, hr_cq->cons_index);
1193 }
1194
1195 void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index,
1196                            spinlock_t *doorbell_lock)
1197
1198 {
1199         u32 doorbell[2];
1200
1201         doorbell[0] = cons_index & ((hr_cq->cq_depth << 1) - 1);
1202         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1203         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1204                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1205         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1206                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
1207         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1208                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
1209
1210         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1211 }
1212
1213 static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1214                                    struct hns_roce_srq *srq)
1215 {
1216         struct hns_roce_cqe *cqe, *dest;
1217         u32 prod_index;
1218         int nfreed = 0;
1219         u8 owner_bit;
1220
1221         for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
1222              ++prod_index) {
1223                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1224                         break;
1225         }
1226
1227         /*
1228         * Now backwards through the CQ, removing CQ entries
1229         * that match our QP by overwriting them with next entries.
1230         */
1231         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1232                 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1233                 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1234                                      CQE_BYTE_16_LOCAL_QPN_S) &
1235                                      HNS_ROCE_CQE_QPN_MASK) == qpn) {
1236                         /* In v1 engine, not support SRQ */
1237                         ++nfreed;
1238                 } else if (nfreed) {
1239                         dest = get_cqe(hr_cq, (prod_index + nfreed) &
1240                                        hr_cq->ib_cq.cqe);
1241                         owner_bit = roce_get_bit(dest->cqe_byte_4,
1242                                                  CQE_BYTE_4_OWNER_S);
1243                         memcpy(dest, cqe, sizeof(*cqe));
1244                         roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
1245                                      owner_bit);
1246                 }
1247         }
1248
1249         if (nfreed) {
1250                 hr_cq->cons_index += nfreed;
1251                 /*
1252                 * Make sure update of buffer contents is done before
1253                 * updating consumer index.
1254                 */
1255                 wmb();
1256
1257                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index,
1258                                    &to_hr_dev(hr_cq->ib_cq.device)->cq_db_lock);
1259         }
1260 }
1261
1262 static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1263                                  struct hns_roce_srq *srq)
1264 {
1265         spin_lock_irq(&hr_cq->lock);
1266         __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
1267         spin_unlock_irq(&hr_cq->lock);
1268 }
1269
1270 void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
1271                            struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
1272                            dma_addr_t dma_handle, int nent, u32 vector)
1273 {
1274         struct hns_roce_cq_context *cq_context = NULL;
1275         void __iomem *tptr_addr;
1276
1277         cq_context = mb_buf;
1278         memset(cq_context, 0, sizeof(*cq_context));
1279
1280         tptr_addr = 0;
1281         hr_dev->priv_addr = tptr_addr;
1282         hr_cq->tptr_addr = tptr_addr;
1283
1284         /* Register cq_context members */
1285         roce_set_field(cq_context->cqc_byte_4,
1286                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
1287                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
1288         roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
1289                        CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
1290         cq_context->cqc_byte_4 = cpu_to_le32(cq_context->cqc_byte_4);
1291
1292         cq_context->cq_bt_l = (u32)dma_handle;
1293         cq_context->cq_bt_l = cpu_to_le32(cq_context->cq_bt_l);
1294
1295         roce_set_field(cq_context->cqc_byte_12,
1296                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
1297                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
1298                        ((u64)dma_handle >> 32));
1299         roce_set_field(cq_context->cqc_byte_12,
1300                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
1301                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
1302                        ilog2((unsigned int)nent));
1303         roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
1304                        CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
1305         cq_context->cqc_byte_12 = cpu_to_le32(cq_context->cqc_byte_12);
1306
1307         cq_context->cur_cqe_ba0_l = (u32)(mtts[0]);
1308         cq_context->cur_cqe_ba0_l = cpu_to_le32(cq_context->cur_cqe_ba0_l);
1309
1310         roce_set_field(cq_context->cqc_byte_20,
1311                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
1312                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S,
1313                        cpu_to_le32((mtts[0]) >> 32));
1314         /* Dedicated hardware, directly set 0 */
1315         roce_set_field(cq_context->cqc_byte_20,
1316                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
1317                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
1318         /**
1319          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1320          * using 4K page, and shift more 32 because of
1321          * caculating the high 32 bit value evaluated to hardware.
1322          */
1323         roce_set_field(cq_context->cqc_byte_20,
1324                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
1325                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
1326                        (u64)tptr_addr >> 44);
1327         cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
1328
1329         cq_context->cqe_tptr_addr_l = (u32)((u64)tptr_addr >> 12);
1330
1331         roce_set_field(cq_context->cqc_byte_32,
1332                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
1333                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
1334         roce_set_bit(cq_context->cqc_byte_32,
1335                      CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
1336         roce_set_bit(cq_context->cqc_byte_32,
1337                      CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
1338         roce_set_bit(cq_context->cqc_byte_32,
1339                      CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
1340         roce_set_bit(cq_context->cqc_byte_32,
1341                      CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
1342                      0);
1343         /*The initial value of cq's ci is 0 */
1344         roce_set_field(cq_context->cqc_byte_32,
1345                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
1346                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
1347         cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32);
1348 }
1349
1350 int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
1351 {
1352         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1353         u32 notification_flag;
1354         u32 doorbell[2];
1355         int ret = 0;
1356
1357         notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
1358                             IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
1359         /*
1360         * flags = 0; Notification Flag = 1, next
1361         * flags = 1; Notification Flag = 0, solocited
1362         */
1363         doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
1364         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1365         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1366                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1367         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1368                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
1369         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1370                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
1371                        hr_cq->cqn | notification_flag);
1372
1373         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1374
1375         return ret;
1376 }
1377
1378 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
1379                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
1380 {
1381         int qpn;
1382         int is_send;
1383         u16 wqe_ctr;
1384         u32 status;
1385         u32 opcode;
1386         struct hns_roce_cqe *cqe;
1387         struct hns_roce_qp *hr_qp;
1388         struct hns_roce_wq *wq;
1389         struct hns_roce_wqe_ctrl_seg *sq_wqe;
1390         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
1391         struct device *dev = &hr_dev->pdev->dev;
1392
1393         /* Find cqe according consumer index */
1394         cqe = next_cqe_sw(hr_cq);
1395         if (!cqe)
1396                 return -EAGAIN;
1397
1398         ++hr_cq->cons_index;
1399         /* Memory barrier */
1400         rmb();
1401         /* 0->SQ, 1->RQ */
1402         is_send  = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
1403
1404         /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
1405         if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1406                            CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
1407                 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
1408                                      CQE_BYTE_20_PORT_NUM_S) +
1409                       roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1410                                      CQE_BYTE_16_LOCAL_QPN_S) *
1411                                      HNS_ROCE_MAX_PORTS;
1412         } else {
1413                 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1414                                      CQE_BYTE_16_LOCAL_QPN_S);
1415         }
1416
1417         if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
1418                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
1419                 if (unlikely(!hr_qp)) {
1420                         dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
1421                                 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
1422                         return -EINVAL;
1423                 }
1424
1425                 *cur_qp = hr_qp;
1426         }
1427
1428         wc->qp = &(*cur_qp)->ibqp;
1429         wc->vendor_err = 0;
1430
1431         status = roce_get_field(cqe->cqe_byte_4,
1432                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
1433                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
1434                                 HNS_ROCE_CQE_STATUS_MASK;
1435         switch (status) {
1436         case HNS_ROCE_CQE_SUCCESS:
1437                 wc->status = IB_WC_SUCCESS;
1438                 break;
1439         case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
1440                 wc->status = IB_WC_LOC_LEN_ERR;
1441                 break;
1442         case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
1443                 wc->status = IB_WC_LOC_QP_OP_ERR;
1444                 break;
1445         case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
1446                 wc->status = IB_WC_LOC_PROT_ERR;
1447                 break;
1448         case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
1449                 wc->status = IB_WC_WR_FLUSH_ERR;
1450                 break;
1451         case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
1452                 wc->status = IB_WC_MW_BIND_ERR;
1453                 break;
1454         case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
1455                 wc->status = IB_WC_BAD_RESP_ERR;
1456                 break;
1457         case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
1458                 wc->status = IB_WC_LOC_ACCESS_ERR;
1459                 break;
1460         case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
1461                 wc->status = IB_WC_REM_INV_REQ_ERR;
1462                 break;
1463         case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
1464                 wc->status = IB_WC_REM_ACCESS_ERR;
1465                 break;
1466         case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
1467                 wc->status = IB_WC_REM_OP_ERR;
1468                 break;
1469         case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
1470                 wc->status = IB_WC_RETRY_EXC_ERR;
1471                 break;
1472         case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
1473                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
1474                 break;
1475         default:
1476                 wc->status = IB_WC_GENERAL_ERR;
1477                 break;
1478         }
1479
1480         /* CQE status error, directly return */
1481         if (wc->status != IB_WC_SUCCESS)
1482                 return 0;
1483
1484         if (is_send) {
1485                 /* SQ conrespond to CQE */
1486                 sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
1487                                                 CQE_BYTE_4_WQE_INDEX_M,
1488                                                 CQE_BYTE_4_WQE_INDEX_S));
1489                 switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
1490                 case HNS_ROCE_WQE_OPCODE_SEND:
1491                         wc->opcode = IB_WC_SEND;
1492                         break;
1493                 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
1494                         wc->opcode = IB_WC_RDMA_READ;
1495                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1496                         break;
1497                 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
1498                         wc->opcode = IB_WC_RDMA_WRITE;
1499                         break;
1500                 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
1501                         wc->opcode = IB_WC_LOCAL_INV;
1502                         break;
1503                 case HNS_ROCE_WQE_OPCODE_UD_SEND:
1504                         wc->opcode = IB_WC_SEND;
1505                         break;
1506                 default:
1507                         wc->status = IB_WC_GENERAL_ERR;
1508                         break;
1509                 }
1510                 wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
1511                                 IB_WC_WITH_IMM : 0);
1512
1513                 wq = &(*cur_qp)->sq;
1514                 if ((*cur_qp)->sq_signal_bits) {
1515                         /*
1516                         * If sg_signal_bit is 1,
1517                         * firstly tail pointer updated to wqe
1518                         * which current cqe correspond to
1519                         */
1520                         wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
1521                                                       CQE_BYTE_4_WQE_INDEX_M,
1522                                                       CQE_BYTE_4_WQE_INDEX_S);
1523                         wq->tail += (wqe_ctr - (u16)wq->tail) &
1524                                     (wq->wqe_cnt - 1);
1525                 }
1526                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1527                 ++wq->tail;
1528                 } else {
1529                 /* RQ conrespond to CQE */
1530                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1531                 opcode = roce_get_field(cqe->cqe_byte_4,
1532                                         CQE_BYTE_4_OPERATION_TYPE_M,
1533                                         CQE_BYTE_4_OPERATION_TYPE_S) &
1534                                         HNS_ROCE_CQE_OPCODE_MASK;
1535                 switch (opcode) {
1536                 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
1537                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
1538                         wc->wc_flags = IB_WC_WITH_IMM;
1539                         wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
1540                         break;
1541                 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
1542                         if (roce_get_bit(cqe->cqe_byte_4,
1543                                          CQE_BYTE_4_IMM_INDICATOR_S)) {
1544                                 wc->opcode = IB_WC_RECV;
1545                                 wc->wc_flags = IB_WC_WITH_IMM;
1546                                 wc->ex.imm_data = le32_to_cpu(
1547                                                   cqe->immediate_data);
1548                         } else {
1549                                 wc->opcode = IB_WC_RECV;
1550                                 wc->wc_flags = 0;
1551                         }
1552                         break;
1553                 default:
1554                         wc->status = IB_WC_GENERAL_ERR;
1555                         break;
1556                 }
1557
1558                 /* Update tail pointer, record wr_id */
1559                 wq = &(*cur_qp)->rq;
1560                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1561                 ++wq->tail;
1562                 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
1563                                             CQE_BYTE_20_SL_S);
1564                 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
1565                                                 CQE_BYTE_20_REMOTE_QPN_M,
1566                                                 CQE_BYTE_20_REMOTE_QPN_S);
1567                 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
1568                                               CQE_BYTE_20_GRH_PRESENT_S) ?
1569                                               IB_WC_GRH : 0);
1570                 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
1571                                                      CQE_BYTE_28_P_KEY_IDX_M,
1572                                                      CQE_BYTE_28_P_KEY_IDX_S);
1573         }
1574
1575         return 0;
1576 }
1577
1578 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
1579 {
1580         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1581         struct hns_roce_qp *cur_qp = NULL;
1582         unsigned long flags;
1583         int npolled;
1584         int ret = 0;
1585
1586         spin_lock_irqsave(&hr_cq->lock, flags);
1587
1588         for (npolled = 0; npolled < num_entries; ++npolled) {
1589                 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
1590                 if (ret)
1591                         break;
1592         }
1593
1594         if (npolled) {
1595                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index,
1596                                       &to_hr_dev(ibcq->device)->cq_db_lock);
1597         }
1598
1599         spin_unlock_irqrestore(&hr_cq->lock, flags);
1600
1601         if (ret == 0 || ret == -EAGAIN)
1602                 return npolled;
1603         else
1604                 return ret;
1605 }
1606
1607 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
1608                                  struct hns_roce_mtt *mtt,
1609                                  enum hns_roce_qp_state cur_state,
1610                                  enum hns_roce_qp_state new_state,
1611                                  struct hns_roce_qp_context *context,
1612                                  struct hns_roce_qp *hr_qp)
1613 {
1614         static const u16
1615         op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
1616                 [HNS_ROCE_QP_STATE_RST] = {
1617                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1618                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1619                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1620                 },
1621                 [HNS_ROCE_QP_STATE_INIT] = {
1622                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1623                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1624                 /* Note: In v1 engine, HW doesn't support RST2INIT.
1625                  * We use RST2INIT cmd instead of INIT2INIT.
1626                  */
1627                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1628                 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
1629                 },
1630                 [HNS_ROCE_QP_STATE_RTR] = {
1631                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1632                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1633                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
1634                 },
1635                 [HNS_ROCE_QP_STATE_RTS] = {
1636                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1637                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1638                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
1639                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
1640                 },
1641                 [HNS_ROCE_QP_STATE_SQD] = {
1642                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1643                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1644                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
1645                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
1646                 },
1647                 [HNS_ROCE_QP_STATE_ERR] = {
1648                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1649                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1650                 }
1651         };
1652
1653         struct hns_roce_cmd_mailbox *mailbox;
1654         struct device *dev = &hr_dev->pdev->dev;
1655         int ret = 0;
1656
1657         if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
1658             new_state >= HNS_ROCE_QP_NUM_STATE ||
1659             !op[cur_state][new_state]) {
1660                 dev_err(dev, "[modify_qp]not support state %d to %d\n",
1661                         cur_state, new_state);
1662                 return -EINVAL;
1663         }
1664
1665         if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
1666                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1667                                          HNS_ROCE_CMD_2RST_QP,
1668                                          HNS_ROCE_CMD_TIME_CLASS_A);
1669
1670         if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
1671                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1672                                          HNS_ROCE_CMD_2ERR_QP,
1673                                          HNS_ROCE_CMD_TIME_CLASS_A);
1674
1675         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1676         if (IS_ERR(mailbox))
1677                 return PTR_ERR(mailbox);
1678
1679         memcpy(mailbox->buf, context, sizeof(*context));
1680
1681         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
1682                                 op[cur_state][new_state],
1683                                 HNS_ROCE_CMD_TIME_CLASS_C);
1684
1685         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1686         return ret;
1687 }
1688
1689 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1690                              int attr_mask, enum ib_qp_state cur_state,
1691                              enum ib_qp_state new_state)
1692 {
1693         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1694         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1695         struct hns_roce_sqp_context *context;
1696         struct device *dev = &hr_dev->pdev->dev;
1697         dma_addr_t dma_handle = 0;
1698         int rq_pa_start;
1699         u32 reg_val;
1700         u64 *mtts;
1701         u32 *addr;
1702
1703         context = kzalloc(sizeof(*context), GFP_KERNEL);
1704         if (!context)
1705                 return -ENOMEM;
1706
1707         /* Search QP buf's MTTs */
1708         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1709                                    hr_qp->mtt.first_seg, &dma_handle);
1710         if (!mtts) {
1711                 dev_err(dev, "qp buf pa find failed\n");
1712                 goto out;
1713         }
1714
1715         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1716                 roce_set_field(context->qp1c_bytes_4,
1717                                QP1C_BYTES_4_SQ_WQE_SHIFT_M,
1718                                QP1C_BYTES_4_SQ_WQE_SHIFT_S,
1719                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
1720                 roce_set_field(context->qp1c_bytes_4,
1721                                QP1C_BYTES_4_RQ_WQE_SHIFT_M,
1722                                QP1C_BYTES_4_RQ_WQE_SHIFT_S,
1723                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
1724                 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
1725                                QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
1726
1727                 context->sq_rq_bt_l = (u32)(dma_handle);
1728                 roce_set_field(context->qp1c_bytes_12,
1729                                QP1C_BYTES_12_SQ_RQ_BT_H_M,
1730                                QP1C_BYTES_12_SQ_RQ_BT_H_S,
1731                                ((u32)(dma_handle >> 32)));
1732
1733                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
1734                                QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
1735                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
1736                                QP1C_BYTES_16_PORT_NUM_S, hr_qp->port);
1737                 roce_set_bit(context->qp1c_bytes_16,
1738                              QP1C_BYTES_16_SIGNALING_TYPE_S,
1739                              hr_qp->sq_signal_bits);
1740                 roce_set_bit(context->qp1c_bytes_16,
1741                              QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S,
1742                              hr_qp->sq_signal_bits);
1743                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
1744                              1);
1745                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
1746                              1);
1747                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
1748                              0);
1749
1750                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
1751                                QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
1752                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
1753                                QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
1754
1755                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
1756                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
1757
1758                 roce_set_field(context->qp1c_bytes_28,
1759                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
1760                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
1761                                (mtts[rq_pa_start]) >> 32);
1762                 roce_set_field(context->qp1c_bytes_28,
1763                                QP1C_BYTES_28_RQ_CUR_IDX_M,
1764                                QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
1765
1766                 roce_set_field(context->qp1c_bytes_32,
1767                                QP1C_BYTES_32_RX_CQ_NUM_M,
1768                                QP1C_BYTES_32_RX_CQ_NUM_S,
1769                                to_hr_cq(ibqp->recv_cq)->cqn);
1770                 roce_set_field(context->qp1c_bytes_32,
1771                                QP1C_BYTES_32_TX_CQ_NUM_M,
1772                                QP1C_BYTES_32_TX_CQ_NUM_S,
1773                                to_hr_cq(ibqp->send_cq)->cqn);
1774
1775                 context->cur_sq_wqe_ba_l  = (u32)mtts[0];
1776
1777                 roce_set_field(context->qp1c_bytes_40,
1778                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
1779                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
1780                                (mtts[0]) >> 32);
1781                 roce_set_field(context->qp1c_bytes_40,
1782                                QP1C_BYTES_40_SQ_CUR_IDX_M,
1783                                QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
1784
1785                 /* Copy context to QP1C register */
1786                 addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG +
1787                         hr_qp->port * sizeof(*context));
1788
1789                 writel(context->qp1c_bytes_4, addr);
1790                 writel(context->sq_rq_bt_l, addr + 1);
1791                 writel(context->qp1c_bytes_12, addr + 2);
1792                 writel(context->qp1c_bytes_16, addr + 3);
1793                 writel(context->qp1c_bytes_20, addr + 4);
1794                 writel(context->cur_rq_wqe_ba_l, addr + 5);
1795                 writel(context->qp1c_bytes_28, addr + 6);
1796                 writel(context->qp1c_bytes_32, addr + 7);
1797                 writel(context->cur_sq_wqe_ba_l, addr + 8);
1798         }
1799
1800         /* Modify QP1C status */
1801         reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1802                             hr_qp->port * sizeof(*context));
1803         roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
1804                        ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
1805         roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1806                     hr_qp->port * sizeof(*context), reg_val);
1807
1808         hr_qp->state = new_state;
1809         if (new_state == IB_QPS_RESET) {
1810                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
1811                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
1812                 if (ibqp->send_cq != ibqp->recv_cq)
1813                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
1814                                              hr_qp->qpn, NULL);
1815
1816                 hr_qp->rq.head = 0;
1817                 hr_qp->rq.tail = 0;
1818                 hr_qp->sq.head = 0;
1819                 hr_qp->sq.tail = 0;
1820                 hr_qp->sq_next_wqe = 0;
1821         }
1822
1823         kfree(context);
1824         return 0;
1825
1826 out:
1827         kfree(context);
1828         return -EINVAL;
1829 }
1830
1831 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1832                             int attr_mask, enum ib_qp_state cur_state,
1833                             enum ib_qp_state new_state)
1834 {
1835         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1836         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1837         struct device *dev = &hr_dev->pdev->dev;
1838         struct hns_roce_qp_context *context;
1839         struct hns_roce_rq_db rq_db;
1840         dma_addr_t dma_handle_2 = 0;
1841         dma_addr_t dma_handle = 0;
1842         uint32_t doorbell[2] = {0};
1843         int rq_pa_start = 0;
1844         u32 reg_val = 0;
1845         u64 *mtts_2 = NULL;
1846         int ret = -EINVAL;
1847         u64 *mtts = NULL;
1848         int port;
1849         u8 *dmac;
1850         u8 *smac;
1851
1852         context = kzalloc(sizeof(*context), GFP_KERNEL);
1853         if (!context)
1854                 return -ENOMEM;
1855
1856         /* Search qp buf's mtts */
1857         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1858                                    hr_qp->mtt.first_seg, &dma_handle);
1859         if (mtts == NULL) {
1860                 dev_err(dev, "qp buf pa find failed\n");
1861                 goto out;
1862         }
1863
1864         /* Search IRRL's mtts */
1865         mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn,
1866                                      &dma_handle_2);
1867         if (mtts_2 == NULL) {
1868                 dev_err(dev, "qp irrl_table find failed\n");
1869                 goto out;
1870         }
1871
1872         /*
1873         *Reset to init
1874         *       Mandatory param:
1875         *       IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
1876         *       Optional param: NA
1877         */
1878         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1879                 roce_set_field(context->qpc_bytes_4,
1880                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
1881                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
1882                                to_hr_qp_type(hr_qp->ibqp.qp_type));
1883
1884                 roce_set_bit(context->qpc_bytes_4,
1885                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
1886                 roce_set_bit(context->qpc_bytes_4,
1887                              QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
1888                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
1889                 roce_set_bit(context->qpc_bytes_4,
1890                              QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
1891                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
1892                              );
1893                 roce_set_bit(context->qpc_bytes_4,
1894                              QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
1895                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
1896                              );
1897                 roce_set_bit(context->qpc_bytes_4,
1898                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
1899                 roce_set_field(context->qpc_bytes_4,
1900                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
1901                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
1902                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
1903                 roce_set_field(context->qpc_bytes_4,
1904                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
1905                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
1906                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
1907                 roce_set_field(context->qpc_bytes_4,
1908                                QP_CONTEXT_QPC_BYTES_4_PD_M,
1909                                QP_CONTEXT_QPC_BYTES_4_PD_S,
1910                                to_hr_pd(ibqp->pd)->pdn);
1911                 hr_qp->access_flags = attr->qp_access_flags;
1912                 roce_set_field(context->qpc_bytes_8,
1913                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
1914                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
1915                                to_hr_cq(ibqp->send_cq)->cqn);
1916                 roce_set_field(context->qpc_bytes_8,
1917                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
1918                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
1919                                to_hr_cq(ibqp->recv_cq)->cqn);
1920
1921                 if (ibqp->srq)
1922                         roce_set_field(context->qpc_bytes_12,
1923                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
1924                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
1925                                        to_hr_srq(ibqp->srq)->srqn);
1926
1927                 roce_set_field(context->qpc_bytes_12,
1928                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
1929                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
1930                                attr->pkey_index);
1931                 hr_qp->pkey_index = attr->pkey_index;
1932                 roce_set_field(context->qpc_bytes_16,
1933                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
1934                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
1935
1936         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
1937                 roce_set_field(context->qpc_bytes_4,
1938                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
1939                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
1940                                to_hr_qp_type(hr_qp->ibqp.qp_type));
1941                 roce_set_bit(context->qpc_bytes_4,
1942                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
1943                 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1944                         roce_set_bit(context->qpc_bytes_4,
1945                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
1946                                      !!(attr->qp_access_flags &
1947                                      IB_ACCESS_REMOTE_READ));
1948                         roce_set_bit(context->qpc_bytes_4,
1949                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
1950                                      !!(attr->qp_access_flags &
1951                                      IB_ACCESS_REMOTE_WRITE));
1952                 } else {
1953                         roce_set_bit(context->qpc_bytes_4,
1954                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
1955                                      !!(hr_qp->access_flags &
1956                                      IB_ACCESS_REMOTE_READ));
1957                         roce_set_bit(context->qpc_bytes_4,
1958                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
1959                                      !!(hr_qp->access_flags &
1960                                      IB_ACCESS_REMOTE_WRITE));
1961                 }
1962
1963                 roce_set_bit(context->qpc_bytes_4,
1964                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
1965                 roce_set_field(context->qpc_bytes_4,
1966                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
1967                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
1968                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
1969                 roce_set_field(context->qpc_bytes_4,
1970                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
1971                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
1972                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
1973                 roce_set_field(context->qpc_bytes_4,
1974                                QP_CONTEXT_QPC_BYTES_4_PD_M,
1975                                QP_CONTEXT_QPC_BYTES_4_PD_S,
1976                                to_hr_pd(ibqp->pd)->pdn);
1977
1978                 roce_set_field(context->qpc_bytes_8,
1979                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
1980                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
1981                                to_hr_cq(ibqp->send_cq)->cqn);
1982                 roce_set_field(context->qpc_bytes_8,
1983                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
1984                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
1985                                to_hr_cq(ibqp->recv_cq)->cqn);
1986
1987                 if (ibqp->srq)
1988                         roce_set_field(context->qpc_bytes_12,
1989                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
1990                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
1991                                        to_hr_srq(ibqp->srq)->srqn);
1992                 if (attr_mask & IB_QP_PKEY_INDEX)
1993                         roce_set_field(context->qpc_bytes_12,
1994                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
1995                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
1996                                        attr->pkey_index);
1997                 else
1998                         roce_set_field(context->qpc_bytes_12,
1999                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2000                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2001                                        hr_qp->pkey_index);
2002
2003                 roce_set_field(context->qpc_bytes_16,
2004                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2005                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2006         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2007                 if ((attr_mask & IB_QP_ALT_PATH) ||
2008                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2009                     (attr_mask & IB_QP_PKEY_INDEX) ||
2010                     (attr_mask & IB_QP_QKEY)) {
2011                         dev_err(dev, "INIT2RTR attr_mask error\n");
2012                         goto out;
2013                 }
2014
2015                 dmac = (u8 *)attr->ah_attr.dmac;
2016
2017                 context->sq_rq_bt_l = (u32)(dma_handle);
2018                 roce_set_field(context->qpc_bytes_24,
2019                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
2020                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
2021                                ((u32)(dma_handle >> 32)));
2022                 roce_set_bit(context->qpc_bytes_24,
2023                              QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
2024                              1);
2025                 roce_set_field(context->qpc_bytes_24,
2026                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2027                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
2028                                attr->min_rnr_timer);
2029                 context->irrl_ba_l = (u32)(dma_handle_2);
2030                 roce_set_field(context->qpc_bytes_32,
2031                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
2032                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
2033                                ((u32)(dma_handle_2 >> 32)) &
2034                                 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
2035                 roce_set_field(context->qpc_bytes_32,
2036                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
2037                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
2038                 roce_set_bit(context->qpc_bytes_32,
2039                              QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
2040                              1);
2041                 roce_set_bit(context->qpc_bytes_32,
2042                              QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
2043                              hr_qp->sq_signal_bits);
2044
2045                 for (port = 0; port < hr_dev->caps.num_ports; port++) {
2046                         smac = (u8 *)hr_dev->dev_addr[port];
2047                         dev_dbg(dev, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
2048                                 smac[0], smac[1], smac[2], smac[3], smac[4],
2049                                 smac[5]);
2050                         if ((dmac[0] == smac[0]) && (dmac[1] == smac[1]) &&
2051                             (dmac[2] == smac[2]) && (dmac[3] == smac[3]) &&
2052                             (dmac[4] == smac[4]) && (dmac[5] == smac[5])) {
2053                                 roce_set_bit(context->qpc_bytes_32,
2054                                     QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S,
2055                                     1);
2056                                 break;
2057                         }
2058                 }
2059
2060                 if (hr_dev->loop_idc == 0x1)
2061                         roce_set_bit(context->qpc_bytes_32,
2062                                 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
2063
2064                 roce_set_bit(context->qpc_bytes_32,
2065                              QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
2066                              attr->ah_attr.ah_flags);
2067                 roce_set_field(context->qpc_bytes_32,
2068                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2069                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
2070                                ilog2((unsigned int)attr->max_dest_rd_atomic));
2071
2072                 roce_set_field(context->qpc_bytes_36,
2073                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2074                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
2075                                attr->dest_qp_num);
2076
2077                 /* Configure GID index */
2078                 roce_set_field(context->qpc_bytes_36,
2079                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2080                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
2081                                hns_get_gid_index(hr_dev,
2082                                                  attr->ah_attr.port_num - 1,
2083                                                  attr->ah_attr.grh.sgid_index));
2084
2085                 memcpy(&(context->dmac_l), dmac, 4);
2086
2087                 roce_set_field(context->qpc_bytes_44,
2088                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2089                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
2090                                *((u16 *)(&dmac[4])));
2091                 roce_set_field(context->qpc_bytes_44,
2092                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
2093                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
2094                                attr->ah_attr.static_rate);
2095                 roce_set_field(context->qpc_bytes_44,
2096                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2097                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
2098                                attr->ah_attr.grh.hop_limit);
2099
2100                 roce_set_field(context->qpc_bytes_48,
2101                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2102                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
2103                                attr->ah_attr.grh.flow_label);
2104                 roce_set_field(context->qpc_bytes_48,
2105                                QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2106                                QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
2107                                attr->ah_attr.grh.traffic_class);
2108                 roce_set_field(context->qpc_bytes_48,
2109                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2110                                QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
2111
2112                 memcpy(context->dgid, attr->ah_attr.grh.dgid.raw,
2113                        sizeof(attr->ah_attr.grh.dgid.raw));
2114
2115                 dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
2116                         roce_get_field(context->qpc_bytes_44,
2117                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2118                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
2119
2120                 roce_set_field(context->qpc_bytes_68,
2121                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
2122                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S, 0);
2123                 roce_set_field(context->qpc_bytes_68,
2124                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
2125                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
2126
2127                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2128                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
2129
2130                 roce_set_field(context->qpc_bytes_76,
2131                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
2132                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
2133                         mtts[rq_pa_start] >> 32);
2134                 roce_set_field(context->qpc_bytes_76,
2135                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
2136                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
2137
2138                 context->rx_rnr_time = 0;
2139
2140                 roce_set_field(context->qpc_bytes_84,
2141                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
2142                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
2143                                attr->rq_psn - 1);
2144                 roce_set_field(context->qpc_bytes_84,
2145                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
2146                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
2147
2148                 roce_set_field(context->qpc_bytes_88,
2149                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2150                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
2151                                attr->rq_psn);
2152                 roce_set_bit(context->qpc_bytes_88,
2153                              QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
2154                 roce_set_bit(context->qpc_bytes_88,
2155                              QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
2156                 roce_set_field(context->qpc_bytes_88,
2157                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
2158                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
2159                         0);
2160                 roce_set_field(context->qpc_bytes_88,
2161                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
2162                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
2163                                0);
2164
2165                 context->dma_length = 0;
2166                 context->r_key = 0;
2167                 context->va_l = 0;
2168                 context->va_h = 0;
2169
2170                 roce_set_field(context->qpc_bytes_108,
2171                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
2172                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
2173                 roce_set_bit(context->qpc_bytes_108,
2174                              QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
2175                 roce_set_bit(context->qpc_bytes_108,
2176                              QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
2177
2178                 roce_set_field(context->qpc_bytes_112,
2179                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
2180                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
2181                 roce_set_field(context->qpc_bytes_112,
2182                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
2183                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
2184
2185                 /* For chip resp ack */
2186                 roce_set_field(context->qpc_bytes_156,
2187                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2188                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2189                                hr_qp->port);
2190                 roce_set_field(context->qpc_bytes_156,
2191                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2192                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2193                 hr_qp->sl = attr->ah_attr.sl;
2194         } else if (cur_state == IB_QPS_RTR &&
2195                 new_state == IB_QPS_RTS) {
2196                 /* If exist optional param, return error */
2197                 if ((attr_mask & IB_QP_ALT_PATH) ||
2198                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2199                     (attr_mask & IB_QP_QKEY) ||
2200                     (attr_mask & IB_QP_PATH_MIG_STATE) ||
2201                     (attr_mask & IB_QP_CUR_STATE) ||
2202                     (attr_mask & IB_QP_MIN_RNR_TIMER)) {
2203                         dev_err(dev, "RTR2RTS attr_mask error\n");
2204                         goto out;
2205                 }
2206
2207                 context->rx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2208
2209                 roce_set_field(context->qpc_bytes_120,
2210                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
2211                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
2212                                (mtts[0]) >> 32);
2213
2214                 roce_set_field(context->qpc_bytes_124,
2215                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
2216                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
2217                 roce_set_field(context->qpc_bytes_124,
2218                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
2219                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
2220
2221                 roce_set_field(context->qpc_bytes_128,
2222                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
2223                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
2224                                attr->sq_psn);
2225                 roce_set_bit(context->qpc_bytes_128,
2226                              QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
2227                 roce_set_field(context->qpc_bytes_128,
2228                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
2229                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
2230                              0);
2231                 roce_set_bit(context->qpc_bytes_128,
2232                              QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
2233
2234                 roce_set_field(context->qpc_bytes_132,
2235                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
2236                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
2237                 roce_set_field(context->qpc_bytes_132,
2238                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
2239                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
2240
2241                 roce_set_field(context->qpc_bytes_136,
2242                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
2243                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
2244                                attr->sq_psn);
2245                 roce_set_field(context->qpc_bytes_136,
2246                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
2247                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
2248                                attr->sq_psn);
2249
2250                 roce_set_field(context->qpc_bytes_140,
2251                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
2252                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
2253                                (attr->sq_psn >> SQ_PSN_SHIFT));
2254                 roce_set_field(context->qpc_bytes_140,
2255                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
2256                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
2257                 roce_set_bit(context->qpc_bytes_140,
2258                              QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
2259
2260                 roce_set_field(context->qpc_bytes_144,
2261                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2262                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_S,
2263                                attr->qp_state);
2264
2265                 roce_set_field(context->qpc_bytes_148,
2266                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
2267                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
2268                 roce_set_field(context->qpc_bytes_148,
2269                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2270                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S, 0);
2271                 roce_set_field(context->qpc_bytes_148,
2272                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
2273                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S, 0);
2274                 roce_set_field(context->qpc_bytes_148,
2275                                QP_CONTEXT_QPC_BYTES_148_LSN_M,
2276                                QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
2277
2278                 context->rnr_retry = 0;
2279
2280                 roce_set_field(context->qpc_bytes_156,
2281                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
2282                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
2283                                attr->retry_cnt);
2284                 roce_set_field(context->qpc_bytes_156,
2285                                QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2286                                QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
2287                                attr->timeout);
2288                 roce_set_field(context->qpc_bytes_156,
2289                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
2290                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
2291                                attr->rnr_retry);
2292                 roce_set_field(context->qpc_bytes_156,
2293                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2294                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2295                                hr_qp->port);
2296                 roce_set_field(context->qpc_bytes_156,
2297                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2298                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2299                 hr_qp->sl = attr->ah_attr.sl;
2300                 roce_set_field(context->qpc_bytes_156,
2301                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2302                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
2303                                ilog2((unsigned int)attr->max_rd_atomic));
2304                 roce_set_field(context->qpc_bytes_156,
2305                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
2306                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
2307                 context->pkt_use_len = 0;
2308
2309                 roce_set_field(context->qpc_bytes_164,
2310                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2311                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
2312                 roce_set_field(context->qpc_bytes_164,
2313                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
2314                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
2315
2316                 roce_set_field(context->qpc_bytes_168,
2317                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
2318                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
2319                                attr->sq_psn);
2320                 roce_set_field(context->qpc_bytes_168,
2321                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
2322                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
2323                 roce_set_field(context->qpc_bytes_168,
2324                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
2325                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
2326                 roce_set_bit(context->qpc_bytes_168,
2327                              QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
2328                 roce_set_bit(context->qpc_bytes_168,
2329                              QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
2330                 roce_set_bit(context->qpc_bytes_168,
2331                              QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
2332                 context->sge_use_len = 0;
2333
2334                 roce_set_field(context->qpc_bytes_176,
2335                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
2336                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
2337                 roce_set_field(context->qpc_bytes_176,
2338                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
2339                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
2340                                0);
2341                 roce_set_field(context->qpc_bytes_180,
2342                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
2343                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
2344                 roce_set_field(context->qpc_bytes_180,
2345                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
2346                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
2347
2348                 context->tx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2349
2350                 roce_set_field(context->qpc_bytes_188,
2351                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
2352                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
2353                                (mtts[0]) >> 32);
2354                 roce_set_bit(context->qpc_bytes_188,
2355                              QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
2356                 roce_set_field(context->qpc_bytes_188,
2357                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
2358                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
2359                                0);
2360         } else if ((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
2361                    (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
2362                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
2363                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
2364                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
2365                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
2366                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
2367                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR)) {
2368                 roce_set_field(context->qpc_bytes_144,
2369                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2370                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_S,
2371                                attr->qp_state);
2372
2373         } else {
2374                 dev_err(dev, "not support this modify\n");
2375                 goto out;
2376         }
2377
2378         /* Every status migrate must change state */
2379         roce_set_field(context->qpc_bytes_144,
2380                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2381                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, attr->qp_state);
2382
2383         /* SW pass context to HW */
2384         ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
2385                                     to_hns_roce_state(cur_state),
2386                                     to_hns_roce_state(new_state), context,
2387                                     hr_qp);
2388         if (ret) {
2389                 dev_err(dev, "hns_roce_qp_modify failed\n");
2390                 goto out;
2391         }
2392
2393         /*
2394         * Use rst2init to instead of init2init with drv,
2395         * need to hw to flash RQ HEAD by DB again
2396         */
2397         if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2398                 /* Memory barrier */
2399                 wmb();
2400                 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
2401                         /* SW update GSI rq header */
2402                         reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG3_0_REG +
2403                                             QP1C_CFGN_OFFSET * hr_qp->port);
2404                         roce_set_field(reg_val,
2405                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
2406                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
2407                                        hr_qp->rq.head);
2408                         roce_write(hr_dev, ROCEE_QP1C_CFG3_0_REG +
2409                                     QP1C_CFGN_OFFSET * hr_qp->port, reg_val);
2410                 } else {
2411                         rq_db.u32_4 = 0;
2412                         rq_db.u32_8 = 0;
2413
2414                         roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
2415                                        RQ_DOORBELL_U32_4_RQ_HEAD_S,
2416                                        hr_qp->rq.head);
2417                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
2418                                        RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
2419                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
2420                                        RQ_DOORBELL_U32_8_CMD_S, 1);
2421                         roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
2422                                      1);
2423
2424                         doorbell[0] = rq_db.u32_4;
2425                         doorbell[1] = rq_db.u32_8;
2426
2427                         hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
2428                 }
2429         }
2430
2431         hr_qp->state = new_state;
2432
2433         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2434                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
2435         if (attr_mask & IB_QP_PORT)
2436                 hr_qp->port = (attr->port_num - 1);
2437
2438         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
2439                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2440                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2441                 if (ibqp->send_cq != ibqp->recv_cq)
2442                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2443                                              hr_qp->qpn, NULL);
2444
2445                 hr_qp->rq.head = 0;
2446                 hr_qp->rq.tail = 0;
2447                 hr_qp->sq.head = 0;
2448                 hr_qp->sq.tail = 0;
2449                 hr_qp->sq_next_wqe = 0;
2450         }
2451 out:
2452         kfree(context);
2453         return ret;
2454 }
2455
2456 int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2457                           int attr_mask, enum ib_qp_state cur_state,
2458                           enum ib_qp_state new_state)
2459 {
2460
2461         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
2462                 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
2463                                          new_state);
2464         else
2465                 return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
2466                                         new_state);
2467 }
2468
2469 static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
2470 {
2471         switch (state) {
2472         case HNS_ROCE_QP_STATE_RST:
2473                 return IB_QPS_RESET;
2474         case HNS_ROCE_QP_STATE_INIT:
2475                 return IB_QPS_INIT;
2476         case HNS_ROCE_QP_STATE_RTR:
2477                 return IB_QPS_RTR;
2478         case HNS_ROCE_QP_STATE_RTS:
2479                 return IB_QPS_RTS;
2480         case HNS_ROCE_QP_STATE_SQD:
2481                 return IB_QPS_SQD;
2482         case HNS_ROCE_QP_STATE_ERR:
2483                 return IB_QPS_ERR;
2484         default:
2485                 return IB_QPS_ERR;
2486         }
2487 }
2488
2489 static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
2490                                  struct hns_roce_qp *hr_qp,
2491                                  struct hns_roce_qp_context *hr_context)
2492 {
2493         struct hns_roce_cmd_mailbox *mailbox;
2494         int ret;
2495
2496         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2497         if (IS_ERR(mailbox))
2498                 return PTR_ERR(mailbox);
2499
2500         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
2501                                 HNS_ROCE_CMD_QUERY_QP,
2502                                 HNS_ROCE_CMD_TIME_CLASS_A);
2503         if (!ret)
2504                 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
2505         else
2506                 dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
2507
2508         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2509
2510         return ret;
2511 }
2512
2513 int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
2514                          int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2515 {
2516         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2517         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2518         struct device *dev = &hr_dev->pdev->dev;
2519         struct hns_roce_qp_context *context;
2520         int tmp_qp_state = 0;
2521         int ret = 0;
2522         int state;
2523
2524         context = kzalloc(sizeof(*context), GFP_KERNEL);
2525         if (!context)
2526                 return -ENOMEM;
2527
2528         memset(qp_attr, 0, sizeof(*qp_attr));
2529         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2530
2531         mutex_lock(&hr_qp->mutex);
2532
2533         if (hr_qp->state == IB_QPS_RESET) {
2534                 qp_attr->qp_state = IB_QPS_RESET;
2535                 goto done;
2536         }
2537
2538         ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
2539         if (ret) {
2540                 dev_err(dev, "query qpc error\n");
2541                 ret = -EINVAL;
2542                 goto out;
2543         }
2544
2545         state = roce_get_field(context->qpc_bytes_144,
2546                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2547                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
2548         tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
2549         if (tmp_qp_state == -1) {
2550                 dev_err(dev, "to_ib_qp_state error\n");
2551                 ret = -EINVAL;
2552                 goto out;
2553         }
2554         hr_qp->state = (u8)tmp_qp_state;
2555         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
2556         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
2557                                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2558                                                QP_CONTEXT_QPC_BYTES_48_MTU_S);
2559         qp_attr->path_mig_state = IB_MIG_ARMED;
2560         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
2561                 qp_attr->qkey = QKEY_VAL;
2562
2563         qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
2564                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2565                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
2566         qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
2567                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2568                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
2569         qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
2570                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2571                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
2572         qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
2573                         QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
2574                                    ((roce_get_bit(context->qpc_bytes_4,
2575                         QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
2576                                    ((roce_get_bit(context->qpc_bytes_4,
2577                         QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
2578
2579         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
2580             hr_qp->ibqp.qp_type == IB_QPT_UC) {
2581                 qp_attr->ah_attr.sl = roce_get_field(context->qpc_bytes_156,
2582                                                 QP_CONTEXT_QPC_BYTES_156_SL_M,
2583                                                 QP_CONTEXT_QPC_BYTES_156_SL_S);
2584                 qp_attr->ah_attr.grh.flow_label = roce_get_field(
2585                                         context->qpc_bytes_48,
2586                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2587                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
2588                 qp_attr->ah_attr.grh.sgid_index = roce_get_field(
2589                                         context->qpc_bytes_36,
2590                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2591                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
2592                 qp_attr->ah_attr.grh.hop_limit = roce_get_field(
2593                                         context->qpc_bytes_44,
2594                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2595                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
2596                 qp_attr->ah_attr.grh.traffic_class = roce_get_field(
2597                                         context->qpc_bytes_48,
2598                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2599                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
2600
2601                 memcpy(qp_attr->ah_attr.grh.dgid.raw, context->dgid,
2602                        sizeof(qp_attr->ah_attr.grh.dgid.raw));
2603         }
2604
2605         qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
2606                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2607                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
2608         qp_attr->port_num = (u8)roce_get_field(context->qpc_bytes_156,
2609                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2610                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + 1;
2611         qp_attr->sq_draining = 0;
2612         qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
2613                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2614                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
2615         qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32,
2616                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2617                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
2618         qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
2619                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2620                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
2621         qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
2622                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2623                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
2624         qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
2625                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2626                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
2627         qp_attr->rnr_retry = context->rnr_retry;
2628
2629 done:
2630         qp_attr->cur_qp_state = qp_attr->qp_state;
2631         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
2632         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
2633
2634         if (!ibqp->uobject) {
2635                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
2636                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
2637         } else {
2638                 qp_attr->cap.max_send_wr = 0;
2639                 qp_attr->cap.max_send_sge = 0;
2640         }
2641
2642         qp_init_attr->cap = qp_attr->cap;
2643
2644 out:
2645         mutex_unlock(&hr_qp->mutex);
2646         kfree(context);
2647         return ret;
2648 }
2649
2650 static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev *hr_dev,
2651                                           struct hns_roce_qp *hr_qp,
2652                                           int is_user)
2653 {
2654         u32 sdbinvcnt;
2655         unsigned long end = 0;
2656         u32 sdbinvcnt_val;
2657         u32 sdbsendptr_val;
2658         u32 sdbisusepr_val;
2659         struct hns_roce_cq *send_cq, *recv_cq;
2660         struct device *dev = &hr_dev->pdev->dev;
2661
2662         if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
2663                 if (hr_qp->state != IB_QPS_RESET) {
2664                         /*
2665                         * Set qp to ERR,
2666                         * waiting for hw complete processing all dbs
2667                         */
2668                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2669                                         to_hns_roce_state(
2670                                                 (enum ib_qp_state)hr_qp->state),
2671                                                 HNS_ROCE_QP_STATE_ERR, NULL,
2672                                                 hr_qp))
2673                                 dev_err(dev, "modify QP %06lx to ERR failed.\n",
2674                                         hr_qp->qpn);
2675
2676                         /* Record issued doorbell */
2677                         sdbisusepr_val = roce_read(hr_dev,
2678                                          ROCEE_SDB_ISSUE_PTR_REG);
2679                         /*
2680                         * Query db process status,
2681                         * until hw process completely
2682                         */
2683                         end = msecs_to_jiffies(
2684                               HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS) + jiffies;
2685                         do {
2686                                 sdbsendptr_val = roce_read(hr_dev,
2687                                                  ROCEE_SDB_SEND_PTR_REG);
2688                                 if (!time_before(jiffies, end)) {
2689                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2690                                                 hr_qp->qpn);
2691                                         break;
2692                                 }
2693                         } while ((short)(roce_get_field(sdbsendptr_val,
2694                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
2695                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) -
2696                                 roce_get_field(sdbisusepr_val,
2697                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
2698                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
2699                                 ) < 0);
2700
2701                         /* Get list pointer */
2702                         sdbinvcnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
2703
2704                         /* Query db's list status, until hw reversal */
2705                         do {
2706                                 sdbinvcnt_val = roce_read(hr_dev,
2707                                                 ROCEE_SDB_INV_CNT_REG);
2708                                 if (!time_before(jiffies, end)) {
2709                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2710                                                 hr_qp->qpn);
2711                                         dev_err(dev, "SdbInvCnt = 0x%x\n",
2712                                                 sdbinvcnt_val);
2713                                         break;
2714                                 }
2715                         } while ((short)(roce_get_field(sdbinvcnt_val,
2716                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_M,
2717                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) -
2718                                   (sdbinvcnt + SDB_INV_CNT_OFFSET)) < 0);
2719
2720                         /* Modify qp to reset before destroying qp */
2721                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2722                                         to_hns_roce_state(
2723                                         (enum ib_qp_state)hr_qp->state),
2724                                         HNS_ROCE_QP_STATE_RST, NULL, hr_qp))
2725                                 dev_err(dev, "modify QP %06lx to RESET failed.\n",
2726                                         hr_qp->qpn);
2727                 }
2728         }
2729
2730         send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
2731         recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
2732
2733         hns_roce_lock_cqs(send_cq, recv_cq);
2734
2735         if (!is_user) {
2736                 __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
2737                                        to_hr_srq(hr_qp->ibqp.srq) : NULL);
2738                 if (send_cq != recv_cq)
2739                         __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
2740         }
2741
2742         hns_roce_qp_remove(hr_dev, hr_qp);
2743
2744         hns_roce_unlock_cqs(send_cq, recv_cq);
2745
2746         hns_roce_qp_free(hr_dev, hr_qp);
2747
2748         /* Not special_QP, free their QPN */
2749         if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
2750             (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
2751             (hr_qp->ibqp.qp_type == IB_QPT_UD))
2752                 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
2753
2754         hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
2755
2756         if (is_user) {
2757                 ib_umem_release(hr_qp->umem);
2758         } else {
2759                 kfree(hr_qp->sq.wrid);
2760                 kfree(hr_qp->rq.wrid);
2761                 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
2762         }
2763 }
2764
2765 int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
2766 {
2767         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2768         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2769
2770         hns_roce_v1_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
2771
2772         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
2773                 kfree(hr_to_hr_sqp(hr_qp));
2774         else
2775                 kfree(hr_qp);
2776
2777         return 0;
2778 }
2779
2780 struct hns_roce_v1_priv hr_v1_priv;
2781
2782 struct hns_roce_hw hns_roce_hw_v1 = {
2783         .reset = hns_roce_v1_reset,
2784         .hw_profile = hns_roce_v1_profile,
2785         .hw_init = hns_roce_v1_init,
2786         .hw_exit = hns_roce_v1_exit,
2787         .set_gid = hns_roce_v1_set_gid,
2788         .set_mac = hns_roce_v1_set_mac,
2789         .set_mtu = hns_roce_v1_set_mtu,
2790         .write_mtpt = hns_roce_v1_write_mtpt,
2791         .write_cqc = hns_roce_v1_write_cqc,
2792         .modify_qp = hns_roce_v1_modify_qp,
2793         .query_qp = hns_roce_v1_query_qp,
2794         .destroy_qp = hns_roce_v1_destroy_qp,
2795         .post_send = hns_roce_v1_post_send,
2796         .post_recv = hns_roce_v1_post_recv,
2797         .req_notify_cq = hns_roce_v1_req_notify_cq,
2798         .poll_cq = hns_roce_v1_poll_cq,
2799         .priv = &hr_v1_priv,
2800 };