drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43 #include <rdma/ib_cache.h>
  44
  45 #include <linux/atomic.h>
  46
  47 #include <scsi/scsi.h>
  48 #include <scsi/scsi_device.h>
  49 #include <scsi/scsi_dbg.h>
  50 #include <scsi/scsi_tcq.h>
  51 #include <scsi/srp.h>
  52 #include <scsi/scsi_transport_srp.h>
  53
  54 #include "ib_srp.h"
  55
  56 #define DRV_NAME        "ib_srp"
  57 #define PFX             DRV_NAME ": "
  58 #define DRV_VERSION     "2.0"
  59 #define DRV_RELDATE     "July 26, 2015"
  60
  61 MODULE_AUTHOR("Roland Dreier");
  62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64 MODULE_VERSION(DRV_VERSION);
  65 MODULE_INFO(release_date, DRV_RELDATE);
  66
  67 static unsigned int srp_sg_tablesize;
  68 static unsigned int cmd_sg_entries;
  69 static unsigned int indirect_sg_entries;
  70 static bool allow_ext_sg;
  71 static bool prefer_fr = true;
  72 static bool register_always = true;
  73 static int topspin_workarounds = 1;
  74
  75 module_param(srp_sg_tablesize, uint, 0444);
  76 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  77
  78 module_param(cmd_sg_entries, uint, 0444);
  79 MODULE_PARM_DESC(cmd_sg_entries,
  80                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  81
  82 module_param(indirect_sg_entries, uint, 0444);
  83 MODULE_PARM_DESC(indirect_sg_entries,
  84                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
  85
  86 module_param(allow_ext_sg, bool, 0444);
  87 MODULE_PARM_DESC(allow_ext_sg,
  88                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  89
  90 module_param(topspin_workarounds, int, 0444);
  91 MODULE_PARM_DESC(topspin_workarounds,
  92                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  93
  94 module_param(prefer_fr, bool, 0444);
  95 MODULE_PARM_DESC(prefer_fr,
  96 "Whether to use fast registration if both FMR and fast registration are supported");
  97
  98 module_param(register_always, bool, 0444);
  99 MODULE_PARM_DESC(register_always,
 100                  "Use memory registration even for contiguous memory regions");
 101
 102 static const struct kernel_param_ops srp_tmo_ops;
 103
 104 static int srp_reconnect_delay = 10;
 105 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 106                 S_IRUGO | S_IWUSR);
 107 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 108
 109 static int srp_fast_io_fail_tmo = 15;
 110 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 111                 S_IRUGO | S_IWUSR);
 112 MODULE_PARM_DESC(fast_io_fail_tmo,
 113                  "Number of seconds between the observation of a transport"
 114                  " layer error and failing all I/O. \"off\" means that this"
 115                  " functionality is disabled.");
 116
 117 static int srp_dev_loss_tmo = 600;
 118 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 119                 S_IRUGO | S_IWUSR);
 120 MODULE_PARM_DESC(dev_loss_tmo,
 121                  "Maximum number of seconds that the SRP transport should"
 122                  " insulate transport layer errors. After this time has been"
 123                  " exceeded the SCSI host is removed. Should be"
 124                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 125                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 126                  " this functionality is disabled.");
 127
 128 static unsigned ch_count;
 129 module_param(ch_count, uint, 0444);
 130 MODULE_PARM_DESC(ch_count,
 131                  "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 132
 133 static void srp_add_one(struct ib_device *device);
 134 static void srp_remove_one(struct ib_device *device, void *client_data);
 135 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 136 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 137                 const char *opname);
 138 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 139
 140 static struct scsi_transport_template *ib_srp_transport_template;
 141 static struct workqueue_struct *srp_remove_wq;
 142
 143 static struct ib_client srp_client = {
 144         .name   = "srp",
 145         .add    = srp_add_one,
 146         .remove = srp_remove_one
 147 };
 148
 149 static struct ib_sa_client srp_sa_client;
 150
 151 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 152 {
 153         int tmo = *(int *)kp->arg;
 154
 155         if (tmo >= 0)
 156                 return sprintf(buffer, "%d", tmo);
 157         else
 158                 return sprintf(buffer, "off");
 159 }
 160
 161 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 162 {
 163         int tmo, res;
 164
 165         res = srp_parse_tmo(&tmo, val);
 166         if (res)
 167                 goto out;
 168
 169         if (kp->arg == &srp_reconnect_delay)
 170                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 171                                     srp_dev_loss_tmo);
 172         else if (kp->arg == &srp_fast_io_fail_tmo)
 173                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 174         else
 175                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 176                                     tmo);
 177         if (res)
 178                 goto out;
 179         *(int *)kp->arg = tmo;
 180
 181 out:
 182         return res;
 183 }
 184
 185 static const struct kernel_param_ops srp_tmo_ops = {
 186         .get = srp_tmo_get,
 187         .set = srp_tmo_set,
 188 };
 189
 190 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 191 {
 192         return (struct srp_target_port *) host->hostdata;
 193 }
 194
 195 static const char *srp_target_info(struct Scsi_Host *host)
 196 {
 197         return host_to_target(host)->target_name;
 198 }
 199
 200 static int srp_target_is_topspin(struct srp_target_port *target)
 201 {
 202         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 203         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 204
 205         return topspin_workarounds &&
 206                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 207                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 208 }
 209
 210 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 211                                    gfp_t gfp_mask,
 212                                    enum dma_data_direction direction)
 213 {
 214         struct srp_iu *iu;
 215
 216         iu = kmalloc(sizeof *iu, gfp_mask);
 217         if (!iu)
 218                 goto out;
 219
 220         iu->buf = kzalloc(size, gfp_mask);
 221         if (!iu->buf)
 222                 goto out_free_iu;
 223
 224         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 225                                     direction);
 226         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 227                 goto out_free_buf;
 228
 229         iu->size      = size;
 230         iu->direction = direction;
 231
 232         return iu;
 233
 234 out_free_buf:
 235         kfree(iu->buf);
 236 out_free_iu:
 237         kfree(iu);
 238 out:
 239         return NULL;
 240 }
 241
 242 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 243 {
 244         if (!iu)
 245                 return;
 246
 247         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 248                             iu->direction);
 249         kfree(iu->buf);
 250         kfree(iu);
 251 }
 252
 253 static void srp_qp_event(struct ib_event *event, void *context)
 254 {
 255         pr_debug("QP event %s (%d)\n",
 256                  ib_event_msg(event->event), event->event);
 257 }
 258
 259 static int srp_init_qp(struct srp_target_port *target,
 260                        struct ib_qp *qp)
 261 {
 262         struct ib_qp_attr *attr;
 263         int ret;
 264
 265         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 266         if (!attr)
 267                 return -ENOMEM;
 268
 269         ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 270                                   target->srp_host->port,
 271                                   be16_to_cpu(target->pkey),
 272                                   &attr->pkey_index);
 273         if (ret)
 274                 goto out;
 275
 276         attr->qp_state        = IB_QPS_INIT;
 277         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 278                                     IB_ACCESS_REMOTE_WRITE);
 279         attr->port_num        = target->srp_host->port;
 280
 281         ret = ib_modify_qp(qp, attr,
 282                            IB_QP_STATE          |
 283                            IB_QP_PKEY_INDEX     |
 284                            IB_QP_ACCESS_FLAGS   |
 285                            IB_QP_PORT);
 286
 287 out:
 288         kfree(attr);
 289         return ret;
 290 }
 291
 292 static int srp_new_cm_id(struct srp_rdma_ch *ch)
 293 {
 294         struct srp_target_port *target = ch->target;
 295         struct ib_cm_id *new_cm_id;
 296
 297         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 298                                     srp_cm_handler, ch);
 299         if (IS_ERR(new_cm_id))
 300                 return PTR_ERR(new_cm_id);
 301
 302         if (ch->cm_id)
 303                 ib_destroy_cm_id(ch->cm_id);
 304         ch->cm_id = new_cm_id;
 305         ch->path.sgid = target->sgid;
 306         ch->path.dgid = target->orig_dgid;
 307         ch->path.pkey = target->pkey;
 308         ch->path.service_id = target->service_id;
 309
 310         return 0;
 311 }
 312
 313 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 314 {
 315         struct srp_device *dev = target->srp_host->srp_dev;
 316         struct ib_fmr_pool_param fmr_param;
 317
 318         memset(&fmr_param, 0, sizeof(fmr_param));
 319         fmr_param.pool_size         = target->scsi_host->can_queue;
 320         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 321         fmr_param.cache             = 1;
 322         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 323         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 324         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 325                                        IB_ACCESS_REMOTE_WRITE |
 326                                        IB_ACCESS_REMOTE_READ);
 327
 328         return ib_create_fmr_pool(dev->pd, &fmr_param);
 329 }
 330
 331 /**
 332  * srp_destroy_fr_pool() - free the resources owned by a pool
 333  * @pool: Fast registration pool to be destroyed.
 334  */
 335 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 336 {
 337         int i;
 338         struct srp_fr_desc *d;
 339
 340         if (!pool)
 341                 return;
 342
 343         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 344                 if (d->mr)
 345                         ib_dereg_mr(d->mr);
 346         }
 347         kfree(pool);
 348 }
 349
 350 /**
 351  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 352  * @device:            IB device to allocate fast registration descriptors for.
 353  * @pd:                Protection domain associated with the FR descriptors.
 354  * @pool_size:         Number of descriptors to allocate.
 355  * @max_page_list_len: Maximum fast registration work request page list length.
 356  */
 357 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 358                                               struct ib_pd *pd, int pool_size,
 359                                               int max_page_list_len)
 360 {
 361         struct srp_fr_pool *pool;
 362         struct srp_fr_desc *d;
 363         struct ib_mr *mr;
 364         int i, ret = -EINVAL;
 365
 366         if (pool_size <= 0)
 367                 goto err;
 368         ret = -ENOMEM;
 369         pool = kzalloc(sizeof(struct srp_fr_pool) +
 370                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 371         if (!pool)
 372                 goto err;
 373         pool->size = pool_size;
 374         pool->max_page_list_len = max_page_list_len;
 375         spin_lock_init(&pool->lock);
 376         INIT_LIST_HEAD(&pool->free_list);
 377
 378         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 379                 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
 380                                  max_page_list_len);
 381                 if (IS_ERR(mr)) {
 382                         ret = PTR_ERR(mr);
 383                         goto destroy_pool;
 384                 }
 385                 d->mr = mr;
 386                 list_add_tail(&d->entry, &pool->free_list);
 387         }
 388
 389 out:
 390         return pool;
 391
 392 destroy_pool:
 393         srp_destroy_fr_pool(pool);
 394
 395 err:
 396         pool = ERR_PTR(ret);
 397         goto out;
 398 }
 399
 400 /**
 401  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 402  * @pool: Pool to obtain descriptor from.
 403  */
 404 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 405 {
 406         struct srp_fr_desc *d = NULL;
 407         unsigned long flags;
 408
 409         spin_lock_irqsave(&pool->lock, flags);
 410         if (!list_empty(&pool->free_list)) {
 411                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 412                 list_del(&d->entry);
 413         }
 414         spin_unlock_irqrestore(&pool->lock, flags);
 415
 416         return d;
 417 }
 418
 419 /**
 420  * srp_fr_pool_put() - put an FR descriptor back in the free list
 421  * @pool: Pool the descriptor was allocated from.
 422  * @desc: Pointer to an array of fast registration descriptor pointers.
 423  * @n:    Number of descriptors to put back.
 424  *
 425  * Note: The caller must already have queued an invalidation request for
 426  * desc->mr->rkey before calling this function.
 427  */
 428 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 429                             int n)
 430 {
 431         unsigned long flags;
 432         int i;
 433
 434         spin_lock_irqsave(&pool->lock, flags);
 435         for (i = 0; i < n; i++)
 436                 list_add(&desc[i]->entry, &pool->free_list);
 437         spin_unlock_irqrestore(&pool->lock, flags);
 438 }
 439
 440 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 441 {
 442         struct srp_device *dev = target->srp_host->srp_dev;
 443
 444         return srp_create_fr_pool(dev->dev, dev->pd,
 445                                   target->scsi_host->can_queue,
 446                                   dev->max_pages_per_mr);
 447 }
 448
 449 static void srp_drain_done(struct ib_cq *cq, struct ib_wc *wc)
 450 {
 451         struct srp_rdma_ch *ch = cq->cq_context;
 452
 453         complete(&ch->done);
 454 }
 455
 456 static struct ib_cqe srp_drain_cqe = {
 457         .done           = srp_drain_done,
 458 };
 459
 460 /**
 461  * srp_destroy_qp() - destroy an RDMA queue pair
 462  * @ch: SRP RDMA channel.
 463  *
 464  * Change a queue pair into the error state and wait until all receive
 465  * completions have been processed before destroying it. This avoids that
 466  * the receive completion handler can access the queue pair while it is
 467  * being destroyed.
 468  */
 469 static void srp_destroy_qp(struct srp_rdma_ch *ch)
 470 {
 471         static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
 472         static struct ib_recv_wr wr = { 0 };
 473         struct ib_recv_wr *bad_wr;
 474         int ret;
 475
 476         wr.wr_cqe = &srp_drain_cqe;
 477         /* Destroying a QP and reusing ch->done is only safe if not connected */
 478         WARN_ON_ONCE(ch->connected);
 479
 480         ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
 481         WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
 482         if (ret)
 483                 goto out;
 484
 485         init_completion(&ch->done);
 486         ret = ib_post_recv(ch->qp, &wr, &bad_wr);
 487         WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
 488         if (ret == 0)
 489                 wait_for_completion(&ch->done);
 490
 491 out:
 492         ib_destroy_qp(ch->qp);
 493 }
 494
 495 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 496 {
 497         struct srp_target_port *target = ch->target;
 498         struct srp_device *dev = target->srp_host->srp_dev;
 499         struct ib_qp_init_attr *init_attr;
 500         struct ib_cq *recv_cq, *send_cq;
 501         struct ib_qp *qp;
 502         struct ib_fmr_pool *fmr_pool = NULL;
 503         struct srp_fr_pool *fr_pool = NULL;
 504         const int m = dev->use_fast_reg ? 3 : 1;
 505         int ret;
 506
 507         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 508         if (!init_attr)
 509                 return -ENOMEM;
 510
 511         /* queue_size + 1 for ib_drain_qp */
 512         recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
 513                                 ch->comp_vector, IB_POLL_SOFTIRQ);
 514         if (IS_ERR(recv_cq)) {
 515                 ret = PTR_ERR(recv_cq);
 516                 goto err;
 517         }
 518
 519         send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
 520                                 ch->comp_vector, IB_POLL_DIRECT);
 521         if (IS_ERR(send_cq)) {
 522                 ret = PTR_ERR(send_cq);
 523                 goto err_recv_cq;
 524         }
 525
 526         init_attr->event_handler       = srp_qp_event;
 527         init_attr->cap.max_send_wr     = m * target->queue_size;
 528         init_attr->cap.max_recv_wr     = target->queue_size + 1;
 529         init_attr->cap.max_recv_sge    = 1;
 530         init_attr->cap.max_send_sge    = 1;
 531         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 532         init_attr->qp_type             = IB_QPT_RC;
 533         init_attr->send_cq             = send_cq;
 534         init_attr->recv_cq             = recv_cq;
 535
 536         qp = ib_create_qp(dev->pd, init_attr);
 537         if (IS_ERR(qp)) {
 538                 ret = PTR_ERR(qp);
 539                 goto err_send_cq;
 540         }
 541
 542         ret = srp_init_qp(target, qp);
 543         if (ret)
 544                 goto err_qp;
 545
 546         if (dev->use_fast_reg) {
 547                 fr_pool = srp_alloc_fr_pool(target);
 548                 if (IS_ERR(fr_pool)) {
 549                         ret = PTR_ERR(fr_pool);
 550                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 551                                      "FR pool allocation failed (%d)\n", ret);
 552                         goto err_qp;
 553                 }
 554         } else if (dev->use_fmr) {
 555                 fmr_pool = srp_alloc_fmr_pool(target);
 556                 if (IS_ERR(fmr_pool)) {
 557                         ret = PTR_ERR(fmr_pool);
 558                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 559                                      "FMR pool allocation failed (%d)\n", ret);
 560                         goto err_qp;
 561                 }
 562         }
 563
 564         if (ch->qp)
 565                 srp_destroy_qp(ch);
 566         if (ch->recv_cq)
 567                 ib_free_cq(ch->recv_cq);
 568         if (ch->send_cq)
 569                 ib_free_cq(ch->send_cq);
 570
 571         ch->qp = qp;
 572         ch->recv_cq = recv_cq;
 573         ch->send_cq = send_cq;
 574
 575         if (dev->use_fast_reg) {
 576                 if (ch->fr_pool)
 577                         srp_destroy_fr_pool(ch->fr_pool);
 578                 ch->fr_pool = fr_pool;
 579         } else if (dev->use_fmr) {
 580                 if (ch->fmr_pool)
 581                         ib_destroy_fmr_pool(ch->fmr_pool);
 582                 ch->fmr_pool = fmr_pool;
 583         }
 584
 585         kfree(init_attr);
 586         return 0;
 587
 588 err_qp:
 589         srp_destroy_qp(ch);
 590
 591 err_send_cq:
 592         ib_free_cq(send_cq);
 593
 594 err_recv_cq:
 595         ib_free_cq(recv_cq);
 596
 597 err:
 598         kfree(init_attr);
 599         return ret;
 600 }
 601
 602 /*
 603  * Note: this function may be called without srp_alloc_iu_bufs() having been
 604  * invoked. Hence the ch->[rt]x_ring checks.
 605  */
 606 static void srp_free_ch_ib(struct srp_target_port *target,
 607                            struct srp_rdma_ch *ch)
 608 {
 609         struct srp_device *dev = target->srp_host->srp_dev;
 610         int i;
 611
 612         if (!ch->target)
 613                 return;
 614
 615         if (ch->cm_id) {
 616                 ib_destroy_cm_id(ch->cm_id);
 617                 ch->cm_id = NULL;
 618         }
 619
 620         /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
 621         if (!ch->qp)
 622                 return;
 623
 624         if (dev->use_fast_reg) {
 625                 if (ch->fr_pool)
 626                         srp_destroy_fr_pool(ch->fr_pool);
 627         } else if (dev->use_fmr) {
 628                 if (ch->fmr_pool)
 629                         ib_destroy_fmr_pool(ch->fmr_pool);
 630         }
 631
 632         srp_destroy_qp(ch);
 633         ib_free_cq(ch->send_cq);
 634         ib_free_cq(ch->recv_cq);
 635
 636         /*
 637          * Avoid that the SCSI error handler tries to use this channel after
 638          * it has been freed. The SCSI error handler can namely continue
 639          * trying to perform recovery actions after scsi_remove_host()
 640          * returned.
 641          */
 642         ch->target = NULL;
 643
 644         ch->qp = NULL;
 645         ch->send_cq = ch->recv_cq = NULL;
 646
 647         if (ch->rx_ring) {
 648                 for (i = 0; i < target->queue_size; ++i)
 649                         srp_free_iu(target->srp_host, ch->rx_ring[i]);
 650                 kfree(ch->rx_ring);
 651                 ch->rx_ring = NULL;
 652         }
 653         if (ch->tx_ring) {
 654                 for (i = 0; i < target->queue_size; ++i)
 655                         srp_free_iu(target->srp_host, ch->tx_ring[i]);
 656                 kfree(ch->tx_ring);
 657                 ch->tx_ring = NULL;
 658         }
 659 }
 660
 661 static void srp_path_rec_completion(int status,
 662                                     struct ib_sa_path_rec *pathrec,
 663                                     void *ch_ptr)
 664 {
 665         struct srp_rdma_ch *ch = ch_ptr;
 666         struct srp_target_port *target = ch->target;
 667
 668         ch->status = status;
 669         if (status)
 670                 shost_printk(KERN_ERR, target->scsi_host,
 671                              PFX "Got failed path rec status %d\n", status);
 672         else
 673                 ch->path = *pathrec;
 674         complete(&ch->done);
 675 }
 676
 677 static int srp_lookup_path(struct srp_rdma_ch *ch)
 678 {
 679         struct srp_target_port *target = ch->target;
 680         int ret;
 681
 682         ch->path.numb_path = 1;
 683
 684         init_completion(&ch->done);
 685
 686         ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 687                                                target->srp_host->srp_dev->dev,
 688                                                target->srp_host->port,
 689                                                &ch->path,
 690                                                IB_SA_PATH_REC_SERVICE_ID |
 691                                                IB_SA_PATH_REC_DGID       |
 692                                                IB_SA_PATH_REC_SGID       |
 693                                                IB_SA_PATH_REC_NUMB_PATH  |
 694                                                IB_SA_PATH_REC_PKEY,
 695                                                SRP_PATH_REC_TIMEOUT_MS,
 696                                                GFP_KERNEL,
 697                                                srp_path_rec_completion,
 698                                                ch, &ch->path_query);
 699         if (ch->path_query_id < 0)
 700                 return ch->path_query_id;
 701
 702         ret = wait_for_completion_interruptible(&ch->done);
 703         if (ret < 0)
 704                 return ret;
 705
 706         if (ch->status < 0)
 707                 shost_printk(KERN_WARNING, target->scsi_host,
 708                              PFX "Path record query failed\n");
 709
 710         return ch->status;
 711 }
 712
 713 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 714 {
 715         struct srp_target_port *target = ch->target;
 716         struct {
 717                 struct ib_cm_req_param param;
 718                 struct srp_login_req   priv;
 719         } *req = NULL;
 720         int status;
 721
 722         req = kzalloc(sizeof *req, GFP_KERNEL);
 723         if (!req)
 724                 return -ENOMEM;
 725
 726         req->param.primary_path               = &ch->path;
 727         req->param.alternate_path             = NULL;
 728         req->param.service_id                 = target->service_id;
 729         req->param.qp_num                     = ch->qp->qp_num;
 730         req->param.qp_type                    = ch->qp->qp_type;
 731         req->param.private_data               = &req->priv;
 732         req->param.private_data_len           = sizeof req->priv;
 733         req->param.flow_control               = 1;
 734
 735         get_random_bytes(&req->param.starting_psn, 4);
 736         req->param.starting_psn              &= 0xffffff;
 737
 738         /*
 739          * Pick some arbitrary defaults here; we could make these
 740          * module parameters if anyone cared about setting them.
 741          */
 742         req->param.responder_resources        = 4;
 743         req->param.remote_cm_response_timeout = 20;
 744         req->param.local_cm_response_timeout  = 20;
 745         req->param.retry_count                = target->tl_retry_count;
 746         req->param.rnr_retry_count            = 7;
 747         req->param.max_cm_retries             = 15;
 748
 749         req->priv.opcode        = SRP_LOGIN_REQ;
 750         req->priv.tag           = 0;
 751         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 752         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 753                                               SRP_BUF_FORMAT_INDIRECT);
 754         req->priv.req_flags     = (multich ? SRP_MULTICHAN_MULTI :
 755                                    SRP_MULTICHAN_SINGLE);
 756         /*
 757          * In the published SRP specification (draft rev. 16a), the
 758          * port identifier format is 8 bytes of ID extension followed
 759          * by 8 bytes of GUID.  Older drafts put the two halves in the
 760          * opposite order, so that the GUID comes first.
 761          *
 762          * Targets conforming to these obsolete drafts can be
 763          * recognized by the I/O Class they report.
 764          */
 765         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 766                 memcpy(req->priv.initiator_port_id,
 767                        &target->sgid.global.interface_id, 8);
 768                 memcpy(req->priv.initiator_port_id + 8,
 769                        &target->initiator_ext, 8);
 770                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 771                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 772         } else {
 773                 memcpy(req->priv.initiator_port_id,
 774                        &target->initiator_ext, 8);
 775                 memcpy(req->priv.initiator_port_id + 8,
 776                        &target->sgid.global.interface_id, 8);
 777                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 778                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 779         }
 780
 781         /*
 782          * Topspin/Cisco SRP targets will reject our login unless we
 783          * zero out the first 8 bytes of our initiator port ID and set
 784          * the second 8 bytes to the local node GUID.
 785          */
 786         if (srp_target_is_topspin(target)) {
 787                 shost_printk(KERN_DEBUG, target->scsi_host,
 788                              PFX "Topspin/Cisco initiator port ID workaround "
 789                              "activated for target GUID %016llx\n",
 790                              be64_to_cpu(target->ioc_guid));
 791                 memset(req->priv.initiator_port_id, 0, 8);
 792                 memcpy(req->priv.initiator_port_id + 8,
 793                        &target->srp_host->srp_dev->dev->node_guid, 8);
 794         }
 795
 796         status = ib_send_cm_req(ch->cm_id, &req->param);
 797
 798         kfree(req);
 799
 800         return status;
 801 }
 802
 803 static bool srp_queue_remove_work(struct srp_target_port *target)
 804 {
 805         bool changed = false;
 806
 807         spin_lock_irq(&target->lock);
 808         if (target->state != SRP_TARGET_REMOVED) {
 809                 target->state = SRP_TARGET_REMOVED;
 810                 changed = true;
 811         }
 812         spin_unlock_irq(&target->lock);
 813
 814         if (changed)
 815                 queue_work(srp_remove_wq, &target->remove_work);
 816
 817         return changed;
 818 }
 819
 820 static void srp_disconnect_target(struct srp_target_port *target)
 821 {
 822         struct srp_rdma_ch *ch;
 823         int i;
 824
 825         /* XXX should send SRP_I_LOGOUT request */
 826
 827         for (i = 0; i < target->ch_count; i++) {
 828                 ch = &target->ch[i];
 829                 ch->connected = false;
 830                 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
 831                         shost_printk(KERN_DEBUG, target->scsi_host,
 832                                      PFX "Sending CM DREQ failed\n");
 833                 }
 834         }
 835 }
 836
 837 static void srp_free_req_data(struct srp_target_port *target,
 838                               struct srp_rdma_ch *ch)
 839 {
 840         struct srp_device *dev = target->srp_host->srp_dev;
 841         struct ib_device *ibdev = dev->dev;
 842         struct srp_request *req;
 843         int i;
 844
 845         if (!ch->req_ring)
 846                 return;
 847
 848         for (i = 0; i < target->req_ring_size; ++i) {
 849                 req = &ch->req_ring[i];
 850                 if (dev->use_fast_reg) {
 851                         kfree(req->fr_list);
 852                 } else {
 853                         kfree(req->fmr_list);
 854                         kfree(req->map_page);
 855                 }
 856                 if (req->indirect_dma_addr) {
 857                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 858                                             target->indirect_size,
 859                                             DMA_TO_DEVICE);
 860                 }
 861                 kfree(req->indirect_desc);
 862         }
 863
 864         kfree(ch->req_ring);
 865         ch->req_ring = NULL;
 866 }
 867
 868 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
 869 {
 870         struct srp_target_port *target = ch->target;
 871         struct srp_device *srp_dev = target->srp_host->srp_dev;
 872         struct ib_device *ibdev = srp_dev->dev;
 873         struct srp_request *req;
 874         void *mr_list;
 875         dma_addr_t dma_addr;
 876         int i, ret = -ENOMEM;
 877
 878         ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
 879                                GFP_KERNEL);
 880         if (!ch->req_ring)
 881                 goto out;
 882
 883         for (i = 0; i < target->req_ring_size; ++i) {
 884                 req = &ch->req_ring[i];
 885                 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
 886                                   GFP_KERNEL);
 887                 if (!mr_list)
 888                         goto out;
 889                 if (srp_dev->use_fast_reg) {
 890                         req->fr_list = mr_list;
 891                 } else {
 892                         req->fmr_list = mr_list;
 893                         req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 894                                                 sizeof(void *), GFP_KERNEL);
 895                         if (!req->map_page)
 896                                 goto out;
 897                 }
 898                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 899                 if (!req->indirect_desc)
 900                         goto out;
 901
 902                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 903                                              target->indirect_size,
 904                                              DMA_TO_DEVICE);
 905                 if (ib_dma_mapping_error(ibdev, dma_addr))
 906                         goto out;
 907
 908                 req->indirect_dma_addr = dma_addr;
 909         }
 910         ret = 0;
 911
 912 out:
 913         return ret;
 914 }
 915
 916 /**
 917  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 918  * @shost: SCSI host whose attributes to remove from sysfs.
 919  *
 920  * Note: Any attributes defined in the host template and that did not exist
 921  * before invocation of this function will be ignored.
 922  */
 923 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 924 {
 925         struct device_attribute **attr;
 926
 927         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 928                 device_remove_file(&shost->shost_dev, *attr);
 929 }
 930
 931 static void srp_remove_target(struct srp_target_port *target)
 932 {
 933         struct srp_rdma_ch *ch;
 934         int i;
 935
 936         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 937
 938         srp_del_scsi_host_attr(target->scsi_host);
 939         srp_rport_get(target->rport);
 940         srp_remove_host(target->scsi_host);
 941         scsi_remove_host(target->scsi_host);
 942         srp_stop_rport_timers(target->rport);
 943         srp_disconnect_target(target);
 944         for (i = 0; i < target->ch_count; i++) {
 945                 ch = &target->ch[i];
 946                 srp_free_ch_ib(target, ch);
 947         }
 948         cancel_work_sync(&target->tl_err_work);
 949         srp_rport_put(target->rport);
 950         for (i = 0; i < target->ch_count; i++) {
 951                 ch = &target->ch[i];
 952                 srp_free_req_data(target, ch);
 953         }
 954         kfree(target->ch);
 955         target->ch = NULL;
 956
 957         spin_lock(&target->srp_host->target_lock);
 958         list_del(&target->list);
 959         spin_unlock(&target->srp_host->target_lock);
 960
 961         scsi_host_put(target->scsi_host);
 962 }
 963
 964 static void srp_remove_work(struct work_struct *work)
 965 {
 966         struct srp_target_port *target =
 967                 container_of(work, struct srp_target_port, remove_work);
 968
 969         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 970
 971         srp_remove_target(target);
 972 }
 973
 974 static void srp_rport_delete(struct srp_rport *rport)
 975 {
 976         struct srp_target_port *target = rport->lld_data;
 977
 978         srp_queue_remove_work(target);
 979 }
 980
 981 /**
 982  * srp_connected_ch() - number of connected channels
 983  * @target: SRP target port.
 984  */
 985 static int srp_connected_ch(struct srp_target_port *target)
 986 {
 987         int i, c = 0;
 988
 989         for (i = 0; i < target->ch_count; i++)
 990                 c += target->ch[i].connected;
 991
 992         return c;
 993 }
 994
 995 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
 996 {
 997         struct srp_target_port *target = ch->target;
 998         int ret;
 999
1000         WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
1001
1002         ret = srp_lookup_path(ch);
1003         if (ret)
1004                 goto out;
1005
1006         while (1) {
1007                 init_completion(&ch->done);
1008                 ret = srp_send_req(ch, multich);
1009                 if (ret)
1010                         goto out;
1011                 ret = wait_for_completion_interruptible(&ch->done);
1012                 if (ret < 0)
1013                         goto out;
1014
1015                 /*
1016                  * The CM event handling code will set status to
1017                  * SRP_PORT_REDIRECT if we get a port redirect REJ
1018                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
1019                  * redirect REJ back.
1020                  */
1021                 ret = ch->status;
1022                 switch (ret) {
1023                 case 0:
1024                         ch->connected = true;
1025                         goto out;
1026
1027                 case SRP_PORT_REDIRECT:
1028                         ret = srp_lookup_path(ch);
1029                         if (ret)
1030                                 goto out;
1031                         break;
1032
1033                 case SRP_DLID_REDIRECT:
1034                         break;
1035
1036                 case SRP_STALE_CONN:
1037                         shost_printk(KERN_ERR, target->scsi_host, PFX
1038                                      "giving up on stale connection\n");
1039                         ret = -ECONNRESET;
1040                         goto out;
1041
1042                 default:
1043                         goto out;
1044                 }
1045         }
1046
1047 out:
1048         return ret <= 0 ? ret : -ENODEV;
1049 }
1050
1051 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1052 {
1053         srp_handle_qp_err(cq, wc, "INV RKEY");
1054 }
1055
1056 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1057                 u32 rkey)
1058 {
1059         struct ib_send_wr *bad_wr;
1060         struct ib_send_wr wr = {
1061                 .opcode             = IB_WR_LOCAL_INV,
1062                 .next               = NULL,
1063                 .num_sge            = 0,
1064                 .send_flags         = 0,
1065                 .ex.invalidate_rkey = rkey,
1066         };
1067
1068         wr.wr_cqe = &req->reg_cqe;
1069         req->reg_cqe.done = srp_inv_rkey_err_done;
1070         return ib_post_send(ch->qp, &wr, &bad_wr);
1071 }
1072
1073 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1074                            struct srp_rdma_ch *ch,
1075                            struct srp_request *req)
1076 {
1077         struct srp_target_port *target = ch->target;
1078         struct srp_device *dev = target->srp_host->srp_dev;
1079         struct ib_device *ibdev = dev->dev;
1080         int i, res;
1081
1082         if (!scsi_sglist(scmnd) ||
1083             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1084              scmnd->sc_data_direction != DMA_FROM_DEVICE))
1085                 return;
1086
1087         if (dev->use_fast_reg) {
1088                 struct srp_fr_desc **pfr;
1089
1090                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1091                         res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1092                         if (res < 0) {
1093                                 shost_printk(KERN_ERR, target->scsi_host, PFX
1094                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1095                                   (*pfr)->mr->rkey, res);
1096                                 queue_work(system_long_wq,
1097                                            &target->tl_err_work);
1098                         }
1099                 }
1100                 if (req->nmdesc)
1101                         srp_fr_pool_put(ch->fr_pool, req->fr_list,
1102                                         req->nmdesc);
1103         } else if (dev->use_fmr) {
1104                 struct ib_pool_fmr **pfmr;
1105
1106                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1107                         ib_fmr_pool_unmap(*pfmr);
1108         }
1109
1110         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1111                         scmnd->sc_data_direction);
1112 }
1113
1114 /**
1115  * srp_claim_req - Take ownership of the scmnd associated with a request.
1116  * @ch: SRP RDMA channel.
1117  * @req: SRP request.
1118  * @sdev: If not NULL, only take ownership for this SCSI device.
1119  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1120  *         ownership of @req->scmnd if it equals @scmnd.
1121  *
1122  * Return value:
1123  * Either NULL or a pointer to the SCSI command the caller became owner of.
1124  */
1125 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1126                                        struct srp_request *req,
1127                                        struct scsi_device *sdev,
1128                                        struct scsi_cmnd *scmnd)
1129 {
1130         unsigned long flags;
1131
1132         spin_lock_irqsave(&ch->lock, flags);
1133         if (req->scmnd &&
1134             (!sdev || req->scmnd->device == sdev) &&
1135             (!scmnd || req->scmnd == scmnd)) {
1136                 scmnd = req->scmnd;
1137                 req->scmnd = NULL;
1138         } else {
1139                 scmnd = NULL;
1140         }
1141         spin_unlock_irqrestore(&ch->lock, flags);
1142
1143         return scmnd;
1144 }
1145
1146 /**
1147  * srp_free_req() - Unmap data and add request to the free request list.
1148  * @ch:     SRP RDMA channel.
1149  * @req:    Request to be freed.
1150  * @scmnd:  SCSI command associated with @req.
1151  * @req_lim_delta: Amount to be added to @target->req_lim.
1152  */
1153 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1154                          struct scsi_cmnd *scmnd, s32 req_lim_delta)
1155 {
1156         unsigned long flags;
1157
1158         srp_unmap_data(scmnd, ch, req);
1159
1160         spin_lock_irqsave(&ch->lock, flags);
1161         ch->req_lim += req_lim_delta;
1162         spin_unlock_irqrestore(&ch->lock, flags);
1163 }
1164
1165 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1166                            struct scsi_device *sdev, int result)
1167 {
1168         struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1169
1170         if (scmnd) {
1171                 srp_free_req(ch, req, scmnd, 0);
1172                 scmnd->result = result;
1173                 scmnd->scsi_done(scmnd);
1174         }
1175 }
1176
1177 static void srp_terminate_io(struct srp_rport *rport)
1178 {
1179         struct srp_target_port *target = rport->lld_data;
1180         struct srp_rdma_ch *ch;
1181         struct Scsi_Host *shost = target->scsi_host;
1182         struct scsi_device *sdev;
1183         int i, j;
1184
1185         /*
1186          * Invoking srp_terminate_io() while srp_queuecommand() is running
1187          * is not safe. Hence the warning statement below.
1188          */
1189         shost_for_each_device(sdev, shost)
1190                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1191
1192         for (i = 0; i < target->ch_count; i++) {
1193                 ch = &target->ch[i];
1194
1195                 for (j = 0; j < target->req_ring_size; ++j) {
1196                         struct srp_request *req = &ch->req_ring[j];
1197
1198                         srp_finish_req(ch, req, NULL,
1199                                        DID_TRANSPORT_FAILFAST << 16);
1200                 }
1201         }
1202 }
1203
1204 /*
1205  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1206  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1207  * srp_reset_device() or srp_reset_host() calls will occur while this function
1208  * is in progress. One way to realize that is not to call this function
1209  * directly but to call srp_reconnect_rport() instead since that last function
1210  * serializes calls of this function via rport->mutex and also blocks
1211  * srp_queuecommand() calls before invoking this function.
1212  */
1213 static int srp_rport_reconnect(struct srp_rport *rport)
1214 {
1215         struct srp_target_port *target = rport->lld_data;
1216         struct srp_rdma_ch *ch;
1217         int i, j, ret = 0;
1218         bool multich = false;
1219
1220         srp_disconnect_target(target);
1221
1222         if (target->state == SRP_TARGET_SCANNING)
1223                 return -ENODEV;
1224
1225         /*
1226          * Now get a new local CM ID so that we avoid confusing the target in
1227          * case things are really fouled up. Doing so also ensures that all CM
1228          * callbacks will have finished before a new QP is allocated.
1229          */
1230         for (i = 0; i < target->ch_count; i++) {
1231                 ch = &target->ch[i];
1232                 ret += srp_new_cm_id(ch);
1233         }
1234         for (i = 0; i < target->ch_count; i++) {
1235                 ch = &target->ch[i];
1236                 for (j = 0; j < target->req_ring_size; ++j) {
1237                         struct srp_request *req = &ch->req_ring[j];
1238
1239                         srp_finish_req(ch, req, NULL, DID_RESET << 16);
1240                 }
1241         }
1242         for (i = 0; i < target->ch_count; i++) {
1243                 ch = &target->ch[i];
1244                 /*
1245                  * Whether or not creating a new CM ID succeeded, create a new
1246                  * QP. This guarantees that all completion callback function
1247                  * invocations have finished before request resetting starts.
1248                  */
1249                 ret += srp_create_ch_ib(ch);
1250
1251                 INIT_LIST_HEAD(&ch->free_tx);
1252                 for (j = 0; j < target->queue_size; ++j)
1253                         list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1254         }
1255
1256         target->qp_in_error = false;
1257
1258         for (i = 0; i < target->ch_count; i++) {
1259                 ch = &target->ch[i];
1260                 if (ret)
1261                         break;
1262                 ret = srp_connect_ch(ch, multich);
1263                 multich = true;
1264         }
1265
1266         if (ret == 0)
1267                 shost_printk(KERN_INFO, target->scsi_host,
1268                              PFX "reconnect succeeded\n");
1269
1270         return ret;
1271 }
1272
1273 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1274                          unsigned int dma_len, u32 rkey)
1275 {
1276         struct srp_direct_buf *desc = state->desc;
1277
1278         WARN_ON_ONCE(!dma_len);
1279
1280         desc->va = cpu_to_be64(dma_addr);
1281         desc->key = cpu_to_be32(rkey);
1282         desc->len = cpu_to_be32(dma_len);
1283
1284         state->total_len += dma_len;
1285         state->desc++;
1286         state->ndesc++;
1287 }
1288
1289 static int srp_map_finish_fmr(struct srp_map_state *state,
1290                               struct srp_rdma_ch *ch)
1291 {
1292         struct srp_target_port *target = ch->target;
1293         struct srp_device *dev = target->srp_host->srp_dev;
1294         struct ib_pool_fmr *fmr;
1295         u64 io_addr = 0;
1296
1297         if (state->fmr.next >= state->fmr.end)
1298                 return -ENOMEM;
1299
1300         WARN_ON_ONCE(!dev->use_fmr);
1301
1302         if (state->npages == 0)
1303                 return 0;
1304
1305         if (state->npages == 1 && target->global_mr) {
1306                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1307                              target->global_mr->rkey);
1308                 goto reset_state;
1309         }
1310
1311         fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1312                                    state->npages, io_addr);
1313         if (IS_ERR(fmr))
1314                 return PTR_ERR(fmr);
1315
1316         *state->fmr.next++ = fmr;
1317         state->nmdesc++;
1318
1319         srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1320                      state->dma_len, fmr->fmr->rkey);
1321
1322 reset_state:
1323         state->npages = 0;
1324         state->dma_len = 0;
1325
1326         return 0;
1327 }
1328
1329 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1330 {
1331         srp_handle_qp_err(cq, wc, "FAST REG");
1332 }
1333
1334 static int srp_map_finish_fr(struct srp_map_state *state,
1335                              struct srp_request *req,
1336                              struct srp_rdma_ch *ch, int sg_nents)
1337 {
1338         struct srp_target_port *target = ch->target;
1339         struct srp_device *dev = target->srp_host->srp_dev;
1340         struct ib_send_wr *bad_wr;
1341         struct ib_reg_wr wr;
1342         struct srp_fr_desc *desc;
1343         u32 rkey;
1344         int n, err;
1345
1346         if (state->fr.next >= state->fr.end)
1347                 return -ENOMEM;
1348
1349         WARN_ON_ONCE(!dev->use_fast_reg);
1350
1351         if (sg_nents == 0)
1352                 return 0;
1353
1354         if (sg_nents == 1 && target->global_mr) {
1355                 srp_map_desc(state, sg_dma_address(state->sg),
1356                              sg_dma_len(state->sg),
1357                              target->global_mr->rkey);
1358                 return 1;
1359         }
1360
1361         desc = srp_fr_pool_get(ch->fr_pool);
1362         if (!desc)
1363                 return -ENOMEM;
1364
1365         rkey = ib_inc_rkey(desc->mr->rkey);
1366         ib_update_fast_reg_key(desc->mr, rkey);
1367
1368         n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, dev->mr_page_size);
1369         if (unlikely(n < 0))
1370                 return n;
1371
1372         req->reg_cqe.done = srp_reg_mr_err_done;
1373
1374         wr.wr.next = NULL;
1375         wr.wr.opcode = IB_WR_REG_MR;
1376         wr.wr.wr_cqe = &req->reg_cqe;
1377         wr.wr.num_sge = 0;
1378         wr.wr.send_flags = 0;
1379         wr.mr = desc->mr;
1380         wr.key = desc->mr->rkey;
1381         wr.access = (IB_ACCESS_LOCAL_WRITE |
1382                      IB_ACCESS_REMOTE_READ |
1383                      IB_ACCESS_REMOTE_WRITE);
1384
1385         *state->fr.next++ = desc;
1386         state->nmdesc++;
1387
1388         srp_map_desc(state, desc->mr->iova,
1389                      desc->mr->length, desc->mr->rkey);
1390
1391         err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1392         if (unlikely(err))
1393                 return err;
1394
1395         return n;
1396 }
1397
1398 static int srp_map_sg_entry(struct srp_map_state *state,
1399                             struct srp_rdma_ch *ch,
1400                             struct scatterlist *sg, int sg_index)
1401 {
1402         struct srp_target_port *target = ch->target;
1403         struct srp_device *dev = target->srp_host->srp_dev;
1404         struct ib_device *ibdev = dev->dev;
1405         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1406         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1407         unsigned int len = 0;
1408         int ret;
1409
1410         WARN_ON_ONCE(!dma_len);
1411
1412         while (dma_len) {
1413                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1414                 if (state->npages == dev->max_pages_per_mr || offset != 0) {
1415                         ret = srp_map_finish_fmr(state, ch);
1416                         if (ret)
1417                                 return ret;
1418                 }
1419
1420                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1421
1422                 if (!state->npages)
1423                         state->base_dma_addr = dma_addr;
1424                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1425                 state->dma_len += len;
1426                 dma_addr += len;
1427                 dma_len -= len;
1428         }
1429
1430         /*
1431          * If the last entry of the MR wasn't a full page, then we need to
1432          * close it out and start a new one -- we can only merge at page
1433          * boundries.
1434          */
1435         ret = 0;
1436         if (len != dev->mr_page_size)
1437                 ret = srp_map_finish_fmr(state, ch);
1438         return ret;
1439 }
1440
1441 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1442                           struct srp_request *req, struct scatterlist *scat,
1443                           int count)
1444 {
1445         struct scatterlist *sg;
1446         int i, ret;
1447
1448         state->desc = req->indirect_desc;
1449         state->pages = req->map_page;
1450         state->fmr.next = req->fmr_list;
1451         state->fmr.end = req->fmr_list + ch->target->cmd_sg_cnt;
1452
1453         for_each_sg(scat, sg, count, i) {
1454                 ret = srp_map_sg_entry(state, ch, sg, i);
1455                 if (ret)
1456                         return ret;
1457         }
1458
1459         ret = srp_map_finish_fmr(state, ch);
1460         if (ret)
1461                 return ret;
1462
1463         req->nmdesc = state->nmdesc;
1464
1465         return 0;
1466 }
1467
1468 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1469                          struct srp_request *req, struct scatterlist *scat,
1470                          int count)
1471 {
1472         state->desc = req->indirect_desc;
1473         state->fr.next = req->fr_list;
1474         state->fr.end = req->fr_list + ch->target->cmd_sg_cnt;
1475         state->sg = scat;
1476
1477         while (count) {
1478                 int i, n;
1479
1480                 n = srp_map_finish_fr(state, req, ch, count);
1481                 if (unlikely(n < 0))
1482                         return n;
1483
1484                 count -= n;
1485                 for (i = 0; i < n; i++)
1486                         state->sg = sg_next(state->sg);
1487         }
1488
1489         req->nmdesc = state->nmdesc;
1490
1491         return 0;
1492 }
1493
1494 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1495                           struct srp_request *req, struct scatterlist *scat,
1496                           int count)
1497 {
1498         struct srp_target_port *target = ch->target;
1499         struct srp_device *dev = target->srp_host->srp_dev;
1500         struct scatterlist *sg;
1501         int i;
1502
1503         state->desc = req->indirect_desc;
1504         for_each_sg(scat, sg, count, i) {
1505                 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1506                              ib_sg_dma_len(dev->dev, sg),
1507                              target->global_mr->rkey);
1508         }
1509
1510         req->nmdesc = state->nmdesc;
1511
1512         return 0;
1513 }
1514
1515 /*
1516  * Register the indirect data buffer descriptor with the HCA.
1517  *
1518  * Note: since the indirect data buffer descriptor has been allocated with
1519  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1520  * memory buffer.
1521  */
1522 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1523                        void **next_mr, void **end_mr, u32 idb_len,
1524                        __be32 *idb_rkey)
1525 {
1526         struct srp_target_port *target = ch->target;
1527         struct srp_device *dev = target->srp_host->srp_dev;
1528         struct srp_map_state state;
1529         struct srp_direct_buf idb_desc;
1530         u64 idb_pages[1];
1531         struct scatterlist idb_sg[1];
1532         int ret;
1533
1534         memset(&state, 0, sizeof(state));
1535         memset(&idb_desc, 0, sizeof(idb_desc));
1536         state.gen.next = next_mr;
1537         state.gen.end = end_mr;
1538         state.desc = &idb_desc;
1539         state.base_dma_addr = req->indirect_dma_addr;
1540         state.dma_len = idb_len;
1541
1542         if (dev->use_fast_reg) {
1543                 state.sg = idb_sg;
1544                 sg_set_buf(idb_sg, req->indirect_desc, idb_len);
1545                 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1546 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1547                 idb_sg->dma_length = idb_sg->length;          /* hack^2 */
1548 #endif
1549                 ret = srp_map_finish_fr(&state, req, ch, 1);
1550                 if (ret < 0)
1551                         return ret;
1552         } else if (dev->use_fmr) {
1553                 state.pages = idb_pages;
1554                 state.pages[0] = (req->indirect_dma_addr &
1555                                   dev->mr_page_mask);
1556                 state.npages = 1;
1557                 ret = srp_map_finish_fmr(&state, ch);
1558                 if (ret < 0)
1559                         return ret;
1560         } else {
1561                 return -EINVAL;
1562         }
1563
1564         *idb_rkey = idb_desc.key;
1565
1566         return 0;
1567 }
1568
1569 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1570                         struct srp_request *req)
1571 {
1572         struct srp_target_port *target = ch->target;
1573         struct scatterlist *scat;
1574         struct srp_cmd *cmd = req->cmd->buf;
1575         int len, nents, count, ret;
1576         struct srp_device *dev;
1577         struct ib_device *ibdev;
1578         struct srp_map_state state;
1579         struct srp_indirect_buf *indirect_hdr;
1580         u32 idb_len, table_len;
1581         __be32 idb_rkey;
1582         u8 fmt;
1583
1584         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1585                 return sizeof (struct srp_cmd);
1586
1587         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1588             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1589                 shost_printk(KERN_WARNING, target->scsi_host,
1590                              PFX "Unhandled data direction %d\n",
1591                              scmnd->sc_data_direction);
1592                 return -EINVAL;
1593         }
1594
1595         nents = scsi_sg_count(scmnd);
1596         scat  = scsi_sglist(scmnd);
1597
1598         dev = target->srp_host->srp_dev;
1599         ibdev = dev->dev;
1600
1601         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1602         if (unlikely(count == 0))
1603                 return -EIO;
1604
1605         fmt = SRP_DATA_DESC_DIRECT;
1606         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1607
1608         if (count == 1 && target->global_mr) {
1609                 /*
1610                  * The midlayer only generated a single gather/scatter
1611                  * entry, or DMA mapping coalesced everything to a
1612                  * single entry.  So a direct descriptor along with
1613                  * the DMA MR suffices.
1614                  */
1615                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1616
1617                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1618                 buf->key = cpu_to_be32(target->global_mr->rkey);
1619                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1620
1621                 req->nmdesc = 0;
1622                 goto map_complete;
1623         }
1624
1625         /*
1626          * We have more than one scatter/gather entry, so build our indirect
1627          * descriptor table, trying to merge as many entries as we can.
1628          */
1629         indirect_hdr = (void *) cmd->add_data;
1630
1631         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1632                                    target->indirect_size, DMA_TO_DEVICE);
1633
1634         memset(&state, 0, sizeof(state));
1635         if (dev->use_fast_reg)
1636                 srp_map_sg_fr(&state, ch, req, scat, count);
1637         else if (dev->use_fmr)
1638                 srp_map_sg_fmr(&state, ch, req, scat, count);
1639         else
1640                 srp_map_sg_dma(&state, ch, req, scat, count);
1641
1642         /* We've mapped the request, now pull as much of the indirect
1643          * descriptor table as we can into the command buffer. If this
1644          * target is not using an external indirect table, we are
1645          * guaranteed to fit into the command, as the SCSI layer won't
1646          * give us more S/G entries than we allow.
1647          */
1648         if (state.ndesc == 1) {
1649                 /*
1650                  * Memory registration collapsed the sg-list into one entry,
1651                  * so use a direct descriptor.
1652                  */
1653                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1654
1655                 *buf = req->indirect_desc[0];
1656                 goto map_complete;
1657         }
1658
1659         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1660                                                 !target->allow_ext_sg)) {
1661                 shost_printk(KERN_ERR, target->scsi_host,
1662                              "Could not fit S/G list into SRP_CMD\n");
1663                 return -EIO;
1664         }
1665
1666         count = min(state.ndesc, target->cmd_sg_cnt);
1667         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1668         idb_len = sizeof(struct srp_indirect_buf) + table_len;
1669
1670         fmt = SRP_DATA_DESC_INDIRECT;
1671         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1672         len += count * sizeof (struct srp_direct_buf);
1673
1674         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1675                count * sizeof (struct srp_direct_buf));
1676
1677         if (!target->global_mr) {
1678                 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1679                                   idb_len, &idb_rkey);
1680                 if (ret < 0)
1681                         return ret;
1682                 req->nmdesc++;
1683         } else {
1684                 idb_rkey = cpu_to_be32(target->global_mr->rkey);
1685         }
1686
1687         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1688         indirect_hdr->table_desc.key = idb_rkey;
1689         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1690         indirect_hdr->len = cpu_to_be32(state.total_len);
1691
1692         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1693                 cmd->data_out_desc_cnt = count;
1694         else
1695                 cmd->data_in_desc_cnt = count;
1696
1697         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1698                                       DMA_TO_DEVICE);
1699
1700 map_complete:
1701         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1702                 cmd->buf_fmt = fmt << 4;
1703         else
1704                 cmd->buf_fmt = fmt;
1705
1706         return len;
1707 }
1708
1709 /*
1710  * Return an IU and possible credit to the free pool
1711  */
1712 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1713                           enum srp_iu_type iu_type)
1714 {
1715         unsigned long flags;
1716
1717         spin_lock_irqsave(&ch->lock, flags);
1718         list_add(&iu->list, &ch->free_tx);
1719         if (iu_type != SRP_IU_RSP)
1720                 ++ch->req_lim;
1721         spin_unlock_irqrestore(&ch->lock, flags);
1722 }
1723
1724 /*
1725  * Must be called with ch->lock held to protect req_lim and free_tx.
1726  * If IU is not sent, it must be returned using srp_put_tx_iu().
1727  *
1728  * Note:
1729  * An upper limit for the number of allocated information units for each
1730  * request type is:
1731  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1732  *   more than Scsi_Host.can_queue requests.
1733  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1734  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1735  *   one unanswered SRP request to an initiator.
1736  */
1737 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1738                                       enum srp_iu_type iu_type)
1739 {
1740         struct srp_target_port *target = ch->target;
1741         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1742         struct srp_iu *iu;
1743
1744         ib_process_cq_direct(ch->send_cq, -1);
1745
1746         if (list_empty(&ch->free_tx))
1747                 return NULL;
1748
1749         /* Initiator responses to target requests do not consume credits */
1750         if (iu_type != SRP_IU_RSP) {
1751                 if (ch->req_lim <= rsv) {
1752                         ++target->zero_req_lim;
1753                         return NULL;
1754                 }
1755
1756                 --ch->req_lim;
1757         }
1758
1759         iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1760         list_del(&iu->list);
1761         return iu;
1762 }
1763
1764 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1765 {
1766         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1767         struct srp_rdma_ch *ch = cq->cq_context;
1768
1769         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1770                 srp_handle_qp_err(cq, wc, "SEND");
1771                 return;
1772         }
1773
1774         list_add(&iu->list, &ch->free_tx);
1775 }
1776
1777 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1778 {
1779         struct srp_target_port *target = ch->target;
1780         struct ib_sge list;
1781         struct ib_send_wr wr, *bad_wr;
1782
1783         list.addr   = iu->dma;
1784         list.length = len;
1785         list.lkey   = target->lkey;
1786
1787         iu->cqe.done = srp_send_done;
1788
1789         wr.next       = NULL;
1790         wr.wr_cqe     = &iu->cqe;
1791         wr.sg_list    = &list;
1792         wr.num_sge    = 1;
1793         wr.opcode     = IB_WR_SEND;
1794         wr.send_flags = IB_SEND_SIGNALED;
1795
1796         return ib_post_send(ch->qp, &wr, &bad_wr);
1797 }
1798
1799 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1800 {
1801         struct srp_target_port *target = ch->target;
1802         struct ib_recv_wr wr, *bad_wr;
1803         struct ib_sge list;
1804
1805         list.addr   = iu->dma;
1806         list.length = iu->size;
1807         list.lkey   = target->lkey;
1808
1809         iu->cqe.done = srp_recv_done;
1810
1811         wr.next     = NULL;
1812         wr.wr_cqe   = &iu->cqe;
1813         wr.sg_list  = &list;
1814         wr.num_sge  = 1;
1815
1816         return ib_post_recv(ch->qp, &wr, &bad_wr);
1817 }
1818
1819 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1820 {
1821         struct srp_target_port *target = ch->target;
1822         struct srp_request *req;
1823         struct scsi_cmnd *scmnd;
1824         unsigned long flags;
1825
1826         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1827                 spin_lock_irqsave(&ch->lock, flags);
1828                 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1829                 spin_unlock_irqrestore(&ch->lock, flags);
1830
1831                 ch->tsk_mgmt_status = -1;
1832                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1833                         ch->tsk_mgmt_status = rsp->data[3];
1834                 complete(&ch->tsk_mgmt_done);
1835         } else {
1836                 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1837                 if (scmnd) {
1838                         req = (void *)scmnd->host_scribble;
1839                         scmnd = srp_claim_req(ch, req, NULL, scmnd);
1840                 }
1841                 if (!scmnd) {
1842                         shost_printk(KERN_ERR, target->scsi_host,
1843                                      "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1844                                      rsp->tag, ch - target->ch, ch->qp->qp_num);
1845
1846                         spin_lock_irqsave(&ch->lock, flags);
1847                         ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1848                         spin_unlock_irqrestore(&ch->lock, flags);
1849
1850                         return;
1851                 }
1852                 scmnd->result = rsp->status;
1853
1854                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1855                         memcpy(scmnd->sense_buffer, rsp->data +
1856                                be32_to_cpu(rsp->resp_data_len),
1857                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1858                                      SCSI_SENSE_BUFFERSIZE));
1859                 }
1860
1861                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1862                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1863                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1864                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1865                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1866                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1867                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1868                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1869
1870                 srp_free_req(ch, req, scmnd,
1871                              be32_to_cpu(rsp->req_lim_delta));
1872
1873                 scmnd->host_scribble = NULL;
1874                 scmnd->scsi_done(scmnd);
1875         }
1876 }
1877
1878 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1879                                void *rsp, int len)
1880 {
1881         struct srp_target_port *target = ch->target;
1882         struct ib_device *dev = target->srp_host->srp_dev->dev;
1883         unsigned long flags;
1884         struct srp_iu *iu;
1885         int err;
1886
1887         spin_lock_irqsave(&ch->lock, flags);
1888         ch->req_lim += req_delta;
1889         iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1890         spin_unlock_irqrestore(&ch->lock, flags);
1891
1892         if (!iu) {
1893                 shost_printk(KERN_ERR, target->scsi_host, PFX
1894                              "no IU available to send response\n");
1895                 return 1;
1896         }
1897
1898         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1899         memcpy(iu->buf, rsp, len);
1900         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1901
1902         err = srp_post_send(ch, iu, len);
1903         if (err) {
1904                 shost_printk(KERN_ERR, target->scsi_host, PFX
1905                              "unable to post response: %d\n", err);
1906                 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1907         }
1908
1909         return err;
1910 }
1911
1912 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1913                                  struct srp_cred_req *req)
1914 {
1915         struct srp_cred_rsp rsp = {
1916                 .opcode = SRP_CRED_RSP,
1917                 .tag = req->tag,
1918         };
1919         s32 delta = be32_to_cpu(req->req_lim_delta);
1920
1921         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1922                 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1923                              "problems processing SRP_CRED_REQ\n");
1924 }
1925
1926 static void srp_process_aer_req(struct srp_rdma_ch *ch,
1927                                 struct srp_aer_req *req)
1928 {
1929         struct srp_target_port *target = ch->target;
1930         struct srp_aer_rsp rsp = {
1931                 .opcode = SRP_AER_RSP,
1932                 .tag = req->tag,
1933         };
1934         s32 delta = be32_to_cpu(req->req_lim_delta);
1935
1936         shost_printk(KERN_ERR, target->scsi_host, PFX
1937                      "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
1938
1939         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1940                 shost_printk(KERN_ERR, target->scsi_host, PFX
1941                              "problems processing SRP_AER_REQ\n");
1942 }
1943
1944 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1945 {
1946         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1947         struct srp_rdma_ch *ch = cq->cq_context;
1948         struct srp_target_port *target = ch->target;
1949         struct ib_device *dev = target->srp_host->srp_dev->dev;
1950         int res;
1951         u8 opcode;
1952
1953         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1954                 srp_handle_qp_err(cq, wc, "RECV");
1955                 return;
1956         }
1957
1958         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
1959                                    DMA_FROM_DEVICE);
1960
1961         opcode = *(u8 *) iu->buf;
1962
1963         if (0) {
1964                 shost_printk(KERN_ERR, target->scsi_host,
1965                              PFX "recv completion, opcode 0x%02x\n", opcode);
1966                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1967                                iu->buf, wc->byte_len, true);
1968         }
1969
1970         switch (opcode) {
1971         case SRP_RSP:
1972                 srp_process_rsp(ch, iu->buf);
1973                 break;
1974
1975         case SRP_CRED_REQ:
1976                 srp_process_cred_req(ch, iu->buf);
1977                 break;
1978
1979         case SRP_AER_REQ:
1980                 srp_process_aer_req(ch, iu->buf);
1981                 break;
1982
1983         case SRP_T_LOGOUT:
1984                 /* XXX Handle target logout */
1985                 shost_printk(KERN_WARNING, target->scsi_host,
1986                              PFX "Got target logout request\n");
1987                 break;
1988
1989         default:
1990                 shost_printk(KERN_WARNING, target->scsi_host,
1991                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1992                 break;
1993         }
1994
1995         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
1996                                       DMA_FROM_DEVICE);
1997
1998         res = srp_post_recv(ch, iu);
1999         if (res != 0)
2000                 shost_printk(KERN_ERR, target->scsi_host,
2001                              PFX "Recv failed with error code %d\n", res);
2002 }
2003
2004 /**
2005  * srp_tl_err_work() - handle a transport layer error
2006  * @work: Work structure embedded in an SRP target port.
2007  *
2008  * Note: This function may get invoked before the rport has been created,
2009  * hence the target->rport test.
2010  */
2011 static void srp_tl_err_work(struct work_struct *work)
2012 {
2013         struct srp_target_port *target;
2014
2015         target = container_of(work, struct srp_target_port, tl_err_work);
2016         if (target->rport)
2017                 srp_start_tl_fail_timers(target->rport);
2018 }
2019
2020 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2021                 const char *opname)
2022 {
2023         struct srp_rdma_ch *ch = cq->cq_context;
2024         struct srp_target_port *target = ch->target;
2025
2026         if (ch->connected && !target->qp_in_error) {
2027                 shost_printk(KERN_ERR, target->scsi_host,
2028                              PFX "failed %s status %s (%d) for CQE %p\n",
2029                              opname, ib_wc_status_msg(wc->status), wc->status,
2030                              wc->wr_cqe);
2031                 queue_work(system_long_wq, &target->tl_err_work);
2032         }
2033         target->qp_in_error = true;
2034 }
2035
2036 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2037 {
2038         struct srp_target_port *target = host_to_target(shost);
2039         struct srp_rport *rport = target->rport;
2040         struct srp_rdma_ch *ch;
2041         struct srp_request *req;
2042         struct srp_iu *iu;
2043         struct srp_cmd *cmd;
2044         struct ib_device *dev;
2045         unsigned long flags;
2046         u32 tag;
2047         u16 idx;
2048         int len, ret;
2049         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2050
2051         /*
2052          * The SCSI EH thread is the only context from which srp_queuecommand()
2053          * can get invoked for blocked devices (SDEV_BLOCK /
2054          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2055          * locking the rport mutex if invoked from inside the SCSI EH.
2056          */
2057         if (in_scsi_eh)
2058                 mutex_lock(&rport->mutex);
2059
2060         scmnd->result = srp_chkready(target->rport);
2061         if (unlikely(scmnd->result))
2062                 goto err;
2063
2064         WARN_ON_ONCE(scmnd->request->tag < 0);
2065         tag = blk_mq_unique_tag(scmnd->request);
2066         ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2067         idx = blk_mq_unique_tag_to_tag(tag);
2068         WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2069                   dev_name(&shost->shost_gendev), tag, idx,
2070                   target->req_ring_size);
2071
2072         spin_lock_irqsave(&ch->lock, flags);
2073         iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2074         spin_unlock_irqrestore(&ch->lock, flags);
2075
2076         if (!iu)
2077                 goto err;
2078
2079         req = &ch->req_ring[idx];
2080         dev = target->srp_host->srp_dev->dev;
2081         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2082                                    DMA_TO_DEVICE);
2083
2084         scmnd->host_scribble = (void *) req;
2085
2086         cmd = iu->buf;
2087         memset(cmd, 0, sizeof *cmd);
2088
2089         cmd->opcode = SRP_CMD;
2090         int_to_scsilun(scmnd->device->lun, &cmd->lun);
2091         cmd->tag    = tag;
2092         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2093
2094         req->scmnd    = scmnd;
2095         req->cmd      = iu;
2096
2097         len = srp_map_data(scmnd, ch, req);
2098         if (len < 0) {
2099                 shost_printk(KERN_ERR, target->scsi_host,
2100                              PFX "Failed to map data (%d)\n", len);
2101                 /*
2102                  * If we ran out of memory descriptors (-ENOMEM) because an
2103                  * application is queuing many requests with more than
2104                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2105                  * to reduce queue depth temporarily.
2106                  */
2107                 scmnd->result = len == -ENOMEM ?
2108                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2109                 goto err_iu;
2110         }
2111
2112         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2113                                       DMA_TO_DEVICE);
2114
2115         if (srp_post_send(ch, iu, len)) {
2116                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2117                 goto err_unmap;
2118         }
2119
2120         ret = 0;
2121
2122 unlock_rport:
2123         if (in_scsi_eh)
2124                 mutex_unlock(&rport->mutex);
2125
2126         return ret;
2127
2128 err_unmap:
2129         srp_unmap_data(scmnd, ch, req);
2130
2131 err_iu:
2132         srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2133
2134         /*
2135          * Avoid that the loops that iterate over the request ring can
2136          * encounter a dangling SCSI command pointer.
2137          */
2138         req->scmnd = NULL;
2139
2140 err:
2141         if (scmnd->result) {
2142                 scmnd->scsi_done(scmnd);
2143                 ret = 0;
2144         } else {
2145                 ret = SCSI_MLQUEUE_HOST_BUSY;
2146         }
2147
2148         goto unlock_rport;
2149 }
2150
2151 /*
2152  * Note: the resources allocated in this function are freed in
2153  * srp_free_ch_ib().
2154  */
2155 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2156 {
2157         struct srp_target_port *target = ch->target;
2158         int i;
2159
2160         ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2161                               GFP_KERNEL);
2162         if (!ch->rx_ring)
2163                 goto err_no_ring;
2164         ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2165                               GFP_KERNEL);
2166         if (!ch->tx_ring)
2167                 goto err_no_ring;
2168
2169         for (i = 0; i < target->queue_size; ++i) {
2170                 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2171                                               ch->max_ti_iu_len,
2172                                               GFP_KERNEL, DMA_FROM_DEVICE);
2173                 if (!ch->rx_ring[i])
2174                         goto err;
2175         }
2176
2177         for (i = 0; i < target->queue_size; ++i) {
2178                 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2179                                               target->max_iu_len,
2180                                               GFP_KERNEL, DMA_TO_DEVICE);
2181                 if (!ch->tx_ring[i])
2182                         goto err;
2183
2184                 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2185         }
2186
2187         return 0;
2188
2189 err:
2190         for (i = 0; i < target->queue_size; ++i) {
2191                 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2192                 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2193         }
2194
2195
2196 err_no_ring:
2197         kfree(ch->tx_ring);
2198         ch->tx_ring = NULL;
2199         kfree(ch->rx_ring);
2200         ch->rx_ring = NULL;
2201
2202         return -ENOMEM;
2203 }
2204
2205 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2206 {
2207         uint64_t T_tr_ns, max_compl_time_ms;
2208         uint32_t rq_tmo_jiffies;
2209
2210         /*
2211          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2212          * table 91), both the QP timeout and the retry count have to be set
2213          * for RC QP's during the RTR to RTS transition.
2214          */
2215         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2216                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2217
2218         /*
2219          * Set target->rq_tmo_jiffies to one second more than the largest time
2220          * it can take before an error completion is generated. See also
2221          * C9-140..142 in the IBTA spec for more information about how to
2222          * convert the QP Local ACK Timeout value to nanoseconds.
2223          */
2224         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2225         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2226         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2227         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2228
2229         return rq_tmo_jiffies;
2230 }
2231
2232 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2233                                const struct srp_login_rsp *lrsp,
2234                                struct srp_rdma_ch *ch)
2235 {
2236         struct srp_target_port *target = ch->target;
2237         struct ib_qp_attr *qp_attr = NULL;
2238         int attr_mask = 0;
2239         int ret;
2240         int i;
2241
2242         if (lrsp->opcode == SRP_LOGIN_RSP) {
2243                 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2244                 ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2245
2246                 /*
2247                  * Reserve credits for task management so we don't
2248                  * bounce requests back to the SCSI mid-layer.
2249                  */
2250                 target->scsi_host->can_queue
2251                         = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2252                               target->scsi_host->can_queue);
2253                 target->scsi_host->cmd_per_lun
2254                         = min_t(int, target->scsi_host->can_queue,
2255                                 target->scsi_host->cmd_per_lun);
2256         } else {
2257                 shost_printk(KERN_WARNING, target->scsi_host,
2258                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2259                 ret = -ECONNRESET;
2260                 goto error;
2261         }
2262
2263         if (!ch->rx_ring) {
2264                 ret = srp_alloc_iu_bufs(ch);
2265                 if (ret)
2266                         goto error;
2267         }
2268
2269         ret = -ENOMEM;
2270         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2271         if (!qp_attr)
2272                 goto error;
2273
2274         qp_attr->qp_state = IB_QPS_RTR;
2275         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2276         if (ret)
2277                 goto error_free;
2278
2279         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2280         if (ret)
2281                 goto error_free;
2282
2283         for (i = 0; i < target->queue_size; i++) {
2284                 struct srp_iu *iu = ch->rx_ring[i];
2285
2286                 ret = srp_post_recv(ch, iu);
2287                 if (ret)
2288                         goto error_free;
2289         }
2290
2291         qp_attr->qp_state = IB_QPS_RTS;
2292         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2293         if (ret)
2294                 goto error_free;
2295
2296         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2297
2298         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2299         if (ret)
2300                 goto error_free;
2301
2302         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2303
2304 error_free:
2305         kfree(qp_attr);
2306
2307 error:
2308         ch->status = ret;
2309 }
2310
2311 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2312                                struct ib_cm_event *event,
2313                                struct srp_rdma_ch *ch)
2314 {
2315         struct srp_target_port *target = ch->target;
2316         struct Scsi_Host *shost = target->scsi_host;
2317         struct ib_class_port_info *cpi;
2318         int opcode;
2319
2320         switch (event->param.rej_rcvd.reason) {
2321         case IB_CM_REJ_PORT_CM_REDIRECT:
2322                 cpi = event->param.rej_rcvd.ari;
2323                 ch->path.dlid = cpi->redirect_lid;
2324                 ch->path.pkey = cpi->redirect_pkey;
2325                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2326                 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2327
2328                 ch->status = ch->path.dlid ?
2329                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2330                 break;
2331
2332         case IB_CM_REJ_PORT_REDIRECT:
2333                 if (srp_target_is_topspin(target)) {
2334                         /*
2335                          * Topspin/Cisco SRP gateways incorrectly send
2336                          * reject reason code 25 when they mean 24
2337                          * (port redirect).
2338                          */
2339                         memcpy(ch->path.dgid.raw,
2340                                event->param.rej_rcvd.ari, 16);
2341
2342                         shost_printk(KERN_DEBUG, shost,
2343                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2344                                      be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2345                                      be64_to_cpu(ch->path.dgid.global.interface_id));
2346
2347                         ch->status = SRP_PORT_REDIRECT;
2348                 } else {
2349                         shost_printk(KERN_WARNING, shost,
2350                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2351                         ch->status = -ECONNRESET;
2352                 }
2353                 break;
2354
2355         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2356                 shost_printk(KERN_WARNING, shost,
2357                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2358                 ch->status = -ECONNRESET;
2359                 break;
2360
2361         case IB_CM_REJ_CONSUMER_DEFINED:
2362                 opcode = *(u8 *) event->private_data;
2363                 if (opcode == SRP_LOGIN_REJ) {
2364                         struct srp_login_rej *rej = event->private_data;
2365                         u32 reason = be32_to_cpu(rej->reason);
2366
2367                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2368                                 shost_printk(KERN_WARNING, shost,
2369                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2370                         else
2371                                 shost_printk(KERN_WARNING, shost, PFX
2372                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2373                                              target->sgid.raw,
2374                                              target->orig_dgid.raw, reason);
2375                 } else
2376                         shost_printk(KERN_WARNING, shost,
2377                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2378                                      " opcode 0x%02x\n", opcode);
2379                 ch->status = -ECONNRESET;
2380                 break;
2381
2382         case IB_CM_REJ_STALE_CONN:
2383                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2384                 ch->status = SRP_STALE_CONN;
2385                 break;
2386
2387         default:
2388                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2389                              event->param.rej_rcvd.reason);
2390                 ch->status = -ECONNRESET;
2391         }
2392 }
2393
2394 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2395 {
2396         struct srp_rdma_ch *ch = cm_id->context;
2397         struct srp_target_port *target = ch->target;
2398         int comp = 0;
2399
2400         switch (event->event) {
2401         case IB_CM_REQ_ERROR:
2402                 shost_printk(KERN_DEBUG, target->scsi_host,
2403                              PFX "Sending CM REQ failed\n");
2404                 comp = 1;
2405                 ch->status = -ECONNRESET;
2406                 break;
2407
2408         case IB_CM_REP_RECEIVED:
2409                 comp = 1;
2410                 srp_cm_rep_handler(cm_id, event->private_data, ch);
2411                 break;
2412
2413         case IB_CM_REJ_RECEIVED:
2414                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2415                 comp = 1;
2416
2417                 srp_cm_rej_handler(cm_id, event, ch);
2418                 break;
2419
2420         case IB_CM_DREQ_RECEIVED:
2421                 shost_printk(KERN_WARNING, target->scsi_host,
2422                              PFX "DREQ received - connection closed\n");
2423                 ch->connected = false;
2424                 if (ib_send_cm_drep(cm_id, NULL, 0))
2425                         shost_printk(KERN_ERR, target->scsi_host,
2426                                      PFX "Sending CM DREP failed\n");
2427                 queue_work(system_long_wq, &target->tl_err_work);
2428                 break;
2429
2430         case IB_CM_TIMEWAIT_EXIT:
2431                 shost_printk(KERN_ERR, target->scsi_host,
2432                              PFX "connection closed\n");
2433                 comp = 1;
2434
2435                 ch->status = 0;
2436                 break;
2437
2438         case IB_CM_MRA_RECEIVED:
2439         case IB_CM_DREQ_ERROR:
2440         case IB_CM_DREP_RECEIVED:
2441                 break;
2442
2443         default:
2444                 shost_printk(KERN_WARNING, target->scsi_host,
2445                              PFX "Unhandled CM event %d\n", event->event);
2446                 break;
2447         }
2448
2449         if (comp)
2450                 complete(&ch->done);
2451
2452         return 0;
2453 }
2454
2455 /**
2456  * srp_change_queue_depth - setting device queue depth
2457  * @sdev: scsi device struct
2458  * @qdepth: requested queue depth
2459  *
2460  * Returns queue depth.
2461  */
2462 static int
2463 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2464 {
2465         if (!sdev->tagged_supported)
2466                 qdepth = 1;
2467         return scsi_change_queue_depth(sdev, qdepth);
2468 }
2469
2470 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2471                              u8 func)
2472 {
2473         struct srp_target_port *target = ch->target;
2474         struct srp_rport *rport = target->rport;
2475         struct ib_device *dev = target->srp_host->srp_dev->dev;
2476         struct srp_iu *iu;
2477         struct srp_tsk_mgmt *tsk_mgmt;
2478
2479         if (!ch->connected || target->qp_in_error)
2480                 return -1;
2481
2482         init_completion(&ch->tsk_mgmt_done);
2483
2484         /*
2485          * Lock the rport mutex to avoid that srp_create_ch_ib() is
2486          * invoked while a task management function is being sent.
2487          */
2488         mutex_lock(&rport->mutex);
2489         spin_lock_irq(&ch->lock);
2490         iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2491         spin_unlock_irq(&ch->lock);
2492
2493         if (!iu) {
2494                 mutex_unlock(&rport->mutex);
2495
2496                 return -1;
2497         }
2498
2499         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2500                                    DMA_TO_DEVICE);
2501         tsk_mgmt = iu->buf;
2502         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2503
2504         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2505         int_to_scsilun(lun, &tsk_mgmt->lun);
2506         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
2507         tsk_mgmt->tsk_mgmt_func = func;
2508         tsk_mgmt->task_tag      = req_tag;
2509
2510         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2511                                       DMA_TO_DEVICE);
2512         if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2513                 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2514                 mutex_unlock(&rport->mutex);
2515
2516                 return -1;
2517         }
2518         mutex_unlock(&rport->mutex);
2519
2520         if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2521                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2522                 return -1;
2523
2524         return 0;
2525 }
2526
2527 static int srp_abort(struct scsi_cmnd *scmnd)
2528 {
2529         struct srp_target_port *target = host_to_target(scmnd->device->host);
2530         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2531         u32 tag;
2532         u16 ch_idx;
2533         struct srp_rdma_ch *ch;
2534         int ret;
2535
2536         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2537
2538         if (!req)
2539                 return SUCCESS;
2540         tag = blk_mq_unique_tag(scmnd->request);
2541         ch_idx = blk_mq_unique_tag_to_hwq(tag);
2542         if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2543                 return SUCCESS;
2544         ch = &target->ch[ch_idx];
2545         if (!srp_claim_req(ch, req, NULL, scmnd))
2546                 return SUCCESS;
2547         shost_printk(KERN_ERR, target->scsi_host,
2548                      "Sending SRP abort for tag %#x\n", tag);
2549         if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2550                               SRP_TSK_ABORT_TASK) == 0)
2551                 ret = SUCCESS;
2552         else if (target->rport->state == SRP_RPORT_LOST)
2553                 ret = FAST_IO_FAIL;
2554         else
2555                 ret = FAILED;
2556         srp_free_req(ch, req, scmnd, 0);
2557         scmnd->result = DID_ABORT << 16;
2558         scmnd->scsi_done(scmnd);
2559
2560         return ret;
2561 }
2562
2563 static int srp_reset_device(struct scsi_cmnd *scmnd)
2564 {
2565         struct srp_target_port *target = host_to_target(scmnd->device->host);
2566         struct srp_rdma_ch *ch;
2567         int i;
2568
2569         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2570
2571         ch = &target->ch[0];
2572         if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2573                               SRP_TSK_LUN_RESET))
2574                 return FAILED;
2575         if (ch->tsk_mgmt_status)
2576                 return FAILED;
2577
2578         for (i = 0; i < target->ch_count; i++) {
2579                 ch = &target->ch[i];
2580                 for (i = 0; i < target->req_ring_size; ++i) {
2581                         struct srp_request *req = &ch->req_ring[i];
2582
2583                         srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2584                 }
2585         }
2586
2587         return SUCCESS;
2588 }
2589
2590 static int srp_reset_host(struct scsi_cmnd *scmnd)
2591 {
2592         struct srp_target_port *target = host_to_target(scmnd->device->host);
2593
2594         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2595
2596         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2597 }
2598
2599 static int srp_slave_configure(struct scsi_device *sdev)
2600 {
2601         struct Scsi_Host *shost = sdev->host;
2602         struct srp_target_port *target = host_to_target(shost);
2603         struct request_queue *q = sdev->request_queue;
2604         unsigned long timeout;
2605
2606         if (sdev->type == TYPE_DISK) {
2607                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2608                 blk_queue_rq_timeout(q, timeout);
2609         }
2610
2611         return 0;
2612 }
2613
2614 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2615                            char *buf)
2616 {
2617         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2618
2619         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2620 }
2621
2622 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2623                              char *buf)
2624 {
2625         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2626
2627         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2628 }
2629
2630 static ssize_t show_service_id(struct device *dev,
2631                                struct device_attribute *attr, char *buf)
2632 {
2633         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2634
2635         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2636 }
2637
2638 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2639                          char *buf)
2640 {
2641         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2642
2643         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2644 }
2645
2646 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2647                          char *buf)
2648 {
2649         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2650
2651         return sprintf(buf, "%pI6\n", target->sgid.raw);
2652 }
2653
2654 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2655                          char *buf)
2656 {
2657         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2658         struct srp_rdma_ch *ch = &target->ch[0];
2659
2660         return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2661 }
2662
2663 static ssize_t show_orig_dgid(struct device *dev,
2664                               struct device_attribute *attr, char *buf)
2665 {
2666         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2667
2668         return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2669 }
2670
2671 static ssize_t show_req_lim(struct device *dev,
2672                             struct device_attribute *attr, char *buf)
2673 {
2674         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2675         struct srp_rdma_ch *ch;
2676         int i, req_lim = INT_MAX;
2677
2678         for (i = 0; i < target->ch_count; i++) {
2679                 ch = &target->ch[i];
2680                 req_lim = min(req_lim, ch->req_lim);
2681         }
2682         return sprintf(buf, "%d\n", req_lim);
2683 }
2684
2685 static ssize_t show_zero_req_lim(struct device *dev,
2686                                  struct device_attribute *attr, char *buf)
2687 {
2688         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2689
2690         return sprintf(buf, "%d\n", target->zero_req_lim);
2691 }
2692
2693 static ssize_t show_local_ib_port(struct device *dev,
2694                                   struct device_attribute *attr, char *buf)
2695 {
2696         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2697
2698         return sprintf(buf, "%d\n", target->srp_host->port);
2699 }
2700
2701 static ssize_t show_local_ib_device(struct device *dev,
2702                                     struct device_attribute *attr, char *buf)
2703 {
2704         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2705
2706         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2707 }
2708
2709 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2710                              char *buf)
2711 {
2712         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2713
2714         return sprintf(buf, "%d\n", target->ch_count);
2715 }
2716
2717 static ssize_t show_comp_vector(struct device *dev,
2718                                 struct device_attribute *attr, char *buf)
2719 {
2720         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2721
2722         return sprintf(buf, "%d\n", target->comp_vector);
2723 }
2724
2725 static ssize_t show_tl_retry_count(struct device *dev,
2726                                    struct device_attribute *attr, char *buf)
2727 {
2728         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2729
2730         return sprintf(buf, "%d\n", target->tl_retry_count);
2731 }
2732
2733 static ssize_t show_cmd_sg_entries(struct device *dev,
2734                                    struct device_attribute *attr, char *buf)
2735 {
2736         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2737
2738         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2739 }
2740
2741 static ssize_t show_allow_ext_sg(struct device *dev,
2742                                  struct device_attribute *attr, char *buf)
2743 {
2744         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2745
2746         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2747 }
2748
2749 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2750 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2751 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2752 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2753 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2754 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2755 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2756 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2757 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2758 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2759 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2760 static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2761 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2762 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2763 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2764 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2765
2766 static struct device_attribute *srp_host_attrs[] = {
2767         &dev_attr_id_ext,
2768         &dev_attr_ioc_guid,
2769         &dev_attr_service_id,
2770         &dev_attr_pkey,
2771         &dev_attr_sgid,
2772         &dev_attr_dgid,
2773         &dev_attr_orig_dgid,
2774         &dev_attr_req_lim,
2775         &dev_attr_zero_req_lim,
2776         &dev_attr_local_ib_port,
2777         &dev_attr_local_ib_device,
2778         &dev_attr_ch_count,
2779         &dev_attr_comp_vector,
2780         &dev_attr_tl_retry_count,
2781         &dev_attr_cmd_sg_entries,
2782         &dev_attr_allow_ext_sg,
2783         NULL
2784 };
2785
2786 static struct scsi_host_template srp_template = {
2787         .module                         = THIS_MODULE,
2788         .name                           = "InfiniBand SRP initiator",
2789         .proc_name                      = DRV_NAME,
2790         .slave_configure                = srp_slave_configure,
2791         .info                           = srp_target_info,
2792         .queuecommand                   = srp_queuecommand,
2793         .change_queue_depth             = srp_change_queue_depth,
2794         .eh_abort_handler               = srp_abort,
2795         .eh_device_reset_handler        = srp_reset_device,
2796         .eh_host_reset_handler          = srp_reset_host,
2797         .skip_settle_delay              = true,
2798         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2799         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2800         .this_id                        = -1,
2801         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2802         .use_clustering                 = ENABLE_CLUSTERING,
2803         .shost_attrs                    = srp_host_attrs,
2804         .track_queue_depth              = 1,
2805 };
2806
2807 static int srp_sdev_count(struct Scsi_Host *host)
2808 {
2809         struct scsi_device *sdev;
2810         int c = 0;
2811
2812         shost_for_each_device(sdev, host)
2813                 c++;
2814
2815         return c;
2816 }
2817
2818 /*
2819  * Return values:
2820  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2821  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2822  *    removal has been scheduled.
2823  * 0 and target->state != SRP_TARGET_REMOVED upon success.
2824  */
2825 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2826 {
2827         struct srp_rport_identifiers ids;
2828         struct srp_rport *rport;
2829
2830         target->state = SRP_TARGET_SCANNING;
2831         sprintf(target->target_name, "SRP.T10:%016llX",
2832                 be64_to_cpu(target->id_ext));
2833
2834         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2835                 return -ENODEV;
2836
2837         memcpy(ids.port_id, &target->id_ext, 8);
2838         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2839         ids.roles = SRP_RPORT_ROLE_TARGET;
2840         rport = srp_rport_add(target->scsi_host, &ids);
2841         if (IS_ERR(rport)) {
2842                 scsi_remove_host(target->scsi_host);
2843                 return PTR_ERR(rport);
2844         }
2845
2846         rport->lld_data = target;
2847         target->rport = rport;
2848
2849         spin_lock(&host->target_lock);
2850         list_add_tail(&target->list, &host->target_list);
2851         spin_unlock(&host->target_lock);
2852
2853         scsi_scan_target(&target->scsi_host->shost_gendev,
2854                          0, target->scsi_id, SCAN_WILD_CARD, 0);
2855
2856         if (srp_connected_ch(target) < target->ch_count ||
2857             target->qp_in_error) {
2858                 shost_printk(KERN_INFO, target->scsi_host,
2859                              PFX "SCSI scan failed - removing SCSI host\n");
2860                 srp_queue_remove_work(target);
2861                 goto out;
2862         }
2863
2864         pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
2865                  dev_name(&target->scsi_host->shost_gendev),
2866                  srp_sdev_count(target->scsi_host));
2867
2868         spin_lock_irq(&target->lock);
2869         if (target->state == SRP_TARGET_SCANNING)
2870                 target->state = SRP_TARGET_LIVE;
2871         spin_unlock_irq(&target->lock);
2872
2873 out:
2874         return 0;
2875 }
2876
2877 static void srp_release_dev(struct device *dev)
2878 {
2879         struct srp_host *host =
2880                 container_of(dev, struct srp_host, dev);
2881
2882         complete(&host->released);
2883 }
2884
2885 static struct class srp_class = {
2886         .name    = "infiniband_srp",
2887         .dev_release = srp_release_dev
2888 };
2889
2890 /**
2891  * srp_conn_unique() - check whether the connection to a target is unique
2892  * @host:   SRP host.
2893  * @target: SRP target port.
2894  */
2895 static bool srp_conn_unique(struct srp_host *host,
2896                             struct srp_target_port *target)
2897 {
2898         struct srp_target_port *t;
2899         bool ret = false;
2900
2901         if (target->state == SRP_TARGET_REMOVED)
2902                 goto out;
2903
2904         ret = true;
2905
2906         spin_lock(&host->target_lock);
2907         list_for_each_entry(t, &host->target_list, list) {
2908                 if (t != target &&
2909                     target->id_ext == t->id_ext &&
2910                     target->ioc_guid == t->ioc_guid &&
2911                     target->initiator_ext == t->initiator_ext) {
2912                         ret = false;
2913                         break;
2914                 }
2915         }
2916         spin_unlock(&host->target_lock);
2917
2918 out:
2919         return ret;
2920 }
2921
2922 /*
2923  * Target ports are added by writing
2924  *
2925  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2926  *     pkey=<P_Key>,service_id=<service ID>
2927  *
2928  * to the add_target sysfs attribute.
2929  */
2930 enum {
2931         SRP_OPT_ERR             = 0,
2932         SRP_OPT_ID_EXT          = 1 << 0,
2933         SRP_OPT_IOC_GUID        = 1 << 1,
2934         SRP_OPT_DGID            = 1 << 2,
2935         SRP_OPT_PKEY            = 1 << 3,
2936         SRP_OPT_SERVICE_ID      = 1 << 4,
2937         SRP_OPT_MAX_SECT        = 1 << 5,
2938         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2939         SRP_OPT_IO_CLASS        = 1 << 7,
2940         SRP_OPT_INITIATOR_EXT   = 1 << 8,
2941         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
2942         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
2943         SRP_OPT_SG_TABLESIZE    = 1 << 11,
2944         SRP_OPT_COMP_VECTOR     = 1 << 12,
2945         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
2946         SRP_OPT_QUEUE_SIZE      = 1 << 14,
2947         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
2948                                    SRP_OPT_IOC_GUID     |
2949                                    SRP_OPT_DGID         |
2950                                    SRP_OPT_PKEY         |
2951                                    SRP_OPT_SERVICE_ID),
2952 };
2953
2954 static const match_table_t srp_opt_tokens = {
2955         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
2956         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
2957         { SRP_OPT_DGID,                 "dgid=%s"               },
2958         { SRP_OPT_PKEY,                 "pkey=%x"               },
2959         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
2960         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
2961         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
2962         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
2963         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
2964         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
2965         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
2966         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
2967         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
2968         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
2969         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
2970         { SRP_OPT_ERR,                  NULL                    }
2971 };
2972
2973 static int srp_parse_options(const char *buf, struct srp_target_port *target)
2974 {
2975         char *options, *sep_opt;
2976         char *p;
2977         char dgid[3];
2978         substring_t args[MAX_OPT_ARGS];
2979         int opt_mask = 0;
2980         int token;
2981         int ret = -EINVAL;
2982         int i;
2983
2984         options = kstrdup(buf, GFP_KERNEL);
2985         if (!options)
2986                 return -ENOMEM;
2987
2988         sep_opt = options;
2989         while ((p = strsep(&sep_opt, ",\n")) != NULL) {
2990                 if (!*p)
2991                         continue;
2992
2993                 token = match_token(p, srp_opt_tokens, args);
2994                 opt_mask |= token;
2995
2996                 switch (token) {
2997                 case SRP_OPT_ID_EXT:
2998                         p = match_strdup(args);
2999                         if (!p) {
3000                                 ret = -ENOMEM;
3001                                 goto out;
3002                         }
3003                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3004                         kfree(p);
3005                         break;
3006
3007                 case SRP_OPT_IOC_GUID:
3008                         p = match_strdup(args);
3009                         if (!p) {
3010                                 ret = -ENOMEM;
3011                                 goto out;
3012                         }
3013                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
3014                         kfree(p);
3015                         break;
3016
3017                 case SRP_OPT_DGID:
3018                         p = match_strdup(args);
3019                         if (!p) {
3020                                 ret = -ENOMEM;
3021                                 goto out;
3022                         }
3023                         if (strlen(p) != 32) {
3024                                 pr_warn("bad dest GID parameter '%s'\n", p);
3025                                 kfree(p);
3026                                 goto out;
3027                         }
3028
3029                         for (i = 0; i < 16; ++i) {
3030                                 strlcpy(dgid, p + i * 2, sizeof(dgid));
3031                                 if (sscanf(dgid, "%hhx",
3032                                            &target->orig_dgid.raw[i]) < 1) {
3033                                         ret = -EINVAL;
3034                                         kfree(p);
3035                                         goto out;
3036                                 }
3037                         }
3038                         kfree(p);
3039                         break;
3040
3041                 case SRP_OPT_PKEY:
3042                         if (match_hex(args, &token)) {
3043                                 pr_warn("bad P_Key parameter '%s'\n", p);
3044                                 goto out;
3045                         }
3046                         target->pkey = cpu_to_be16(token);
3047                         break;
3048
3049                 case SRP_OPT_SERVICE_ID:
3050                         p = match_strdup(args);
3051                         if (!p) {
3052                                 ret = -ENOMEM;
3053                                 goto out;
3054                         }
3055                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3056                         kfree(p);
3057                         break;
3058
3059                 case SRP_OPT_MAX_SECT:
3060                         if (match_int(args, &token)) {
3061                                 pr_warn("bad max sect parameter '%s'\n", p);
3062                                 goto out;
3063                         }
3064                         target->scsi_host->max_sectors = token;
3065                         break;
3066
3067                 case SRP_OPT_QUEUE_SIZE:
3068                         if (match_int(args, &token) || token < 1) {
3069                                 pr_warn("bad queue_size parameter '%s'\n", p);
3070                                 goto out;
3071                         }
3072                         target->scsi_host->can_queue = token;
3073                         target->queue_size = token + SRP_RSP_SQ_SIZE +
3074                                              SRP_TSK_MGMT_SQ_SIZE;
3075                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3076                                 target->scsi_host->cmd_per_lun = token;
3077                         break;
3078
3079                 case SRP_OPT_MAX_CMD_PER_LUN:
3080                         if (match_int(args, &token) || token < 1) {
3081                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3082                                         p);
3083                                 goto out;
3084                         }
3085                         target->scsi_host->cmd_per_lun = token;
3086                         break;
3087
3088                 case SRP_OPT_IO_CLASS:
3089                         if (match_hex(args, &token)) {
3090                                 pr_warn("bad IO class parameter '%s'\n", p);
3091                                 goto out;
3092                         }
3093                         if (token != SRP_REV10_IB_IO_CLASS &&
3094                             token != SRP_REV16A_IB_IO_CLASS) {
3095                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3096                                         token, SRP_REV10_IB_IO_CLASS,
3097                                         SRP_REV16A_IB_IO_CLASS);
3098                                 goto out;
3099                         }
3100                         target->io_class = token;
3101                         break;
3102
3103                 case SRP_OPT_INITIATOR_EXT:
3104                         p = match_strdup(args);
3105                         if (!p) {
3106                                 ret = -ENOMEM;
3107                                 goto out;
3108                         }
3109                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3110                         kfree(p);
3111                         break;
3112
3113                 case SRP_OPT_CMD_SG_ENTRIES:
3114                         if (match_int(args, &token) || token < 1 || token > 255) {
3115                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3116                                         p);
3117                                 goto out;
3118                         }
3119                         target->cmd_sg_cnt = token;
3120                         break;
3121
3122                 case SRP_OPT_ALLOW_EXT_SG:
3123                         if (match_int(args, &token)) {
3124                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3125                                 goto out;
3126                         }
3127                         target->allow_ext_sg = !!token;
3128                         break;
3129
3130                 case SRP_OPT_SG_TABLESIZE:
3131                         if (match_int(args, &token) || token < 1 ||
3132                                         token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
3133                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
3134                                         p);
3135                                 goto out;
3136                         }
3137                         target->sg_tablesize = token;
3138                         break;
3139
3140                 case SRP_OPT_COMP_VECTOR:
3141                         if (match_int(args, &token) || token < 0) {
3142                                 pr_warn("bad comp_vector parameter '%s'\n", p);
3143                                 goto out;
3144                         }
3145                         target->comp_vector = token;
3146                         break;
3147
3148                 case SRP_OPT_TL_RETRY_COUNT:
3149                         if (match_int(args, &token) || token < 2 || token > 7) {
3150                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3151                                         p);
3152                                 goto out;
3153                         }
3154                         target->tl_retry_count = token;
3155                         break;
3156
3157                 default:
3158                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3159                                 p);
3160                         goto out;
3161                 }
3162         }
3163
3164         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3165                 ret = 0;
3166         else
3167                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3168                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3169                             !(srp_opt_tokens[i].token & opt_mask))
3170                                 pr_warn("target creation request is missing parameter '%s'\n",
3171                                         srp_opt_tokens[i].pattern);
3172
3173         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3174             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3175                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3176                         target->scsi_host->cmd_per_lun,
3177                         target->scsi_host->can_queue);
3178
3179 out:
3180         kfree(options);
3181         return ret;
3182 }
3183
3184 static ssize_t srp_create_target(struct device *dev,
3185                                  struct device_attribute *attr,
3186                                  const char *buf, size_t count)
3187 {
3188         struct srp_host *host =
3189                 container_of(dev, struct srp_host, dev);
3190         struct Scsi_Host *target_host;
3191         struct srp_target_port *target;
3192         struct srp_rdma_ch *ch;
3193         struct srp_device *srp_dev = host->srp_dev;
3194         struct ib_device *ibdev = srp_dev->dev;
3195         int ret, node_idx, node, cpu, i;
3196         bool multich = false;
3197
3198         target_host = scsi_host_alloc(&srp_template,
3199                                       sizeof (struct srp_target_port));
3200         if (!target_host)
3201                 return -ENOMEM;
3202
3203         target_host->transportt  = ib_srp_transport_template;
3204         target_host->max_channel = 0;
3205         target_host->max_id      = 1;
3206         target_host->max_lun     = -1LL;
3207         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3208
3209         target = host_to_target(target_host);
3210
3211         target->io_class        = SRP_REV16A_IB_IO_CLASS;
3212         target->scsi_host       = target_host;
3213         target->srp_host        = host;
3214         target->lkey            = host->srp_dev->pd->local_dma_lkey;
3215         target->global_mr       = host->srp_dev->global_mr;
3216         target->cmd_sg_cnt      = cmd_sg_entries;
3217         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
3218         target->allow_ext_sg    = allow_ext_sg;
3219         target->tl_retry_count  = 7;
3220         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
3221
3222         /*
3223          * Avoid that the SCSI host can be removed by srp_remove_target()
3224          * before this function returns.
3225          */
3226         scsi_host_get(target->scsi_host);
3227
3228         mutex_lock(&host->add_target_mutex);
3229
3230         ret = srp_parse_options(buf, target);
3231         if (ret)
3232                 goto out;
3233
3234         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3235
3236         if (!srp_conn_unique(target->srp_host, target)) {
3237                 shost_printk(KERN_INFO, target->scsi_host,
3238                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3239                              be64_to_cpu(target->id_ext),
3240                              be64_to_cpu(target->ioc_guid),
3241                              be64_to_cpu(target->initiator_ext));
3242                 ret = -EEXIST;
3243                 goto out;
3244         }
3245
3246         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3247             target->cmd_sg_cnt < target->sg_tablesize) {
3248                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3249                 target->sg_tablesize = target->cmd_sg_cnt;
3250         }
3251
3252         target_host->sg_tablesize = target->sg_tablesize;
3253         target->indirect_size = target->sg_tablesize *
3254                                 sizeof (struct srp_direct_buf);
3255         target->max_iu_len = sizeof (struct srp_cmd) +
3256                              sizeof (struct srp_indirect_buf) +
3257                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3258
3259         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3260         INIT_WORK(&target->remove_work, srp_remove_work);
3261         spin_lock_init(&target->lock);
3262         ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3263         if (ret)
3264                 goto out;
3265
3266         ret = -ENOMEM;
3267         target->ch_count = max_t(unsigned, num_online_nodes(),
3268                                  min(ch_count ? :
3269                                      min(4 * num_online_nodes(),
3270                                          ibdev->num_comp_vectors),
3271                                      num_online_cpus()));
3272         target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3273                              GFP_KERNEL);
3274         if (!target->ch)
3275                 goto out;
3276
3277         node_idx = 0;
3278         for_each_online_node(node) {
3279                 const int ch_start = (node_idx * target->ch_count /
3280                                       num_online_nodes());
3281                 const int ch_end = ((node_idx + 1) * target->ch_count /
3282                                     num_online_nodes());
3283                 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3284                                       num_online_nodes() + target->comp_vector)
3285                                      % ibdev->num_comp_vectors;
3286                 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3287                                     num_online_nodes() + target->comp_vector)
3288                                    % ibdev->num_comp_vectors;
3289                 int cpu_idx = 0;
3290
3291                 for_each_online_cpu(cpu) {
3292                         if (cpu_to_node(cpu) != node)
3293                                 continue;
3294                         if (ch_start + cpu_idx >= ch_end)
3295                                 continue;
3296                         ch = &target->ch[ch_start + cpu_idx];
3297                         ch->target = target;
3298                         ch->comp_vector = cv_start == cv_end ? cv_start :
3299                                 cv_start + cpu_idx % (cv_end - cv_start);
3300                         spin_lock_init(&ch->lock);
3301                         INIT_LIST_HEAD(&ch->free_tx);
3302                         ret = srp_new_cm_id(ch);
3303                         if (ret)
3304                                 goto err_disconnect;
3305
3306                         ret = srp_create_ch_ib(ch);
3307                         if (ret)
3308                                 goto err_disconnect;
3309
3310                         ret = srp_alloc_req_data(ch);
3311                         if (ret)
3312                                 goto err_disconnect;
3313
3314                         ret = srp_connect_ch(ch, multich);
3315                         if (ret) {
3316                                 shost_printk(KERN_ERR, target->scsi_host,
3317                                              PFX "Connection %d/%d failed\n",
3318                                              ch_start + cpu_idx,
3319                                              target->ch_count);
3320                                 if (node_idx == 0 && cpu_idx == 0) {
3321                                         goto err_disconnect;
3322                                 } else {
3323                                         srp_free_ch_ib(target, ch);
3324                                         srp_free_req_data(target, ch);
3325                                         target->ch_count = ch - target->ch;
3326                                         goto connected;
3327                                 }
3328                         }
3329
3330                         multich = true;
3331                         cpu_idx++;
3332                 }
3333                 node_idx++;
3334         }
3335
3336 connected:
3337         target->scsi_host->nr_hw_queues = target->ch_count;
3338
3339         ret = srp_add_target(host, target);
3340         if (ret)
3341                 goto err_disconnect;
3342
3343         if (target->state != SRP_TARGET_REMOVED) {
3344                 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3345                              "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3346                              be64_to_cpu(target->id_ext),
3347                              be64_to_cpu(target->ioc_guid),
3348                              be16_to_cpu(target->pkey),
3349                              be64_to_cpu(target->service_id),
3350                              target->sgid.raw, target->orig_dgid.raw);
3351         }
3352
3353         ret = count;
3354
3355 out:
3356         mutex_unlock(&host->add_target_mutex);
3357
3358         scsi_host_put(target->scsi_host);
3359         if (ret < 0)
3360                 scsi_host_put(target->scsi_host);
3361
3362         return ret;
3363
3364 err_disconnect:
3365         srp_disconnect_target(target);
3366
3367         for (i = 0; i < target->ch_count; i++) {
3368                 ch = &target->ch[i];
3369                 srp_free_ch_ib(target, ch);
3370                 srp_free_req_data(target, ch);
3371         }
3372
3373         kfree(target->ch);
3374         goto out;
3375 }
3376
3377 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3378
3379 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3380                           char *buf)
3381 {
3382         struct srp_host *host = container_of(dev, struct srp_host, dev);
3383
3384         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3385 }
3386
3387 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3388
3389 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3390                          char *buf)
3391 {
3392         struct srp_host *host = container_of(dev, struct srp_host, dev);
3393
3394         return sprintf(buf, "%d\n", host->port);
3395 }
3396
3397 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3398
3399 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3400 {
3401         struct srp_host *host;
3402
3403         host = kzalloc(sizeof *host, GFP_KERNEL);
3404         if (!host)
3405                 return NULL;
3406
3407         INIT_LIST_HEAD(&host->target_list);
3408         spin_lock_init(&host->target_lock);
3409         init_completion(&host->released);
3410         mutex_init(&host->add_target_mutex);
3411         host->srp_dev = device;
3412         host->port = port;
3413
3414         host->dev.class = &srp_class;
3415         host->dev.parent = device->dev->dma_device;
3416         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3417
3418         if (device_register(&host->dev))
3419                 goto free_host;
3420         if (device_create_file(&host->dev, &dev_attr_add_target))
3421                 goto err_class;
3422         if (device_create_file(&host->dev, &dev_attr_ibdev))
3423                 goto err_class;
3424         if (device_create_file(&host->dev, &dev_attr_port))
3425                 goto err_class;
3426
3427         return host;
3428
3429 err_class:
3430         device_unregister(&host->dev);
3431
3432 free_host:
3433         kfree(host);
3434
3435         return NULL;
3436 }
3437
3438 static void srp_add_one(struct ib_device *device)
3439 {
3440         struct srp_device *srp_dev;
3441         struct ib_device_attr *dev_attr;
3442         struct srp_host *host;
3443         int mr_page_shift, p;
3444         u64 max_pages_per_mr;
3445
3446         dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3447         if (!dev_attr)
3448                 return;
3449
3450         if (ib_query_device(device, dev_attr)) {
3451                 pr_warn("Query device failed for %s\n", device->name);
3452                 goto free_attr;
3453         }
3454
3455         srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3456         if (!srp_dev)
3457                 goto free_attr;
3458
3459         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3460                             device->map_phys_fmr && device->unmap_fmr);
3461         srp_dev->has_fr = (dev_attr->device_cap_flags &
3462                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3463         if (!srp_dev->has_fmr && !srp_dev->has_fr)
3464                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3465
3466         srp_dev->use_fast_reg = (srp_dev->has_fr &&
3467                                  (!srp_dev->has_fmr || prefer_fr));
3468         srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3469
3470         /*
3471          * Use the smallest page size supported by the HCA, down to a
3472          * minimum of 4096 bytes. We're unlikely to build large sglists
3473          * out of smaller entries.
3474          */
3475         mr_page_shift           = max(12, ffs(dev_attr->page_size_cap) - 1);
3476         srp_dev->mr_page_size   = 1 << mr_page_shift;
3477         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3478         max_pages_per_mr        = dev_attr->max_mr_size;
3479         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3480         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3481                                           max_pages_per_mr);
3482         if (srp_dev->use_fast_reg) {
3483                 srp_dev->max_pages_per_mr =
3484                         min_t(u32, srp_dev->max_pages_per_mr,
3485                               dev_attr->max_fast_reg_page_list_len);
3486         }
3487         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3488                                    srp_dev->max_pages_per_mr;
3489         pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3490                  device->name, mr_page_shift, dev_attr->max_mr_size,
3491                  dev_attr->max_fast_reg_page_list_len,
3492                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3493
3494         INIT_LIST_HEAD(&srp_dev->dev_list);
3495
3496         srp_dev->dev = device;
3497         srp_dev->pd  = ib_alloc_pd(device);
3498         if (IS_ERR(srp_dev->pd))
3499                 goto free_dev;
3500
3501         if (!register_always || (!srp_dev->has_fmr && !srp_dev->has_fr)) {
3502                 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
3503                                                    IB_ACCESS_LOCAL_WRITE |
3504                                                    IB_ACCESS_REMOTE_READ |
3505                                                    IB_ACCESS_REMOTE_WRITE);
3506                 if (IS_ERR(srp_dev->global_mr))
3507                         goto err_pd;
3508         } else {
3509                 srp_dev->global_mr = NULL;
3510         }
3511
3512         for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3513                 host = srp_add_port(srp_dev, p);
3514                 if (host)
3515                         list_add_tail(&host->list, &srp_dev->dev_list);
3516         }
3517
3518         ib_set_client_data(device, &srp_client, srp_dev);
3519
3520         goto free_attr;
3521
3522 err_pd:
3523         ib_dealloc_pd(srp_dev->pd);
3524
3525 free_dev:
3526         kfree(srp_dev);
3527
3528 free_attr:
3529         kfree(dev_attr);
3530 }
3531
3532 static void srp_remove_one(struct ib_device *device, void *client_data)
3533 {
3534         struct srp_device *srp_dev;
3535         struct srp_host *host, *tmp_host;
3536         struct srp_target_port *target;
3537
3538         srp_dev = client_data;
3539         if (!srp_dev)
3540                 return;
3541
3542         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3543                 device_unregister(&host->dev);
3544                 /*
3545                  * Wait for the sysfs entry to go away, so that no new
3546                  * target ports can be created.
3547                  */
3548                 wait_for_completion(&host->released);
3549
3550                 /*
3551                  * Remove all target ports.
3552                  */
3553                 spin_lock(&host->target_lock);
3554                 list_for_each_entry(target, &host->target_list, list)
3555                         srp_queue_remove_work(target);
3556                 spin_unlock(&host->target_lock);
3557
3558                 /*
3559                  * Wait for tl_err and target port removal tasks.
3560                  */
3561                 flush_workqueue(system_long_wq);
3562                 flush_workqueue(srp_remove_wq);
3563
3564                 kfree(host);
3565         }
3566
3567         if (srp_dev->global_mr)
3568                 ib_dereg_mr(srp_dev->global_mr);
3569         ib_dealloc_pd(srp_dev->pd);
3570
3571         kfree(srp_dev);
3572 }
3573
3574 static struct srp_function_template ib_srp_transport_functions = {
3575         .has_rport_state         = true,
3576         .reset_timer_if_blocked  = true,
3577         .reconnect_delay         = &srp_reconnect_delay,
3578         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3579         .dev_loss_tmo            = &srp_dev_loss_tmo,
3580         .reconnect               = srp_rport_reconnect,
3581         .rport_delete            = srp_rport_delete,
3582         .terminate_rport_io      = srp_terminate_io,
3583 };
3584
3585 static int __init srp_init_module(void)
3586 {
3587         int ret;
3588
3589         if (srp_sg_tablesize) {
3590                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3591                 if (!cmd_sg_entries)
3592                         cmd_sg_entries = srp_sg_tablesize;
3593         }
3594
3595         if (!cmd_sg_entries)
3596                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3597
3598         if (cmd_sg_entries > 255) {
3599                 pr_warn("Clamping cmd_sg_entries to 255\n");
3600                 cmd_sg_entries = 255;
3601         }
3602
3603         if (!indirect_sg_entries)
3604                 indirect_sg_entries = cmd_sg_entries;
3605         else if (indirect_sg_entries < cmd_sg_entries) {
3606                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3607                         cmd_sg_entries);
3608                 indirect_sg_entries = cmd_sg_entries;
3609         }
3610
3611         srp_remove_wq = create_workqueue("srp_remove");
3612         if (!srp_remove_wq) {
3613                 ret = -ENOMEM;
3614                 goto out;
3615         }
3616
3617         ret = -ENOMEM;
3618         ib_srp_transport_template =
3619                 srp_attach_transport(&ib_srp_transport_functions);
3620         if (!ib_srp_transport_template)
3621                 goto destroy_wq;
3622
3623         ret = class_register(&srp_class);
3624         if (ret) {
3625                 pr_err("couldn't register class infiniband_srp\n");
3626                 goto release_tr;
3627         }
3628
3629         ib_sa_register_client(&srp_sa_client);
3630
3631         ret = ib_register_client(&srp_client);
3632         if (ret) {
3633                 pr_err("couldn't register IB client\n");
3634                 goto unreg_sa;
3635         }
3636
3637 out:
3638         return ret;
3639
3640 unreg_sa:
3641         ib_sa_unregister_client(&srp_sa_client);
3642         class_unregister(&srp_class);
3643
3644 release_tr:
3645         srp_release_transport(ib_srp_transport_template);
3646
3647 destroy_wq:
3648         destroy_workqueue(srp_remove_wq);
3649         goto out;
3650 }
3651
3652 static void __exit srp_cleanup_module(void)
3653 {
3654         ib_unregister_client(&srp_client);
3655         ib_sa_unregister_client(&srp_sa_client);
3656         class_unregister(&srp_class);
3657         srp_release_transport(ib_srp_transport_template);
3658         destroy_workqueue(srp_remove_wq);
3659 }
3660
3661 module_init(srp_init_module);
3662 module_exit(srp_cleanup_module);