drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   5  *
   6  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   7  *
   8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  ******************************************************************************/
  20
  21 #include <linux/string.h>
  22 #include <linux/kthread.h>
  23 #include <linux/crypto.h>
  24 #include <linux/completion.h>
  25 #include <linux/module.h>
  26 #include <linux/idr.h>
  27 #include <asm/unaligned.h>
  28 #include <scsi/scsi_device.h>
  29 #include <scsi/iscsi_proto.h>
  30 #include <target/target_core_base.h>
  31 #include <target/target_core_fabric.h>
  32
  33 #include "iscsi_target_core.h"
  34 #include "iscsi_target_parameters.h"
  35 #include "iscsi_target_seq_pdu_list.h"
  36 #include "iscsi_target_tq.h"
  37 #include "iscsi_target_configfs.h"
  38 #include "iscsi_target_datain_values.h"
  39 #include "iscsi_target_erl0.h"
  40 #include "iscsi_target_erl1.h"
  41 #include "iscsi_target_erl2.h"
  42 #include "iscsi_target_login.h"
  43 #include "iscsi_target_tmr.h"
  44 #include "iscsi_target_tpg.h"
  45 #include "iscsi_target_util.h"
  46 #include "iscsi_target.h"
  47 #include "iscsi_target_device.h"
  48 #include "iscsi_target_stat.h"
  49
  50 static LIST_HEAD(g_tiqn_list);
  51 static LIST_HEAD(g_np_list);
  52 static DEFINE_SPINLOCK(tiqn_lock);
  53 static DEFINE_SPINLOCK(np_lock);
  54
  55 static struct idr tiqn_idr;
  56 struct idr sess_idr;
  57 struct mutex auth_id_lock;
  58 spinlock_t sess_idr_lock;
  59
  60 struct iscsit_global *iscsit_global;
  61
  62 struct kmem_cache *lio_cmd_cache;
  63 struct kmem_cache *lio_qr_cache;
  64 struct kmem_cache *lio_dr_cache;
  65 struct kmem_cache *lio_ooo_cache;
  66 struct kmem_cache *lio_r2t_cache;
  67
  68 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  69                         unsigned char *buf, u32);
  70 static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
  71
  72 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  73 {
  74         struct iscsi_tiqn *tiqn = NULL;
  75
  76         spin_lock(&tiqn_lock);
  77         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  78                 if (!strcmp(tiqn->tiqn, buf)) {
  79
  80                         spin_lock(&tiqn->tiqn_state_lock);
  81                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  82                                 tiqn->tiqn_access_count++;
  83                                 spin_unlock(&tiqn->tiqn_state_lock);
  84                                 spin_unlock(&tiqn_lock);
  85                                 return tiqn;
  86                         }
  87                         spin_unlock(&tiqn->tiqn_state_lock);
  88                 }
  89         }
  90         spin_unlock(&tiqn_lock);
  91
  92         return NULL;
  93 }
  94
  95 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  96 {
  97         spin_lock(&tiqn->tiqn_state_lock);
  98         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  99                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 100                 spin_unlock(&tiqn->tiqn_state_lock);
 101                 return 0;
 102         }
 103         spin_unlock(&tiqn->tiqn_state_lock);
 104
 105         return -1;
 106 }
 107
 108 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 109 {
 110         spin_lock(&tiqn->tiqn_state_lock);
 111         tiqn->tiqn_access_count--;
 112         spin_unlock(&tiqn->tiqn_state_lock);
 113 }
 114
 115 /*
 116  * Note that IQN formatting is expected to be done in userspace, and
 117  * no explict IQN format checks are done here.
 118  */
 119 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 120 {
 121         struct iscsi_tiqn *tiqn = NULL;
 122         int ret;
 123
 124         if (strlen(buf) >= ISCSI_IQN_LEN) {
 125                 pr_err("Target IQN exceeds %d bytes\n",
 126                                 ISCSI_IQN_LEN);
 127                 return ERR_PTR(-EINVAL);
 128         }
 129
 130         tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
 131         if (!tiqn) {
 132                 pr_err("Unable to allocate struct iscsi_tiqn\n");
 133                 return ERR_PTR(-ENOMEM);
 134         }
 135
 136         sprintf(tiqn->tiqn, "%s", buf);
 137         INIT_LIST_HEAD(&tiqn->tiqn_list);
 138         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 139         spin_lock_init(&tiqn->tiqn_state_lock);
 140         spin_lock_init(&tiqn->tiqn_tpg_lock);
 141         spin_lock_init(&tiqn->sess_err_stats.lock);
 142         spin_lock_init(&tiqn->login_stats.lock);
 143         spin_lock_init(&tiqn->logout_stats.lock);
 144
 145         if (!idr_pre_get(&tiqn_idr, GFP_KERNEL)) {
 146                 pr_err("idr_pre_get() for tiqn_idr failed\n");
 147                 kfree(tiqn);
 148                 return ERR_PTR(-ENOMEM);
 149         }
 150         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 151
 152         spin_lock(&tiqn_lock);
 153         ret = idr_get_new(&tiqn_idr, NULL, &tiqn->tiqn_index);
 154         if (ret < 0) {
 155                 pr_err("idr_get_new() failed for tiqn->tiqn_index\n");
 156                 spin_unlock(&tiqn_lock);
 157                 kfree(tiqn);
 158                 return ERR_PTR(ret);
 159         }
 160         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 161         spin_unlock(&tiqn_lock);
 162
 163         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 164
 165         return tiqn;
 166
 167 }
 168
 169 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 170 {
 171         /*
 172          * Wait for accesses to said struct iscsi_tiqn to end.
 173          */
 174         spin_lock(&tiqn->tiqn_state_lock);
 175         while (tiqn->tiqn_access_count != 0) {
 176                 spin_unlock(&tiqn->tiqn_state_lock);
 177                 msleep(10);
 178                 spin_lock(&tiqn->tiqn_state_lock);
 179         }
 180         spin_unlock(&tiqn->tiqn_state_lock);
 181 }
 182
 183 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 184 {
 185         /*
 186          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 187          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 188          * attempts to access this struct iscsi_tiqn will fail from both transport
 189          * fabric and control code paths.
 190          */
 191         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 192                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 193                 return;
 194         }
 195
 196         iscsit_wait_for_tiqn(tiqn);
 197
 198         spin_lock(&tiqn_lock);
 199         list_del(&tiqn->tiqn_list);
 200         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 201         spin_unlock(&tiqn_lock);
 202
 203         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 204                         tiqn->tiqn);
 205         kfree(tiqn);
 206 }
 207
 208 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 209 {
 210         int ret;
 211         /*
 212          * Determine if the network portal is accepting storage traffic.
 213          */
 214         spin_lock_bh(&np->np_thread_lock);
 215         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 216                 spin_unlock_bh(&np->np_thread_lock);
 217                 return -1;
 218         }
 219         if (np->np_login_tpg) {
 220                 pr_err("np->np_login_tpg() is not NULL!\n");
 221                 spin_unlock_bh(&np->np_thread_lock);
 222                 return -1;
 223         }
 224         spin_unlock_bh(&np->np_thread_lock);
 225         /*
 226          * Determine if the portal group is accepting storage traffic.
 227          */
 228         spin_lock_bh(&tpg->tpg_state_lock);
 229         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 230                 spin_unlock_bh(&tpg->tpg_state_lock);
 231                 return -1;
 232         }
 233         spin_unlock_bh(&tpg->tpg_state_lock);
 234
 235         /*
 236          * Here we serialize access across the TIQN+TPG Tuple.
 237          */
 238         ret = mutex_lock_interruptible(&tpg->np_login_lock);
 239         if ((ret != 0) || signal_pending(current))
 240                 return -1;
 241
 242         spin_lock_bh(&np->np_thread_lock);
 243         np->np_login_tpg = tpg;
 244         spin_unlock_bh(&np->np_thread_lock);
 245
 246         return 0;
 247 }
 248
 249 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 250 {
 251         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 252
 253         spin_lock_bh(&np->np_thread_lock);
 254         np->np_login_tpg = NULL;
 255         spin_unlock_bh(&np->np_thread_lock);
 256
 257         mutex_unlock(&tpg->np_login_lock);
 258
 259         if (tiqn)
 260                 iscsit_put_tiqn_for_login(tiqn);
 261
 262         return 0;
 263 }
 264
 265 static struct iscsi_np *iscsit_get_np(
 266         struct __kernel_sockaddr_storage *sockaddr,
 267         int network_transport)
 268 {
 269         struct sockaddr_in *sock_in, *sock_in_e;
 270         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 271         struct iscsi_np *np;
 272         int ip_match = 0;
 273         u16 port;
 274
 275         spin_lock_bh(&np_lock);
 276         list_for_each_entry(np, &g_np_list, np_list) {
 277                 spin_lock(&np->np_thread_lock);
 278                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 279                         spin_unlock(&np->np_thread_lock);
 280                         continue;
 281                 }
 282
 283                 if (sockaddr->ss_family == AF_INET6) {
 284                         sock_in6 = (struct sockaddr_in6 *)sockaddr;
 285                         sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 286
 287                         if (!memcmp(&sock_in6->sin6_addr.in6_u,
 288                                     &sock_in6_e->sin6_addr.in6_u,
 289                                     sizeof(struct in6_addr)))
 290                                 ip_match = 1;
 291
 292                         port = ntohs(sock_in6->sin6_port);
 293                 } else {
 294                         sock_in = (struct sockaddr_in *)sockaddr;
 295                         sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 296
 297                         if (sock_in->sin_addr.s_addr ==
 298                             sock_in_e->sin_addr.s_addr)
 299                                 ip_match = 1;
 300
 301                         port = ntohs(sock_in->sin_port);
 302                 }
 303
 304                 if ((ip_match == 1) && (np->np_port == port) &&
 305                     (np->np_network_transport == network_transport)) {
 306                         /*
 307                          * Increment the np_exports reference count now to
 308                          * prevent iscsit_del_np() below from being called
 309                          * while iscsi_tpg_add_network_portal() is called.
 310                          */
 311                         np->np_exports++;
 312                         spin_unlock(&np->np_thread_lock);
 313                         spin_unlock_bh(&np_lock);
 314                         return np;
 315                 }
 316                 spin_unlock(&np->np_thread_lock);
 317         }
 318         spin_unlock_bh(&np_lock);
 319
 320         return NULL;
 321 }
 322
 323 struct iscsi_np *iscsit_add_np(
 324         struct __kernel_sockaddr_storage *sockaddr,
 325         char *ip_str,
 326         int network_transport)
 327 {
 328         struct sockaddr_in *sock_in;
 329         struct sockaddr_in6 *sock_in6;
 330         struct iscsi_np *np;
 331         int ret;
 332         /*
 333          * Locate the existing struct iscsi_np if already active..
 334          */
 335         np = iscsit_get_np(sockaddr, network_transport);
 336         if (np)
 337                 return np;
 338
 339         np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
 340         if (!np) {
 341                 pr_err("Unable to allocate memory for struct iscsi_np\n");
 342                 return ERR_PTR(-ENOMEM);
 343         }
 344
 345         np->np_flags |= NPF_IP_NETWORK;
 346         if (sockaddr->ss_family == AF_INET6) {
 347                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 348                 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str);
 349                 np->np_port = ntohs(sock_in6->sin6_port);
 350         } else {
 351                 sock_in = (struct sockaddr_in *)sockaddr;
 352                 sprintf(np->np_ip, "%s", ip_str);
 353                 np->np_port = ntohs(sock_in->sin_port);
 354         }
 355
 356         np->np_network_transport = network_transport;
 357         spin_lock_init(&np->np_thread_lock);
 358         init_completion(&np->np_restart_comp);
 359         INIT_LIST_HEAD(&np->np_list);
 360
 361         ret = iscsi_target_setup_login_socket(np, sockaddr);
 362         if (ret != 0) {
 363                 kfree(np);
 364                 return ERR_PTR(ret);
 365         }
 366
 367         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 368         if (IS_ERR(np->np_thread)) {
 369                 pr_err("Unable to create kthread: iscsi_np\n");
 370                 ret = PTR_ERR(np->np_thread);
 371                 kfree(np);
 372                 return ERR_PTR(ret);
 373         }
 374         /*
 375          * Increment the np_exports reference count now to prevent
 376          * iscsit_del_np() below from being run while a new call to
 377          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 378          * active.  We don't need to hold np->np_thread_lock at this
 379          * point because iscsi_np has not been added to g_np_list yet.
 380          */
 381         np->np_exports = 1;
 382
 383         spin_lock_bh(&np_lock);
 384         list_add_tail(&np->np_list, &g_np_list);
 385         spin_unlock_bh(&np_lock);
 386
 387         pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
 388                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 389                 "TCP" : "SCTP");
 390
 391         return np;
 392 }
 393
 394 int iscsit_reset_np_thread(
 395         struct iscsi_np *np,
 396         struct iscsi_tpg_np *tpg_np,
 397         struct iscsi_portal_group *tpg)
 398 {
 399         spin_lock_bh(&np->np_thread_lock);
 400         if (tpg && tpg_np) {
 401                 /*
 402                  * The reset operation need only be performed when the
 403                  * passed struct iscsi_portal_group has a login in progress
 404                  * to one of the network portals.
 405                  */
 406                 if (tpg_np->tpg_np->np_login_tpg != tpg) {
 407                         spin_unlock_bh(&np->np_thread_lock);
 408                         return 0;
 409                 }
 410         }
 411         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 412                 spin_unlock_bh(&np->np_thread_lock);
 413                 return 0;
 414         }
 415         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 416
 417         if (np->np_thread) {
 418                 spin_unlock_bh(&np->np_thread_lock);
 419                 send_sig(SIGINT, np->np_thread, 1);
 420                 wait_for_completion(&np->np_restart_comp);
 421                 spin_lock_bh(&np->np_thread_lock);
 422         }
 423         spin_unlock_bh(&np->np_thread_lock);
 424
 425         return 0;
 426 }
 427
 428 int iscsit_del_np_comm(struct iscsi_np *np)
 429 {
 430         if (!np->np_socket)
 431                 return 0;
 432
 433         /*
 434          * Some network transports allocate their own struct sock->file,
 435          * see  if we need to free any additional allocated resources.
 436          */
 437         if (np->np_flags & NPF_SCTP_STRUCT_FILE) {
 438                 kfree(np->np_socket->file);
 439                 np->np_socket->file = NULL;
 440         }
 441
 442         sock_release(np->np_socket);
 443         return 0;
 444 }
 445
 446 int iscsit_del_np(struct iscsi_np *np)
 447 {
 448         spin_lock_bh(&np->np_thread_lock);
 449         np->np_exports--;
 450         if (np->np_exports) {
 451                 spin_unlock_bh(&np->np_thread_lock);
 452                 return 0;
 453         }
 454         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 455         spin_unlock_bh(&np->np_thread_lock);
 456
 457         if (np->np_thread) {
 458                 /*
 459                  * We need to send the signal to wakeup Linux/Net
 460                  * which may be sleeping in sock_accept()..
 461                  */
 462                 send_sig(SIGINT, np->np_thread, 1);
 463                 kthread_stop(np->np_thread);
 464         }
 465         iscsit_del_np_comm(np);
 466
 467         spin_lock_bh(&np_lock);
 468         list_del(&np->np_list);
 469         spin_unlock_bh(&np_lock);
 470
 471         pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
 472                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 473                 "TCP" : "SCTP");
 474
 475         kfree(np);
 476         return 0;
 477 }
 478
 479 static int __init iscsi_target_init_module(void)
 480 {
 481         int ret = 0;
 482
 483         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 484
 485         iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
 486         if (!iscsit_global) {
 487                 pr_err("Unable to allocate memory for iscsit_global\n");
 488                 return -1;
 489         }
 490         mutex_init(&auth_id_lock);
 491         spin_lock_init(&sess_idr_lock);
 492         idr_init(&tiqn_idr);
 493         idr_init(&sess_idr);
 494
 495         ret = iscsi_target_register_configfs();
 496         if (ret < 0)
 497                 goto out;
 498
 499         ret = iscsi_thread_set_init();
 500         if (ret < 0)
 501                 goto configfs_out;
 502
 503         if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
 504                         TARGET_THREAD_SET_COUNT) {
 505                 pr_err("iscsi_allocate_thread_sets() returned"
 506                         " unexpected value!\n");
 507                 goto ts_out1;
 508         }
 509
 510         lio_cmd_cache = kmem_cache_create("lio_cmd_cache",
 511                         sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd),
 512                         0, NULL);
 513         if (!lio_cmd_cache) {
 514                 pr_err("Unable to kmem_cache_create() for"
 515                                 " lio_cmd_cache\n");
 516                 goto ts_out2;
 517         }
 518
 519         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 520                         sizeof(struct iscsi_queue_req),
 521                         __alignof__(struct iscsi_queue_req), 0, NULL);
 522         if (!lio_qr_cache) {
 523                 pr_err("nable to kmem_cache_create() for"
 524                                 " lio_qr_cache\n");
 525                 goto cmd_out;
 526         }
 527
 528         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 529                         sizeof(struct iscsi_datain_req),
 530                         __alignof__(struct iscsi_datain_req), 0, NULL);
 531         if (!lio_dr_cache) {
 532                 pr_err("Unable to kmem_cache_create() for"
 533                                 " lio_dr_cache\n");
 534                 goto qr_out;
 535         }
 536
 537         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 538                         sizeof(struct iscsi_ooo_cmdsn),
 539                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 540         if (!lio_ooo_cache) {
 541                 pr_err("Unable to kmem_cache_create() for"
 542                                 " lio_ooo_cache\n");
 543                 goto dr_out;
 544         }
 545
 546         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 547                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 548                         0, NULL);
 549         if (!lio_r2t_cache) {
 550                 pr_err("Unable to kmem_cache_create() for"
 551                                 " lio_r2t_cache\n");
 552                 goto ooo_out;
 553         }
 554
 555         if (iscsit_load_discovery_tpg() < 0)
 556                 goto r2t_out;
 557
 558         return ret;
 559 r2t_out:
 560         kmem_cache_destroy(lio_r2t_cache);
 561 ooo_out:
 562         kmem_cache_destroy(lio_ooo_cache);
 563 dr_out:
 564         kmem_cache_destroy(lio_dr_cache);
 565 qr_out:
 566         kmem_cache_destroy(lio_qr_cache);
 567 cmd_out:
 568         kmem_cache_destroy(lio_cmd_cache);
 569 ts_out2:
 570         iscsi_deallocate_thread_sets();
 571 ts_out1:
 572         iscsi_thread_set_free();
 573 configfs_out:
 574         iscsi_target_deregister_configfs();
 575 out:
 576         kfree(iscsit_global);
 577         return -ENOMEM;
 578 }
 579
 580 static void __exit iscsi_target_cleanup_module(void)
 581 {
 582         iscsi_deallocate_thread_sets();
 583         iscsi_thread_set_free();
 584         iscsit_release_discovery_tpg();
 585         kmem_cache_destroy(lio_cmd_cache);
 586         kmem_cache_destroy(lio_qr_cache);
 587         kmem_cache_destroy(lio_dr_cache);
 588         kmem_cache_destroy(lio_ooo_cache);
 589         kmem_cache_destroy(lio_r2t_cache);
 590
 591         iscsi_target_deregister_configfs();
 592
 593         kfree(iscsit_global);
 594 }
 595
 596 int iscsit_add_reject(
 597         u8 reason,
 598         int fail_conn,
 599         unsigned char *buf,
 600         struct iscsi_conn *conn)
 601 {
 602         struct iscsi_cmd *cmd;
 603         struct iscsi_reject *hdr;
 604         int ret;
 605
 606         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
 607         if (!cmd)
 608                 return -1;
 609
 610         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 611         if (fail_conn)
 612                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 613
 614         hdr     = (struct iscsi_reject *) cmd->pdu;
 615         hdr->reason = reason;
 616
 617         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 618         if (!cmd->buf_ptr) {
 619                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 620                 iscsit_release_cmd(cmd);
 621                 return -1;
 622         }
 623
 624         spin_lock_bh(&conn->cmd_lock);
 625         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 626         spin_unlock_bh(&conn->cmd_lock);
 627
 628         cmd->i_state = ISTATE_SEND_REJECT;
 629         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 630
 631         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 632         if (ret != 0)
 633                 return -1;
 634
 635         return (!fail_conn) ? 0 : -1;
 636 }
 637
 638 int iscsit_add_reject_from_cmd(
 639         u8 reason,
 640         int fail_conn,
 641         int add_to_conn,
 642         unsigned char *buf,
 643         struct iscsi_cmd *cmd)
 644 {
 645         struct iscsi_conn *conn;
 646         struct iscsi_reject *hdr;
 647         int ret;
 648
 649         if (!cmd->conn) {
 650                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 651                                 cmd->init_task_tag);
 652                 return -1;
 653         }
 654         conn = cmd->conn;
 655
 656         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 657         if (fail_conn)
 658                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 659
 660         hdr     = (struct iscsi_reject *) cmd->pdu;
 661         hdr->reason = reason;
 662
 663         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 664         if (!cmd->buf_ptr) {
 665                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 666                 iscsit_release_cmd(cmd);
 667                 return -1;
 668         }
 669
 670         if (add_to_conn) {
 671                 spin_lock_bh(&conn->cmd_lock);
 672                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 673                 spin_unlock_bh(&conn->cmd_lock);
 674         }
 675
 676         cmd->i_state = ISTATE_SEND_REJECT;
 677         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 678
 679         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 680         if (ret != 0)
 681                 return -1;
 682
 683         return (!fail_conn) ? 0 : -1;
 684 }
 685
 686 /*
 687  * Map some portion of the allocated scatterlist to an iovec, suitable for
 688  * kernel sockets to copy data in/out. This handles both pages and slab-allocated
 689  * buffers, since we have been tricky and mapped t_mem_sg to the buffer in
 690  * either case (see iscsit_alloc_buffs)
 691  */
 692 static int iscsit_map_iovec(
 693         struct iscsi_cmd *cmd,
 694         struct kvec *iov,
 695         u32 data_offset,
 696         u32 data_length)
 697 {
 698         u32 i = 0;
 699         struct scatterlist *sg;
 700         unsigned int page_off;
 701
 702         /*
 703          * We have a private mapping of the allocated pages in t_mem_sg.
 704          * At this point, we also know each contains a page.
 705          */
 706         sg = &cmd->t_mem_sg[data_offset / PAGE_SIZE];
 707         page_off = (data_offset % PAGE_SIZE);
 708
 709         cmd->first_data_sg = sg;
 710         cmd->first_data_sg_off = page_off;
 711
 712         while (data_length) {
 713                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 714
 715                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 716                 iov[i].iov_len = cur_len;
 717
 718                 data_length -= cur_len;
 719                 page_off = 0;
 720                 sg = sg_next(sg);
 721                 i++;
 722         }
 723
 724         cmd->kmapped_nents = i;
 725
 726         return i;
 727 }
 728
 729 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 730 {
 731         u32 i;
 732         struct scatterlist *sg;
 733
 734         sg = cmd->first_data_sg;
 735
 736         for (i = 0; i < cmd->kmapped_nents; i++)
 737                 kunmap(sg_page(&sg[i]));
 738 }
 739
 740 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 741 {
 742         struct iscsi_cmd *cmd;
 743
 744         conn->exp_statsn = exp_statsn;
 745
 746         spin_lock_bh(&conn->cmd_lock);
 747         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 748                 spin_lock(&cmd->istate_lock);
 749                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 750                     (cmd->stat_sn < exp_statsn)) {
 751                         cmd->i_state = ISTATE_REMOVE;
 752                         spin_unlock(&cmd->istate_lock);
 753                         iscsit_add_cmd_to_immediate_queue(cmd, conn,
 754                                                 cmd->i_state);
 755                         continue;
 756                 }
 757                 spin_unlock(&cmd->istate_lock);
 758         }
 759         spin_unlock_bh(&conn->cmd_lock);
 760 }
 761
 762 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 763 {
 764         u32 iov_count = (cmd->se_cmd.t_data_nents == 0) ? 1 :
 765                                 cmd->se_cmd.t_data_nents;
 766
 767         iov_count += ISCSI_IOV_DATA_BUFFER;
 768
 769         cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
 770         if (!cmd->iov_data) {
 771                 pr_err("Unable to allocate cmd->iov_data\n");
 772                 return -ENOMEM;
 773         }
 774
 775         cmd->orig_iov_data_count = iov_count;
 776         return 0;
 777 }
 778
 779 static int iscsit_alloc_buffs(struct iscsi_cmd *cmd)
 780 {
 781         struct scatterlist *sgl;
 782         u32 length = cmd->se_cmd.data_length;
 783         int nents = DIV_ROUND_UP(length, PAGE_SIZE);
 784         int i = 0, j = 0, ret;
 785         /*
 786          * If no SCSI payload is present, allocate the default iovecs used for
 787          * iSCSI PDU Header
 788          */
 789         if (!length)
 790                 return iscsit_allocate_iovecs(cmd);
 791
 792         sgl = kzalloc(sizeof(*sgl) * nents, GFP_KERNEL);
 793         if (!sgl)
 794                 return -ENOMEM;
 795
 796         sg_init_table(sgl, nents);
 797
 798         while (length) {
 799                 int buf_size = min_t(int, length, PAGE_SIZE);
 800                 struct page *page;
 801
 802                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 803                 if (!page)
 804                         goto page_alloc_failed;
 805
 806                 sg_set_page(&sgl[i], page, buf_size, 0);
 807
 808                 length -= buf_size;
 809                 i++;
 810         }
 811
 812         cmd->t_mem_sg = sgl;
 813         cmd->t_mem_sg_nents = nents;
 814
 815         /* BIDI ops not supported */
 816
 817         /* Tell the core about our preallocated memory */
 818         transport_generic_map_mem_to_cmd(&cmd->se_cmd, sgl, nents, NULL, 0);
 819         /*
 820          * Allocate iovecs for SCSI payload after transport_generic_map_mem_to_cmd
 821          * so that cmd->se_cmd.t_tasks_se_num has been set.
 822          */
 823         ret = iscsit_allocate_iovecs(cmd);
 824         if (ret < 0)
 825                 return -ENOMEM;
 826
 827         return 0;
 828
 829 page_alloc_failed:
 830         while (j < i)
 831                 __free_page(sg_page(&sgl[j++]));
 832
 833         kfree(sgl);
 834         return -ENOMEM;
 835 }
 836
 837 static int iscsit_handle_scsi_cmd(
 838         struct iscsi_conn *conn,
 839         unsigned char *buf)
 840 {
 841         int     data_direction, cmdsn_ret = 0, immed_ret, ret, transport_ret;
 842         int     dump_immediate_data = 0, send_check_condition = 0, payload_length;
 843         struct iscsi_cmd        *cmd = NULL;
 844         struct iscsi_scsi_req *hdr;
 845
 846         spin_lock_bh(&conn->sess->session_stats_lock);
 847         conn->sess->cmd_pdus++;
 848         if (conn->sess->se_sess->se_node_acl) {
 849                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
 850                 conn->sess->se_sess->se_node_acl->num_cmds++;
 851                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
 852         }
 853         spin_unlock_bh(&conn->sess->session_stats_lock);
 854
 855         hdr                     = (struct iscsi_scsi_req *) buf;
 856         payload_length          = ntoh24(hdr->dlength);
 857         hdr->itt                = be32_to_cpu(hdr->itt);
 858         hdr->data_length        = be32_to_cpu(hdr->data_length);
 859         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
 860         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
 861
 862         /* FIXME; Add checks for AdditionalHeaderSegment */
 863
 864         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
 865             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
 866                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
 867                                 " not set. Bad iSCSI Initiator.\n");
 868                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 869                                 buf, conn);
 870         }
 871
 872         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
 873              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
 874                 /*
 875                  * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2)
 876                  * that adds support for RESERVE/RELEASE.  There is a bug
 877                  * add with this new functionality that sets R/W bits when
 878                  * neither CDB carries any READ or WRITE datapayloads.
 879                  */
 880                 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) {
 881                         hdr->flags &= ~ISCSI_FLAG_CMD_READ;
 882                         hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
 883                         goto done;
 884                 }
 885
 886                 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
 887                         " set when Expected Data Transfer Length is 0 for"
 888                         " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]);
 889                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 890                                 buf, conn);
 891         }
 892 done:
 893
 894         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
 895             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
 896                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
 897                         " MUST be set if Expected Data Transfer Length is not 0."
 898                         " Bad iSCSI Initiator\n");
 899                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 900                                 buf, conn);
 901         }
 902
 903         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
 904             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
 905                 pr_err("Bidirectional operations not supported!\n");
 906                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 907                                 buf, conn);
 908         }
 909
 910         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
 911                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
 912                                 " Scsi Command PDU.\n");
 913                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 914                                 buf, conn);
 915         }
 916
 917         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
 918                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
 919                         " protocol error.\n", payload_length);
 920                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 921                                 buf, conn);
 922         }
 923
 924         if ((hdr->data_length == payload_length) &&
 925             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
 926                 pr_err("Expected Data Transfer Length and Length of"
 927                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
 928                         " bit is not set protocol error\n");
 929                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 930                                 buf, conn);
 931         }
 932
 933         if (payload_length > hdr->data_length) {
 934                 pr_err("DataSegmentLength: %u is greater than"
 935                         " EDTL: %u, protocol error.\n", payload_length,
 936                                 hdr->data_length);
 937                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 938                                 buf, conn);
 939         }
 940
 941         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
 942                 pr_err("DataSegmentLength: %u is greater than"
 943                         " MaxRecvDataSegmentLength: %u, protocol error.\n",
 944                         payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
 945                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 946                                 buf, conn);
 947         }
 948
 949         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
 950                 pr_err("DataSegmentLength: %u is greater than"
 951                         " FirstBurstLength: %u, protocol error.\n",
 952                         payload_length, conn->sess->sess_ops->FirstBurstLength);
 953                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 954                                         buf, conn);
 955         }
 956
 957         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
 958                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
 959                           DMA_NONE;
 960
 961         cmd = iscsit_allocate_se_cmd(conn, hdr->data_length, data_direction,
 962                                 (hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK));
 963         if (!cmd)
 964                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
 965                                         buf, conn);
 966
 967         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 968                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
 969                 hdr->cmdsn, hdr->data_length, payload_length, conn->cid);
 970
 971         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
 972         cmd->i_state            = ISTATE_NEW_CMD;
 973         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
 974         cmd->immediate_data     = (payload_length) ? 1 : 0;
 975         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
 976                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
 977         if (cmd->unsolicited_data)
 978                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
 979
 980         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
 981         if (hdr->flags & ISCSI_FLAG_CMD_READ) {
 982                 spin_lock_bh(&conn->sess->ttt_lock);
 983                 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 984                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
 985                         cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 986                 spin_unlock_bh(&conn->sess->ttt_lock);
 987         } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE)
 988                 cmd->targ_xfer_tag = 0xFFFFFFFF;
 989         cmd->cmd_sn             = hdr->cmdsn;
 990         cmd->exp_stat_sn        = hdr->exp_statsn;
 991         cmd->first_burst_len    = payload_length;
 992
 993         if (cmd->data_direction == DMA_FROM_DEVICE) {
 994                 struct iscsi_datain_req *dr;
 995
 996                 dr = iscsit_allocate_datain_req();
 997                 if (!dr)
 998                         return iscsit_add_reject_from_cmd(
 999                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1000                                         1, 1, buf, cmd);
1001
1002                 iscsit_attach_datain_req(cmd, dr);
1003         }
1004
1005         /*
1006          * The CDB is going to an se_device_t.
1007          */
1008         ret = transport_lookup_cmd_lun(&cmd->se_cmd,
1009                                        scsilun_to_int(&hdr->lun));
1010         if (ret < 0) {
1011                 if (cmd->se_cmd.scsi_sense_reason == TCM_NON_EXISTENT_LUN) {
1012                         pr_debug("Responding to non-acl'ed,"
1013                                 " non-existent or non-exported iSCSI LUN:"
1014                                 " 0x%016Lx\n", get_unaligned_le64(&hdr->lun));
1015                 }
1016                 send_check_condition = 1;
1017                 goto attach_cmd;
1018         }
1019         /*
1020          * The Initiator Node has access to the LUN (the addressing method
1021          * is handled inside of iscsit_get_lun_for_cmd()).  Now it's time to
1022          * allocate 1->N transport tasks (depending on sector count and
1023          * maximum request size the physical HBA(s) can handle.
1024          */
1025         transport_ret = transport_generic_allocate_tasks(&cmd->se_cmd, hdr->cdb);
1026         if (transport_ret == -ENOMEM) {
1027                 return iscsit_add_reject_from_cmd(
1028                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1029                                 1, 1, buf, cmd);
1030         } else if (transport_ret < 0) {
1031                 /*
1032                  * Unsupported SAM Opcode.  CHECK_CONDITION will be sent
1033                  * in iscsit_execute_cmd() during the CmdSN OOO Execution
1034                  * Mechinism.
1035                  */
1036                 send_check_condition = 1;
1037         } else {
1038                 cmd->data_length = cmd->se_cmd.data_length;
1039
1040                 if (iscsit_decide_list_to_build(cmd, payload_length) < 0)
1041                         return iscsit_add_reject_from_cmd(
1042                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1043                                 1, 1, buf, cmd);
1044         }
1045
1046 attach_cmd:
1047         spin_lock_bh(&conn->cmd_lock);
1048         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1049         spin_unlock_bh(&conn->cmd_lock);
1050         /*
1051          * Check if we need to delay processing because of ALUA
1052          * Active/NonOptimized primary access state..
1053          */
1054         core_alua_check_nonop_delay(&cmd->se_cmd);
1055         /*
1056          * Allocate and setup SGL used with transport_generic_map_mem_to_cmd().
1057          * also call iscsit_allocate_iovecs()
1058          */
1059         ret = iscsit_alloc_buffs(cmd);
1060         if (ret < 0)
1061                 return iscsit_add_reject_from_cmd(
1062                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1063                                 1, 0, buf, cmd);
1064         /*
1065          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1066          * the Immediate Bit is not set, and no Immediate
1067          * Data is attached.
1068          *
1069          * A PDU/CmdSN carrying Immediate Data can only
1070          * be processed after the DataCRC has passed.
1071          * If the DataCRC fails, the CmdSN MUST NOT
1072          * be acknowledged. (See below)
1073          */
1074         if (!cmd->immediate_data) {
1075                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1076                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1077                         return 0;
1078                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1079                         return iscsit_add_reject_from_cmd(
1080                                 ISCSI_REASON_PROTOCOL_ERROR,
1081                                 1, 0, buf, cmd);
1082         }
1083
1084         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1085
1086         /*
1087          * If no Immediate Data is attached, it's OK to return now.
1088          */
1089         if (!cmd->immediate_data) {
1090                 if (send_check_condition)
1091                         return 0;
1092
1093                 if (cmd->unsolicited_data) {
1094                         iscsit_set_dataout_sequence_values(cmd);
1095
1096                         spin_lock_bh(&cmd->dataout_timeout_lock);
1097                         iscsit_start_dataout_timer(cmd, cmd->conn);
1098                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1099                 }
1100
1101                 return 0;
1102         }
1103
1104         /*
1105          * Early CHECK_CONDITIONs never make it to the transport processing
1106          * thread.  They are processed in CmdSN order by
1107          * iscsit_check_received_cmdsn() below.
1108          */
1109         if (send_check_condition) {
1110                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1111                 dump_immediate_data = 1;
1112                 goto after_immediate_data;
1113         }
1114         /*
1115          * Call directly into transport_generic_new_cmd() to perform
1116          * the backend memory allocation.
1117          */
1118         ret = transport_generic_new_cmd(&cmd->se_cmd);
1119         if (ret < 0) {
1120                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1121                 dump_immediate_data = 1;
1122                 goto after_immediate_data;
1123         }
1124
1125         immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length);
1126 after_immediate_data:
1127         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1128                 /*
1129                  * A PDU/CmdSN carrying Immediate Data passed
1130                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1131                  * Immediate Bit is not set.
1132                  */
1133                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1134                 /*
1135                  * Special case for Unsupported SAM WRITE Opcodes
1136                  * and ImmediateData=Yes.
1137                  */
1138                 if (dump_immediate_data) {
1139                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
1140                                 return -1;
1141                 } else if (cmd->unsolicited_data) {
1142                         iscsit_set_dataout_sequence_values(cmd);
1143
1144                         spin_lock_bh(&cmd->dataout_timeout_lock);
1145                         iscsit_start_dataout_timer(cmd, cmd->conn);
1146                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1147                 }
1148
1149                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1150                         return iscsit_add_reject_from_cmd(
1151                                 ISCSI_REASON_PROTOCOL_ERROR,
1152                                 1, 0, buf, cmd);
1153
1154         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1155                 /*
1156                  * Immediate Data failed DataCRC and ERL>=1,
1157                  * silently drop this PDU and let the initiator
1158                  * plug the CmdSN gap.
1159                  *
1160                  * FIXME: Send Unsolicited NOPIN with reserved
1161                  * TTT here to help the initiator figure out
1162                  * the missing CmdSN, although they should be
1163                  * intelligent enough to determine the missing
1164                  * CmdSN and issue a retry to plug the sequence.
1165                  */
1166                 cmd->i_state = ISTATE_REMOVE;
1167                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1168         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1169                 return -1;
1170
1171         return 0;
1172 }
1173
1174 static u32 iscsit_do_crypto_hash_sg(
1175         struct hash_desc *hash,
1176         struct iscsi_cmd *cmd,
1177         u32 data_offset,
1178         u32 data_length,
1179         u32 padding,
1180         u8 *pad_bytes)
1181 {
1182         u32 data_crc;
1183         u32 i;
1184         struct scatterlist *sg;
1185         unsigned int page_off;
1186
1187         crypto_hash_init(hash);
1188
1189         sg = cmd->first_data_sg;
1190         page_off = cmd->first_data_sg_off;
1191
1192         i = 0;
1193         while (data_length) {
1194                 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off));
1195
1196                 crypto_hash_update(hash, &sg[i], cur_len);
1197
1198                 data_length -= cur_len;
1199                 page_off = 0;
1200                 i++;
1201         }
1202
1203         if (padding) {
1204                 struct scatterlist pad_sg;
1205
1206                 sg_init_one(&pad_sg, pad_bytes, padding);
1207                 crypto_hash_update(hash, &pad_sg, padding);
1208         }
1209         crypto_hash_final(hash, (u8 *) &data_crc);
1210
1211         return data_crc;
1212 }
1213
1214 static void iscsit_do_crypto_hash_buf(
1215         struct hash_desc *hash,
1216         unsigned char *buf,
1217         u32 payload_length,
1218         u32 padding,
1219         u8 *pad_bytes,
1220         u8 *data_crc)
1221 {
1222         struct scatterlist sg;
1223
1224         crypto_hash_init(hash);
1225
1226         sg_init_one(&sg, buf, payload_length);
1227         crypto_hash_update(hash, &sg, payload_length);
1228
1229         if (padding) {
1230                 sg_init_one(&sg, pad_bytes, padding);
1231                 crypto_hash_update(hash, &sg, padding);
1232         }
1233         crypto_hash_final(hash, data_crc);
1234 }
1235
1236 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1237 {
1238         int iov_ret, ooo_cmdsn = 0, ret;
1239         u8 data_crc_failed = 0;
1240         u32 checksum, iov_count = 0, padding = 0, rx_got = 0;
1241         u32 rx_size = 0, payload_length;
1242         struct iscsi_cmd *cmd = NULL;
1243         struct se_cmd *se_cmd;
1244         struct iscsi_data *hdr;
1245         struct kvec *iov;
1246         unsigned long flags;
1247
1248         hdr                     = (struct iscsi_data *) buf;
1249         payload_length          = ntoh24(hdr->dlength);
1250         hdr->itt                = be32_to_cpu(hdr->itt);
1251         hdr->ttt                = be32_to_cpu(hdr->ttt);
1252         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1253         hdr->datasn             = be32_to_cpu(hdr->datasn);
1254         hdr->offset             = be32_to_cpu(hdr->offset);
1255
1256         if (!payload_length) {
1257                 pr_err("DataOUT payload is ZERO, protocol error.\n");
1258                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1259                                         buf, conn);
1260         }
1261
1262         /* iSCSI write */
1263         spin_lock_bh(&conn->sess->session_stats_lock);
1264         conn->sess->rx_data_octets += payload_length;
1265         if (conn->sess->se_sess->se_node_acl) {
1266                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
1267                 conn->sess->se_sess->se_node_acl->write_bytes += payload_length;
1268                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
1269         }
1270         spin_unlock_bh(&conn->sess->session_stats_lock);
1271
1272         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1273                 pr_err("DataSegmentLength: %u is greater than"
1274                         " MaxRecvDataSegmentLength: %u\n", payload_length,
1275                         conn->conn_ops->MaxRecvDataSegmentLength);
1276                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1277                                         buf, conn);
1278         }
1279
1280         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
1281                         payload_length);
1282         if (!cmd)
1283                 return 0;
1284
1285         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1286                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1287                 hdr->itt, hdr->ttt, hdr->datasn, hdr->offset,
1288                 payload_length, conn->cid);
1289
1290         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1291                 pr_err("Command ITT: 0x%08x received DataOUT after"
1292                         " last DataOUT received, dumping payload\n",
1293                         cmd->init_task_tag);
1294                 return iscsit_dump_data_payload(conn, payload_length, 1);
1295         }
1296
1297         if (cmd->data_direction != DMA_TO_DEVICE) {
1298                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1299                         " NON-WRITE command.\n", cmd->init_task_tag);
1300                 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
1301                                 1, 0, buf, cmd);
1302         }
1303         se_cmd = &cmd->se_cmd;
1304         iscsit_mod_dataout_timer(cmd);
1305
1306         if ((hdr->offset + payload_length) > cmd->data_length) {
1307                 pr_err("DataOut Offset: %u, Length %u greater than"
1308                         " iSCSI Command EDTL %u, protocol error.\n",
1309                         hdr->offset, payload_length, cmd->data_length);
1310                 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
1311                                 1, 0, buf, cmd);
1312         }
1313
1314         if (cmd->unsolicited_data) {
1315                 int dump_unsolicited_data = 0;
1316
1317                 if (conn->sess->sess_ops->InitialR2T) {
1318                         pr_err("Received unexpected unsolicited data"
1319                                 " while InitialR2T=Yes, protocol error.\n");
1320                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1321                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1322                         return -1;
1323                 }
1324                 /*
1325                  * Special case for dealing with Unsolicited DataOUT
1326                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1327                  * failures;
1328                  */
1329
1330                 /* Something's amiss if we're not in WRITE_PENDING state... */
1331                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1332                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1333                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1334
1335                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1336                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) ||
1337                      (se_cmd->se_cmd_flags & SCF_SCSI_CDB_EXCEPTION))
1338                         dump_unsolicited_data = 1;
1339                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1340
1341                 if (dump_unsolicited_data) {
1342                         /*
1343                          * Check if a delayed TASK_ABORTED status needs to
1344                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1345                          * received with the unsolicitied data out.
1346                          */
1347                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1348                                 iscsit_stop_dataout_timer(cmd);
1349
1350                         transport_check_aborted_status(se_cmd,
1351                                         (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1352                         return iscsit_dump_data_payload(conn, payload_length, 1);
1353                 }
1354         } else {
1355                 /*
1356                  * For the normal solicited data path:
1357                  *
1358                  * Check for a delayed TASK_ABORTED status and dump any
1359                  * incoming data out payload if one exists.  Also, when the
1360                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1361                  * data out sequence, we decrement outstanding_r2ts.  Once
1362                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1363                  * TASK_ABORTED status.
1364                  */
1365                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1366                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1367                                 if (--cmd->outstanding_r2ts < 1) {
1368                                         iscsit_stop_dataout_timer(cmd);
1369                                         transport_check_aborted_status(
1370                                                         se_cmd, 1);
1371                                 }
1372
1373                         return iscsit_dump_data_payload(conn, payload_length, 1);
1374                 }
1375         }
1376         /*
1377          * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1378          * within-command recovery checks before receiving the payload.
1379          */
1380         ret = iscsit_check_pre_dataout(cmd, buf);
1381         if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)
1382                 return 0;
1383         else if (ret == DATAOUT_CANNOT_RECOVER)
1384                 return -1;
1385
1386         rx_size += payload_length;
1387         iov = &cmd->iov_data[0];
1388
1389         iov_ret = iscsit_map_iovec(cmd, iov, hdr->offset, payload_length);
1390         if (iov_ret < 0)
1391                 return -1;
1392
1393         iov_count += iov_ret;
1394
1395         padding = ((-payload_length) & 3);
1396         if (padding != 0) {
1397                 iov[iov_count].iov_base = cmd->pad_bytes;
1398                 iov[iov_count++].iov_len = padding;
1399                 rx_size += padding;
1400                 pr_debug("Receiving %u padding bytes.\n", padding);
1401         }
1402
1403         if (conn->conn_ops->DataDigest) {
1404                 iov[iov_count].iov_base = &checksum;
1405                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1406                 rx_size += ISCSI_CRC_LEN;
1407         }
1408
1409         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1410
1411         iscsit_unmap_iovec(cmd);
1412
1413         if (rx_got != rx_size)
1414                 return -1;
1415
1416         if (conn->conn_ops->DataDigest) {
1417                 u32 data_crc;
1418
1419                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
1420                                                     hdr->offset, payload_length, padding,
1421                                                     cmd->pad_bytes);
1422
1423                 if (checksum != data_crc) {
1424                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1425                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1426                                 " does not match computed 0x%08x\n",
1427                                 hdr->itt, hdr->offset, payload_length,
1428                                 hdr->datasn, checksum, data_crc);
1429                         data_crc_failed = 1;
1430                 } else {
1431                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1432                                 " %u bytes of Data Out\n", checksum,
1433                                 payload_length);
1434                 }
1435         }
1436         /*
1437          * Increment post receive data and CRC values or perform
1438          * within-command recovery.
1439          */
1440         ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed);
1441         if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY))
1442                 return 0;
1443         else if (ret == DATAOUT_SEND_R2T) {
1444                 iscsit_set_dataout_sequence_values(cmd);
1445                 iscsit_build_r2ts_for_cmd(cmd, conn, 0);
1446         } else if (ret == DATAOUT_SEND_TO_TRANSPORT) {
1447                 /*
1448                  * Handle extra special case for out of order
1449                  * Unsolicited Data Out.
1450                  */
1451                 spin_lock_bh(&cmd->istate_lock);
1452                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1453                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1454                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1455                 spin_unlock_bh(&cmd->istate_lock);
1456
1457                 iscsit_stop_dataout_timer(cmd);
1458                 return (!ooo_cmdsn) ? transport_generic_handle_data(
1459                                         &cmd->se_cmd) : 0;
1460         } else /* DATAOUT_CANNOT_RECOVER */
1461                 return -1;
1462
1463         return 0;
1464 }
1465
1466 static int iscsit_handle_nop_out(
1467         struct iscsi_conn *conn,
1468         unsigned char *buf)
1469 {
1470         unsigned char *ping_data = NULL;
1471         int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
1472         u32 checksum, data_crc, padding = 0, payload_length;
1473         struct iscsi_cmd *cmd = NULL;
1474         struct kvec *iov = NULL;
1475         struct iscsi_nopout *hdr;
1476
1477         hdr                     = (struct iscsi_nopout *) buf;
1478         payload_length          = ntoh24(hdr->dlength);
1479         hdr->itt                = be32_to_cpu(hdr->itt);
1480         hdr->ttt                = be32_to_cpu(hdr->ttt);
1481         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1482         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1483
1484         if ((hdr->itt == 0xFFFFFFFF) && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1485                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1486                         " not set, protocol error.\n");
1487                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1488                                         buf, conn);
1489         }
1490
1491         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1492                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1493                         " greater than MaxRecvDataSegmentLength: %u, protocol"
1494                         " error.\n", payload_length,
1495                         conn->conn_ops->MaxRecvDataSegmentLength);
1496                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1497                                         buf, conn);
1498         }
1499
1500         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x,"
1501                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1502                 (hdr->itt == 0xFFFFFFFF) ? "Response" : "Request",
1503                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1504                 payload_length);
1505         /*
1506          * This is not a response to a Unsolicited NopIN, which means
1507          * it can either be a NOPOUT ping request (with a valid ITT),
1508          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1509          * Either way, make sure we allocate an struct iscsi_cmd, as both
1510          * can contain ping data.
1511          */
1512         if (hdr->ttt == 0xFFFFFFFF) {
1513                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1514                 if (!cmd)
1515                         return iscsit_add_reject(
1516                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1517                                         1, buf, conn);
1518
1519                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1520                 cmd->i_state            = ISTATE_SEND_NOPIN;
1521                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1522                                                 1 : 0);
1523                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1524                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1525                 cmd->cmd_sn             = hdr->cmdsn;
1526                 cmd->exp_stat_sn        = hdr->exp_statsn;
1527                 cmd->data_direction     = DMA_NONE;
1528         }
1529
1530         if (payload_length && (hdr->ttt == 0xFFFFFFFF)) {
1531                 rx_size = payload_length;
1532                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1533                 if (!ping_data) {
1534                         pr_err("Unable to allocate memory for"
1535                                 " NOPOUT ping data.\n");
1536                         ret = -1;
1537                         goto out;
1538                 }
1539
1540                 iov = &cmd->iov_misc[0];
1541                 iov[niov].iov_base      = ping_data;
1542                 iov[niov++].iov_len     = payload_length;
1543
1544                 padding = ((-payload_length) & 3);
1545                 if (padding != 0) {
1546                         pr_debug("Receiving %u additional bytes"
1547                                 " for padding.\n", padding);
1548                         iov[niov].iov_base      = &cmd->pad_bytes;
1549                         iov[niov++].iov_len     = padding;
1550                         rx_size += padding;
1551                 }
1552                 if (conn->conn_ops->DataDigest) {
1553                         iov[niov].iov_base      = &checksum;
1554                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1555                         rx_size += ISCSI_CRC_LEN;
1556                 }
1557
1558                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1559                 if (rx_got != rx_size) {
1560                         ret = -1;
1561                         goto out;
1562                 }
1563
1564                 if (conn->conn_ops->DataDigest) {
1565                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1566                                         ping_data, payload_length,
1567                                         padding, cmd->pad_bytes,
1568                                         (u8 *)&data_crc);
1569
1570                         if (checksum != data_crc) {
1571                                 pr_err("Ping data CRC32C DataDigest"
1572                                 " 0x%08x does not match computed 0x%08x\n",
1573                                         checksum, data_crc);
1574                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1575                                         pr_err("Unable to recover from"
1576                                         " NOPOUT Ping DataCRC failure while in"
1577                                                 " ERL=0.\n");
1578                                         ret = -1;
1579                                         goto out;
1580                                 } else {
1581                                         /*
1582                                          * Silently drop this PDU and let the
1583                                          * initiator plug the CmdSN gap.
1584                                          */
1585                                         pr_debug("Dropping NOPOUT"
1586                                         " Command CmdSN: 0x%08x due to"
1587                                         " DataCRC error.\n", hdr->cmdsn);
1588                                         ret = 0;
1589                                         goto out;
1590                                 }
1591                         } else {
1592                                 pr_debug("Got CRC32C DataDigest"
1593                                 " 0x%08x for %u bytes of ping data.\n",
1594                                         checksum, payload_length);
1595                         }
1596                 }
1597
1598                 ping_data[payload_length] = '\0';
1599                 /*
1600                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1601                  */
1602                 cmd->buf_ptr = ping_data;
1603                 cmd->buf_ptr_size = payload_length;
1604
1605                 pr_debug("Got %u bytes of NOPOUT ping"
1606                         " data.\n", payload_length);
1607                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1608         }
1609
1610         if (hdr->itt != 0xFFFFFFFF) {
1611                 if (!cmd) {
1612                         pr_err("Checking CmdSN for NOPOUT,"
1613                                 " but cmd is NULL!\n");
1614                         return -1;
1615                 }
1616                 /*
1617                  * Initiator is expecting a NopIN ping reply,
1618                  */
1619                 spin_lock_bh(&conn->cmd_lock);
1620                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1621                 spin_unlock_bh(&conn->cmd_lock);
1622
1623                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1624
1625                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1626                         iscsit_add_cmd_to_response_queue(cmd, conn,
1627                                         cmd->i_state);
1628                         return 0;
1629                 }
1630
1631                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1632                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1633                         ret = 0;
1634                         goto ping_out;
1635                 }
1636                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1637                         return iscsit_add_reject_from_cmd(
1638                                         ISCSI_REASON_PROTOCOL_ERROR,
1639                                         1, 0, buf, cmd);
1640
1641                 return 0;
1642         }
1643
1644         if (hdr->ttt != 0xFFFFFFFF) {
1645                 /*
1646                  * This was a response to a unsolicited NOPIN ping.
1647                  */
1648                 cmd = iscsit_find_cmd_from_ttt(conn, hdr->ttt);
1649                 if (!cmd)
1650                         return -1;
1651
1652                 iscsit_stop_nopin_response_timer(conn);
1653
1654                 cmd->i_state = ISTATE_REMOVE;
1655                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1656                 iscsit_start_nopin_timer(conn);
1657         } else {
1658                 /*
1659                  * Initiator is not expecting a NOPIN is response.
1660                  * Just ignore for now.
1661                  *
1662                  * iSCSI v19-91 10.18
1663                  * "A NOP-OUT may also be used to confirm a changed
1664                  *  ExpStatSN if another PDU will not be available
1665                  *  for a long time."
1666                  */
1667                 ret = 0;
1668                 goto out;
1669         }
1670
1671         return 0;
1672 out:
1673         if (cmd)
1674                 iscsit_release_cmd(cmd);
1675 ping_out:
1676         kfree(ping_data);
1677         return ret;
1678 }
1679
1680 static int iscsit_handle_task_mgt_cmd(
1681         struct iscsi_conn *conn,
1682         unsigned char *buf)
1683 {
1684         struct iscsi_cmd *cmd;
1685         struct se_tmr_req *se_tmr;
1686         struct iscsi_tmr_req *tmr_req;
1687         struct iscsi_tm *hdr;
1688         int out_of_order_cmdsn = 0;
1689         int ret;
1690         u8 function;
1691
1692         hdr                     = (struct iscsi_tm *) buf;
1693         hdr->itt                = be32_to_cpu(hdr->itt);
1694         hdr->rtt                = be32_to_cpu(hdr->rtt);
1695         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1696         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1697         hdr->refcmdsn           = be32_to_cpu(hdr->refcmdsn);
1698         hdr->exp_datasn         = be32_to_cpu(hdr->exp_datasn);
1699         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1700         function = hdr->flags;
1701
1702         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1703                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1704                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1705                 hdr->rtt, hdr->refcmdsn, conn->cid);
1706
1707         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1708             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1709              (hdr->rtt != ISCSI_RESERVED_TAG))) {
1710                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1711                 hdr->rtt = ISCSI_RESERVED_TAG;
1712         }
1713
1714         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1715                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1716                 pr_err("Task Management Request TASK_REASSIGN not"
1717                         " issued as immediate command, bad iSCSI Initiator"
1718                                 "implementation\n");
1719                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1720                                         buf, conn);
1721         }
1722         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1723             (hdr->refcmdsn != ISCSI_RESERVED_TAG))
1724                 hdr->refcmdsn = ISCSI_RESERVED_TAG;
1725
1726         cmd = iscsit_allocate_se_cmd_for_tmr(conn, function);
1727         if (!cmd)
1728                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1729                                         1, buf, conn);
1730
1731         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
1732         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
1733         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1734         cmd->init_task_tag      = hdr->itt;
1735         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1736         cmd->cmd_sn             = hdr->cmdsn;
1737         cmd->exp_stat_sn        = hdr->exp_statsn;
1738         se_tmr                  = cmd->se_cmd.se_tmr_req;
1739         tmr_req                 = cmd->tmr_req;
1740         /*
1741          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
1742          */
1743         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1744                 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
1745                                                scsilun_to_int(&hdr->lun));
1746                 if (ret < 0) {
1747                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1748                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
1749                         goto attach;
1750                 }
1751         }
1752
1753         switch (function) {
1754         case ISCSI_TM_FUNC_ABORT_TASK:
1755                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
1756                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) {
1757                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1758                         goto attach;
1759                 }
1760                 break;
1761         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1762         case ISCSI_TM_FUNC_CLEAR_ACA:
1763         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1764         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1765                 break;
1766         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1767                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
1768                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1769                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1770                         goto attach;
1771                 }
1772                 break;
1773         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1774                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
1775                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1776                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1777                         goto attach;
1778                 }
1779                 break;
1780         case ISCSI_TM_FUNC_TASK_REASSIGN:
1781                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
1782                 /*
1783                  * Perform sanity checks on the ExpDataSN only if the
1784                  * TASK_REASSIGN was successful.
1785                  */
1786                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE)
1787                         break;
1788
1789                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
1790                         return iscsit_add_reject_from_cmd(
1791                                         ISCSI_REASON_BOOKMARK_INVALID, 1, 1,
1792                                         buf, cmd);
1793                 break;
1794         default:
1795                 pr_err("Unknown TMR function: 0x%02x, protocol"
1796                         " error.\n", function);
1797                 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1798                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
1799                 goto attach;
1800         }
1801
1802         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1803             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
1804                 se_tmr->call_transport = 1;
1805 attach:
1806         spin_lock_bh(&conn->cmd_lock);
1807         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1808         spin_unlock_bh(&conn->cmd_lock);
1809
1810         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1811                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1812                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
1813                         out_of_order_cmdsn = 1;
1814                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1815                         return 0;
1816                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1817                         return iscsit_add_reject_from_cmd(
1818                                         ISCSI_REASON_PROTOCOL_ERROR,
1819                                         1, 0, buf, cmd);
1820         }
1821         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1822
1823         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
1824                 return 0;
1825         /*
1826          * Found the referenced task, send to transport for processing.
1827          */
1828         if (se_tmr->call_transport)
1829                 return transport_generic_handle_tmr(&cmd->se_cmd);
1830
1831         /*
1832          * Could not find the referenced LUN, task, or Task Management
1833          * command not authorized or supported.  Change state and
1834          * let the tx_thread send the response.
1835          *
1836          * For connection recovery, this is also the default action for
1837          * TMR TASK_REASSIGN.
1838          */
1839         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
1840         return 0;
1841 }
1842
1843 /* #warning FIXME: Support Text Command parameters besides SendTargets */
1844 static int iscsit_handle_text_cmd(
1845         struct iscsi_conn *conn,
1846         unsigned char *buf)
1847 {
1848         char *text_ptr, *text_in;
1849         int cmdsn_ret, niov = 0, rx_got, rx_size;
1850         u32 checksum = 0, data_crc = 0, payload_length;
1851         u32 padding = 0, pad_bytes = 0, text_length = 0;
1852         struct iscsi_cmd *cmd;
1853         struct kvec iov[3];
1854         struct iscsi_text *hdr;
1855
1856         hdr                     = (struct iscsi_text *) buf;
1857         payload_length          = ntoh24(hdr->dlength);
1858         hdr->itt                = be32_to_cpu(hdr->itt);
1859         hdr->ttt                = be32_to_cpu(hdr->ttt);
1860         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1861         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1862
1863         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1864                 pr_err("Unable to accept text parameter length: %u"
1865                         "greater than MaxRecvDataSegmentLength %u.\n",
1866                        payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
1867                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1868                                         buf, conn);
1869         }
1870
1871         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
1872                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
1873                 hdr->exp_statsn, payload_length);
1874
1875         rx_size = text_length = payload_length;
1876         if (text_length) {
1877                 text_in = kzalloc(text_length, GFP_KERNEL);
1878                 if (!text_in) {
1879                         pr_err("Unable to allocate memory for"
1880                                 " incoming text parameters\n");
1881                         return -1;
1882                 }
1883
1884                 memset(iov, 0, 3 * sizeof(struct kvec));
1885                 iov[niov].iov_base      = text_in;
1886                 iov[niov++].iov_len     = text_length;
1887
1888                 padding = ((-payload_length) & 3);
1889                 if (padding != 0) {
1890                         iov[niov].iov_base = &pad_bytes;
1891                         iov[niov++].iov_len  = padding;
1892                         rx_size += padding;
1893                         pr_debug("Receiving %u additional bytes"
1894                                         " for padding.\n", padding);
1895                 }
1896                 if (conn->conn_ops->DataDigest) {
1897                         iov[niov].iov_base      = &checksum;
1898                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1899                         rx_size += ISCSI_CRC_LEN;
1900                 }
1901
1902                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
1903                 if (rx_got != rx_size) {
1904                         kfree(text_in);
1905                         return -1;
1906                 }
1907
1908                 if (conn->conn_ops->DataDigest) {
1909                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1910                                         text_in, text_length,
1911                                         padding, (u8 *)&pad_bytes,
1912                                         (u8 *)&data_crc);
1913
1914                         if (checksum != data_crc) {
1915                                 pr_err("Text data CRC32C DataDigest"
1916                                         " 0x%08x does not match computed"
1917                                         " 0x%08x\n", checksum, data_crc);
1918                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1919                                         pr_err("Unable to recover from"
1920                                         " Text Data digest failure while in"
1921                                                 " ERL=0.\n");
1922                                         kfree(text_in);
1923                                         return -1;
1924                                 } else {
1925                                         /*
1926                                          * Silently drop this PDU and let the
1927                                          * initiator plug the CmdSN gap.
1928                                          */
1929                                         pr_debug("Dropping Text"
1930                                         " Command CmdSN: 0x%08x due to"
1931                                         " DataCRC error.\n", hdr->cmdsn);
1932                                         kfree(text_in);
1933                                         return 0;
1934                                 }
1935                         } else {
1936                                 pr_debug("Got CRC32C DataDigest"
1937                                         " 0x%08x for %u bytes of text data.\n",
1938                                                 checksum, text_length);
1939                         }
1940                 }
1941                 text_in[text_length - 1] = '\0';
1942                 pr_debug("Successfully read %d bytes of text"
1943                                 " data.\n", text_length);
1944
1945                 if (strncmp("SendTargets", text_in, 11) != 0) {
1946                         pr_err("Received Text Data that is not"
1947                                 " SendTargets, cannot continue.\n");
1948                         kfree(text_in);
1949                         return -1;
1950                 }
1951                 text_ptr = strchr(text_in, '=');
1952                 if (!text_ptr) {
1953                         pr_err("No \"=\" separator found in Text Data,"
1954                                 "  cannot continue.\n");
1955                         kfree(text_in);
1956                         return -1;
1957                 }
1958                 if (strncmp("=All", text_ptr, 4) != 0) {
1959                         pr_err("Unable to locate All value for"
1960                                 " SendTargets key,  cannot continue.\n");
1961                         kfree(text_in);
1962                         return -1;
1963                 }
1964 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */
1965                 kfree(text_in);
1966         }
1967
1968         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1969         if (!cmd)
1970                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1971                                         1, buf, conn);
1972
1973         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
1974         cmd->i_state            = ISTATE_SEND_TEXTRSP;
1975         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1976         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
1977         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1978         cmd->cmd_sn             = hdr->cmdsn;
1979         cmd->exp_stat_sn        = hdr->exp_statsn;
1980         cmd->data_direction     = DMA_NONE;
1981
1982         spin_lock_bh(&conn->cmd_lock);
1983         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1984         spin_unlock_bh(&conn->cmd_lock);
1985
1986         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1987
1988         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1989                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1990                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1991                         return iscsit_add_reject_from_cmd(
1992                                         ISCSI_REASON_PROTOCOL_ERROR,
1993                                         1, 0, buf, cmd);
1994
1995                 return 0;
1996         }
1997
1998         return iscsit_execute_cmd(cmd, 0);
1999 }
2000
2001 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2002 {
2003         struct iscsi_conn *conn_p;
2004         struct iscsi_session *sess = conn->sess;
2005
2006         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2007                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2008
2009         atomic_set(&sess->session_logout, 1);
2010         atomic_set(&conn->conn_logout_remove, 1);
2011         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2012
2013         iscsit_inc_conn_usage_count(conn);
2014         iscsit_inc_session_usage_count(sess);
2015
2016         spin_lock_bh(&sess->conn_lock);
2017         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2018                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2019                         continue;
2020
2021                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2022                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2023         }
2024         spin_unlock_bh(&sess->conn_lock);
2025
2026         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2027
2028         return 0;
2029 }
2030
2031 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2032 {
2033         struct iscsi_conn *l_conn;
2034         struct iscsi_session *sess = conn->sess;
2035
2036         pr_debug("Received logout request CLOSECONNECTION for CID:"
2037                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2038
2039         /*
2040          * A Logout Request with a CLOSECONNECTION reason code for a CID
2041          * can arrive on a connection with a differing CID.
2042          */
2043         if (conn->cid == cmd->logout_cid) {
2044                 spin_lock_bh(&conn->state_lock);
2045                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2046                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2047
2048                 atomic_set(&conn->conn_logout_remove, 1);
2049                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2050                 iscsit_inc_conn_usage_count(conn);
2051
2052                 spin_unlock_bh(&conn->state_lock);
2053         } else {
2054                 /*
2055                  * Handle all different cid CLOSECONNECTION requests in
2056                  * iscsit_logout_post_handler_diffcid() as to give enough
2057                  * time for any non immediate command's CmdSN to be
2058                  * acknowledged on the connection in question.
2059                  *
2060                  * Here we simply make sure the CID is still around.
2061                  */
2062                 l_conn = iscsit_get_conn_from_cid(sess,
2063                                 cmd->logout_cid);
2064                 if (!l_conn) {
2065                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2066                         iscsit_add_cmd_to_response_queue(cmd, conn,
2067                                         cmd->i_state);
2068                         return 0;
2069                 }
2070
2071                 iscsit_dec_conn_usage_count(l_conn);
2072         }
2073
2074         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2075
2076         return 0;
2077 }
2078
2079 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2080 {
2081         struct iscsi_session *sess = conn->sess;
2082
2083         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2084                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2085
2086         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2087                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2088                         " while ERL!=2.\n");
2089                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2090                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2091                 return 0;
2092         }
2093
2094         if (conn->cid == cmd->logout_cid) {
2095                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2096                         " with CID: %hu on CID: %hu, implementation error.\n",
2097                                 cmd->logout_cid, conn->cid);
2098                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2099                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2100                 return 0;
2101         }
2102
2103         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2104
2105         return 0;
2106 }
2107
2108 static int iscsit_handle_logout_cmd(
2109         struct iscsi_conn *conn,
2110         unsigned char *buf)
2111 {
2112         int cmdsn_ret, logout_remove = 0;
2113         u8 reason_code = 0;
2114         struct iscsi_cmd *cmd;
2115         struct iscsi_logout *hdr;
2116         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2117
2118         hdr                     = (struct iscsi_logout *) buf;
2119         reason_code             = (hdr->flags & 0x7f);
2120         hdr->itt                = be32_to_cpu(hdr->itt);
2121         hdr->cid                = be16_to_cpu(hdr->cid);
2122         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
2123         hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn);
2124
2125         if (tiqn) {
2126                 spin_lock(&tiqn->logout_stats.lock);
2127                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2128                         tiqn->logout_stats.normal_logouts++;
2129                 else
2130                         tiqn->logout_stats.abnormal_logouts++;
2131                 spin_unlock(&tiqn->logout_stats.lock);
2132         }
2133
2134         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2135                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2136                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2137                 hdr->cid, conn->cid);
2138
2139         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2140                 pr_err("Received logout request on connection that"
2141                         " is not in logged in state, ignoring request.\n");
2142                 return 0;
2143         }
2144
2145         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
2146         if (!cmd)
2147                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
2148                                         buf, conn);
2149
2150         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2151         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2152         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2153         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2154         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2155         cmd->cmd_sn             = hdr->cmdsn;
2156         cmd->exp_stat_sn        = hdr->exp_statsn;
2157         cmd->logout_cid         = hdr->cid;
2158         cmd->logout_reason      = reason_code;
2159         cmd->data_direction     = DMA_NONE;
2160
2161         /*
2162          * We need to sleep in these cases (by returning 1) until the Logout
2163          * Response gets sent in the tx thread.
2164          */
2165         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2166            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2167             (hdr->cid == conn->cid)))
2168                 logout_remove = 1;
2169
2170         spin_lock_bh(&conn->cmd_lock);
2171         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2172         spin_unlock_bh(&conn->cmd_lock);
2173
2174         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2175                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
2176
2177         /*
2178          * Immediate commands are executed, well, immediately.
2179          * Non-Immediate Logout Commands are executed in CmdSN order.
2180          */
2181         if (cmd->immediate_cmd) {
2182                 int ret = iscsit_execute_cmd(cmd, 0);
2183
2184                 if (ret < 0)
2185                         return ret;
2186         } else {
2187                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
2188                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2189                         logout_remove = 0;
2190                 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2191                         return iscsit_add_reject_from_cmd(
2192                                 ISCSI_REASON_PROTOCOL_ERROR,
2193                                 1, 0, buf, cmd);
2194                 }
2195         }
2196
2197         return logout_remove;
2198 }
2199
2200 static int iscsit_handle_snack(
2201         struct iscsi_conn *conn,
2202         unsigned char *buf)
2203 {
2204         struct iscsi_snack *hdr;
2205
2206         hdr                     = (struct iscsi_snack *) buf;
2207         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2208         hdr->itt                = be32_to_cpu(hdr->itt);
2209         hdr->ttt                = be32_to_cpu(hdr->ttt);
2210         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
2211         hdr->begrun             = be32_to_cpu(hdr->begrun);
2212         hdr->runlength          = be32_to_cpu(hdr->runlength);
2213
2214         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2215                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2216                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2217                         hdr->begrun, hdr->runlength, conn->cid);
2218
2219         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2220                 pr_err("Initiator sent SNACK request while in"
2221                         " ErrorRecoveryLevel=0.\n");
2222                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2223                                         buf, conn);
2224         }
2225         /*
2226          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2227          * call from inside iscsi_send_recovery_datain_or_r2t().
2228          */
2229         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2230         case 0:
2231                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2232                         hdr->itt, hdr->ttt, hdr->begrun, hdr->runlength);
2233         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2234                 return iscsit_handle_status_snack(conn, hdr->itt, hdr->ttt,
2235                         hdr->begrun, hdr->runlength);
2236         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2237                 return iscsit_handle_data_ack(conn, hdr->ttt, hdr->begrun,
2238                         hdr->runlength);
2239         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2240                 /* FIXME: Support R-Data SNACK */
2241                 pr_err("R-Data SNACK Not Supported.\n");
2242                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2243                                         buf, conn);
2244         default:
2245                 pr_err("Unknown SNACK type 0x%02x, protocol"
2246                         " error.\n", hdr->flags & 0x0f);
2247                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2248                                         buf, conn);
2249         }
2250
2251         return 0;
2252 }
2253
2254 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2255 {
2256         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2257             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2258                 wait_for_completion_interruptible_timeout(
2259                                         &conn->rx_half_close_comp,
2260                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2261         }
2262 }
2263
2264 static int iscsit_handle_immediate_data(
2265         struct iscsi_cmd *cmd,
2266         unsigned char *buf,
2267         u32 length)
2268 {
2269         int iov_ret, rx_got = 0, rx_size = 0;
2270         u32 checksum, iov_count = 0, padding = 0;
2271         struct iscsi_conn *conn = cmd->conn;
2272         struct kvec *iov;
2273
2274         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2275         if (iov_ret < 0)
2276                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2277
2278         rx_size = length;
2279         iov_count = iov_ret;
2280         iov = &cmd->iov_data[0];
2281
2282         padding = ((-length) & 3);
2283         if (padding != 0) {
2284                 iov[iov_count].iov_base = cmd->pad_bytes;
2285                 iov[iov_count++].iov_len = padding;
2286                 rx_size += padding;
2287         }
2288
2289         if (conn->conn_ops->DataDigest) {
2290                 iov[iov_count].iov_base         = &checksum;
2291                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2292                 rx_size += ISCSI_CRC_LEN;
2293         }
2294
2295         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2296
2297         iscsit_unmap_iovec(cmd);
2298
2299         if (rx_got != rx_size) {
2300                 iscsit_rx_thread_wait_for_tcp(conn);
2301                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2302         }
2303
2304         if (conn->conn_ops->DataDigest) {
2305                 u32 data_crc;
2306
2307                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
2308                                                     cmd->write_data_done, length, padding,
2309                                                     cmd->pad_bytes);
2310
2311                 if (checksum != data_crc) {
2312                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2313                                 " does not match computed 0x%08x\n", checksum,
2314                                 data_crc);
2315
2316                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2317                                 pr_err("Unable to recover from"
2318                                         " Immediate Data digest failure while"
2319                                         " in ERL=0.\n");
2320                                 iscsit_add_reject_from_cmd(
2321                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2322                                                 1, 0, buf, cmd);
2323                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2324                         } else {
2325                                 iscsit_add_reject_from_cmd(
2326                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2327                                                 0, 0, buf, cmd);
2328                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2329                         }
2330                 } else {
2331                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2332                                 " %u bytes of Immediate Data\n", checksum,
2333                                 length);
2334                 }
2335         }
2336
2337         cmd->write_data_done += length;
2338
2339         if (cmd->write_data_done == cmd->data_length) {
2340                 spin_lock_bh(&cmd->istate_lock);
2341                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2342                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2343                 spin_unlock_bh(&cmd->istate_lock);
2344         }
2345
2346         return IMMEDIATE_DATA_NORMAL_OPERATION;
2347 }
2348
2349 /*
2350  *      Called with sess->conn_lock held.
2351  */
2352 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2353         with active network interface */
2354 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2355 {
2356         struct iscsi_cmd *cmd;
2357         struct iscsi_conn *conn_p;
2358
2359         /*
2360          * Only send a Asynchronous Message on connections whos network
2361          * interface is still functional.
2362          */
2363         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2364                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2365                         iscsit_inc_conn_usage_count(conn_p);
2366                         break;
2367                 }
2368         }
2369
2370         if (!conn_p)
2371                 return;
2372
2373         cmd = iscsit_allocate_cmd(conn_p, GFP_KERNEL);
2374         if (!cmd) {
2375                 iscsit_dec_conn_usage_count(conn_p);
2376                 return;
2377         }
2378
2379         cmd->logout_cid = conn->cid;
2380         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2381         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2382
2383         spin_lock_bh(&conn_p->cmd_lock);
2384         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2385         spin_unlock_bh(&conn_p->cmd_lock);
2386
2387         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2388         iscsit_dec_conn_usage_count(conn_p);
2389 }
2390
2391 static int iscsit_send_conn_drop_async_message(
2392         struct iscsi_cmd *cmd,
2393         struct iscsi_conn *conn)
2394 {
2395         struct iscsi_async *hdr;
2396
2397         cmd->tx_size = ISCSI_HDR_LEN;
2398         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2399
2400         hdr                     = (struct iscsi_async *) cmd->pdu;
2401         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2402         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2403         cmd->init_task_tag      = 0xFFFFFFFF;
2404         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2405         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2406         cmd->stat_sn            = conn->stat_sn++;
2407         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2408         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2409         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2410         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2411         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2412         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2413         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2414
2415         if (conn->conn_ops->HeaderDigest) {
2416                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2417
2418                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2419                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2420                                 0, NULL, (u8 *)header_digest);
2421
2422                 cmd->tx_size += ISCSI_CRC_LEN;
2423                 pr_debug("Attaching CRC32C HeaderDigest to"
2424                         " Async Message 0x%08x\n", *header_digest);
2425         }
2426
2427         cmd->iov_misc[0].iov_base       = cmd->pdu;
2428         cmd->iov_misc[0].iov_len        = cmd->tx_size;
2429         cmd->iov_misc_count             = 1;
2430
2431         pr_debug("Sending Connection Dropped Async Message StatSN:"
2432                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2433                         cmd->logout_cid, conn->cid);
2434         return 0;
2435 }
2436
2437 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2438 {
2439         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2440             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2441                 wait_for_completion_interruptible_timeout(
2442                                         &conn->tx_half_close_comp,
2443                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2444         }
2445 }
2446
2447 static int iscsit_send_data_in(
2448         struct iscsi_cmd *cmd,
2449         struct iscsi_conn *conn)
2450 {
2451         int iov_ret = 0, set_statsn = 0;
2452         u32 iov_count = 0, tx_size = 0;
2453         struct iscsi_datain datain;
2454         struct iscsi_datain_req *dr;
2455         struct iscsi_data_rsp *hdr;
2456         struct kvec *iov;
2457         int eodr = 0;
2458         int ret;
2459
2460         memset(&datain, 0, sizeof(struct iscsi_datain));
2461         dr = iscsit_get_datain_values(cmd, &datain);
2462         if (!dr) {
2463                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2464                                 cmd->init_task_tag);
2465                 return -1;
2466         }
2467
2468         /*
2469          * Be paranoid and double check the logic for now.
2470          */
2471         if ((datain.offset + datain.length) > cmd->data_length) {
2472                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2473                         " datain.length: %u exceeds cmd->data_length: %u\n",
2474                         cmd->init_task_tag, datain.offset, datain.length,
2475                                 cmd->data_length);
2476                 return -1;
2477         }
2478
2479         spin_lock_bh(&conn->sess->session_stats_lock);
2480         conn->sess->tx_data_octets += datain.length;
2481         if (conn->sess->se_sess->se_node_acl) {
2482                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
2483                 conn->sess->se_sess->se_node_acl->read_bytes += datain.length;
2484                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
2485         }
2486         spin_unlock_bh(&conn->sess->session_stats_lock);
2487         /*
2488          * Special case for successfully execution w/ both DATAIN
2489          * and Sense Data.
2490          */
2491         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2492             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2493                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2494         else {
2495                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2496                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2497                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2498                         cmd->stat_sn = conn->stat_sn++;
2499                         set_statsn = 1;
2500                 } else if (dr->dr_complete ==
2501                                 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2502                         set_statsn = 1;
2503         }
2504
2505         hdr     = (struct iscsi_data_rsp *) cmd->pdu;
2506         memset(hdr, 0, ISCSI_HDR_LEN);
2507         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2508         hdr->flags              = datain.flags;
2509         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2510                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2511                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2512                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2513                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2514                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2515                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2516                 }
2517         }
2518         hton24(hdr->dlength, datain.length);
2519         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2520                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2521                                 (struct scsi_lun *)&hdr->lun);
2522         else
2523                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2524
2525         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2526         hdr->ttt                = (hdr->flags & ISCSI_FLAG_DATA_ACK) ?
2527                                    cpu_to_be32(cmd->targ_xfer_tag) :
2528                                    0xFFFFFFFF;
2529         hdr->statsn             = (set_statsn) ? cpu_to_be32(cmd->stat_sn) :
2530                                                 0xFFFFFFFF;
2531         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2532         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2533         hdr->datasn             = cpu_to_be32(datain.data_sn);
2534         hdr->offset             = cpu_to_be32(datain.offset);
2535
2536         iov = &cmd->iov_data[0];
2537         iov[iov_count].iov_base = cmd->pdu;
2538         iov[iov_count++].iov_len        = ISCSI_HDR_LEN;
2539         tx_size += ISCSI_HDR_LEN;
2540
2541         if (conn->conn_ops->HeaderDigest) {
2542                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2543
2544                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2545                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2546                                 0, NULL, (u8 *)header_digest);
2547
2548                 iov[0].iov_len += ISCSI_CRC_LEN;
2549                 tx_size += ISCSI_CRC_LEN;
2550
2551                 pr_debug("Attaching CRC32 HeaderDigest"
2552                         " for DataIN PDU 0x%08x\n", *header_digest);
2553         }
2554
2555         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length);
2556         if (iov_ret < 0)
2557                 return -1;
2558
2559         iov_count += iov_ret;
2560         tx_size += datain.length;
2561
2562         cmd->padding = ((-datain.length) & 3);
2563         if (cmd->padding) {
2564                 iov[iov_count].iov_base         = cmd->pad_bytes;
2565                 iov[iov_count++].iov_len        = cmd->padding;
2566                 tx_size += cmd->padding;
2567
2568                 pr_debug("Attaching %u padding bytes\n",
2569                                 cmd->padding);
2570         }
2571         if (conn->conn_ops->DataDigest) {
2572                 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd,
2573                          datain.offset, datain.length, cmd->padding, cmd->pad_bytes);
2574
2575                 iov[iov_count].iov_base = &cmd->data_crc;
2576                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2577                 tx_size += ISCSI_CRC_LEN;
2578
2579                 pr_debug("Attached CRC32C DataDigest %d bytes, crc"
2580                         " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc);
2581         }
2582
2583         cmd->iov_data_count = iov_count;
2584         cmd->tx_size = tx_size;
2585
2586         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2587                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2588                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2589                 ntohl(hdr->offset), datain.length, conn->cid);
2590
2591         /* sendpage is preferred but can't insert markers */
2592         if (!conn->conn_ops->IFMarker)
2593                 ret = iscsit_fe_sendpage_sg(cmd, conn);
2594         else
2595                 ret = iscsit_send_tx_data(cmd, conn, 0);
2596
2597         iscsit_unmap_iovec(cmd);
2598
2599         if (ret < 0) {
2600                 iscsit_tx_thread_wait_for_tcp(conn);
2601                 return ret;
2602         }
2603
2604         if (dr->dr_complete) {
2605                 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2606                                 2 : 1;
2607                 iscsit_free_datain_req(cmd, dr);
2608         }
2609
2610         return eodr;
2611 }
2612
2613 static int iscsit_send_logout_response(
2614         struct iscsi_cmd *cmd,
2615         struct iscsi_conn *conn)
2616 {
2617         int niov = 0, tx_size;
2618         struct iscsi_conn *logout_conn = NULL;
2619         struct iscsi_conn_recovery *cr = NULL;
2620         struct iscsi_session *sess = conn->sess;
2621         struct kvec *iov;
2622         struct iscsi_logout_rsp *hdr;
2623         /*
2624          * The actual shutting down of Sessions and/or Connections
2625          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2626          * is done in scsi_logout_post_handler().
2627          */
2628         switch (cmd->logout_reason) {
2629         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2630                 pr_debug("iSCSI session logout successful, setting"
2631                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2632                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2633                 break;
2634         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2635                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2636                         break;
2637                 /*
2638                  * For CLOSECONNECTION logout requests carrying
2639                  * a matching logout CID -> local CID, the reference
2640                  * for the local CID will have been incremented in
2641                  * iscsi_logout_closeconnection().
2642                  *
2643                  * For CLOSECONNECTION logout requests carrying
2644                  * a different CID than the connection it arrived
2645                  * on, the connection responding to cmd->logout_cid
2646                  * is stopped in iscsit_logout_post_handler_diffcid().
2647                  */
2648
2649                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2650                         " successful.\n", cmd->logout_cid, conn->cid);
2651                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2652                 break;
2653         case ISCSI_LOGOUT_REASON_RECOVERY:
2654                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2655                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2656                         break;
2657                 /*
2658                  * If the connection is still active from our point of view
2659                  * force connection recovery to occur.
2660                  */
2661                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2662                                 cmd->logout_cid);
2663                 if ((logout_conn)) {
2664                         iscsit_connection_reinstatement_rcfr(logout_conn);
2665                         iscsit_dec_conn_usage_count(logout_conn);
2666                 }
2667
2668                 cr = iscsit_get_inactive_connection_recovery_entry(
2669                                 conn->sess, cmd->logout_cid);
2670                 if (!cr) {
2671                         pr_err("Unable to locate CID: %hu for"
2672                         " REMOVECONNFORRECOVERY Logout Request.\n",
2673                                 cmd->logout_cid);
2674                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2675                         break;
2676                 }
2677
2678                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2679
2680                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2681                         " for recovery for CID: %hu on CID: %hu successful.\n",
2682                                 cmd->logout_cid, conn->cid);
2683                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2684                 break;
2685         default:
2686                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2687                                 cmd->logout_reason);
2688                 return -1;
2689         }
2690
2691         tx_size = ISCSI_HDR_LEN;
2692         hdr                     = (struct iscsi_logout_rsp *)cmd->pdu;
2693         memset(hdr, 0, ISCSI_HDR_LEN);
2694         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2695         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2696         hdr->response           = cmd->logout_response;
2697         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2698         cmd->stat_sn            = conn->stat_sn++;
2699         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2700
2701         iscsit_increment_maxcmdsn(cmd, conn->sess);
2702         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2703         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2704
2705         iov = &cmd->iov_misc[0];
2706         iov[niov].iov_base      = cmd->pdu;
2707         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2708
2709         if (conn->conn_ops->HeaderDigest) {
2710                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2711
2712                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2713                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2714                                 0, NULL, (u8 *)header_digest);
2715
2716                 iov[0].iov_len += ISCSI_CRC_LEN;
2717                 tx_size += ISCSI_CRC_LEN;
2718                 pr_debug("Attaching CRC32C HeaderDigest to"
2719                         " Logout Response 0x%08x\n", *header_digest);
2720         }
2721         cmd->iov_misc_count = niov;
2722         cmd->tx_size = tx_size;
2723
2724         pr_debug("Sending Logout Response ITT: 0x%08x StatSN:"
2725                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2726                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2727                 cmd->logout_cid, conn->cid);
2728
2729         return 0;
2730 }
2731
2732 /*
2733  *      Unsolicited NOPIN, either requesting a response or not.
2734  */
2735 static int iscsit_send_unsolicited_nopin(
2736         struct iscsi_cmd *cmd,
2737         struct iscsi_conn *conn,
2738         int want_response)
2739 {
2740         int tx_size = ISCSI_HDR_LEN;
2741         struct iscsi_nopin *hdr;
2742         int ret;
2743
2744         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2745         memset(hdr, 0, ISCSI_HDR_LEN);
2746         hdr->opcode             = ISCSI_OP_NOOP_IN;
2747         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2748         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2749         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2750         cmd->stat_sn            = conn->stat_sn;
2751         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2752         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2753         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2754
2755         if (conn->conn_ops->HeaderDigest) {
2756                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2757
2758                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2759                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2760                                 0, NULL, (u8 *)header_digest);
2761
2762                 tx_size += ISCSI_CRC_LEN;
2763                 pr_debug("Attaching CRC32C HeaderDigest to"
2764                         " NopIN 0x%08x\n", *header_digest);
2765         }
2766
2767         cmd->iov_misc[0].iov_base       = cmd->pdu;
2768         cmd->iov_misc[0].iov_len        = tx_size;
2769         cmd->iov_misc_count     = 1;
2770         cmd->tx_size            = tx_size;
2771
2772         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
2773                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
2774
2775         ret = iscsit_send_tx_data(cmd, conn, 1);
2776         if (ret < 0) {
2777                 iscsit_tx_thread_wait_for_tcp(conn);
2778                 return ret;
2779         }
2780
2781         spin_lock_bh(&cmd->istate_lock);
2782         cmd->i_state = want_response ?
2783                 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
2784         spin_unlock_bh(&cmd->istate_lock);
2785
2786         return 0;
2787 }
2788
2789 static int iscsit_send_nopin_response(
2790         struct iscsi_cmd *cmd,
2791         struct iscsi_conn *conn)
2792 {
2793         int niov = 0, tx_size;
2794         u32 padding = 0;
2795         struct kvec *iov;
2796         struct iscsi_nopin *hdr;
2797
2798         tx_size = ISCSI_HDR_LEN;
2799         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2800         memset(hdr, 0, ISCSI_HDR_LEN);
2801         hdr->opcode             = ISCSI_OP_NOOP_IN;
2802         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2803         hton24(hdr->dlength, cmd->buf_ptr_size);
2804         put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2805         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2806         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2807         cmd->stat_sn            = conn->stat_sn++;
2808         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2809
2810         iscsit_increment_maxcmdsn(cmd, conn->sess);
2811         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2812         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2813
2814         iov = &cmd->iov_misc[0];
2815         iov[niov].iov_base      = cmd->pdu;
2816         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2817
2818         if (conn->conn_ops->HeaderDigest) {
2819                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2820
2821                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2822                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2823                                 0, NULL, (u8 *)header_digest);
2824
2825                 iov[0].iov_len += ISCSI_CRC_LEN;
2826                 tx_size += ISCSI_CRC_LEN;
2827                 pr_debug("Attaching CRC32C HeaderDigest"
2828                         " to NopIn 0x%08x\n", *header_digest);
2829         }
2830
2831         /*
2832          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
2833          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
2834          */
2835         if (cmd->buf_ptr_size) {
2836                 iov[niov].iov_base      = cmd->buf_ptr;
2837                 iov[niov++].iov_len     = cmd->buf_ptr_size;
2838                 tx_size += cmd->buf_ptr_size;
2839
2840                 pr_debug("Echoing back %u bytes of ping"
2841                         " data.\n", cmd->buf_ptr_size);
2842
2843                 padding = ((-cmd->buf_ptr_size) & 3);
2844                 if (padding != 0) {
2845                         iov[niov].iov_base = &cmd->pad_bytes;
2846                         iov[niov++].iov_len = padding;
2847                         tx_size += padding;
2848                         pr_debug("Attaching %u additional"
2849                                 " padding bytes.\n", padding);
2850                 }
2851                 if (conn->conn_ops->DataDigest) {
2852                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2853                                 cmd->buf_ptr, cmd->buf_ptr_size,
2854                                 padding, (u8 *)&cmd->pad_bytes,
2855                                 (u8 *)&cmd->data_crc);
2856
2857                         iov[niov].iov_base = &cmd->data_crc;
2858                         iov[niov++].iov_len = ISCSI_CRC_LEN;
2859                         tx_size += ISCSI_CRC_LEN;
2860                         pr_debug("Attached DataDigest for %u"
2861                                 " bytes of ping data, CRC 0x%08x\n",
2862                                 cmd->buf_ptr_size, cmd->data_crc);
2863                 }
2864         }
2865
2866         cmd->iov_misc_count = niov;
2867         cmd->tx_size = tx_size;
2868
2869         pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:"
2870                 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag,
2871                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
2872
2873         return 0;
2874 }
2875
2876 static int iscsit_send_r2t(
2877         struct iscsi_cmd *cmd,
2878         struct iscsi_conn *conn)
2879 {
2880         int tx_size = 0;
2881         struct iscsi_r2t *r2t;
2882         struct iscsi_r2t_rsp *hdr;
2883         int ret;
2884
2885         r2t = iscsit_get_r2t_from_list(cmd);
2886         if (!r2t)
2887                 return -1;
2888
2889         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
2890         memset(hdr, 0, ISCSI_HDR_LEN);
2891         hdr->opcode             = ISCSI_OP_R2T;
2892         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2893         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2894                         (struct scsi_lun *)&hdr->lun);
2895         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2896         spin_lock_bh(&conn->sess->ttt_lock);
2897         r2t->targ_xfer_tag      = conn->sess->targ_xfer_tag++;
2898         if (r2t->targ_xfer_tag == 0xFFFFFFFF)
2899                 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
2900         spin_unlock_bh(&conn->sess->ttt_lock);
2901         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
2902         hdr->statsn             = cpu_to_be32(conn->stat_sn);
2903         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2904         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2905         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
2906         hdr->data_offset        = cpu_to_be32(r2t->offset);
2907         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
2908
2909         cmd->iov_misc[0].iov_base       = cmd->pdu;
2910         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
2911         tx_size += ISCSI_HDR_LEN;
2912
2913         if (conn->conn_ops->HeaderDigest) {
2914                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2915
2916                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2917                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2918                                 0, NULL, (u8 *)header_digest);
2919
2920                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
2921                 tx_size += ISCSI_CRC_LEN;
2922                 pr_debug("Attaching CRC32 HeaderDigest for R2T"
2923                         " PDU 0x%08x\n", *header_digest);
2924         }
2925
2926         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
2927                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
2928                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
2929                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
2930                         r2t->offset, r2t->xfer_len, conn->cid);
2931
2932         cmd->iov_misc_count = 1;
2933         cmd->tx_size = tx_size;
2934
2935         spin_lock_bh(&cmd->r2t_lock);
2936         r2t->sent_r2t = 1;
2937         spin_unlock_bh(&cmd->r2t_lock);
2938
2939         ret = iscsit_send_tx_data(cmd, conn, 1);
2940         if (ret < 0) {
2941                 iscsit_tx_thread_wait_for_tcp(conn);
2942                 return ret;
2943         }
2944
2945         spin_lock_bh(&cmd->dataout_timeout_lock);
2946         iscsit_start_dataout_timer(cmd, conn);
2947         spin_unlock_bh(&cmd->dataout_timeout_lock);
2948
2949         return 0;
2950 }
2951
2952 /*
2953  *      type 0: Normal Operation.
2954  *      type 1: Called from Storage Transport.
2955  *      type 2: Called from iscsi_task_reassign_complete_write() for
2956  *              connection recovery.
2957  */
2958 int iscsit_build_r2ts_for_cmd(
2959         struct iscsi_cmd *cmd,
2960         struct iscsi_conn *conn,
2961         int type)
2962 {
2963         int first_r2t = 1;
2964         u32 offset = 0, xfer_len = 0;
2965
2966         spin_lock_bh(&cmd->r2t_lock);
2967         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
2968                 spin_unlock_bh(&cmd->r2t_lock);
2969                 return 0;
2970         }
2971
2972         if (conn->sess->sess_ops->DataSequenceInOrder && (type != 2))
2973                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
2974
2975         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
2976                 if (conn->sess->sess_ops->DataSequenceInOrder) {
2977                         offset = cmd->r2t_offset;
2978
2979                         if (first_r2t && (type == 2)) {
2980                                 xfer_len = ((offset +
2981                                              (conn->sess->sess_ops->MaxBurstLength -
2982                                              cmd->next_burst_len) >
2983                                              cmd->data_length) ?
2984                                             (cmd->data_length - offset) :
2985                                             (conn->sess->sess_ops->MaxBurstLength -
2986                                              cmd->next_burst_len));
2987                         } else {
2988                                 xfer_len = ((offset +
2989                                              conn->sess->sess_ops->MaxBurstLength) >
2990                                              cmd->data_length) ?
2991                                              (cmd->data_length - offset) :
2992                                              conn->sess->sess_ops->MaxBurstLength;
2993                         }
2994                         cmd->r2t_offset += xfer_len;
2995
2996                         if (cmd->r2t_offset == cmd->data_length)
2997                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
2998                 } else {
2999                         struct iscsi_seq *seq;
3000
3001                         seq = iscsit_get_seq_holder_for_r2t(cmd);
3002                         if (!seq) {
3003                                 spin_unlock_bh(&cmd->r2t_lock);
3004                                 return -1;
3005                         }
3006
3007                         offset = seq->offset;
3008                         xfer_len = seq->xfer_len;
3009
3010                         if (cmd->seq_send_order == cmd->seq_count)
3011                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3012                 }
3013                 cmd->outstanding_r2ts++;
3014                 first_r2t = 0;
3015
3016                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3017                         spin_unlock_bh(&cmd->r2t_lock);
3018                         return -1;
3019                 }
3020
3021                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3022                         break;
3023         }
3024         spin_unlock_bh(&cmd->r2t_lock);
3025
3026         return 0;
3027 }
3028
3029 static int iscsit_send_status(
3030         struct iscsi_cmd *cmd,
3031         struct iscsi_conn *conn)
3032 {
3033         u8 iov_count = 0, recovery;
3034         u32 padding = 0, tx_size = 0;
3035         struct iscsi_scsi_rsp *hdr;
3036         struct kvec *iov;
3037
3038         recovery = (cmd->i_state != ISTATE_SEND_STATUS);
3039         if (!recovery)
3040                 cmd->stat_sn = conn->stat_sn++;
3041
3042         spin_lock_bh(&conn->sess->session_stats_lock);
3043         conn->sess->rsp_pdus++;
3044         spin_unlock_bh(&conn->sess->session_stats_lock);
3045
3046         hdr                     = (struct iscsi_scsi_rsp *) cmd->pdu;
3047         memset(hdr, 0, ISCSI_HDR_LEN);
3048         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3049         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3050         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3051                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3052                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3053         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3054                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3055                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3056         }
3057         hdr->response           = cmd->iscsi_response;
3058         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3059         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3060         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3061
3062         iscsit_increment_maxcmdsn(cmd, conn->sess);
3063         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3064         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3065
3066         iov = &cmd->iov_misc[0];
3067         iov[iov_count].iov_base = cmd->pdu;
3068         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3069         tx_size += ISCSI_HDR_LEN;
3070
3071         /*
3072          * Attach SENSE DATA payload to iSCSI Response PDU
3073          */
3074         if (cmd->se_cmd.sense_buffer &&
3075            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3076             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3077                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3078                 hton24(hdr->dlength, cmd->se_cmd.scsi_sense_length);
3079                 iov[iov_count].iov_base = cmd->se_cmd.sense_buffer;
3080                 iov[iov_count++].iov_len =
3081                                 (cmd->se_cmd.scsi_sense_length + padding);
3082                 tx_size += cmd->se_cmd.scsi_sense_length;
3083
3084                 if (padding) {
3085                         memset(cmd->se_cmd.sense_buffer +
3086                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3087                         tx_size += padding;
3088                         pr_debug("Adding %u bytes of padding to"
3089                                 " SENSE.\n", padding);
3090                 }
3091
3092                 if (conn->conn_ops->DataDigest) {
3093                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3094                                 cmd->se_cmd.sense_buffer,
3095                                 (cmd->se_cmd.scsi_sense_length + padding),
3096                                 0, NULL, (u8 *)&cmd->data_crc);
3097
3098                         iov[iov_count].iov_base    = &cmd->data_crc;
3099                         iov[iov_count++].iov_len     = ISCSI_CRC_LEN;
3100                         tx_size += ISCSI_CRC_LEN;
3101
3102                         pr_debug("Attaching CRC32 DataDigest for"
3103                                 " SENSE, %u bytes CRC 0x%08x\n",
3104                                 (cmd->se_cmd.scsi_sense_length + padding),
3105                                 cmd->data_crc);
3106                 }
3107
3108                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3109                                 " Response PDU\n",
3110                                 cmd->se_cmd.scsi_sense_length);
3111         }
3112
3113         if (conn->conn_ops->HeaderDigest) {
3114                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3115
3116                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3117                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3118                                 0, NULL, (u8 *)header_digest);
3119
3120                 iov[0].iov_len += ISCSI_CRC_LEN;
3121                 tx_size += ISCSI_CRC_LEN;
3122                 pr_debug("Attaching CRC32 HeaderDigest for Response"
3123                                 " PDU 0x%08x\n", *header_digest);
3124         }
3125
3126         cmd->iov_misc_count = iov_count;
3127         cmd->tx_size = tx_size;
3128
3129         pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3130                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3131                 (!recovery) ? "" : "Recovery ", cmd->init_task_tag,
3132                 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid);
3133
3134         return 0;
3135 }
3136
3137 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3138 {
3139         switch (se_tmr->response) {
3140         case TMR_FUNCTION_COMPLETE:
3141                 return ISCSI_TMF_RSP_COMPLETE;
3142         case TMR_TASK_DOES_NOT_EXIST:
3143                 return ISCSI_TMF_RSP_NO_TASK;
3144         case TMR_LUN_DOES_NOT_EXIST:
3145                 return ISCSI_TMF_RSP_NO_LUN;
3146         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3147                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3148         case TMR_FUNCTION_AUTHORIZATION_FAILED:
3149                 return ISCSI_TMF_RSP_AUTH_FAILED;
3150         case TMR_FUNCTION_REJECTED:
3151         default:
3152                 return ISCSI_TMF_RSP_REJECTED;
3153         }
3154 }
3155
3156 static int iscsit_send_task_mgt_rsp(
3157         struct iscsi_cmd *cmd,
3158         struct iscsi_conn *conn)
3159 {
3160         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3161         struct iscsi_tm_rsp *hdr;
3162         u32 tx_size = 0;
3163
3164         hdr                     = (struct iscsi_tm_rsp *) cmd->pdu;
3165         memset(hdr, 0, ISCSI_HDR_LEN);
3166         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3167         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3168         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3169         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3170         cmd->stat_sn            = conn->stat_sn++;
3171         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3172
3173         iscsit_increment_maxcmdsn(cmd, conn->sess);
3174         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3175         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3176
3177         cmd->iov_misc[0].iov_base       = cmd->pdu;
3178         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
3179         tx_size += ISCSI_HDR_LEN;
3180
3181         if (conn->conn_ops->HeaderDigest) {
3182                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3183
3184                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3185                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3186                                 0, NULL, (u8 *)header_digest);
3187
3188                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3189                 tx_size += ISCSI_CRC_LEN;
3190                 pr_debug("Attaching CRC32 HeaderDigest for Task"
3191                         " Mgmt Response PDU 0x%08x\n", *header_digest);
3192         }
3193
3194         cmd->iov_misc_count = 1;
3195         cmd->tx_size = tx_size;
3196
3197         pr_debug("Built Task Management Response ITT: 0x%08x,"
3198                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3199                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3200
3201         return 0;
3202 }
3203
3204 static bool iscsit_check_inaddr_any(struct iscsi_np *np)
3205 {
3206         bool ret = false;
3207
3208         if (np->np_sockaddr.ss_family == AF_INET6) {
3209                 const struct sockaddr_in6 sin6 = {
3210                         .sin6_addr = IN6ADDR_ANY_INIT };
3211                 struct sockaddr_in6 *sock_in6 =
3212                          (struct sockaddr_in6 *)&np->np_sockaddr;
3213
3214                 if (!memcmp(sock_in6->sin6_addr.s6_addr,
3215                                 sin6.sin6_addr.s6_addr, 16))
3216                         ret = true;
3217         } else {
3218                 struct sockaddr_in * sock_in =
3219                         (struct sockaddr_in *)&np->np_sockaddr;
3220
3221                 if (sock_in->sin_addr.s_addr == INADDR_ANY)
3222                         ret = true;
3223         }
3224
3225         return ret;
3226 }
3227
3228 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd)
3229 {
3230         char *payload = NULL;
3231         struct iscsi_conn *conn = cmd->conn;
3232         struct iscsi_portal_group *tpg;
3233         struct iscsi_tiqn *tiqn;
3234         struct iscsi_tpg_np *tpg_np;
3235         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3236         unsigned char buf[256];
3237
3238         buffer_len = (conn->conn_ops->MaxRecvDataSegmentLength > 32768) ?
3239                         32768 : conn->conn_ops->MaxRecvDataSegmentLength;
3240
3241         memset(buf, 0, 256);
3242
3243         payload = kzalloc(buffer_len, GFP_KERNEL);
3244         if (!payload) {
3245                 pr_err("Unable to allocate memory for sendtargets"
3246                                 " response.\n");
3247                 return -ENOMEM;
3248         }
3249
3250         spin_lock(&tiqn_lock);
3251         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3252                 len = sprintf(buf, "TargetName=%s", tiqn->tiqn);
3253                 len += 1;
3254
3255                 if ((len + payload_len) > buffer_len) {
3256                         spin_unlock(&tiqn->tiqn_tpg_lock);
3257                         end_of_buf = 1;
3258                         goto eob;
3259                 }
3260                 memcpy(payload + payload_len, buf, len);
3261                 payload_len += len;
3262
3263                 spin_lock(&tiqn->tiqn_tpg_lock);
3264                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3265
3266                         spin_lock(&tpg->tpg_state_lock);
3267                         if ((tpg->tpg_state == TPG_STATE_FREE) ||
3268                             (tpg->tpg_state == TPG_STATE_INACTIVE)) {
3269                                 spin_unlock(&tpg->tpg_state_lock);
3270                                 continue;
3271                         }
3272                         spin_unlock(&tpg->tpg_state_lock);
3273
3274                         spin_lock(&tpg->tpg_np_lock);
3275                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3276                                                 tpg_np_list) {
3277                                 struct iscsi_np *np = tpg_np->tpg_np;
3278                                 bool inaddr_any = iscsit_check_inaddr_any(np);
3279
3280                                 len = sprintf(buf, "TargetAddress="
3281                                         "%s%s%s:%hu,%hu",
3282                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3283                                         "[" : "", (inaddr_any == false) ?
3284                                                 np->np_ip : conn->local_ip,
3285                                         (np->np_sockaddr.ss_family == AF_INET6) ?
3286                                         "]" : "", (inaddr_any == false) ?
3287                                                 np->np_port : conn->local_port,
3288                                         tpg->tpgt);
3289                                 len += 1;
3290
3291                                 if ((len + payload_len) > buffer_len) {
3292                                         spin_unlock(&tpg->tpg_np_lock);
3293                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3294                                         end_of_buf = 1;
3295                                         goto eob;
3296                                 }
3297                                 memcpy(payload + payload_len, buf, len);
3298                                 payload_len += len;
3299                         }
3300                         spin_unlock(&tpg->tpg_np_lock);
3301                 }
3302                 spin_unlock(&tiqn->tiqn_tpg_lock);
3303 eob:
3304                 if (end_of_buf)
3305                         break;
3306         }
3307         spin_unlock(&tiqn_lock);
3308
3309         cmd->buf_ptr = payload;
3310
3311         return payload_len;
3312 }
3313
3314 /*
3315  *      FIXME: Add support for F_BIT and C_BIT when the length is longer than
3316  *      MaxRecvDataSegmentLength.
3317  */
3318 static int iscsit_send_text_rsp(
3319         struct iscsi_cmd *cmd,
3320         struct iscsi_conn *conn)
3321 {
3322         struct iscsi_text_rsp *hdr;
3323         struct kvec *iov;
3324         u32 padding = 0, tx_size = 0;
3325         int text_length, iov_count = 0;
3326
3327         text_length = iscsit_build_sendtargets_response(cmd);
3328         if (text_length < 0)
3329                 return text_length;
3330
3331         padding = ((-text_length) & 3);
3332         if (padding != 0) {
3333                 memset(cmd->buf_ptr + text_length, 0, padding);
3334                 pr_debug("Attaching %u additional bytes for"
3335                         " padding.\n", padding);
3336         }
3337
3338         hdr                     = (struct iscsi_text_rsp *) cmd->pdu;
3339         memset(hdr, 0, ISCSI_HDR_LEN);
3340         hdr->opcode             = ISCSI_OP_TEXT_RSP;
3341         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3342         hton24(hdr->dlength, text_length);
3343         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3344         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3345         cmd->stat_sn            = conn->stat_sn++;
3346         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3347
3348         iscsit_increment_maxcmdsn(cmd, conn->sess);
3349         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3350         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3351
3352         iov = &cmd->iov_misc[0];
3353
3354         iov[iov_count].iov_base = cmd->pdu;
3355         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3356         iov[iov_count].iov_base = cmd->buf_ptr;
3357         iov[iov_count++].iov_len = text_length + padding;
3358
3359         tx_size += (ISCSI_HDR_LEN + text_length + padding);
3360
3361         if (conn->conn_ops->HeaderDigest) {
3362                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3363
3364                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3365                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3366                                 0, NULL, (u8 *)header_digest);
3367
3368                 iov[0].iov_len += ISCSI_CRC_LEN;
3369                 tx_size += ISCSI_CRC_LEN;
3370                 pr_debug("Attaching CRC32 HeaderDigest for"
3371                         " Text Response PDU 0x%08x\n", *header_digest);
3372         }
3373
3374         if (conn->conn_ops->DataDigest) {
3375                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3376                                 cmd->buf_ptr, (text_length + padding),
3377                                 0, NULL, (u8 *)&cmd->data_crc);
3378
3379                 iov[iov_count].iov_base = &cmd->data_crc;
3380                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3381                 tx_size += ISCSI_CRC_LEN;
3382
3383                 pr_debug("Attaching DataDigest for %u bytes of text"
3384                         " data, CRC 0x%08x\n", (text_length + padding),
3385                         cmd->data_crc);
3386         }
3387
3388         cmd->iov_misc_count = iov_count;
3389         cmd->tx_size = tx_size;
3390
3391         pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x,"
3392                 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn,
3393                         text_length, conn->cid);
3394         return 0;
3395 }
3396
3397 static int iscsit_send_reject(
3398         struct iscsi_cmd *cmd,
3399         struct iscsi_conn *conn)
3400 {
3401         u32 iov_count = 0, tx_size = 0;
3402         struct iscsi_reject *hdr;
3403         struct kvec *iov;
3404
3405         hdr                     = (struct iscsi_reject *) cmd->pdu;
3406         hdr->opcode             = ISCSI_OP_REJECT;
3407         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3408         hton24(hdr->dlength, ISCSI_HDR_LEN);
3409         cmd->stat_sn            = conn->stat_sn++;
3410         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3411         hdr->exp_cmdsn  = cpu_to_be32(conn->sess->exp_cmd_sn);
3412         hdr->max_cmdsn  = cpu_to_be32(conn->sess->max_cmd_sn);
3413
3414         iov = &cmd->iov_misc[0];
3415
3416         iov[iov_count].iov_base = cmd->pdu;
3417         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3418         iov[iov_count].iov_base = cmd->buf_ptr;
3419         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3420
3421         tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN);
3422
3423         if (conn->conn_ops->HeaderDigest) {
3424                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3425
3426                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3427                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3428                                 0, NULL, (u8 *)header_digest);
3429
3430                 iov[0].iov_len += ISCSI_CRC_LEN;
3431                 tx_size += ISCSI_CRC_LEN;
3432                 pr_debug("Attaching CRC32 HeaderDigest for"
3433                         " REJECT PDU 0x%08x\n", *header_digest);
3434         }
3435
3436         if (conn->conn_ops->DataDigest) {
3437                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3438                                 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN,
3439                                 0, NULL, (u8 *)&cmd->data_crc);
3440
3441                 iov[iov_count].iov_base = &cmd->data_crc;
3442                 iov[iov_count++].iov_len  = ISCSI_CRC_LEN;
3443                 tx_size += ISCSI_CRC_LEN;
3444                 pr_debug("Attaching CRC32 DataDigest for REJECT"
3445                                 " PDU 0x%08x\n", cmd->data_crc);
3446         }
3447
3448         cmd->iov_misc_count = iov_count;
3449         cmd->tx_size = tx_size;
3450
3451         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3452                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3453
3454         return 0;
3455 }
3456
3457 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3458 {
3459         struct iscsi_thread_set *ts = conn->thread_set;
3460         int ord, cpu;
3461         /*
3462          * thread_id is assigned from iscsit_global->ts_bitmap from
3463          * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
3464          *
3465          * Here we use thread_id to determine which CPU that this
3466          * iSCSI connection's iscsi_thread_set will be scheduled to
3467          * execute upon.
3468          */
3469         ord = ts->thread_id % cpumask_weight(cpu_online_mask);
3470         for_each_online_cpu(cpu) {
3471                 if (ord-- == 0) {
3472                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3473                         return;
3474                 }
3475         }
3476         /*
3477          * This should never be reached..
3478          */
3479         dump_stack();
3480         cpumask_setall(conn->conn_cpumask);
3481 }
3482
3483 static inline void iscsit_thread_check_cpumask(
3484         struct iscsi_conn *conn,
3485         struct task_struct *p,
3486         int mode)
3487 {
3488         char buf[128];
3489         /*
3490          * mode == 1 signals iscsi_target_tx_thread() usage.
3491          * mode == 0 signals iscsi_target_rx_thread() usage.
3492          */
3493         if (mode == 1) {
3494                 if (!conn->conn_tx_reset_cpumask)
3495                         return;
3496                 conn->conn_tx_reset_cpumask = 0;
3497         } else {
3498                 if (!conn->conn_rx_reset_cpumask)
3499                         return;
3500                 conn->conn_rx_reset_cpumask = 0;
3501         }
3502         /*
3503          * Update the CPU mask for this single kthread so that
3504          * both TX and RX kthreads are scheduled to run on the
3505          * same CPU.
3506          */
3507         memset(buf, 0, 128);
3508         cpumask_scnprintf(buf, 128, conn->conn_cpumask);
3509         set_cpus_allowed_ptr(p, conn->conn_cpumask);
3510 }
3511
3512 static int handle_immediate_queue(struct iscsi_conn *conn)
3513 {
3514         struct iscsi_queue_req *qr;
3515         struct iscsi_cmd *cmd;
3516         u8 state;
3517         int ret;
3518
3519         while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3520                 atomic_set(&conn->check_immediate_queue, 0);
3521                 cmd = qr->cmd;
3522                 state = qr->state;
3523                 kmem_cache_free(lio_qr_cache, qr);
3524
3525                 switch (state) {
3526                 case ISTATE_SEND_R2T:
3527                         ret = iscsit_send_r2t(cmd, conn);
3528                         if (ret < 0)
3529                                 goto err;
3530                         break;
3531                 case ISTATE_REMOVE:
3532                         if (cmd->data_direction == DMA_TO_DEVICE)
3533                                 iscsit_stop_dataout_timer(cmd);
3534
3535                         spin_lock_bh(&conn->cmd_lock);
3536                         list_del(&cmd->i_conn_node);
3537                         spin_unlock_bh(&conn->cmd_lock);
3538
3539                         iscsit_free_cmd(cmd);
3540                         continue;
3541                 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3542                         iscsit_mod_nopin_response_timer(conn);
3543                         ret = iscsit_send_unsolicited_nopin(cmd,
3544                                                             conn, 1);
3545                         if (ret < 0)
3546                                 goto err;
3547                         break;
3548                 case ISTATE_SEND_NOPIN_NO_RESPONSE:
3549                         ret = iscsit_send_unsolicited_nopin(cmd,
3550                                                             conn, 0);
3551                         if (ret < 0)
3552                                 goto err;
3553                         break;
3554                 default:
3555                         pr_err("Unknown Opcode: 0x%02x ITT:"
3556                                " 0x%08x, i_state: %d on CID: %hu\n",
3557                                cmd->iscsi_opcode, cmd->init_task_tag, state,
3558                                conn->cid);
3559                         goto err;
3560                 }
3561         }
3562
3563         return 0;
3564
3565 err:
3566         return -1;
3567 }
3568
3569 static int handle_response_queue(struct iscsi_conn *conn)
3570 {
3571         struct iscsi_queue_req *qr;
3572         struct iscsi_cmd *cmd;
3573         u8 state;
3574         int ret;
3575
3576         while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3577                 cmd = qr->cmd;
3578                 state = qr->state;
3579                 kmem_cache_free(lio_qr_cache, qr);
3580
3581 check_rsp_state:
3582                 switch (state) {
3583                 case ISTATE_SEND_DATAIN:
3584                         ret = iscsit_send_data_in(cmd, conn);
3585                         if (ret < 0)
3586                                 goto err;
3587                         else if (!ret)
3588                                 /* more drs */
3589                                 goto check_rsp_state;
3590                         else if (ret == 1) {
3591                                 /* all done */
3592                                 spin_lock_bh(&cmd->istate_lock);
3593                                 cmd->i_state = ISTATE_SENT_STATUS;
3594                                 spin_unlock_bh(&cmd->istate_lock);
3595                                 continue;
3596                         } else if (ret == 2) {
3597                                 /* Still must send status,
3598                                    SCF_TRANSPORT_TASK_SENSE was set */
3599                                 spin_lock_bh(&cmd->istate_lock);
3600                                 cmd->i_state = ISTATE_SEND_STATUS;
3601                                 spin_unlock_bh(&cmd->istate_lock);
3602                                 state = ISTATE_SEND_STATUS;
3603                                 goto check_rsp_state;
3604                         }
3605
3606                         break;
3607                 case ISTATE_SEND_STATUS:
3608                 case ISTATE_SEND_STATUS_RECOVERY:
3609                         ret = iscsit_send_status(cmd, conn);
3610                         break;
3611                 case ISTATE_SEND_LOGOUTRSP:
3612                         ret = iscsit_send_logout_response(cmd, conn);
3613                         break;
3614                 case ISTATE_SEND_ASYNCMSG:
3615                         ret = iscsit_send_conn_drop_async_message(
3616                                 cmd, conn);
3617                         break;
3618                 case ISTATE_SEND_NOPIN:
3619                         ret = iscsit_send_nopin_response(cmd, conn);
3620                         break;
3621                 case ISTATE_SEND_REJECT:
3622                         ret = iscsit_send_reject(cmd, conn);
3623                         break;
3624                 case ISTATE_SEND_TASKMGTRSP:
3625                         ret = iscsit_send_task_mgt_rsp(cmd, conn);
3626                         if (ret != 0)
3627                                 break;
3628                         ret = iscsit_tmr_post_handler(cmd, conn);
3629                         if (ret != 0)
3630                                 iscsit_fall_back_to_erl0(conn->sess);
3631                         break;
3632                 case ISTATE_SEND_TEXTRSP:
3633                         ret = iscsit_send_text_rsp(cmd, conn);
3634                         break;
3635                 default:
3636                         pr_err("Unknown Opcode: 0x%02x ITT:"
3637                                " 0x%08x, i_state: %d on CID: %hu\n",
3638                                cmd->iscsi_opcode, cmd->init_task_tag,
3639                                state, conn->cid);
3640                         goto err;
3641                 }
3642                 if (ret < 0)
3643                         goto err;
3644
3645                 if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
3646                         iscsit_tx_thread_wait_for_tcp(conn);
3647                         iscsit_unmap_iovec(cmd);
3648                         goto err;
3649                 }
3650                 iscsit_unmap_iovec(cmd);
3651
3652                 switch (state) {
3653                 case ISTATE_SEND_LOGOUTRSP:
3654                         if (!iscsit_logout_post_handler(cmd, conn))
3655                                 goto restart;
3656                         /* fall through */
3657                 case ISTATE_SEND_STATUS:
3658                 case ISTATE_SEND_ASYNCMSG:
3659                 case ISTATE_SEND_NOPIN:
3660                 case ISTATE_SEND_STATUS_RECOVERY:
3661                 case ISTATE_SEND_TEXTRSP:
3662                 case ISTATE_SEND_TASKMGTRSP:
3663                         spin_lock_bh(&cmd->istate_lock);
3664                         cmd->i_state = ISTATE_SENT_STATUS;
3665                         spin_unlock_bh(&cmd->istate_lock);
3666                         break;
3667                 case ISTATE_SEND_REJECT:
3668                         if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
3669                                 cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
3670                                 complete(&cmd->reject_comp);
3671                                 goto err;
3672                         }
3673                         complete(&cmd->reject_comp);
3674                         break;
3675                 default:
3676                         pr_err("Unknown Opcode: 0x%02x ITT:"
3677                                " 0x%08x, i_state: %d on CID: %hu\n",
3678                                cmd->iscsi_opcode, cmd->init_task_tag,
3679                                cmd->i_state, conn->cid);
3680                         goto err;
3681                 }
3682
3683                 if (atomic_read(&conn->check_immediate_queue))
3684                         break;
3685         }
3686
3687         return 0;
3688
3689 err:
3690         return -1;
3691 restart:
3692         return -EAGAIN;
3693 }
3694
3695 int iscsi_target_tx_thread(void *arg)
3696 {
3697         int ret = 0;
3698         struct iscsi_conn *conn;
3699         struct iscsi_thread_set *ts = arg;
3700         /*
3701          * Allow ourselves to be interrupted by SIGINT so that a
3702          * connection recovery / failure event can be triggered externally.
3703          */
3704         allow_signal(SIGINT);
3705
3706 restart:
3707         conn = iscsi_tx_thread_pre_handler(ts);
3708         if (!conn)
3709                 goto out;
3710
3711         ret = 0;
3712
3713         while (!kthread_should_stop()) {
3714                 /*
3715                  * Ensure that both TX and RX per connection kthreads
3716                  * are scheduled to run on the same CPU.
3717                  */
3718                 iscsit_thread_check_cpumask(conn, current, 1);
3719
3720                 schedule_timeout_interruptible(MAX_SCHEDULE_TIMEOUT);
3721
3722                 if ((ts->status == ISCSI_THREAD_SET_RESET) ||
3723                      signal_pending(current))
3724                         goto transport_err;
3725
3726                 ret = handle_immediate_queue(conn);
3727                 if (ret < 0)
3728                         goto transport_err;
3729
3730                 ret = handle_response_queue(conn);
3731                 if (ret == -EAGAIN)
3732                         goto restart;
3733                 else if (ret < 0)
3734                         goto transport_err;
3735         }
3736
3737 transport_err:
3738         iscsit_take_action_for_connection_exit(conn);
3739         goto restart;
3740 out:
3741         return 0;
3742 }
3743
3744 int iscsi_target_rx_thread(void *arg)
3745 {
3746         int ret;
3747         u8 buffer[ISCSI_HDR_LEN], opcode;
3748         u32 checksum = 0, digest = 0;
3749         struct iscsi_conn *conn = NULL;
3750         struct iscsi_thread_set *ts = arg;
3751         struct kvec iov;
3752         /*
3753          * Allow ourselves to be interrupted by SIGINT so that a
3754          * connection recovery / failure event can be triggered externally.
3755          */
3756         allow_signal(SIGINT);
3757
3758 restart:
3759         conn = iscsi_rx_thread_pre_handler(ts);
3760         if (!conn)
3761                 goto out;
3762
3763         while (!kthread_should_stop()) {
3764                 /*
3765                  * Ensure that both TX and RX per connection kthreads
3766                  * are scheduled to run on the same CPU.
3767                  */
3768                 iscsit_thread_check_cpumask(conn, current, 0);
3769
3770                 memset(buffer, 0, ISCSI_HDR_LEN);
3771                 memset(&iov, 0, sizeof(struct kvec));
3772
3773                 iov.iov_base    = buffer;
3774                 iov.iov_len     = ISCSI_HDR_LEN;
3775
3776                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3777                 if (ret != ISCSI_HDR_LEN) {
3778                         iscsit_rx_thread_wait_for_tcp(conn);
3779                         goto transport_err;
3780                 }
3781
3782                 /*
3783                  * Set conn->bad_hdr for use with REJECT PDUs.
3784                  */
3785                 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN);
3786
3787                 if (conn->conn_ops->HeaderDigest) {
3788                         iov.iov_base    = &digest;
3789                         iov.iov_len     = ISCSI_CRC_LEN;
3790
3791                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
3792                         if (ret != ISCSI_CRC_LEN) {
3793                                 iscsit_rx_thread_wait_for_tcp(conn);
3794                                 goto transport_err;
3795                         }
3796
3797                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
3798                                         buffer, ISCSI_HDR_LEN,
3799                                         0, NULL, (u8 *)&checksum);
3800
3801                         if (digest != checksum) {
3802                                 pr_err("HeaderDigest CRC32C failed,"
3803                                         " received 0x%08x, computed 0x%08x\n",
3804                                         digest, checksum);
3805                                 /*
3806                                  * Set the PDU to 0xff so it will intentionally
3807                                  * hit default in the switch below.
3808                                  */
3809                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
3810                                 spin_lock_bh(&conn->sess->session_stats_lock);
3811                                 conn->sess->conn_digest_errors++;
3812                                 spin_unlock_bh(&conn->sess->session_stats_lock);
3813                         } else {
3814                                 pr_debug("Got HeaderDigest CRC32C"
3815                                                 " 0x%08x\n", checksum);
3816                         }
3817                 }
3818
3819                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
3820                         goto transport_err;
3821
3822                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
3823
3824                 if (conn->sess->sess_ops->SessionType &&
3825                    ((!(opcode & ISCSI_OP_TEXT)) ||
3826                     (!(opcode & ISCSI_OP_LOGOUT)))) {
3827                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
3828                         " while in Discovery Session, rejecting.\n", opcode);
3829                         iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
3830                                         buffer, conn);
3831                         goto transport_err;
3832                 }
3833
3834                 switch (opcode) {
3835                 case ISCSI_OP_SCSI_CMD:
3836                         if (iscsit_handle_scsi_cmd(conn, buffer) < 0)
3837                                 goto transport_err;
3838                         break;
3839                 case ISCSI_OP_SCSI_DATA_OUT:
3840                         if (iscsit_handle_data_out(conn, buffer) < 0)
3841                                 goto transport_err;
3842                         break;
3843                 case ISCSI_OP_NOOP_OUT:
3844                         if (iscsit_handle_nop_out(conn, buffer) < 0)
3845                                 goto transport_err;
3846                         break;
3847                 case ISCSI_OP_SCSI_TMFUNC:
3848                         if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0)
3849                                 goto transport_err;
3850                         break;
3851                 case ISCSI_OP_TEXT:
3852                         if (iscsit_handle_text_cmd(conn, buffer) < 0)
3853                                 goto transport_err;
3854                         break;
3855                 case ISCSI_OP_LOGOUT:
3856                         ret = iscsit_handle_logout_cmd(conn, buffer);
3857                         if (ret > 0) {
3858                                 wait_for_completion_timeout(&conn->conn_logout_comp,
3859                                                 SECONDS_FOR_LOGOUT_COMP * HZ);
3860                                 goto transport_err;
3861                         } else if (ret < 0)
3862                                 goto transport_err;
3863                         break;
3864                 case ISCSI_OP_SNACK:
3865                         if (iscsit_handle_snack(conn, buffer) < 0)
3866                                 goto transport_err;
3867                         break;
3868                 default:
3869                         pr_err("Got unknown iSCSI OpCode: 0x%02x\n",
3870                                         opcode);
3871                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3872                                 pr_err("Cannot recover from unknown"
3873                                 " opcode while ERL=0, closing iSCSI connection"
3874                                 ".\n");
3875                                 goto transport_err;
3876                         }
3877                         if (!conn->conn_ops->OFMarker) {
3878                                 pr_err("Unable to recover from unknown"
3879                                 " opcode while OFMarker=No, closing iSCSI"
3880                                         " connection.\n");
3881                                 goto transport_err;
3882                         }
3883                         if (iscsit_recover_from_unknown_opcode(conn) < 0) {
3884                                 pr_err("Unable to recover from unknown"
3885                                         " opcode, closing iSCSI connection.\n");
3886                                 goto transport_err;
3887                         }
3888                         break;
3889                 }
3890         }
3891
3892 transport_err:
3893         if (!signal_pending(current))
3894                 atomic_set(&conn->transport_failed, 1);
3895         iscsit_take_action_for_connection_exit(conn);
3896         goto restart;
3897 out:
3898         return 0;
3899 }
3900
3901 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
3902 {
3903         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
3904         struct iscsi_session *sess = conn->sess;
3905         /*
3906          * We expect this function to only ever be called from either RX or TX
3907          * thread context via iscsit_close_connection() once the other context
3908          * has been reset -> returned sleeping pre-handler state.
3909          */
3910         spin_lock_bh(&conn->cmd_lock);
3911         list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
3912
3913                 list_del(&cmd->i_conn_node);
3914                 spin_unlock_bh(&conn->cmd_lock);
3915
3916                 iscsit_increment_maxcmdsn(cmd, sess);
3917
3918                 iscsit_free_cmd(cmd);
3919
3920                 spin_lock_bh(&conn->cmd_lock);
3921         }
3922         spin_unlock_bh(&conn->cmd_lock);
3923 }
3924
3925 static void iscsit_stop_timers_for_cmds(
3926         struct iscsi_conn *conn)
3927 {
3928         struct iscsi_cmd *cmd;
3929
3930         spin_lock_bh(&conn->cmd_lock);
3931         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
3932                 if (cmd->data_direction == DMA_TO_DEVICE)
3933                         iscsit_stop_dataout_timer(cmd);
3934         }
3935         spin_unlock_bh(&conn->cmd_lock);
3936 }
3937
3938 int iscsit_close_connection(
3939         struct iscsi_conn *conn)
3940 {
3941         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
3942         struct iscsi_session    *sess = conn->sess;
3943
3944         pr_debug("Closing iSCSI connection CID %hu on SID:"
3945                 " %u\n", conn->cid, sess->sid);
3946         /*
3947          * Always up conn_logout_comp just in case the RX Thread is sleeping
3948          * and the logout response never got sent because the connection
3949          * failed.
3950          */
3951         complete(&conn->conn_logout_comp);
3952
3953         iscsi_release_thread_set(conn);
3954
3955         iscsit_stop_timers_for_cmds(conn);
3956         iscsit_stop_nopin_response_timer(conn);
3957         iscsit_stop_nopin_timer(conn);
3958         iscsit_free_queue_reqs_for_conn(conn);
3959
3960         /*
3961          * During Connection recovery drop unacknowledged out of order
3962          * commands for this connection, and prepare the other commands
3963          * for realligence.
3964          *
3965          * During normal operation clear the out of order commands (but
3966          * do not free the struct iscsi_ooo_cmdsn's) and release all
3967          * struct iscsi_cmds.
3968          */
3969         if (atomic_read(&conn->connection_recovery)) {
3970                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
3971                 iscsit_prepare_cmds_for_realligance(conn);
3972         } else {
3973                 iscsit_clear_ooo_cmdsns_for_conn(conn);
3974                 iscsit_release_commands_from_conn(conn);
3975         }
3976
3977         /*
3978          * Handle decrementing session or connection usage count if
3979          * a logout response was not able to be sent because the
3980          * connection failed.  Fall back to Session Recovery here.
3981          */
3982         if (atomic_read(&conn->conn_logout_remove)) {
3983                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
3984                         iscsit_dec_conn_usage_count(conn);
3985                         iscsit_dec_session_usage_count(sess);
3986                 }
3987                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
3988                         iscsit_dec_conn_usage_count(conn);
3989
3990                 atomic_set(&conn->conn_logout_remove, 0);
3991                 atomic_set(&sess->session_reinstatement, 0);
3992                 atomic_set(&sess->session_fall_back_to_erl0, 1);
3993         }
3994
3995         spin_lock_bh(&sess->conn_lock);
3996         list_del(&conn->conn_list);
3997
3998         /*
3999          * Attempt to let the Initiator know this connection failed by
4000          * sending an Connection Dropped Async Message on another
4001          * active connection.
4002          */
4003         if (atomic_read(&conn->connection_recovery))
4004                 iscsit_build_conn_drop_async_message(conn);
4005
4006         spin_unlock_bh(&sess->conn_lock);
4007
4008         /*
4009          * If connection reinstatement is being performed on this connection,
4010          * up the connection reinstatement semaphore that is being blocked on
4011          * in iscsit_cause_connection_reinstatement().
4012          */
4013         spin_lock_bh(&conn->state_lock);
4014         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4015                 spin_unlock_bh(&conn->state_lock);
4016                 complete(&conn->conn_wait_comp);
4017                 wait_for_completion(&conn->conn_post_wait_comp);
4018                 spin_lock_bh(&conn->state_lock);
4019         }
4020
4021         /*
4022          * If connection reinstatement is being performed on this connection
4023          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4024          * connection wait rcfr semaphore that is being blocked on
4025          * an iscsit_connection_reinstatement_rcfr().
4026          */
4027         if (atomic_read(&conn->connection_wait_rcfr)) {
4028                 spin_unlock_bh(&conn->state_lock);
4029                 complete(&conn->conn_wait_rcfr_comp);
4030                 wait_for_completion(&conn->conn_post_wait_comp);
4031                 spin_lock_bh(&conn->state_lock);
4032         }
4033         atomic_set(&conn->connection_reinstatement, 1);
4034         spin_unlock_bh(&conn->state_lock);
4035
4036         /*
4037          * If any other processes are accessing this connection pointer we
4038          * must wait until they have completed.
4039          */
4040         iscsit_check_conn_usage_count(conn);
4041
4042         if (conn->conn_rx_hash.tfm)
4043                 crypto_free_hash(conn->conn_rx_hash.tfm);
4044         if (conn->conn_tx_hash.tfm)
4045                 crypto_free_hash(conn->conn_tx_hash.tfm);
4046
4047         if (conn->conn_cpumask)
4048                 free_cpumask_var(conn->conn_cpumask);
4049
4050         kfree(conn->conn_ops);
4051         conn->conn_ops = NULL;
4052
4053         if (conn->sock) {
4054                 if (conn->conn_flags & CONNFLAG_SCTP_STRUCT_FILE) {
4055                         kfree(conn->sock->file);
4056                         conn->sock->file = NULL;
4057                 }
4058                 sock_release(conn->sock);
4059         }
4060         conn->thread_set = NULL;
4061
4062         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4063         conn->conn_state = TARG_CONN_STATE_FREE;
4064         kfree(conn);
4065
4066         spin_lock_bh(&sess->conn_lock);
4067         atomic_dec(&sess->nconn);
4068         pr_debug("Decremented iSCSI connection count to %hu from node:"
4069                 " %s\n", atomic_read(&sess->nconn),
4070                 sess->sess_ops->InitiatorName);
4071         /*
4072          * Make sure that if one connection fails in an non ERL=2 iSCSI
4073          * Session that they all fail.
4074          */
4075         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4076              !atomic_read(&sess->session_logout))
4077                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4078
4079         /*
4080          * If this was not the last connection in the session, and we are
4081          * performing session reinstatement or falling back to ERL=0, call
4082          * iscsit_stop_session() without sleeping to shutdown the other
4083          * active connections.
4084          */
4085         if (atomic_read(&sess->nconn)) {
4086                 if (!atomic_read(&sess->session_reinstatement) &&
4087                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4088                         spin_unlock_bh(&sess->conn_lock);
4089                         return 0;
4090                 }
4091                 if (!atomic_read(&sess->session_stop_active)) {
4092                         atomic_set(&sess->session_stop_active, 1);
4093                         spin_unlock_bh(&sess->conn_lock);
4094                         iscsit_stop_session(sess, 0, 0);
4095                         return 0;
4096                 }
4097                 spin_unlock_bh(&sess->conn_lock);
4098                 return 0;
4099         }
4100
4101         /*
4102          * If this was the last connection in the session and one of the
4103          * following is occurring:
4104          *
4105          * Session Reinstatement is not being performed, and are falling back
4106          * to ERL=0 call iscsit_close_session().
4107          *
4108          * Session Logout was requested.  iscsit_close_session() will be called
4109          * elsewhere.
4110          *
4111          * Session Continuation is not being performed, start the Time2Retain
4112          * handler and check if sleep_on_sess_wait_sem is active.
4113          */
4114         if (!atomic_read(&sess->session_reinstatement) &&
4115              atomic_read(&sess->session_fall_back_to_erl0)) {
4116                 spin_unlock_bh(&sess->conn_lock);
4117                 target_put_session(sess->se_sess);
4118
4119                 return 0;
4120         } else if (atomic_read(&sess->session_logout)) {
4121                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4122                 sess->session_state = TARG_SESS_STATE_FREE;
4123                 spin_unlock_bh(&sess->conn_lock);
4124
4125                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4126                         complete(&sess->session_wait_comp);
4127
4128                 return 0;
4129         } else {
4130                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4131                 sess->session_state = TARG_SESS_STATE_FAILED;
4132
4133                 if (!atomic_read(&sess->session_continuation)) {
4134                         spin_unlock_bh(&sess->conn_lock);
4135                         iscsit_start_time2retain_handler(sess);
4136                 } else
4137                         spin_unlock_bh(&sess->conn_lock);
4138
4139                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4140                         complete(&sess->session_wait_comp);
4141
4142                 return 0;
4143         }
4144         spin_unlock_bh(&sess->conn_lock);
4145
4146         return 0;
4147 }
4148
4149 int iscsit_close_session(struct iscsi_session *sess)
4150 {
4151         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
4152         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4153
4154         if (atomic_read(&sess->nconn)) {
4155                 pr_err("%d connection(s) still exist for iSCSI session"
4156                         " to %s\n", atomic_read(&sess->nconn),
4157                         sess->sess_ops->InitiatorName);
4158                 BUG();
4159         }
4160
4161         spin_lock_bh(&se_tpg->session_lock);
4162         atomic_set(&sess->session_logout, 1);
4163         atomic_set(&sess->session_reinstatement, 1);
4164         iscsit_stop_time2retain_timer(sess);
4165         spin_unlock_bh(&se_tpg->session_lock);
4166
4167         /*
4168          * transport_deregister_session_configfs() will clear the
4169          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4170          * can be setting it again with __transport_register_session() in
4171          * iscsi_post_login_handler() again after the iscsit_stop_session()
4172          * completes in iscsi_np context.
4173          */
4174         transport_deregister_session_configfs(sess->se_sess);
4175
4176         /*
4177          * If any other processes are accessing this session pointer we must
4178          * wait until they have completed.  If we are in an interrupt (the
4179          * time2retain handler) and contain and active session usage count we
4180          * restart the timer and exit.
4181          */
4182         if (!in_interrupt()) {
4183                 if (iscsit_check_session_usage_count(sess) == 1)
4184                         iscsit_stop_session(sess, 1, 1);
4185         } else {
4186                 if (iscsit_check_session_usage_count(sess) == 2) {
4187                         atomic_set(&sess->session_logout, 0);
4188                         iscsit_start_time2retain_handler(sess);
4189                         return 0;
4190                 }
4191         }
4192
4193         transport_deregister_session(sess->se_sess);
4194
4195         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4196                 iscsit_free_connection_recovery_entires(sess);
4197
4198         iscsit_free_all_ooo_cmdsns(sess);
4199
4200         spin_lock_bh(&se_tpg->session_lock);
4201         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4202         sess->session_state = TARG_SESS_STATE_FREE;
4203         pr_debug("Released iSCSI session from node: %s\n",
4204                         sess->sess_ops->InitiatorName);
4205         tpg->nsessions--;
4206         if (tpg->tpg_tiqn)
4207                 tpg->tpg_tiqn->tiqn_nsessions--;
4208
4209         pr_debug("Decremented number of active iSCSI Sessions on"
4210                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4211
4212         spin_lock(&sess_idr_lock);
4213         idr_remove(&sess_idr, sess->session_index);
4214         spin_unlock(&sess_idr_lock);
4215
4216         kfree(sess->sess_ops);
4217         sess->sess_ops = NULL;
4218         spin_unlock_bh(&se_tpg->session_lock);
4219
4220         kfree(sess);
4221         return 0;
4222 }
4223
4224 static void iscsit_logout_post_handler_closesession(
4225         struct iscsi_conn *conn)
4226 {
4227         struct iscsi_session *sess = conn->sess;
4228
4229         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4230         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4231
4232         atomic_set(&conn->conn_logout_remove, 0);
4233         complete(&conn->conn_logout_comp);
4234
4235         iscsit_dec_conn_usage_count(conn);
4236         iscsit_stop_session(sess, 1, 1);
4237         iscsit_dec_session_usage_count(sess);
4238         target_put_session(sess->se_sess);
4239 }
4240
4241 static void iscsit_logout_post_handler_samecid(
4242         struct iscsi_conn *conn)
4243 {
4244         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4245         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4246
4247         atomic_set(&conn->conn_logout_remove, 0);
4248         complete(&conn->conn_logout_comp);
4249
4250         iscsit_cause_connection_reinstatement(conn, 1);
4251         iscsit_dec_conn_usage_count(conn);
4252 }
4253
4254 static void iscsit_logout_post_handler_diffcid(
4255         struct iscsi_conn *conn,
4256         u16 cid)
4257 {
4258         struct iscsi_conn *l_conn;
4259         struct iscsi_session *sess = conn->sess;
4260
4261         if (!sess)
4262                 return;
4263
4264         spin_lock_bh(&sess->conn_lock);
4265         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4266                 if (l_conn->cid == cid) {
4267                         iscsit_inc_conn_usage_count(l_conn);
4268                         break;
4269                 }
4270         }
4271         spin_unlock_bh(&sess->conn_lock);
4272
4273         if (!l_conn)
4274                 return;
4275
4276         if (l_conn->sock)
4277                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4278
4279         spin_lock_bh(&l_conn->state_lock);
4280         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4281         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4282         spin_unlock_bh(&l_conn->state_lock);
4283
4284         iscsit_cause_connection_reinstatement(l_conn, 1);
4285         iscsit_dec_conn_usage_count(l_conn);
4286 }
4287
4288 /*
4289  *      Return of 0 causes the TX thread to restart.
4290  */
4291 static int iscsit_logout_post_handler(
4292         struct iscsi_cmd *cmd,
4293         struct iscsi_conn *conn)
4294 {
4295         int ret = 0;
4296
4297         switch (cmd->logout_reason) {
4298         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4299                 switch (cmd->logout_response) {
4300                 case ISCSI_LOGOUT_SUCCESS:
4301                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4302                 default:
4303                         iscsit_logout_post_handler_closesession(conn);
4304                         break;
4305                 }
4306                 ret = 0;
4307                 break;
4308         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4309                 if (conn->cid == cmd->logout_cid) {
4310                         switch (cmd->logout_response) {
4311                         case ISCSI_LOGOUT_SUCCESS:
4312                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4313                         default:
4314                                 iscsit_logout_post_handler_samecid(conn);
4315                                 break;
4316                         }
4317                         ret = 0;
4318                 } else {
4319                         switch (cmd->logout_response) {
4320                         case ISCSI_LOGOUT_SUCCESS:
4321                                 iscsit_logout_post_handler_diffcid(conn,
4322                                         cmd->logout_cid);
4323                                 break;
4324                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4325                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4326                         default:
4327                                 break;
4328                         }
4329                         ret = 1;
4330                 }
4331                 break;
4332         case ISCSI_LOGOUT_REASON_RECOVERY:
4333                 switch (cmd->logout_response) {
4334                 case ISCSI_LOGOUT_SUCCESS:
4335                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4336                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4337                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4338                 default:
4339                         break;
4340                 }
4341                 ret = 1;
4342                 break;
4343         default:
4344                 break;
4345
4346         }
4347         return ret;
4348 }
4349
4350 void iscsit_fail_session(struct iscsi_session *sess)
4351 {
4352         struct iscsi_conn *conn;
4353
4354         spin_lock_bh(&sess->conn_lock);
4355         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4356                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4357                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4358         }
4359         spin_unlock_bh(&sess->conn_lock);
4360
4361         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4362         sess->session_state = TARG_SESS_STATE_FAILED;
4363 }
4364
4365 int iscsit_free_session(struct iscsi_session *sess)
4366 {
4367         u16 conn_count = atomic_read(&sess->nconn);
4368         struct iscsi_conn *conn, *conn_tmp = NULL;
4369         int is_last;
4370
4371         spin_lock_bh(&sess->conn_lock);
4372         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4373
4374         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4375                         conn_list) {
4376                 if (conn_count == 0)
4377                         break;
4378
4379                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4380                         is_last = 1;
4381                 } else {
4382                         iscsit_inc_conn_usage_count(conn_tmp);
4383                         is_last = 0;
4384                 }
4385                 iscsit_inc_conn_usage_count(conn);
4386
4387                 spin_unlock_bh(&sess->conn_lock);
4388                 iscsit_cause_connection_reinstatement(conn, 1);
4389                 spin_lock_bh(&sess->conn_lock);
4390
4391                 iscsit_dec_conn_usage_count(conn);
4392                 if (is_last == 0)
4393                         iscsit_dec_conn_usage_count(conn_tmp);
4394
4395                 conn_count--;
4396         }
4397
4398         if (atomic_read(&sess->nconn)) {
4399                 spin_unlock_bh(&sess->conn_lock);
4400                 wait_for_completion(&sess->session_wait_comp);
4401         } else
4402                 spin_unlock_bh(&sess->conn_lock);
4403
4404         target_put_session(sess->se_sess);
4405         return 0;
4406 }
4407
4408 void iscsit_stop_session(
4409         struct iscsi_session *sess,
4410         int session_sleep,
4411         int connection_sleep)
4412 {
4413         u16 conn_count = atomic_read(&sess->nconn);
4414         struct iscsi_conn *conn, *conn_tmp = NULL;
4415         int is_last;
4416
4417         spin_lock_bh(&sess->conn_lock);
4418         if (session_sleep)
4419                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4420
4421         if (connection_sleep) {
4422                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4423                                 conn_list) {
4424                         if (conn_count == 0)
4425                                 break;
4426
4427                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4428                                 is_last = 1;
4429                         } else {
4430                                 iscsit_inc_conn_usage_count(conn_tmp);
4431                                 is_last = 0;
4432                         }
4433                         iscsit_inc_conn_usage_count(conn);
4434
4435                         spin_unlock_bh(&sess->conn_lock);
4436                         iscsit_cause_connection_reinstatement(conn, 1);
4437                         spin_lock_bh(&sess->conn_lock);
4438
4439                         iscsit_dec_conn_usage_count(conn);
4440                         if (is_last == 0)
4441                                 iscsit_dec_conn_usage_count(conn_tmp);
4442                         conn_count--;
4443                 }
4444         } else {
4445                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4446                         iscsit_cause_connection_reinstatement(conn, 0);
4447         }
4448
4449         if (session_sleep && atomic_read(&sess->nconn)) {
4450                 spin_unlock_bh(&sess->conn_lock);
4451                 wait_for_completion(&sess->session_wait_comp);
4452         } else
4453                 spin_unlock_bh(&sess->conn_lock);
4454 }
4455
4456 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4457 {
4458         struct iscsi_session *sess;
4459         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4460         struct se_session *se_sess, *se_sess_tmp;
4461         int session_count = 0;
4462
4463         spin_lock_bh(&se_tpg->session_lock);
4464         if (tpg->nsessions && !force) {
4465                 spin_unlock_bh(&se_tpg->session_lock);
4466                 return -1;
4467         }
4468
4469         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4470                         sess_list) {
4471                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4472
4473                 spin_lock(&sess->conn_lock);
4474                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4475                     atomic_read(&sess->session_logout) ||
4476                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4477                         spin_unlock(&sess->conn_lock);
4478                         continue;
4479                 }
4480                 atomic_set(&sess->session_reinstatement, 1);
4481                 spin_unlock(&sess->conn_lock);
4482                 spin_unlock_bh(&se_tpg->session_lock);
4483
4484                 iscsit_free_session(sess);
4485                 spin_lock_bh(&se_tpg->session_lock);
4486
4487                 session_count++;
4488         }
4489         spin_unlock_bh(&se_tpg->session_lock);
4490
4491         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4492                         " Group: %hu\n", session_count, tpg->tpgt);
4493         return 0;
4494 }
4495
4496 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4497 MODULE_VERSION("4.1.x");
4498 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4499 MODULE_LICENSE("GPL");
4500
4501 module_init(iscsi_target_init_module);
4502 module_exit(iscsi_target_cleanup_module);