drivers/target/iscsi/cxgbit/cxgbit_target.c

   1 /*
   2  * Copyright (c) 2016 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #include <linux/workqueue.h>
  10 #include <linux/kthread.h>
  11 #include <asm/unaligned.h>
  12 #include <target/target_core_base.h>
  13 #include <target/target_core_fabric.h>
  14 #include "cxgbit.h"
  15
  16 struct sge_opaque_hdr {
  17         void *dev;
  18         dma_addr_t addr[MAX_SKB_FRAGS + 1];
  19 };
  20
  21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
  22
  23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
  24                     sizeof(struct fw_ofld_tx_data_wr))
  25
  26 static struct sk_buff *
  27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
  28 {
  29         struct sk_buff *skb = NULL;
  30         u8 submode = 0;
  31         int errcode;
  32         static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
  33
  34         if (len) {
  35                 skb = alloc_skb_with_frags(hdr_len, len,
  36                                            0, &errcode,
  37                                            GFP_KERNEL);
  38                 if (!skb)
  39                         return NULL;
  40
  41                 skb_reserve(skb, TX_HDR_LEN);
  42                 skb_reset_transport_header(skb);
  43                 __skb_put(skb, ISCSI_HDR_LEN);
  44                 skb->data_len = len;
  45                 skb->len += len;
  46                 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
  47
  48         } else {
  49                 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
  50
  51                 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
  52                 if (!skb)
  53                         return NULL;
  54
  55                 skb_reserve(skb, TX_HDR_LEN + iso_len);
  56                 skb_reset_transport_header(skb);
  57                 __skb_put(skb, ISCSI_HDR_LEN);
  58         }
  59
  60         submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
  61         cxgbit_skcb_submode(skb) = submode;
  62         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
  63         cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
  64         return skb;
  65 }
  66
  67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
  68 {
  69         return __cxgbit_alloc_skb(csk, len, false);
  70 }
  71
  72 /*
  73  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
  74  * @skb: the packet
  75  *
  76  * Returns true if a packet can be sent as an offload WR with immediate
  77  * data.  We currently use the same limit as for Ethernet packets.
  78  */
  79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
  80 {
  81         int length = skb->len;
  82
  83         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
  84                 length += sizeof(struct fw_ofld_tx_data_wr);
  85
  86         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
  87                 length += sizeof(struct cpl_tx_data_iso);
  88
  89 #define MAX_IMM_TX_PKT_LEN      256
  90         return length <= MAX_IMM_TX_PKT_LEN;
  91 }
  92
  93 /*
  94  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
  95  * @n: the number of SGL entries
  96  * Calculates the number of flits needed for a scatter/gather list that
  97  * can hold the given number of entries.
  98  */
  99 static inline unsigned int cxgbit_sgl_len(unsigned int n)
 100 {
 101         n--;
 102         return (3 * n) / 2 + (n & 1) + 2;
 103 }
 104
 105 /*
 106  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
 107  * @skb: the packet
 108  *
 109  * Returns the number of flits needed for the given offload packet.
 110  * These packets are already fully constructed and no additional headers
 111  * will be added.
 112  */
 113 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
 114 {
 115         unsigned int flits, cnt;
 116
 117         if (cxgbit_is_ofld_imm(skb))
 118                 return DIV_ROUND_UP(skb->len, 8);
 119         flits = skb_transport_offset(skb) / 8;
 120         cnt = skb_shinfo(skb)->nr_frags;
 121         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 122                 cnt++;
 123         return flits + cxgbit_sgl_len(cnt);
 124 }
 125
 126 #define CXGBIT_ISO_FSLICE 0x1
 127 #define CXGBIT_ISO_LSLICE 0x2
 128 static void
 129 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
 130 {
 131         struct cpl_tx_data_iso *cpl;
 132         unsigned int submode = cxgbit_skcb_submode(skb);
 133         unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
 134         unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
 135
 136         cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl));
 137
 138         cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
 139                         CPL_TX_DATA_ISO_FIRST_V(fslice) |
 140                         CPL_TX_DATA_ISO_LAST_V(lslice) |
 141                         CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
 142                         CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
 143                         CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
 144                         CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
 145                         CPL_TX_DATA_ISO_SCSI_V(2));
 146
 147         cpl->ahs_len = 0;
 148         cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
 149         cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
 150         cpl->len = htonl(iso_info->len);
 151         cpl->reserved2_seglen_offset = htonl(0);
 152         cpl->datasn_offset = htonl(0);
 153         cpl->buffer_offset = htonl(0);
 154         cpl->reserved3 = 0;
 155
 156         __skb_pull(skb, sizeof(*cpl));
 157 }
 158
 159 static void
 160 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
 161                   u32 len, u32 credits, u32 compl)
 162 {
 163         struct fw_ofld_tx_data_wr *req;
 164         u32 submode = cxgbit_skcb_submode(skb);
 165         u32 wr_ulp_mode = 0;
 166         u32 hdr_size = sizeof(*req);
 167         u32 opcode = FW_OFLD_TX_DATA_WR;
 168         u32 immlen = 0;
 169         u32 force = TX_FORCE_V(!submode);
 170
 171         if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
 172                 opcode = FW_ISCSI_TX_DATA_WR;
 173                 immlen += sizeof(struct cpl_tx_data_iso);
 174                 hdr_size += sizeof(struct cpl_tx_data_iso);
 175                 submode |= 8;
 176         }
 177
 178         if (cxgbit_is_ofld_imm(skb))
 179                 immlen += dlen;
 180
 181         req = (struct fw_ofld_tx_data_wr *)__skb_push(skb,
 182                                                         hdr_size);
 183         req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
 184                                         FW_WR_COMPL_V(compl) |
 185                                         FW_WR_IMMDLEN_V(immlen));
 186         req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 187                                         FW_WR_LEN16_V(credits));
 188         req->plen = htonl(len);
 189         wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
 190                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 191
 192         req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
 193                  FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
 194 }
 195
 196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 197 {
 198         kfree_skb(skb);
 199 }
 200
 201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
 202 {
 203         struct sk_buff *skb;
 204
 205         while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
 206                 u32 dlen = skb->len;
 207                 u32 len = skb->len;
 208                 u32 credits_needed;
 209                 u32 compl = 0;
 210                 u32 flowclen16 = 0;
 211                 u32 iso_cpl_len = 0;
 212
 213                 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
 214                         iso_cpl_len = sizeof(struct cpl_tx_data_iso);
 215
 216                 if (cxgbit_is_ofld_imm(skb))
 217                         credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
 218                 else
 219                         credits_needed = DIV_ROUND_UP((8 *
 220                                         cxgbit_calc_tx_flits_ofld(skb)) +
 221                                         iso_cpl_len, 16);
 222
 223                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
 224                         credits_needed += DIV_ROUND_UP(
 225                                 sizeof(struct fw_ofld_tx_data_wr), 16);
 226                 /*
 227                  * Assumes the initial credits is large enough to support
 228                  * fw_flowc_wr plus largest possible first payload
 229                  */
 230
 231                 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
 232                         flowclen16 = cxgbit_send_tx_flowc_wr(csk);
 233                         csk->wr_cred -= flowclen16;
 234                         csk->wr_una_cred += flowclen16;
 235                 }
 236
 237                 if (csk->wr_cred < credits_needed) {
 238                         pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
 239                                  csk, skb->len, skb->data_len,
 240                                  credits_needed, csk->wr_cred);
 241                         break;
 242                 }
 243                 __skb_unlink(skb, &csk->txq);
 244                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
 245                 skb->csum = credits_needed + flowclen16;
 246                 csk->wr_cred -= credits_needed;
 247                 csk->wr_una_cred += credits_needed;
 248
 249                 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 250                          csk, skb->len, skb->data_len, credits_needed,
 251                          csk->wr_cred, csk->wr_una_cred);
 252
 253                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
 254                         len += cxgbit_skcb_tx_extralen(skb);
 255
 256                         if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
 257                             (!before(csk->write_seq,
 258                                      csk->snd_una + csk->snd_win))) {
 259                                 compl = 1;
 260                                 csk->wr_una_cred = 0;
 261                         }
 262
 263                         cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
 264                                           compl);
 265                         csk->snd_nxt += len;
 266
 267                 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
 268                            (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
 269                         struct cpl_close_con_req *req =
 270                                 (struct cpl_close_con_req *)skb->data;
 271                         req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
 272                         csk->wr_una_cred = 0;
 273                 }
 274
 275                 cxgbit_sock_enqueue_wr(csk, skb);
 276                 t4_set_arp_err_handler(skb, csk,
 277                                        cxgbit_arp_failure_skb_discard);
 278
 279                 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
 280                          csk, csk->tid, skb, len);
 281
 282                 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 283         }
 284 }
 285
 286 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
 287 {
 288         spin_lock_bh(&csk->lock);
 289
 290         if (before(csk->write_seq, csk->snd_una + csk->snd_win))
 291                 csk->lock_owner = true;
 292
 293         spin_unlock_bh(&csk->lock);
 294
 295         return csk->lock_owner;
 296 }
 297
 298 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
 299 {
 300         struct sk_buff_head backlogq;
 301         struct sk_buff *skb;
 302         void (*fn)(struct cxgbit_sock *, struct sk_buff *);
 303
 304         skb_queue_head_init(&backlogq);
 305
 306         spin_lock_bh(&csk->lock);
 307         while (skb_queue_len(&csk->backlogq)) {
 308                 skb_queue_splice_init(&csk->backlogq, &backlogq);
 309                 spin_unlock_bh(&csk->lock);
 310
 311                 while ((skb = __skb_dequeue(&backlogq))) {
 312                         fn = cxgbit_skcb_rx_backlog_fn(skb);
 313                         fn(csk, skb);
 314                 }
 315
 316                 spin_lock_bh(&csk->lock);
 317         }
 318
 319         csk->lock_owner = false;
 320         spin_unlock_bh(&csk->lock);
 321 }
 322
 323 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 324 {
 325         int ret = 0;
 326
 327         wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
 328
 329         if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
 330                      signal_pending(current))) {
 331                 __kfree_skb(skb);
 332                 __skb_queue_purge(&csk->ppodq);
 333                 ret = -1;
 334                 spin_lock_bh(&csk->lock);
 335                 if (csk->lock_owner) {
 336                         spin_unlock_bh(&csk->lock);
 337                         goto unlock;
 338                 }
 339                 spin_unlock_bh(&csk->lock);
 340                 return ret;
 341         }
 342
 343         csk->write_seq += skb->len +
 344                           cxgbit_skcb_tx_extralen(skb);
 345
 346         skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
 347         __skb_queue_tail(&csk->txq, skb);
 348         cxgbit_push_tx_frames(csk);
 349
 350 unlock:
 351         cxgbit_unlock_sock(csk);
 352         return ret;
 353 }
 354
 355 static int
 356 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
 357                u32 data_length)
 358 {
 359         u32 i = 0, nr_frags = MAX_SKB_FRAGS;
 360         u32 padding = ((-data_length) & 3);
 361         struct scatterlist *sg;
 362         struct page *page;
 363         unsigned int page_off;
 364
 365         if (padding)
 366                 nr_frags--;
 367
 368         /*
 369          * We know each entry in t_data_sg contains a page.
 370          */
 371         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 372         page_off = (data_offset % PAGE_SIZE);
 373
 374         while (data_length && (i < nr_frags)) {
 375                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 376
 377                 page = sg_page(sg);
 378
 379                 get_page(page);
 380                 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
 381                                    cur_len);
 382                 skb->data_len += cur_len;
 383                 skb->len += cur_len;
 384                 skb->truesize += cur_len;
 385
 386                 data_length -= cur_len;
 387                 page_off = 0;
 388                 sg = sg_next(sg);
 389                 i++;
 390         }
 391
 392         if (data_length)
 393                 return -1;
 394
 395         if (padding) {
 396                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 397                 if (!page)
 398                         return -1;
 399                 skb_fill_page_desc(skb, i, page, 0, padding);
 400                 skb->data_len += padding;
 401                 skb->len += padding;
 402                 skb->truesize += padding;
 403         }
 404
 405         return 0;
 406 }
 407
 408 static int
 409 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 410                      struct iscsi_datain_req *dr)
 411 {
 412         struct iscsi_conn *conn = csk->conn;
 413         struct sk_buff *skb;
 414         struct iscsi_datain datain;
 415         struct cxgbit_iso_info iso_info;
 416         u32 data_length = cmd->se_cmd.data_length;
 417         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 418         u32 num_pdu, plen, tx_data = 0;
 419         bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
 420                 SCF_TRANSPORT_TASK_SENSE);
 421         bool set_statsn = false;
 422         int ret = -1;
 423
 424         while (data_length) {
 425                 num_pdu = (data_length + mrdsl - 1) / mrdsl;
 426                 if (num_pdu > csk->max_iso_npdu)
 427                         num_pdu = csk->max_iso_npdu;
 428
 429                 plen = num_pdu * mrdsl;
 430                 if (plen > data_length)
 431                         plen = data_length;
 432
 433                 skb = __cxgbit_alloc_skb(csk, 0, true);
 434                 if (unlikely(!skb))
 435                         return -ENOMEM;
 436
 437                 memset(skb->data, 0, ISCSI_HDR_LEN);
 438                 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
 439                 cxgbit_skcb_submode(skb) |= (csk->submode &
 440                                 CXGBIT_SUBMODE_DCRC);
 441                 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
 442                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
 443                                                 ((num_pdu - 1) * ISCSI_HDR_LEN);
 444
 445                 memset(&datain, 0, sizeof(struct iscsi_datain));
 446                 memset(&iso_info, 0, sizeof(iso_info));
 447
 448                 if (!tx_data)
 449                         iso_info.flags |= CXGBIT_ISO_FSLICE;
 450
 451                 if (!(data_length - plen)) {
 452                         iso_info.flags |= CXGBIT_ISO_LSLICE;
 453                         if (!task_sense) {
 454                                 datain.flags = ISCSI_FLAG_DATA_STATUS;
 455                                 iscsit_increment_maxcmdsn(cmd, conn->sess);
 456                                 cmd->stat_sn = conn->stat_sn++;
 457                                 set_statsn = true;
 458                         }
 459                 }
 460
 461                 iso_info.burst_len = num_pdu * mrdsl;
 462                 iso_info.mpdu = mrdsl;
 463                 iso_info.len = ISCSI_HDR_LEN + plen;
 464
 465                 cxgbit_cpl_tx_data_iso(skb, &iso_info);
 466
 467                 datain.offset = tx_data;
 468                 datain.data_sn = cmd->data_sn - 1;
 469
 470                 iscsit_build_datain_pdu(cmd, conn, &datain,
 471                                         (struct iscsi_data_rsp *)skb->data,
 472                                         set_statsn);
 473
 474                 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
 475                 if (unlikely(ret)) {
 476                         __kfree_skb(skb);
 477                         goto out;
 478                 }
 479
 480                 ret = cxgbit_queue_skb(csk, skb);
 481                 if (unlikely(ret))
 482                         goto out;
 483
 484                 tx_data += plen;
 485                 data_length -= plen;
 486
 487                 cmd->read_data_done += plen;
 488                 cmd->data_sn += num_pdu;
 489         }
 490
 491         dr->dr_complete = DATAIN_COMPLETE_NORMAL;
 492
 493         return 0;
 494
 495 out:
 496         return ret;
 497 }
 498
 499 static int
 500 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 501                  const struct iscsi_datain *datain)
 502 {
 503         struct sk_buff *skb;
 504         int ret = 0;
 505
 506         skb = cxgbit_alloc_skb(csk, 0);
 507         if (unlikely(!skb))
 508                 return -ENOMEM;
 509
 510         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 511
 512         if (datain->length) {
 513                 cxgbit_skcb_submode(skb) |= (csk->submode &
 514                                 CXGBIT_SUBMODE_DCRC);
 515                 cxgbit_skcb_tx_extralen(skb) =
 516                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
 517         }
 518
 519         ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
 520         if (ret < 0) {
 521                 __kfree_skb(skb);
 522                 return ret;
 523         }
 524
 525         return cxgbit_queue_skb(csk, skb);
 526 }
 527
 528 static int
 529 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 530                        struct iscsi_datain_req *dr,
 531                        const struct iscsi_datain *datain)
 532 {
 533         struct cxgbit_sock *csk = conn->context;
 534         u32 data_length = cmd->se_cmd.data_length;
 535         u32 padding = ((-data_length) & 3);
 536         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 537
 538         if ((data_length > mrdsl) && (!dr->recovery) &&
 539             (!padding) && (!datain->offset) && csk->max_iso_npdu) {
 540                 atomic_long_add(data_length - datain->length,
 541                                 &conn->sess->tx_data_octets);
 542                 return cxgbit_tx_datain_iso(csk, cmd, dr);
 543         }
 544
 545         return cxgbit_tx_datain(csk, cmd, datain);
 546 }
 547
 548 static int
 549 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 550                           const void *data_buf, u32 data_buf_len)
 551 {
 552         struct cxgbit_sock *csk = conn->context;
 553         struct sk_buff *skb;
 554         u32 padding = ((-data_buf_len) & 3);
 555
 556         skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
 557         if (unlikely(!skb))
 558                 return -ENOMEM;
 559
 560         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 561
 562         if (data_buf_len) {
 563                 u32 pad_bytes = 0;
 564
 565                 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
 566
 567                 if (padding)
 568                         skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
 569                                        &pad_bytes, padding);
 570         }
 571
 572         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
 573                                        cxgbit_skcb_submode(skb)];
 574
 575         return cxgbit_queue_skb(csk, skb);
 576 }
 577
 578 int
 579 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 580                 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
 581 {
 582         if (dr)
 583                 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
 584         else
 585                 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 586 }
 587
 588 int cxgbit_validate_params(struct iscsi_conn *conn)
 589 {
 590         struct cxgbit_sock *csk = conn->context;
 591         struct cxgbit_device *cdev = csk->com.cdev;
 592         struct iscsi_param *param;
 593         u32 max_xmitdsl;
 594
 595         param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
 596                                           conn->param_list);
 597         if (!param)
 598                 return -1;
 599
 600         if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
 601                 return -1;
 602
 603         if (max_xmitdsl > cdev->mdsl) {
 604                 if (iscsi_change_param_sprintf(
 605                         conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
 606                         return -1;
 607         }
 608
 609         return 0;
 610 }
 611
 612 static int cxgbit_set_digest(struct cxgbit_sock *csk)
 613 {
 614         struct iscsi_conn *conn = csk->conn;
 615         struct iscsi_param *param;
 616
 617         param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
 618         if (!param) {
 619                 pr_err("param not found key %s\n", HEADERDIGEST);
 620                 return -1;
 621         }
 622
 623         if (!strcmp(param->value, CRC32C))
 624                 csk->submode |= CXGBIT_SUBMODE_HCRC;
 625
 626         param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
 627         if (!param) {
 628                 csk->submode = 0;
 629                 pr_err("param not found key %s\n", DATADIGEST);
 630                 return -1;
 631         }
 632
 633         if (!strcmp(param->value, CRC32C))
 634                 csk->submode |= CXGBIT_SUBMODE_DCRC;
 635
 636         if (cxgbit_setup_conn_digest(csk)) {
 637                 csk->submode = 0;
 638                 return -1;
 639         }
 640
 641         return 0;
 642 }
 643
 644 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
 645 {
 646         struct iscsi_conn *conn = csk->conn;
 647         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 648         struct iscsi_param *param;
 649         u32 mrdsl, mbl;
 650         u32 max_npdu, max_iso_npdu;
 651
 652         if (conn->login->leading_connection) {
 653                 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
 654                                                   conn->param_list);
 655                 if (!param) {
 656                         pr_err("param not found key %s\n", DATASEQUENCEINORDER);
 657                         return -1;
 658                 }
 659
 660                 if (strcmp(param->value, YES))
 661                         return 0;
 662
 663                 param = iscsi_find_param_from_key(DATAPDUINORDER,
 664                                                   conn->param_list);
 665                 if (!param) {
 666                         pr_err("param not found key %s\n", DATAPDUINORDER);
 667                         return -1;
 668                 }
 669
 670                 if (strcmp(param->value, YES))
 671                         return 0;
 672
 673                 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
 674                                                   conn->param_list);
 675                 if (!param) {
 676                         pr_err("param not found key %s\n", MAXBURSTLENGTH);
 677                         return -1;
 678                 }
 679
 680                 if (kstrtou32(param->value, 0, &mbl) < 0)
 681                         return -1;
 682         } else {
 683                 if (!conn->sess->sess_ops->DataSequenceInOrder)
 684                         return 0;
 685                 if (!conn->sess->sess_ops->DataPDUInOrder)
 686                         return 0;
 687
 688                 mbl = conn->sess->sess_ops->MaxBurstLength;
 689         }
 690
 691         mrdsl = conn_ops->MaxRecvDataSegmentLength;
 692         max_npdu = mbl / mrdsl;
 693
 694         max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD /
 695                         (ISCSI_HDR_LEN + mrdsl +
 696                         cxgbit_digest_len[csk->submode]);
 697
 698         csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
 699
 700         if (csk->max_iso_npdu <= 1)
 701                 csk->max_iso_npdu = 0;
 702
 703         return 0;
 704 }
 705
 706 static int cxgbit_set_params(struct iscsi_conn *conn)
 707 {
 708         struct cxgbit_sock *csk = conn->context;
 709         struct cxgbit_device *cdev = csk->com.cdev;
 710         struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
 711         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 712         struct iscsi_param *param;
 713         u8 erl;
 714
 715         if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
 716                 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
 717
 718         if (conn->login->leading_connection) {
 719                 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
 720                                                   conn->param_list);
 721                 if (!param) {
 722                         pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
 723                         return -1;
 724                 }
 725                 if (kstrtou8(param->value, 0, &erl) < 0)
 726                         return -1;
 727         } else {
 728                 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
 729         }
 730
 731         if (!erl) {
 732                 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
 733                         if (cxgbit_set_iso_npdu(csk))
 734                                 return -1;
 735                 }
 736
 737                 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
 738                         if (cxgbit_setup_conn_pgidx(csk,
 739                                                     ppm->tformat.pgsz_idx_dflt))
 740                                 return -1;
 741                         set_bit(CSK_DDP_ENABLE, &csk->com.flags);
 742                 }
 743         }
 744
 745         if (cxgbit_set_digest(csk))
 746                 return -1;
 747
 748         return 0;
 749 }
 750
 751 int
 752 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 753                     u32 length)
 754 {
 755         struct cxgbit_sock *csk = conn->context;
 756         struct sk_buff *skb;
 757         u32 padding_buf = 0;
 758         u8 padding = ((-length) & 3);
 759
 760         skb = cxgbit_alloc_skb(csk, length + padding);
 761         if (!skb)
 762                 return -ENOMEM;
 763         skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
 764         skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
 765
 766         if (padding)
 767                 skb_store_bits(skb, ISCSI_HDR_LEN + length,
 768                                &padding_buf, padding);
 769
 770         if (login->login_complete) {
 771                 if (cxgbit_set_params(conn)) {
 772                         kfree_skb(skb);
 773                         return -1;
 774                 }
 775
 776                 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
 777         }
 778
 779         if (cxgbit_queue_skb(csk, skb))
 780                 return -1;
 781
 782         if ((!login->login_complete) && (!login->login_failed))
 783                 schedule_delayed_work(&conn->login_work, 0);
 784
 785         return 0;
 786 }
 787
 788 static void
 789 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
 790                       unsigned int nents)
 791 {
 792         struct skb_seq_state st;
 793         const u8 *buf;
 794         unsigned int consumed = 0, buf_len;
 795         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
 796
 797         skb_prepare_seq_read(skb, pdu_cb->doffset,
 798                              pdu_cb->doffset + pdu_cb->dlen,
 799                              &st);
 800
 801         while (true) {
 802                 buf_len = skb_seq_read(consumed, &buf, &st);
 803                 if (!buf_len) {
 804                         skb_abort_seq_read(&st);
 805                         break;
 806                 }
 807
 808                 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
 809                                                  buf_len, consumed);
 810         }
 811 }
 812
 813 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
 814 {
 815         struct iscsi_conn *conn = csk->conn;
 816         struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
 817         struct cxgbit_cmd *ccmd;
 818         struct iscsi_cmd *cmd;
 819
 820         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 821         if (!cmd) {
 822                 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
 823                 return NULL;
 824         }
 825
 826         ccmd = iscsit_priv_cmd(cmd);
 827         ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
 828         ccmd->setup_ddp = true;
 829
 830         return cmd;
 831 }
 832
 833 static int
 834 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 835                              u32 length)
 836 {
 837         struct iscsi_conn *conn = cmd->conn;
 838         struct cxgbit_sock *csk = conn->context;
 839         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 840
 841         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 842                 pr_err("ImmediateData CRC32C DataDigest error\n");
 843                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 844                         pr_err("Unable to recover from"
 845                                " Immediate Data digest failure while"
 846                                " in ERL=0.\n");
 847                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 848                                           (unsigned char *)hdr);
 849                         return IMMEDIATE_DATA_CANNOT_RECOVER;
 850                 }
 851
 852                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 853                                   (unsigned char *)hdr);
 854                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
 855         }
 856
 857         if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
 858                 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
 859                 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
 860                 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
 861
 862                 sg_init_table(&ccmd->sg, 1);
 863                 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
 864                             dfrag->page_offset);
 865                 get_page(dfrag->page.p);
 866
 867                 cmd->se_cmd.t_data_sg = &ccmd->sg;
 868                 cmd->se_cmd.t_data_nents = 1;
 869
 870                 ccmd->release = true;
 871         } else {
 872                 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
 873                 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
 874
 875                 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents);
 876         }
 877
 878         cmd->write_data_done += pdu_cb->dlen;
 879
 880         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 881                 spin_lock_bh(&cmd->istate_lock);
 882                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 883                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 884                 spin_unlock_bh(&cmd->istate_lock);
 885         }
 886
 887         return IMMEDIATE_DATA_NORMAL_OPERATION;
 888 }
 889
 890 static int
 891 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 892                           bool dump_payload)
 893 {
 894         struct iscsi_conn *conn = cmd->conn;
 895         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
 896         /*
 897          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 898          */
 899         if (dump_payload)
 900                 goto after_immediate_data;
 901
 902         immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
 903                                                  cmd->first_burst_len);
 904 after_immediate_data:
 905         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 906                 /*
 907                  * A PDU/CmdSN carrying Immediate Data passed
 908                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
 909                  * Immediate Bit is not set.
 910                  */
 911                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
 912                                                 (unsigned char *)hdr,
 913                                                 hdr->cmdsn);
 914                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
 915                         return -1;
 916
 917                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
 918                         target_put_sess_cmd(&cmd->se_cmd);
 919                         return 0;
 920                 } else if (cmd->unsolicited_data) {
 921                         iscsit_set_unsoliticed_dataout(cmd);
 922                 }
 923
 924         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
 925                 /*
 926                  * Immediate Data failed DataCRC and ERL>=1,
 927                  * silently drop this PDU and let the initiator
 928                  * plug the CmdSN gap.
 929                  *
 930                  * FIXME: Send Unsolicited NOPIN with reserved
 931                  * TTT here to help the initiator figure out
 932                  * the missing CmdSN, although they should be
 933                  * intelligent enough to determine the missing
 934                  * CmdSN and issue a retry to plug the sequence.
 935                  */
 936                 cmd->i_state = ISTATE_REMOVE;
 937                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 938         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
 939                 return -1;
 940
 941         return 0;
 942 }
 943
 944 static int
 945 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
 946 {
 947         struct iscsi_conn *conn = csk->conn;
 948         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 949         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
 950         int rc;
 951         bool dump_payload = false;
 952
 953         rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
 954         if (rc < 0)
 955                 return rc;
 956
 957         if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
 958             (pdu_cb->nr_dfrags == 1))
 959                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
 960
 961         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
 962         if (rc < 0)
 963                 return 0;
 964         else if (rc > 0)
 965                 dump_payload = true;
 966
 967         if (!pdu_cb->dlen)
 968                 return 0;
 969
 970         return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
 971 }
 972
 973 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
 974 {
 975         struct scatterlist *sg_start;
 976         struct iscsi_conn *conn = csk->conn;
 977         struct iscsi_cmd *cmd = NULL;
 978         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 979         struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
 980         u32 data_offset = be32_to_cpu(hdr->offset);
 981         u32 data_len = pdu_cb->dlen;
 982         int rc, sg_nents, sg_off;
 983         bool dcrc_err = false;
 984
 985         rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
 986         if (rc < 0)
 987                 return rc;
 988         else if (!cmd)
 989                 return 0;
 990
 991         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 992                 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
 993                        " DataSN: 0x%08x\n",
 994                        hdr->itt, hdr->offset, data_len,
 995                        hdr->datasn);
 996
 997                 dcrc_err = true;
 998                 goto check_payload;
 999         }
1000
1001         pr_debug("DataOut data_len: %u, "
1002                 "write_data_done: %u, data_length: %u\n",
1003                   data_len,  cmd->write_data_done,
1004                   cmd->se_cmd.data_length);
1005
1006         if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1007                 sg_off = data_offset / PAGE_SIZE;
1008                 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1009                 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE));
1010
1011                 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents);
1012         }
1013
1014 check_payload:
1015
1016         rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1017         if (rc < 0)
1018                 return rc;
1019
1020         return 0;
1021 }
1022
1023 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1024 {
1025         struct iscsi_conn *conn = csk->conn;
1026         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1027         struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1028         unsigned char *ping_data = NULL;
1029         u32 payload_length = pdu_cb->dlen;
1030         int ret;
1031
1032         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1033         if (ret < 0)
1034                 return 0;
1035
1036         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1037                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1038                         pr_err("Unable to recover from"
1039                                " NOPOUT Ping DataCRC failure while in"
1040                                " ERL=0.\n");
1041                         ret = -1;
1042                         goto out;
1043                 } else {
1044                         /*
1045                          * drop this PDU and let the
1046                          * initiator plug the CmdSN gap.
1047                          */
1048                         pr_info("Dropping NOPOUT"
1049                                 " Command CmdSN: 0x%08x due to"
1050                                 " DataCRC error.\n", hdr->cmdsn);
1051                         ret = 0;
1052                         goto out;
1053                 }
1054         }
1055
1056         /*
1057          * Handle NOP-OUT payload for traditional iSCSI sockets
1058          */
1059         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1060                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1061                 if (!ping_data) {
1062                         pr_err("Unable to allocate memory for"
1063                                 " NOPOUT ping data.\n");
1064                         ret = -1;
1065                         goto out;
1066                 }
1067
1068                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1069                               ping_data, payload_length);
1070
1071                 ping_data[payload_length] = '\0';
1072                 /*
1073                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1074                  */
1075                 cmd->buf_ptr = ping_data;
1076                 cmd->buf_ptr_size = payload_length;
1077
1078                 pr_debug("Got %u bytes of NOPOUT ping"
1079                         " data.\n", payload_length);
1080                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1081         }
1082
1083         return iscsit_process_nop_out(conn, cmd, hdr);
1084 out:
1085         if (cmd)
1086                 iscsit_free_cmd(cmd, false);
1087         return ret;
1088 }
1089
1090 static int
1091 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1092 {
1093         struct iscsi_conn *conn = csk->conn;
1094         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1095         struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1096         u32 payload_length = pdu_cb->dlen;
1097         int rc;
1098         unsigned char *text_in = NULL;
1099
1100         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1101         if (rc < 0)
1102                 return rc;
1103
1104         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1105                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1106                         pr_err("Unable to recover from"
1107                                " Text Data digest failure while in"
1108                                " ERL=0.\n");
1109                         goto reject;
1110                 } else {
1111                         /*
1112                          * drop this PDU and let the
1113                          * initiator plug the CmdSN gap.
1114                          */
1115                         pr_info("Dropping Text"
1116                                 " Command CmdSN: 0x%08x due to"
1117                                 " DataCRC error.\n", hdr->cmdsn);
1118                         return 0;
1119                 }
1120         }
1121
1122         if (payload_length) {
1123                 text_in = kzalloc(payload_length, GFP_KERNEL);
1124                 if (!text_in) {
1125                         pr_err("Unable to allocate text_in of payload_length: %u\n",
1126                                payload_length);
1127                         return -ENOMEM;
1128                 }
1129                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1130                               text_in, payload_length);
1131
1132                 text_in[payload_length - 1] = '\0';
1133
1134                 cmd->text_in_ptr = text_in;
1135         }
1136
1137         return iscsit_process_text_cmd(conn, cmd, hdr);
1138
1139 reject:
1140         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1141                                  pdu_cb->hdr);
1142 }
1143
1144 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1145 {
1146         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1147         struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1148         struct iscsi_conn *conn = csk->conn;
1149         struct iscsi_cmd *cmd = NULL;
1150         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1151         int ret = -EINVAL;
1152
1153         switch (opcode) {
1154         case ISCSI_OP_SCSI_CMD:
1155                 cmd = cxgbit_allocate_cmd(csk);
1156                 if (!cmd)
1157                         goto reject;
1158
1159                 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1160                 break;
1161         case ISCSI_OP_SCSI_DATA_OUT:
1162                 ret = cxgbit_handle_iscsi_dataout(csk);
1163                 break;
1164         case ISCSI_OP_NOOP_OUT:
1165                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1166                         cmd = cxgbit_allocate_cmd(csk);
1167                         if (!cmd)
1168                                 goto reject;
1169                 }
1170
1171                 ret = cxgbit_handle_nop_out(csk, cmd);
1172                 break;
1173         case ISCSI_OP_SCSI_TMFUNC:
1174                 cmd = cxgbit_allocate_cmd(csk);
1175                 if (!cmd)
1176                         goto reject;
1177
1178                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1179                                                  (unsigned char *)hdr);
1180                 break;
1181         case ISCSI_OP_TEXT:
1182                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1183                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1184                         if (!cmd)
1185                                 goto reject;
1186                 } else {
1187                         cmd = cxgbit_allocate_cmd(csk);
1188                         if (!cmd)
1189                                 goto reject;
1190                 }
1191
1192                 ret = cxgbit_handle_text_cmd(csk, cmd);
1193                 break;
1194         case ISCSI_OP_LOGOUT:
1195                 cmd = cxgbit_allocate_cmd(csk);
1196                 if (!cmd)
1197                         goto reject;
1198
1199                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1200                 if (ret > 0)
1201                         wait_for_completion_timeout(&conn->conn_logout_comp,
1202                                                     SECONDS_FOR_LOGOUT_COMP
1203                                                     * HZ);
1204                 break;
1205         case ISCSI_OP_SNACK:
1206                 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1207                 break;
1208         default:
1209                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1210                 dump_stack();
1211                 break;
1212         }
1213
1214         return ret;
1215
1216 reject:
1217         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1218                                  (unsigned char *)hdr);
1219         return ret;
1220 }
1221
1222 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1223 {
1224         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1225         struct iscsi_conn *conn = csk->conn;
1226         struct iscsi_hdr *hdr = pdu_cb->hdr;
1227         u8 opcode;
1228
1229         if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1230                 atomic_long_inc(&conn->sess->conn_digest_errors);
1231                 goto transport_err;
1232         }
1233
1234         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1235                 goto transport_err;
1236
1237         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1238
1239         if (conn->sess->sess_ops->SessionType &&
1240             ((!(opcode & ISCSI_OP_TEXT)) ||
1241              (!(opcode & ISCSI_OP_LOGOUT)))) {
1242                 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1243                         " while in Discovery Session, rejecting.\n", opcode);
1244                 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1245                                   (unsigned char *)hdr);
1246                 goto transport_err;
1247         }
1248
1249         if (cxgbit_target_rx_opcode(csk) < 0)
1250                 goto transport_err;
1251
1252         return 0;
1253
1254 transport_err:
1255         return -1;
1256 }
1257
1258 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1259 {
1260         struct iscsi_conn *conn = csk->conn;
1261         struct iscsi_login *login = conn->login;
1262         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1263         struct iscsi_login_req *login_req;
1264
1265         login_req = (struct iscsi_login_req *)login->req;
1266         memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1267
1268         pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1269                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1270                 login_req->flags, login_req->itt, login_req->cmdsn,
1271                 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1272         /*
1273          * Setup the initial iscsi_login values from the leading
1274          * login request PDU.
1275          */
1276         if (login->first_request) {
1277                 login_req = (struct iscsi_login_req *)login->req;
1278                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1279                 login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(
1280                                 login_req->flags);
1281                 login->version_min      = login_req->min_version;
1282                 login->version_max      = login_req->max_version;
1283                 memcpy(login->isid, login_req->isid, 6);
1284                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1285                 login->init_task_tag    = login_req->itt;
1286                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1287                 login->cid              = be16_to_cpu(login_req->cid);
1288                 login->tsih             = be16_to_cpu(login_req->tsih);
1289         }
1290
1291         if (iscsi_target_check_login_request(conn, login) < 0)
1292                 return -1;
1293
1294         memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1295         skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1296
1297         return 0;
1298 }
1299
1300 static int
1301 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1302 {
1303         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1304         int ret;
1305
1306         cxgbit_rx_pdu_cb(skb) = pdu_cb;
1307
1308         csk->skb = skb;
1309
1310         if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1311                 ret = cxgbit_rx_login_pdu(csk);
1312                 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1313         } else {
1314                 ret = cxgbit_rx_opcode(csk);
1315         }
1316
1317         return ret;
1318 }
1319
1320 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1321 {
1322         struct skb_shared_info *ssi = skb_shinfo(skb);
1323         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1324         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1325         u8 i;
1326
1327         pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1328                 skb, skb->head, skb->data, skb->len, skb->data_len,
1329                 ssi->nr_frags);
1330         pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1331                 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1332
1333         for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1334                 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1335                         "frags %u.\n",
1336                         skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1337                         pdu_cb->ddigest, pdu_cb->frags);
1338         for (i = 0; i < ssi->nr_frags; i++)
1339                 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1340                         skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1341 }
1342
1343 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1344 {
1345         struct sk_buff *skb = csk->lro_hskb;
1346         struct skb_shared_info *ssi = skb_shinfo(skb);
1347         u8 i;
1348
1349         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1350         for (i = 0; i < ssi->nr_frags; i++)
1351                 put_page(skb_frag_page(&ssi->frags[i]));
1352         ssi->nr_frags = 0;
1353 }
1354
1355 static void
1356 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1357 {
1358         struct sk_buff *hskb = csk->lro_hskb;
1359         struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1360         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1361         struct skb_shared_info *hssi = skb_shinfo(hskb);
1362         struct skb_shared_info *ssi = skb_shinfo(skb);
1363         unsigned int len = 0;
1364
1365         if (pdu_cb->flags & PDUCBF_RX_HDR) {
1366                 hpdu_cb->flags = pdu_cb->flags;
1367                 hpdu_cb->seq = pdu_cb->seq;
1368                 hpdu_cb->hdr = pdu_cb->hdr;
1369                 hpdu_cb->hlen = pdu_cb->hlen;
1370
1371                 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx],
1372                        sizeof(skb_frag_t));
1373
1374                 get_page(skb_frag_page(&hssi->frags[0]));
1375                 hssi->nr_frags = 1;
1376                 hpdu_cb->frags = 1;
1377                 hpdu_cb->hfrag_idx = 0;
1378
1379                 len = hssi->frags[0].size;
1380                 hskb->len = len;
1381                 hskb->data_len = len;
1382                 hskb->truesize = len;
1383         }
1384
1385         if (pdu_cb->flags & PDUCBF_RX_DATA) {
1386                 u8 hfrag_idx = 1, i;
1387
1388                 hpdu_cb->flags |= pdu_cb->flags;
1389
1390                 len = 0;
1391                 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) {
1392                         memcpy(&hssi->frags[hfrag_idx],
1393                                &ssi->frags[pdu_cb->dfrag_idx + i],
1394                                sizeof(skb_frag_t));
1395
1396                         get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1397
1398                         len += hssi->frags[hfrag_idx].size;
1399
1400                         hssi->nr_frags++;
1401                         hpdu_cb->frags++;
1402                 }
1403
1404                 hpdu_cb->dlen = pdu_cb->dlen;
1405                 hpdu_cb->doffset = hpdu_cb->hlen;
1406                 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1407                 hpdu_cb->dfrag_idx = 1;
1408                 hskb->len += len;
1409                 hskb->data_len += len;
1410                 hskb->truesize += len;
1411         }
1412
1413         if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1414                 hpdu_cb->flags |= pdu_cb->flags;
1415
1416                 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1417                         hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1418
1419                 hpdu_cb->ddigest = pdu_cb->ddigest;
1420                 hpdu_cb->pdulen = pdu_cb->pdulen;
1421         }
1422 }
1423
1424 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1425 {
1426         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1427         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1428         u8 pdu_idx = 0, last_idx = 0;
1429         int ret = 0;
1430
1431         if (!pdu_cb->complete) {
1432                 cxgbit_lro_skb_merge(csk, skb, 0);
1433
1434                 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1435                         struct sk_buff *hskb = csk->lro_hskb;
1436
1437                         ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1438
1439                         cxgbit_lro_hskb_reset(csk);
1440
1441                         if (ret < 0)
1442                                 goto out;
1443                 }
1444
1445                 pdu_idx = 1;
1446         }
1447
1448         if (lro_cb->pdu_idx)
1449                 last_idx = lro_cb->pdu_idx - 1;
1450
1451         for (; pdu_idx <= last_idx; pdu_idx++) {
1452                 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1453                 if (ret < 0)
1454                         goto out;
1455         }
1456
1457         if ((!lro_cb->complete) && lro_cb->pdu_idx)
1458                 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1459
1460 out:
1461         return ret;
1462 }
1463
1464 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1465 {
1466         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1467         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1468         int ret = -1;
1469
1470         if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1471             (pdu_cb->seq != csk->rcv_nxt)) {
1472                 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1473                         csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1474                 cxgbit_lro_skb_dump(skb);
1475                 return ret;
1476         }
1477
1478         csk->rcv_nxt += lro_cb->pdu_totallen;
1479
1480         ret = cxgbit_process_lro_skb(csk, skb);
1481
1482         csk->rx_credits += lro_cb->pdu_totallen;
1483
1484         if (csk->rx_credits >= (csk->rcv_win / 4))
1485                 cxgbit_rx_data_ack(csk);
1486
1487         return ret;
1488 }
1489
1490 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1491 {
1492         int ret = -1;
1493
1494         if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO))
1495                 ret = cxgbit_rx_lro_skb(csk, skb);
1496
1497         __kfree_skb(skb);
1498         return ret;
1499 }
1500
1501 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1502 {
1503         spin_lock_bh(&csk->rxq.lock);
1504         if (skb_queue_len(&csk->rxq)) {
1505                 skb_queue_splice_init(&csk->rxq, rxq);
1506                 spin_unlock_bh(&csk->rxq.lock);
1507                 return true;
1508         }
1509         spin_unlock_bh(&csk->rxq.lock);
1510         return false;
1511 }
1512
1513 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1514 {
1515         struct sk_buff *skb;
1516         struct sk_buff_head rxq;
1517
1518         skb_queue_head_init(&rxq);
1519
1520         wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1521
1522         if (signal_pending(current))
1523                 goto out;
1524
1525         while ((skb = __skb_dequeue(&rxq))) {
1526                 if (cxgbit_rx_skb(csk, skb))
1527                         goto out;
1528         }
1529
1530         return 0;
1531 out:
1532         __skb_queue_purge(&rxq);
1533         return -1;
1534 }
1535
1536 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1537 {
1538         struct cxgbit_sock *csk = conn->context;
1539         int ret = -1;
1540
1541         while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1542                 ret = cxgbit_wait_rxq(csk);
1543                 if (ret) {
1544                         clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1545                         break;
1546                 }
1547         }
1548
1549         return ret;
1550 }
1551
1552 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1553 {
1554         struct cxgbit_sock *csk = conn->context;
1555
1556         while (!kthread_should_stop()) {
1557                 iscsit_thread_check_cpumask(conn, current, 0);
1558                 if (cxgbit_wait_rxq(csk))
1559                         return;
1560         }
1561 }