drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <asm/uaccess.h>
  64
  65 #include "cxgb4.h"
  66 #include "t4_regs.h"
  67 #include "t4_msg.h"
  68 #include "t4fw_api.h"
  69 #include "l2t.h"
  70
  71 #define DRV_VERSION "1.3.0-ko"
  72 #define DRV_DESC "Chelsio T4 Network Driver"
  73
  74 /*
  75  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  76  * under extreme memory pressure so it's largish to give the system time to
  77  * recover.
  78  */
  79 #define MAX_SGE_TIMERVAL 200U
  80
  81 enum {
  82         /*
  83          * Physical Function provisioning constants.
  84          */
  85         PFRES_NVI = 4,                  /* # of Virtual Interfaces */
  86         PFRES_NETHCTRL = 128,           /* # of EQs used for ETH or CTRL Qs */
  87         PFRES_NIQFLINT = 128,           /* # of ingress Qs/w Free List(s)/intr
  88                                          */
  89         PFRES_NEQ = 256,                /* # of egress queues */
  90         PFRES_NIQ = 0,                  /* # of ingress queues */
  91         PFRES_TC = 0,                   /* PCI-E traffic class */
  92         PFRES_NEXACTF = 128,            /* # of exact MPS filters */
  93
  94         PFRES_R_CAPS = FW_CMD_CAP_PF,
  95         PFRES_WX_CAPS = FW_CMD_CAP_PF,
  96
  97 #ifdef CONFIG_PCI_IOV
  98         /*
  99          * Virtual Function provisioning constants.  We need two extra Ingress
 100          * Queues with Interrupt capability to serve as the VF's Firmware
 101          * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
 102          * neither will have Free Lists associated with them).  For each
 103          * Ethernet/Control Egress Queue and for each Free List, we need an
 104          * Egress Context.
 105          */
 106         VFRES_NPORTS = 1,               /* # of "ports" per VF */
 107         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
 108
 109         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
 110         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
 111         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
 112         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
 113         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
 114         VFRES_TC = 0,                   /* PCI-E traffic class */
 115         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
 116
 117         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 118         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 119 #endif
 120 };
 121
 122 /*
 123  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 124  * static and likely not to be useful in the long run.  We really need to
 125  * implement some form of persistent configuration which the firmware
 126  * controls.
 127  */
 128 static unsigned int pfvfres_pmask(struct adapter *adapter,
 129                                   unsigned int pf, unsigned int vf)
 130 {
 131         unsigned int portn, portvec;
 132
 133         /*
 134          * Give PF's access to all of the ports.
 135          */
 136         if (vf == 0)
 137                 return FW_PFVF_CMD_PMASK_MASK;
 138
 139         /*
 140          * For VFs, we'll assign them access to the ports based purely on the
 141          * PF.  We assign active ports in order, wrapping around if there are
 142          * fewer active ports than PFs: e.g. active port[pf % nports].
 143          * Unfortunately the adapter's port_info structs haven't been
 144          * initialized yet so we have to compute this.
 145          */
 146         if (adapter->params.nports == 0)
 147                 return 0;
 148
 149         portn = pf % adapter->params.nports;
 150         portvec = adapter->params.portvec;
 151         for (;;) {
 152                 /*
 153                  * Isolate the lowest set bit in the port vector.  If we're at
 154                  * the port number that we want, return that as the pmask.
 155                  * otherwise mask that bit out of the port vector and
 156                  * decrement our port number ...
 157                  */
 158                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 159                 if (portn == 0)
 160                         return pmask;
 161                 portn--;
 162                 portvec &= ~pmask;
 163         }
 164         /*NOTREACHED*/
 165 }
 166
 167 enum {
 168         MAX_TXQ_ENTRIES      = 16384,
 169         MAX_CTRL_TXQ_ENTRIES = 1024,
 170         MAX_RSPQ_ENTRIES     = 16384,
 171         MAX_RX_BUFFERS       = 16384,
 172         MIN_TXQ_ENTRIES      = 32,
 173         MIN_CTRL_TXQ_ENTRIES = 32,
 174         MIN_RSPQ_ENTRIES     = 128,
 175         MIN_FL_ENTRIES       = 16
 176 };
 177
 178 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 179                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 180                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 181
 182 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 183
 184 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 185         CH_DEVICE(0xa000, 0),  /* PE10K */
 186         CH_DEVICE(0x4001, -1),
 187         CH_DEVICE(0x4002, -1),
 188         CH_DEVICE(0x4003, -1),
 189         CH_DEVICE(0x4004, -1),
 190         CH_DEVICE(0x4005, -1),
 191         CH_DEVICE(0x4006, -1),
 192         CH_DEVICE(0x4007, -1),
 193         CH_DEVICE(0x4008, -1),
 194         CH_DEVICE(0x4009, -1),
 195         CH_DEVICE(0x400a, -1),
 196         CH_DEVICE(0x4401, 4),
 197         CH_DEVICE(0x4402, 4),
 198         CH_DEVICE(0x4403, 4),
 199         CH_DEVICE(0x4404, 4),
 200         CH_DEVICE(0x4405, 4),
 201         CH_DEVICE(0x4406, 4),
 202         CH_DEVICE(0x4407, 4),
 203         CH_DEVICE(0x4408, 4),
 204         CH_DEVICE(0x4409, 4),
 205         CH_DEVICE(0x440a, 4),
 206         CH_DEVICE(0x440d, 4),
 207         CH_DEVICE(0x440e, 4),
 208         { 0, }
 209 };
 210
 211 #define FW_FNAME "cxgb4/t4fw.bin"
 212 #define FW_CFNAME "cxgb4/t4-config.txt"
 213
 214 MODULE_DESCRIPTION(DRV_DESC);
 215 MODULE_AUTHOR("Chelsio Communications");
 216 MODULE_LICENSE("Dual BSD/GPL");
 217 MODULE_VERSION(DRV_VERSION);
 218 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 219 MODULE_FIRMWARE(FW_FNAME);
 220
 221 /*
 222  * Normally we're willing to become the firmware's Master PF but will be happy
 223  * if another PF has already become the Master and initialized the adapter.
 224  * Setting "force_init" will cause this driver to forcibly establish itself as
 225  * the Master PF and initialize the adapter.
 226  */
 227 static uint force_init;
 228
 229 module_param(force_init, uint, 0644);
 230 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 231
 232 /*
 233  * Normally if the firmware we connect to has Configuration File support, we
 234  * use that and only fall back to the old Driver-based initialization if the
 235  * Configuration File fails for some reason.  If force_old_init is set, then
 236  * we'll always use the old Driver-based initialization sequence.
 237  */
 238 static uint force_old_init;
 239
 240 module_param(force_old_init, uint, 0644);
 241 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
 242
 243 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 244
 245 module_param(dflt_msg_enable, int, 0644);
 246 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 247
 248 /*
 249  * The driver uses the best interrupt scheme available on a platform in the
 250  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 251  * of these schemes the driver may consider as follows:
 252  *
 253  * msi = 2: choose from among all three options
 254  * msi = 1: only consider MSI and INTx interrupts
 255  * msi = 0: force INTx interrupts
 256  */
 257 static int msi = 2;
 258
 259 module_param(msi, int, 0644);
 260 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 261
 262 /*
 263  * Queue interrupt hold-off timer values.  Queues default to the first of these
 264  * upon creation.
 265  */
 266 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 267
 268 module_param_array(intr_holdoff, uint, NULL, 0644);
 269 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 270                  "0..4 in microseconds");
 271
 272 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 273
 274 module_param_array(intr_cnt, uint, NULL, 0644);
 275 MODULE_PARM_DESC(intr_cnt,
 276                  "thresholds 1..3 for queue interrupt packet counters");
 277
 278 /*
 279  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 280  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 281  * boundaries.  This is a requirement for many architectures which will throw
 282  * a machine check fault if an attempt is made to access one of the 4-byte IP
 283  * header fields on a non-4-byte boundary.  And it's a major performance issue
 284  * even on some architectures which allow it like some implementations of the
 285  * x86 ISA.  However, some architectures don't mind this and for some very
 286  * edge-case performance sensitive applications (like forwarding large volumes
 287  * of small packets), setting this DMA offset to 0 will decrease the number of
 288  * PCI-E Bus transfers enough to measurably affect performance.
 289  */
 290 static int rx_dma_offset = 2;
 291
 292 static bool vf_acls;
 293
 294 #ifdef CONFIG_PCI_IOV
 295 module_param(vf_acls, bool, 0644);
 296 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 297
 298 static unsigned int num_vf[4];
 299
 300 module_param_array(num_vf, uint, NULL, 0644);
 301 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 302 #endif
 303
 304 /*
 305  * The filter TCAM has a fixed portion and a variable portion.  The fixed
 306  * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
 307  * ports.  The variable portion is 36 bits which can include things like Exact
 308  * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
 309  * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
 310  * far exceed the 36-bit budget for this "compressed" header portion of the
 311  * filter.  Thus, we have a scarce resource which must be carefully managed.
 312  *
 313  * By default we set this up to mostly match the set of filter matching
 314  * capabilities of T3 but with accommodations for some of T4's more
 315  * interesting features:
 316  *
 317  *   { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
 318  *     [Inner] VLAN (17), Port (3), FCoE (1) }
 319  */
 320 enum {
 321         TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
 322         TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
 323         TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
 324 };
 325
 326 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
 327
 328 static struct dentry *cxgb4_debugfs_root;
 329
 330 static LIST_HEAD(adapter_list);
 331 static DEFINE_MUTEX(uld_mutex);
 332 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 333 static const char *uld_str[] = { "RDMA", "iSCSI" };
 334
 335 static void link_report(struct net_device *dev)
 336 {
 337         if (!netif_carrier_ok(dev))
 338                 netdev_info(dev, "link down\n");
 339         else {
 340                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 341
 342                 const char *s = "10Mbps";
 343                 const struct port_info *p = netdev_priv(dev);
 344
 345                 switch (p->link_cfg.speed) {
 346                 case SPEED_10000:
 347                         s = "10Gbps";
 348                         break;
 349                 case SPEED_1000:
 350                         s = "1000Mbps";
 351                         break;
 352                 case SPEED_100:
 353                         s = "100Mbps";
 354                         break;
 355                 }
 356
 357                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 358                             fc[p->link_cfg.fc]);
 359         }
 360 }
 361
 362 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 363 {
 364         struct net_device *dev = adapter->port[port_id];
 365
 366         /* Skip changes from disabled ports. */
 367         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 368                 if (link_stat)
 369                         netif_carrier_on(dev);
 370                 else
 371                         netif_carrier_off(dev);
 372
 373                 link_report(dev);
 374         }
 375 }
 376
 377 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 378 {
 379         static const char *mod_str[] = {
 380                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 381         };
 382
 383         const struct net_device *dev = adap->port[port_id];
 384         const struct port_info *pi = netdev_priv(dev);
 385
 386         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 387                 netdev_info(dev, "port module unplugged\n");
 388         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 389                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 390 }
 391
 392 /*
 393  * Configure the exact and hash address filters to handle a port's multicast
 394  * and secondary unicast MAC addresses.
 395  */
 396 static int set_addr_filters(const struct net_device *dev, bool sleep)
 397 {
 398         u64 mhash = 0;
 399         u64 uhash = 0;
 400         bool free = true;
 401         u16 filt_idx[7];
 402         const u8 *addr[7];
 403         int ret, naddr = 0;
 404         const struct netdev_hw_addr *ha;
 405         int uc_cnt = netdev_uc_count(dev);
 406         int mc_cnt = netdev_mc_count(dev);
 407         const struct port_info *pi = netdev_priv(dev);
 408         unsigned int mb = pi->adapter->fn;
 409
 410         /* first do the secondary unicast addresses */
 411         netdev_for_each_uc_addr(ha, dev) {
 412                 addr[naddr++] = ha->addr;
 413                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 414                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 415                                         naddr, addr, filt_idx, &uhash, sleep);
 416                         if (ret < 0)
 417                                 return ret;
 418
 419                         free = false;
 420                         naddr = 0;
 421                 }
 422         }
 423
 424         /* next set up the multicast addresses */
 425         netdev_for_each_mc_addr(ha, dev) {
 426                 addr[naddr++] = ha->addr;
 427                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 428                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 429                                         naddr, addr, filt_idx, &mhash, sleep);
 430                         if (ret < 0)
 431                                 return ret;
 432
 433                         free = false;
 434                         naddr = 0;
 435                 }
 436         }
 437
 438         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 439                                 uhash | mhash, sleep);
 440 }
 441
 442 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 443 module_param(dbfifo_int_thresh, int, 0644);
 444 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 445
 446 int dbfifo_drain_delay = 1000; /* usecs to sleep while draining the dbfifo */
 447 module_param(dbfifo_drain_delay, int, 0644);
 448 MODULE_PARM_DESC(dbfifo_drain_delay,
 449                  "usecs to sleep while draining the dbfifo");
 450
 451 /*
 452  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 453  * If @mtu is -1 it is left unchanged.
 454  */
 455 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 456 {
 457         int ret;
 458         struct port_info *pi = netdev_priv(dev);
 459
 460         ret = set_addr_filters(dev, sleep_ok);
 461         if (ret == 0)
 462                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 463                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 464                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 465                                     sleep_ok);
 466         return ret;
 467 }
 468
 469 static struct workqueue_struct *workq;
 470
 471 /**
 472  *      link_start - enable a port
 473  *      @dev: the port to enable
 474  *
 475  *      Performs the MAC and PHY actions needed to enable a port.
 476  */
 477 static int link_start(struct net_device *dev)
 478 {
 479         int ret;
 480         struct port_info *pi = netdev_priv(dev);
 481         unsigned int mb = pi->adapter->fn;
 482
 483         /*
 484          * We do not set address filters and promiscuity here, the stack does
 485          * that step explicitly.
 486          */
 487         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 488                             !!(dev->features & NETIF_F_HW_VLAN_RX), true);
 489         if (ret == 0) {
 490                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 491                                     pi->xact_addr_filt, dev->dev_addr, true,
 492                                     true);
 493                 if (ret >= 0) {
 494                         pi->xact_addr_filt = ret;
 495                         ret = 0;
 496                 }
 497         }
 498         if (ret == 0)
 499                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 500                                     &pi->link_cfg);
 501         if (ret == 0)
 502                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 503         return ret;
 504 }
 505
 506 /*
 507  * Response queue handler for the FW event queue.
 508  */
 509 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 510                           const struct pkt_gl *gl)
 511 {
 512         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 513
 514         rsp++;                                          /* skip RSS header */
 515         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 516                 const struct cpl_sge_egr_update *p = (void *)rsp;
 517                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 518                 struct sge_txq *txq;
 519
 520                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 521                 txq->restarts++;
 522                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 523                         struct sge_eth_txq *eq;
 524
 525                         eq = container_of(txq, struct sge_eth_txq, q);
 526                         netif_tx_wake_queue(eq->txq);
 527                 } else {
 528                         struct sge_ofld_txq *oq;
 529
 530                         oq = container_of(txq, struct sge_ofld_txq, q);
 531                         tasklet_schedule(&oq->qresume_tsk);
 532                 }
 533         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 534                 const struct cpl_fw6_msg *p = (void *)rsp;
 535
 536                 if (p->type == 0)
 537                         t4_handle_fw_rpl(q->adap, p->data);
 538         } else if (opcode == CPL_L2T_WRITE_RPL) {
 539                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 540
 541                 do_l2t_write_rpl(q->adap, p);
 542         } else
 543                 dev_err(q->adap->pdev_dev,
 544                         "unexpected CPL %#x on FW event queue\n", opcode);
 545         return 0;
 546 }
 547
 548 /**
 549  *      uldrx_handler - response queue handler for ULD queues
 550  *      @q: the response queue that received the packet
 551  *      @rsp: the response queue descriptor holding the offload message
 552  *      @gl: the gather list of packet fragments
 553  *
 554  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 555  *      the ULD, we just maintain statistics.
 556  */
 557 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 558                          const struct pkt_gl *gl)
 559 {
 560         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 561
 562         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 563                 rxq->stats.nomem++;
 564                 return -1;
 565         }
 566         if (gl == NULL)
 567                 rxq->stats.imm++;
 568         else if (gl == CXGB4_MSG_AN)
 569                 rxq->stats.an++;
 570         else
 571                 rxq->stats.pkts++;
 572         return 0;
 573 }
 574
 575 static void disable_msi(struct adapter *adapter)
 576 {
 577         if (adapter->flags & USING_MSIX) {
 578                 pci_disable_msix(adapter->pdev);
 579                 adapter->flags &= ~USING_MSIX;
 580         } else if (adapter->flags & USING_MSI) {
 581                 pci_disable_msi(adapter->pdev);
 582                 adapter->flags &= ~USING_MSI;
 583         }
 584 }
 585
 586 /*
 587  * Interrupt handler for non-data events used with MSI-X.
 588  */
 589 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 590 {
 591         struct adapter *adap = cookie;
 592
 593         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 594         if (v & PFSW) {
 595                 adap->swintr = 1;
 596                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 597         }
 598         t4_slow_intr_handler(adap);
 599         return IRQ_HANDLED;
 600 }
 601
 602 /*
 603  * Name the MSI-X interrupts.
 604  */
 605 static void name_msix_vecs(struct adapter *adap)
 606 {
 607         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 608
 609         /* non-data interrupts */
 610         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 611
 612         /* FW events */
 613         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 614                  adap->port[0]->name);
 615
 616         /* Ethernet queues */
 617         for_each_port(adap, j) {
 618                 struct net_device *d = adap->port[j];
 619                 const struct port_info *pi = netdev_priv(d);
 620
 621                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 622                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 623                                  d->name, i);
 624         }
 625
 626         /* offload queues */
 627         for_each_ofldrxq(&adap->sge, i)
 628                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 629                          adap->port[0]->name, i);
 630
 631         for_each_rdmarxq(&adap->sge, i)
 632                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 633                          adap->port[0]->name, i);
 634 }
 635
 636 static int request_msix_queue_irqs(struct adapter *adap)
 637 {
 638         struct sge *s = &adap->sge;
 639         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
 640
 641         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 642                           adap->msix_info[1].desc, &s->fw_evtq);
 643         if (err)
 644                 return err;
 645
 646         for_each_ethrxq(s, ethqidx) {
 647                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 648                                   adap->msix_info[msi].desc,
 649                                   &s->ethrxq[ethqidx].rspq);
 650                 if (err)
 651                         goto unwind;
 652                 msi++;
 653         }
 654         for_each_ofldrxq(s, ofldqidx) {
 655                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 656                                   adap->msix_info[msi].desc,
 657                                   &s->ofldrxq[ofldqidx].rspq);
 658                 if (err)
 659                         goto unwind;
 660                 msi++;
 661         }
 662         for_each_rdmarxq(s, rdmaqidx) {
 663                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 664                                   adap->msix_info[msi].desc,
 665                                   &s->rdmarxq[rdmaqidx].rspq);
 666                 if (err)
 667                         goto unwind;
 668                 msi++;
 669         }
 670         return 0;
 671
 672 unwind:
 673         while (--rdmaqidx >= 0)
 674                 free_irq(adap->msix_info[--msi].vec,
 675                          &s->rdmarxq[rdmaqidx].rspq);
 676         while (--ofldqidx >= 0)
 677                 free_irq(adap->msix_info[--msi].vec,
 678                          &s->ofldrxq[ofldqidx].rspq);
 679         while (--ethqidx >= 0)
 680                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
 681         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 682         return err;
 683 }
 684
 685 static void free_msix_queue_irqs(struct adapter *adap)
 686 {
 687         int i, msi = 2;
 688         struct sge *s = &adap->sge;
 689
 690         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 691         for_each_ethrxq(s, i)
 692                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
 693         for_each_ofldrxq(s, i)
 694                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
 695         for_each_rdmarxq(s, i)
 696                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
 697 }
 698
 699 /**
 700  *      write_rss - write the RSS table for a given port
 701  *      @pi: the port
 702  *      @queues: array of queue indices for RSS
 703  *
 704  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 705  *      packets to the Rx queues in @queues.
 706  */
 707 static int write_rss(const struct port_info *pi, const u16 *queues)
 708 {
 709         u16 *rss;
 710         int i, err;
 711         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 712
 713         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 714         if (!rss)
 715                 return -ENOMEM;
 716
 717         /* map the queue indices to queue ids */
 718         for (i = 0; i < pi->rss_size; i++, queues++)
 719                 rss[i] = q[*queues].rspq.abs_id;
 720
 721         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 722                                   pi->rss_size, rss, pi->rss_size);
 723         kfree(rss);
 724         return err;
 725 }
 726
 727 /**
 728  *      setup_rss - configure RSS
 729  *      @adap: the adapter
 730  *
 731  *      Sets up RSS for each port.
 732  */
 733 static int setup_rss(struct adapter *adap)
 734 {
 735         int i, err;
 736
 737         for_each_port(adap, i) {
 738                 const struct port_info *pi = adap2pinfo(adap, i);
 739
 740                 err = write_rss(pi, pi->rss);
 741                 if (err)
 742                         return err;
 743         }
 744         return 0;
 745 }
 746
 747 /*
 748  * Return the channel of the ingress queue with the given qid.
 749  */
 750 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 751 {
 752         qid -= p->ingr_start;
 753         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 754 }
 755
 756 /*
 757  * Wait until all NAPI handlers are descheduled.
 758  */
 759 static void quiesce_rx(struct adapter *adap)
 760 {
 761         int i;
 762
 763         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 764                 struct sge_rspq *q = adap->sge.ingr_map[i];
 765
 766                 if (q && q->handler)
 767                         napi_disable(&q->napi);
 768         }
 769 }
 770
 771 /*
 772  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 773  */
 774 static void enable_rx(struct adapter *adap)
 775 {
 776         int i;
 777
 778         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 779                 struct sge_rspq *q = adap->sge.ingr_map[i];
 780
 781                 if (!q)
 782                         continue;
 783                 if (q->handler)
 784                         napi_enable(&q->napi);
 785                 /* 0-increment GTS to start the timer and enable interrupts */
 786                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 787                              SEINTARM(q->intr_params) |
 788                              INGRESSQID(q->cntxt_id));
 789         }
 790 }
 791
 792 /**
 793  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 794  *      @adap: the adapter
 795  *
 796  *      Determines how many sets of SGE queues to use and initializes them.
 797  *      We support multiple queue sets per port if we have MSI-X, otherwise
 798  *      just one queue set per port.
 799  */
 800 static int setup_sge_queues(struct adapter *adap)
 801 {
 802         int err, msi_idx, i, j;
 803         struct sge *s = &adap->sge;
 804
 805         bitmap_zero(s->starving_fl, MAX_EGRQ);
 806         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 807
 808         if (adap->flags & USING_MSIX)
 809                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 810         else {
 811                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 812                                        NULL, NULL);
 813                 if (err)
 814                         return err;
 815                 msi_idx = -((int)s->intrq.abs_id + 1);
 816         }
 817
 818         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 819                                msi_idx, NULL, fwevtq_handler);
 820         if (err) {
 821 freeout:        t4_free_sge_resources(adap);
 822                 return err;
 823         }
 824
 825         for_each_port(adap, i) {
 826                 struct net_device *dev = adap->port[i];
 827                 struct port_info *pi = netdev_priv(dev);
 828                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 829                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 830
 831                 for (j = 0; j < pi->nqsets; j++, q++) {
 832                         if (msi_idx > 0)
 833                                 msi_idx++;
 834                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 835                                                msi_idx, &q->fl,
 836                                                t4_ethrx_handler);
 837                         if (err)
 838                                 goto freeout;
 839                         q->rspq.idx = j;
 840                         memset(&q->stats, 0, sizeof(q->stats));
 841                 }
 842                 for (j = 0; j < pi->nqsets; j++, t++) {
 843                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 844                                         netdev_get_tx_queue(dev, j),
 845                                         s->fw_evtq.cntxt_id);
 846                         if (err)
 847                                 goto freeout;
 848                 }
 849         }
 850
 851         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 852         for_each_ofldrxq(s, i) {
 853                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 854                 struct net_device *dev = adap->port[i / j];
 855
 856                 if (msi_idx > 0)
 857                         msi_idx++;
 858                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 859                                        &q->fl, uldrx_handler);
 860                 if (err)
 861                         goto freeout;
 862                 memset(&q->stats, 0, sizeof(q->stats));
 863                 s->ofld_rxq[i] = q->rspq.abs_id;
 864                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 865                                             s->fw_evtq.cntxt_id);
 866                 if (err)
 867                         goto freeout;
 868         }
 869
 870         for_each_rdmarxq(s, i) {
 871                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 872
 873                 if (msi_idx > 0)
 874                         msi_idx++;
 875                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 876                                        msi_idx, &q->fl, uldrx_handler);
 877                 if (err)
 878                         goto freeout;
 879                 memset(&q->stats, 0, sizeof(q->stats));
 880                 s->rdma_rxq[i] = q->rspq.abs_id;
 881         }
 882
 883         for_each_port(adap, i) {
 884                 /*
 885                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 886                  * have RDMA queues, and that's the right value.
 887                  */
 888                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 889                                             s->fw_evtq.cntxt_id,
 890                                             s->rdmarxq[i].rspq.cntxt_id);
 891                 if (err)
 892                         goto freeout;
 893         }
 894
 895         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
 896                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
 897                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
 898         return 0;
 899 }
 900
 901 /*
 902  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
 903  * started but failed, and a negative errno if flash load couldn't start.
 904  */
 905 static int upgrade_fw(struct adapter *adap)
 906 {
 907         int ret;
 908         u32 vers;
 909         const struct fw_hdr *hdr;
 910         const struct firmware *fw;
 911         struct device *dev = adap->pdev_dev;
 912
 913         ret = request_firmware(&fw, FW_FNAME, dev);
 914         if (ret < 0) {
 915                 dev_err(dev, "unable to load firmware image " FW_FNAME
 916                         ", error %d\n", ret);
 917                 return ret;
 918         }
 919
 920         hdr = (const struct fw_hdr *)fw->data;
 921         vers = ntohl(hdr->fw_ver);
 922         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
 923                 ret = -EINVAL;              /* wrong major version, won't do */
 924                 goto out;
 925         }
 926
 927         /*
 928          * If the flash FW is unusable or we found something newer, load it.
 929          */
 930         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
 931             vers > adap->params.fw_vers) {
 932                 dev_info(dev, "upgrading firmware ...\n");
 933                 ret = t4_fw_upgrade(adap, adap->mbox, fw->data, fw->size,
 934                                     /*force=*/false);
 935                 if (!ret)
 936                         dev_info(dev, "firmware successfully upgraded to "
 937                                  FW_FNAME " (%d.%d.%d.%d)\n",
 938                                  FW_HDR_FW_VER_MAJOR_GET(vers),
 939                                  FW_HDR_FW_VER_MINOR_GET(vers),
 940                                  FW_HDR_FW_VER_MICRO_GET(vers),
 941                                  FW_HDR_FW_VER_BUILD_GET(vers));
 942                 else
 943                         dev_err(dev, "firmware upgrade failed! err=%d\n", -ret);
 944         } else {
 945                 /*
 946                  * Tell our caller that we didn't upgrade the firmware.
 947                  */
 948                 ret = -EINVAL;
 949         }
 950
 951 out:    release_firmware(fw);
 952         return ret;
 953 }
 954
 955 /*
 956  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 957  * The allocated memory is cleared.
 958  */
 959 void *t4_alloc_mem(size_t size)
 960 {
 961         void *p = kzalloc(size, GFP_KERNEL);
 962
 963         if (!p)
 964                 p = vzalloc(size);
 965         return p;
 966 }
 967
 968 /*
 969  * Free memory allocated through alloc_mem().
 970  */
 971 static void t4_free_mem(void *addr)
 972 {
 973         if (is_vmalloc_addr(addr))
 974                 vfree(addr);
 975         else
 976                 kfree(addr);
 977 }
 978
 979 static inline int is_offload(const struct adapter *adap)
 980 {
 981         return adap->params.offload;
 982 }
 983
 984 /*
 985  * Implementation of ethtool operations.
 986  */
 987
 988 static u32 get_msglevel(struct net_device *dev)
 989 {
 990         return netdev2adap(dev)->msg_enable;
 991 }
 992
 993 static void set_msglevel(struct net_device *dev, u32 val)
 994 {
 995         netdev2adap(dev)->msg_enable = val;
 996 }
 997
 998 static char stats_strings[][ETH_GSTRING_LEN] = {
 999         "TxOctetsOK         ",
1000         "TxFramesOK         ",
1001         "TxBroadcastFrames  ",
1002         "TxMulticastFrames  ",
1003         "TxUnicastFrames    ",
1004         "TxErrorFrames      ",
1005
1006         "TxFrames64         ",
1007         "TxFrames65To127    ",
1008         "TxFrames128To255   ",
1009         "TxFrames256To511   ",
1010         "TxFrames512To1023  ",
1011         "TxFrames1024To1518 ",
1012         "TxFrames1519ToMax  ",
1013
1014         "TxFramesDropped    ",
1015         "TxPauseFrames      ",
1016         "TxPPP0Frames       ",
1017         "TxPPP1Frames       ",
1018         "TxPPP2Frames       ",
1019         "TxPPP3Frames       ",
1020         "TxPPP4Frames       ",
1021         "TxPPP5Frames       ",
1022         "TxPPP6Frames       ",
1023         "TxPPP7Frames       ",
1024
1025         "RxOctetsOK         ",
1026         "RxFramesOK         ",
1027         "RxBroadcastFrames  ",
1028         "RxMulticastFrames  ",
1029         "RxUnicastFrames    ",
1030
1031         "RxFramesTooLong    ",
1032         "RxJabberErrors     ",
1033         "RxFCSErrors        ",
1034         "RxLengthErrors     ",
1035         "RxSymbolErrors     ",
1036         "RxRuntFrames       ",
1037
1038         "RxFrames64         ",
1039         "RxFrames65To127    ",
1040         "RxFrames128To255   ",
1041         "RxFrames256To511   ",
1042         "RxFrames512To1023  ",
1043         "RxFrames1024To1518 ",
1044         "RxFrames1519ToMax  ",
1045
1046         "RxPauseFrames      ",
1047         "RxPPP0Frames       ",
1048         "RxPPP1Frames       ",
1049         "RxPPP2Frames       ",
1050         "RxPPP3Frames       ",
1051         "RxPPP4Frames       ",
1052         "RxPPP5Frames       ",
1053         "RxPPP6Frames       ",
1054         "RxPPP7Frames       ",
1055
1056         "RxBG0FramesDropped ",
1057         "RxBG1FramesDropped ",
1058         "RxBG2FramesDropped ",
1059         "RxBG3FramesDropped ",
1060         "RxBG0FramesTrunc   ",
1061         "RxBG1FramesTrunc   ",
1062         "RxBG2FramesTrunc   ",
1063         "RxBG3FramesTrunc   ",
1064
1065         "TSO                ",
1066         "TxCsumOffload      ",
1067         "RxCsumGood         ",
1068         "VLANextractions    ",
1069         "VLANinsertions     ",
1070         "GROpackets         ",
1071         "GROmerged          ",
1072 };
1073
1074 static int get_sset_count(struct net_device *dev, int sset)
1075 {
1076         switch (sset) {
1077         case ETH_SS_STATS:
1078                 return ARRAY_SIZE(stats_strings);
1079         default:
1080                 return -EOPNOTSUPP;
1081         }
1082 }
1083
1084 #define T4_REGMAP_SIZE (160 * 1024)
1085
1086 static int get_regs_len(struct net_device *dev)
1087 {
1088         return T4_REGMAP_SIZE;
1089 }
1090
1091 static int get_eeprom_len(struct net_device *dev)
1092 {
1093         return EEPROMSIZE;
1094 }
1095
1096 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1097 {
1098         struct adapter *adapter = netdev2adap(dev);
1099
1100         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1101         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1102         strlcpy(info->bus_info, pci_name(adapter->pdev),
1103                 sizeof(info->bus_info));
1104
1105         if (adapter->params.fw_vers)
1106                 snprintf(info->fw_version, sizeof(info->fw_version),
1107                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1108                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1109                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1110                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1111                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1112                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1113                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1114                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1115                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1116 }
1117
1118 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1119 {
1120         if (stringset == ETH_SS_STATS)
1121                 memcpy(data, stats_strings, sizeof(stats_strings));
1122 }
1123
1124 /*
1125  * port stats maintained per queue of the port.  They should be in the same
1126  * order as in stats_strings above.
1127  */
1128 struct queue_port_stats {
1129         u64 tso;
1130         u64 tx_csum;
1131         u64 rx_csum;
1132         u64 vlan_ex;
1133         u64 vlan_ins;
1134         u64 gro_pkts;
1135         u64 gro_merged;
1136 };
1137
1138 static void collect_sge_port_stats(const struct adapter *adap,
1139                 const struct port_info *p, struct queue_port_stats *s)
1140 {
1141         int i;
1142         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1143         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1144
1145         memset(s, 0, sizeof(*s));
1146         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1147                 s->tso += tx->tso;
1148                 s->tx_csum += tx->tx_cso;
1149                 s->rx_csum += rx->stats.rx_cso;
1150                 s->vlan_ex += rx->stats.vlan_ex;
1151                 s->vlan_ins += tx->vlan_ins;
1152                 s->gro_pkts += rx->stats.lro_pkts;
1153                 s->gro_merged += rx->stats.lro_merged;
1154         }
1155 }
1156
1157 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1158                       u64 *data)
1159 {
1160         struct port_info *pi = netdev_priv(dev);
1161         struct adapter *adapter = pi->adapter;
1162
1163         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1164
1165         data += sizeof(struct port_stats) / sizeof(u64);
1166         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1167 }
1168
1169 /*
1170  * Return a version number to identify the type of adapter.  The scheme is:
1171  * - bits 0..9: chip version
1172  * - bits 10..15: chip revision
1173  * - bits 16..23: register dump version
1174  */
1175 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1176 {
1177         return 4 | (ap->params.rev << 10) | (1 << 16);
1178 }
1179
1180 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1181                            unsigned int end)
1182 {
1183         u32 *p = buf + start;
1184
1185         for ( ; start <= end; start += sizeof(u32))
1186                 *p++ = t4_read_reg(ap, start);
1187 }
1188
1189 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1190                      void *buf)
1191 {
1192         static const unsigned int reg_ranges[] = {
1193                 0x1008, 0x1108,
1194                 0x1180, 0x11b4,
1195                 0x11fc, 0x123c,
1196                 0x1300, 0x173c,
1197                 0x1800, 0x18fc,
1198                 0x3000, 0x30d8,
1199                 0x30e0, 0x5924,
1200                 0x5960, 0x59d4,
1201                 0x5a00, 0x5af8,
1202                 0x6000, 0x6098,
1203                 0x6100, 0x6150,
1204                 0x6200, 0x6208,
1205                 0x6240, 0x6248,
1206                 0x6280, 0x6338,
1207                 0x6370, 0x638c,
1208                 0x6400, 0x643c,
1209                 0x6500, 0x6524,
1210                 0x6a00, 0x6a38,
1211                 0x6a60, 0x6a78,
1212                 0x6b00, 0x6b84,
1213                 0x6bf0, 0x6c84,
1214                 0x6cf0, 0x6d84,
1215                 0x6df0, 0x6e84,
1216                 0x6ef0, 0x6f84,
1217                 0x6ff0, 0x7084,
1218                 0x70f0, 0x7184,
1219                 0x71f0, 0x7284,
1220                 0x72f0, 0x7384,
1221                 0x73f0, 0x7450,
1222                 0x7500, 0x7530,
1223                 0x7600, 0x761c,
1224                 0x7680, 0x76cc,
1225                 0x7700, 0x7798,
1226                 0x77c0, 0x77fc,
1227                 0x7900, 0x79fc,
1228                 0x7b00, 0x7c38,
1229                 0x7d00, 0x7efc,
1230                 0x8dc0, 0x8e1c,
1231                 0x8e30, 0x8e78,
1232                 0x8ea0, 0x8f6c,
1233                 0x8fc0, 0x9074,
1234                 0x90fc, 0x90fc,
1235                 0x9400, 0x9458,
1236                 0x9600, 0x96bc,
1237                 0x9800, 0x9808,
1238                 0x9820, 0x983c,
1239                 0x9850, 0x9864,
1240                 0x9c00, 0x9c6c,
1241                 0x9c80, 0x9cec,
1242                 0x9d00, 0x9d6c,
1243                 0x9d80, 0x9dec,
1244                 0x9e00, 0x9e6c,
1245                 0x9e80, 0x9eec,
1246                 0x9f00, 0x9f6c,
1247                 0x9f80, 0x9fec,
1248                 0xd004, 0xd03c,
1249                 0xdfc0, 0xdfe0,
1250                 0xe000, 0xea7c,
1251                 0xf000, 0x11190,
1252                 0x19040, 0x1906c,
1253                 0x19078, 0x19080,
1254                 0x1908c, 0x19124,
1255                 0x19150, 0x191b0,
1256                 0x191d0, 0x191e8,
1257                 0x19238, 0x1924c,
1258                 0x193f8, 0x19474,
1259                 0x19490, 0x194f8,
1260                 0x19800, 0x19f30,
1261                 0x1a000, 0x1a06c,
1262                 0x1a0b0, 0x1a120,
1263                 0x1a128, 0x1a138,
1264                 0x1a190, 0x1a1c4,
1265                 0x1a1fc, 0x1a1fc,
1266                 0x1e040, 0x1e04c,
1267                 0x1e284, 0x1e28c,
1268                 0x1e2c0, 0x1e2c0,
1269                 0x1e2e0, 0x1e2e0,
1270                 0x1e300, 0x1e384,
1271                 0x1e3c0, 0x1e3c8,
1272                 0x1e440, 0x1e44c,
1273                 0x1e684, 0x1e68c,
1274                 0x1e6c0, 0x1e6c0,
1275                 0x1e6e0, 0x1e6e0,
1276                 0x1e700, 0x1e784,
1277                 0x1e7c0, 0x1e7c8,
1278                 0x1e840, 0x1e84c,
1279                 0x1ea84, 0x1ea8c,
1280                 0x1eac0, 0x1eac0,
1281                 0x1eae0, 0x1eae0,
1282                 0x1eb00, 0x1eb84,
1283                 0x1ebc0, 0x1ebc8,
1284                 0x1ec40, 0x1ec4c,
1285                 0x1ee84, 0x1ee8c,
1286                 0x1eec0, 0x1eec0,
1287                 0x1eee0, 0x1eee0,
1288                 0x1ef00, 0x1ef84,
1289                 0x1efc0, 0x1efc8,
1290                 0x1f040, 0x1f04c,
1291                 0x1f284, 0x1f28c,
1292                 0x1f2c0, 0x1f2c0,
1293                 0x1f2e0, 0x1f2e0,
1294                 0x1f300, 0x1f384,
1295                 0x1f3c0, 0x1f3c8,
1296                 0x1f440, 0x1f44c,
1297                 0x1f684, 0x1f68c,
1298                 0x1f6c0, 0x1f6c0,
1299                 0x1f6e0, 0x1f6e0,
1300                 0x1f700, 0x1f784,
1301                 0x1f7c0, 0x1f7c8,
1302                 0x1f840, 0x1f84c,
1303                 0x1fa84, 0x1fa8c,
1304                 0x1fac0, 0x1fac0,
1305                 0x1fae0, 0x1fae0,
1306                 0x1fb00, 0x1fb84,
1307                 0x1fbc0, 0x1fbc8,
1308                 0x1fc40, 0x1fc4c,
1309                 0x1fe84, 0x1fe8c,
1310                 0x1fec0, 0x1fec0,
1311                 0x1fee0, 0x1fee0,
1312                 0x1ff00, 0x1ff84,
1313                 0x1ffc0, 0x1ffc8,
1314                 0x20000, 0x2002c,
1315                 0x20100, 0x2013c,
1316                 0x20190, 0x201c8,
1317                 0x20200, 0x20318,
1318                 0x20400, 0x20528,
1319                 0x20540, 0x20614,
1320                 0x21000, 0x21040,
1321                 0x2104c, 0x21060,
1322                 0x210c0, 0x210ec,
1323                 0x21200, 0x21268,
1324                 0x21270, 0x21284,
1325                 0x212fc, 0x21388,
1326                 0x21400, 0x21404,
1327                 0x21500, 0x21518,
1328                 0x2152c, 0x2153c,
1329                 0x21550, 0x21554,
1330                 0x21600, 0x21600,
1331                 0x21608, 0x21628,
1332                 0x21630, 0x2163c,
1333                 0x21700, 0x2171c,
1334                 0x21780, 0x2178c,
1335                 0x21800, 0x21c38,
1336                 0x21c80, 0x21d7c,
1337                 0x21e00, 0x21e04,
1338                 0x22000, 0x2202c,
1339                 0x22100, 0x2213c,
1340                 0x22190, 0x221c8,
1341                 0x22200, 0x22318,
1342                 0x22400, 0x22528,
1343                 0x22540, 0x22614,
1344                 0x23000, 0x23040,
1345                 0x2304c, 0x23060,
1346                 0x230c0, 0x230ec,
1347                 0x23200, 0x23268,
1348                 0x23270, 0x23284,
1349                 0x232fc, 0x23388,
1350                 0x23400, 0x23404,
1351                 0x23500, 0x23518,
1352                 0x2352c, 0x2353c,
1353                 0x23550, 0x23554,
1354                 0x23600, 0x23600,
1355                 0x23608, 0x23628,
1356                 0x23630, 0x2363c,
1357                 0x23700, 0x2371c,
1358                 0x23780, 0x2378c,
1359                 0x23800, 0x23c38,
1360                 0x23c80, 0x23d7c,
1361                 0x23e00, 0x23e04,
1362                 0x24000, 0x2402c,
1363                 0x24100, 0x2413c,
1364                 0x24190, 0x241c8,
1365                 0x24200, 0x24318,
1366                 0x24400, 0x24528,
1367                 0x24540, 0x24614,
1368                 0x25000, 0x25040,
1369                 0x2504c, 0x25060,
1370                 0x250c0, 0x250ec,
1371                 0x25200, 0x25268,
1372                 0x25270, 0x25284,
1373                 0x252fc, 0x25388,
1374                 0x25400, 0x25404,
1375                 0x25500, 0x25518,
1376                 0x2552c, 0x2553c,
1377                 0x25550, 0x25554,
1378                 0x25600, 0x25600,
1379                 0x25608, 0x25628,
1380                 0x25630, 0x2563c,
1381                 0x25700, 0x2571c,
1382                 0x25780, 0x2578c,
1383                 0x25800, 0x25c38,
1384                 0x25c80, 0x25d7c,
1385                 0x25e00, 0x25e04,
1386                 0x26000, 0x2602c,
1387                 0x26100, 0x2613c,
1388                 0x26190, 0x261c8,
1389                 0x26200, 0x26318,
1390                 0x26400, 0x26528,
1391                 0x26540, 0x26614,
1392                 0x27000, 0x27040,
1393                 0x2704c, 0x27060,
1394                 0x270c0, 0x270ec,
1395                 0x27200, 0x27268,
1396                 0x27270, 0x27284,
1397                 0x272fc, 0x27388,
1398                 0x27400, 0x27404,
1399                 0x27500, 0x27518,
1400                 0x2752c, 0x2753c,
1401                 0x27550, 0x27554,
1402                 0x27600, 0x27600,
1403                 0x27608, 0x27628,
1404                 0x27630, 0x2763c,
1405                 0x27700, 0x2771c,
1406                 0x27780, 0x2778c,
1407                 0x27800, 0x27c38,
1408                 0x27c80, 0x27d7c,
1409                 0x27e00, 0x27e04
1410         };
1411
1412         int i;
1413         struct adapter *ap = netdev2adap(dev);
1414
1415         regs->version = mk_adap_vers(ap);
1416
1417         memset(buf, 0, T4_REGMAP_SIZE);
1418         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1419                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1420 }
1421
1422 static int restart_autoneg(struct net_device *dev)
1423 {
1424         struct port_info *p = netdev_priv(dev);
1425
1426         if (!netif_running(dev))
1427                 return -EAGAIN;
1428         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1429                 return -EINVAL;
1430         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1431         return 0;
1432 }
1433
1434 static int identify_port(struct net_device *dev,
1435                          enum ethtool_phys_id_state state)
1436 {
1437         unsigned int val;
1438         struct adapter *adap = netdev2adap(dev);
1439
1440         if (state == ETHTOOL_ID_ACTIVE)
1441                 val = 0xffff;
1442         else if (state == ETHTOOL_ID_INACTIVE)
1443                 val = 0;
1444         else
1445                 return -EINVAL;
1446
1447         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1448 }
1449
1450 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1451 {
1452         unsigned int v = 0;
1453
1454         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1455             type == FW_PORT_TYPE_BT_XAUI) {
1456                 v |= SUPPORTED_TP;
1457                 if (caps & FW_PORT_CAP_SPEED_100M)
1458                         v |= SUPPORTED_100baseT_Full;
1459                 if (caps & FW_PORT_CAP_SPEED_1G)
1460                         v |= SUPPORTED_1000baseT_Full;
1461                 if (caps & FW_PORT_CAP_SPEED_10G)
1462                         v |= SUPPORTED_10000baseT_Full;
1463         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1464                 v |= SUPPORTED_Backplane;
1465                 if (caps & FW_PORT_CAP_SPEED_1G)
1466                         v |= SUPPORTED_1000baseKX_Full;
1467                 if (caps & FW_PORT_CAP_SPEED_10G)
1468                         v |= SUPPORTED_10000baseKX4_Full;
1469         } else if (type == FW_PORT_TYPE_KR)
1470                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1471         else if (type == FW_PORT_TYPE_BP_AP)
1472                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1473                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1474         else if (type == FW_PORT_TYPE_BP4_AP)
1475                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1476                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1477                      SUPPORTED_10000baseKX4_Full;
1478         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1479                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1480                 v |= SUPPORTED_FIBRE;
1481
1482         if (caps & FW_PORT_CAP_ANEG)
1483                 v |= SUPPORTED_Autoneg;
1484         return v;
1485 }
1486
1487 static unsigned int to_fw_linkcaps(unsigned int caps)
1488 {
1489         unsigned int v = 0;
1490
1491         if (caps & ADVERTISED_100baseT_Full)
1492                 v |= FW_PORT_CAP_SPEED_100M;
1493         if (caps & ADVERTISED_1000baseT_Full)
1494                 v |= FW_PORT_CAP_SPEED_1G;
1495         if (caps & ADVERTISED_10000baseT_Full)
1496                 v |= FW_PORT_CAP_SPEED_10G;
1497         return v;
1498 }
1499
1500 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1501 {
1502         const struct port_info *p = netdev_priv(dev);
1503
1504         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1505             p->port_type == FW_PORT_TYPE_BT_XFI ||
1506             p->port_type == FW_PORT_TYPE_BT_XAUI)
1507                 cmd->port = PORT_TP;
1508         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1509                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1510                 cmd->port = PORT_FIBRE;
1511         else if (p->port_type == FW_PORT_TYPE_SFP) {
1512                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1513                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1514                         cmd->port = PORT_DA;
1515                 else
1516                         cmd->port = PORT_FIBRE;
1517         } else
1518                 cmd->port = PORT_OTHER;
1519
1520         if (p->mdio_addr >= 0) {
1521                 cmd->phy_address = p->mdio_addr;
1522                 cmd->transceiver = XCVR_EXTERNAL;
1523                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1524                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1525         } else {
1526                 cmd->phy_address = 0;  /* not really, but no better option */
1527                 cmd->transceiver = XCVR_INTERNAL;
1528                 cmd->mdio_support = 0;
1529         }
1530
1531         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1532         cmd->advertising = from_fw_linkcaps(p->port_type,
1533                                             p->link_cfg.advertising);
1534         ethtool_cmd_speed_set(cmd,
1535                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1536         cmd->duplex = DUPLEX_FULL;
1537         cmd->autoneg = p->link_cfg.autoneg;
1538         cmd->maxtxpkt = 0;
1539         cmd->maxrxpkt = 0;
1540         return 0;
1541 }
1542
1543 static unsigned int speed_to_caps(int speed)
1544 {
1545         if (speed == SPEED_100)
1546                 return FW_PORT_CAP_SPEED_100M;
1547         if (speed == SPEED_1000)
1548                 return FW_PORT_CAP_SPEED_1G;
1549         if (speed == SPEED_10000)
1550                 return FW_PORT_CAP_SPEED_10G;
1551         return 0;
1552 }
1553
1554 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1555 {
1556         unsigned int cap;
1557         struct port_info *p = netdev_priv(dev);
1558         struct link_config *lc = &p->link_cfg;
1559         u32 speed = ethtool_cmd_speed(cmd);
1560
1561         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1562                 return -EINVAL;
1563
1564         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1565                 /*
1566                  * PHY offers a single speed.  See if that's what's
1567                  * being requested.
1568                  */
1569                 if (cmd->autoneg == AUTONEG_DISABLE &&
1570                     (lc->supported & speed_to_caps(speed)))
1571                         return 0;
1572                 return -EINVAL;
1573         }
1574
1575         if (cmd->autoneg == AUTONEG_DISABLE) {
1576                 cap = speed_to_caps(speed);
1577
1578                 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1579                     (speed == SPEED_10000))
1580                         return -EINVAL;
1581                 lc->requested_speed = cap;
1582                 lc->advertising = 0;
1583         } else {
1584                 cap = to_fw_linkcaps(cmd->advertising);
1585                 if (!(lc->supported & cap))
1586                         return -EINVAL;
1587                 lc->requested_speed = 0;
1588                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1589         }
1590         lc->autoneg = cmd->autoneg;
1591
1592         if (netif_running(dev))
1593                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1594                                      lc);
1595         return 0;
1596 }
1597
1598 static void get_pauseparam(struct net_device *dev,
1599                            struct ethtool_pauseparam *epause)
1600 {
1601         struct port_info *p = netdev_priv(dev);
1602
1603         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1604         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1605         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1606 }
1607
1608 static int set_pauseparam(struct net_device *dev,
1609                           struct ethtool_pauseparam *epause)
1610 {
1611         struct port_info *p = netdev_priv(dev);
1612         struct link_config *lc = &p->link_cfg;
1613
1614         if (epause->autoneg == AUTONEG_DISABLE)
1615                 lc->requested_fc = 0;
1616         else if (lc->supported & FW_PORT_CAP_ANEG)
1617                 lc->requested_fc = PAUSE_AUTONEG;
1618         else
1619                 return -EINVAL;
1620
1621         if (epause->rx_pause)
1622                 lc->requested_fc |= PAUSE_RX;
1623         if (epause->tx_pause)
1624                 lc->requested_fc |= PAUSE_TX;
1625         if (netif_running(dev))
1626                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1627                                      lc);
1628         return 0;
1629 }
1630
1631 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1632 {
1633         const struct port_info *pi = netdev_priv(dev);
1634         const struct sge *s = &pi->adapter->sge;
1635
1636         e->rx_max_pending = MAX_RX_BUFFERS;
1637         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1638         e->rx_jumbo_max_pending = 0;
1639         e->tx_max_pending = MAX_TXQ_ENTRIES;
1640
1641         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1642         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1643         e->rx_jumbo_pending = 0;
1644         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1645 }
1646
1647 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1648 {
1649         int i;
1650         const struct port_info *pi = netdev_priv(dev);
1651         struct adapter *adapter = pi->adapter;
1652         struct sge *s = &adapter->sge;
1653
1654         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1655             e->tx_pending > MAX_TXQ_ENTRIES ||
1656             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1657             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1658             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1659                 return -EINVAL;
1660
1661         if (adapter->flags & FULL_INIT_DONE)
1662                 return -EBUSY;
1663
1664         for (i = 0; i < pi->nqsets; ++i) {
1665                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1666                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1667                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1668         }
1669         return 0;
1670 }
1671
1672 static int closest_timer(const struct sge *s, int time)
1673 {
1674         int i, delta, match = 0, min_delta = INT_MAX;
1675
1676         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1677                 delta = time - s->timer_val[i];
1678                 if (delta < 0)
1679                         delta = -delta;
1680                 if (delta < min_delta) {
1681                         min_delta = delta;
1682                         match = i;
1683                 }
1684         }
1685         return match;
1686 }
1687
1688 static int closest_thres(const struct sge *s, int thres)
1689 {
1690         int i, delta, match = 0, min_delta = INT_MAX;
1691
1692         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1693                 delta = thres - s->counter_val[i];
1694                 if (delta < 0)
1695                         delta = -delta;
1696                 if (delta < min_delta) {
1697                         min_delta = delta;
1698                         match = i;
1699                 }
1700         }
1701         return match;
1702 }
1703
1704 /*
1705  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1706  */
1707 static unsigned int qtimer_val(const struct adapter *adap,
1708                                const struct sge_rspq *q)
1709 {
1710         unsigned int idx = q->intr_params >> 1;
1711
1712         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1713 }
1714
1715 /**
1716  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1717  *      @adap: the adapter
1718  *      @q: the Rx queue
1719  *      @us: the hold-off time in us, or 0 to disable timer
1720  *      @cnt: the hold-off packet count, or 0 to disable counter
1721  *
1722  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1723  *      one of the two needs to be enabled for the queue to generate interrupts.
1724  */
1725 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1726                                unsigned int us, unsigned int cnt)
1727 {
1728         if ((us | cnt) == 0)
1729                 cnt = 1;
1730
1731         if (cnt) {
1732                 int err;
1733                 u32 v, new_idx;
1734
1735                 new_idx = closest_thres(&adap->sge, cnt);
1736                 if (q->desc && q->pktcnt_idx != new_idx) {
1737                         /* the queue has already been created, update it */
1738                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1739                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1740                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1741                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1742                                             &new_idx);
1743                         if (err)
1744                                 return err;
1745                 }
1746                 q->pktcnt_idx = new_idx;
1747         }
1748
1749         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1750         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1751         return 0;
1752 }
1753
1754 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1755 {
1756         const struct port_info *pi = netdev_priv(dev);
1757         struct adapter *adap = pi->adapter;
1758
1759         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1760                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1761 }
1762
1763 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1764 {
1765         const struct port_info *pi = netdev_priv(dev);
1766         const struct adapter *adap = pi->adapter;
1767         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1768
1769         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1770         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1771                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1772         return 0;
1773 }
1774
1775 /**
1776  *      eeprom_ptov - translate a physical EEPROM address to virtual
1777  *      @phys_addr: the physical EEPROM address
1778  *      @fn: the PCI function number
1779  *      @sz: size of function-specific area
1780  *
1781  *      Translate a physical EEPROM address to virtual.  The first 1K is
1782  *      accessed through virtual addresses starting at 31K, the rest is
1783  *      accessed through virtual addresses starting at 0.
1784  *
1785  *      The mapping is as follows:
1786  *      [0..1K) -> [31K..32K)
1787  *      [1K..1K+A) -> [31K-A..31K)
1788  *      [1K+A..ES) -> [0..ES-A-1K)
1789  *
1790  *      where A = @fn * @sz, and ES = EEPROM size.
1791  */
1792 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1793 {
1794         fn *= sz;
1795         if (phys_addr < 1024)
1796                 return phys_addr + (31 << 10);
1797         if (phys_addr < 1024 + fn)
1798                 return 31744 - fn + phys_addr - 1024;
1799         if (phys_addr < EEPROMSIZE)
1800                 return phys_addr - 1024 - fn;
1801         return -EINVAL;
1802 }
1803
1804 /*
1805  * The next two routines implement eeprom read/write from physical addresses.
1806  */
1807 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1808 {
1809         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1810
1811         if (vaddr >= 0)
1812                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1813         return vaddr < 0 ? vaddr : 0;
1814 }
1815
1816 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1817 {
1818         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1819
1820         if (vaddr >= 0)
1821                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1822         return vaddr < 0 ? vaddr : 0;
1823 }
1824
1825 #define EEPROM_MAGIC 0x38E2F10C
1826
1827 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1828                       u8 *data)
1829 {
1830         int i, err = 0;
1831         struct adapter *adapter = netdev2adap(dev);
1832
1833         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1834         if (!buf)
1835                 return -ENOMEM;
1836
1837         e->magic = EEPROM_MAGIC;
1838         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1839                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1840
1841         if (!err)
1842                 memcpy(data, buf + e->offset, e->len);
1843         kfree(buf);
1844         return err;
1845 }
1846
1847 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1848                       u8 *data)
1849 {
1850         u8 *buf;
1851         int err = 0;
1852         u32 aligned_offset, aligned_len, *p;
1853         struct adapter *adapter = netdev2adap(dev);
1854
1855         if (eeprom->magic != EEPROM_MAGIC)
1856                 return -EINVAL;
1857
1858         aligned_offset = eeprom->offset & ~3;
1859         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1860
1861         if (adapter->fn > 0) {
1862                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1863
1864                 if (aligned_offset < start ||
1865                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
1866                         return -EPERM;
1867         }
1868
1869         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1870                 /*
1871                  * RMW possibly needed for first or last words.
1872                  */
1873                 buf = kmalloc(aligned_len, GFP_KERNEL);
1874                 if (!buf)
1875                         return -ENOMEM;
1876                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1877                 if (!err && aligned_len > 4)
1878                         err = eeprom_rd_phys(adapter,
1879                                              aligned_offset + aligned_len - 4,
1880                                              (u32 *)&buf[aligned_len - 4]);
1881                 if (err)
1882                         goto out;
1883                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1884         } else
1885                 buf = data;
1886
1887         err = t4_seeprom_wp(adapter, false);
1888         if (err)
1889                 goto out;
1890
1891         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1892                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1893                 aligned_offset += 4;
1894         }
1895
1896         if (!err)
1897                 err = t4_seeprom_wp(adapter, true);
1898 out:
1899         if (buf != data)
1900                 kfree(buf);
1901         return err;
1902 }
1903
1904 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1905 {
1906         int ret;
1907         const struct firmware *fw;
1908         struct adapter *adap = netdev2adap(netdev);
1909
1910         ef->data[sizeof(ef->data) - 1] = '\0';
1911         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1912         if (ret < 0)
1913                 return ret;
1914
1915         ret = t4_load_fw(adap, fw->data, fw->size);
1916         release_firmware(fw);
1917         if (!ret)
1918                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1919         return ret;
1920 }
1921
1922 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1923 #define BCAST_CRC 0xa0ccc1a6
1924
1925 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1926 {
1927         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1928         wol->wolopts = netdev2adap(dev)->wol;
1929         memset(&wol->sopass, 0, sizeof(wol->sopass));
1930 }
1931
1932 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1933 {
1934         int err = 0;
1935         struct port_info *pi = netdev_priv(dev);
1936
1937         if (wol->wolopts & ~WOL_SUPPORTED)
1938                 return -EINVAL;
1939         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1940                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1941         if (wol->wolopts & WAKE_BCAST) {
1942                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1943                                         ~0ULL, 0, false);
1944                 if (!err)
1945                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1946                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1947         } else
1948                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1949         return err;
1950 }
1951
1952 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
1953 {
1954         const struct port_info *pi = netdev_priv(dev);
1955         netdev_features_t changed = dev->features ^ features;
1956         int err;
1957
1958         if (!(changed & NETIF_F_HW_VLAN_RX))
1959                 return 0;
1960
1961         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1962                             -1, -1, -1,
1963                             !!(features & NETIF_F_HW_VLAN_RX), true);
1964         if (unlikely(err))
1965                 dev->features = features ^ NETIF_F_HW_VLAN_RX;
1966         return err;
1967 }
1968
1969 static u32 get_rss_table_size(struct net_device *dev)
1970 {
1971         const struct port_info *pi = netdev_priv(dev);
1972
1973         return pi->rss_size;
1974 }
1975
1976 static int get_rss_table(struct net_device *dev, u32 *p)
1977 {
1978         const struct port_info *pi = netdev_priv(dev);
1979         unsigned int n = pi->rss_size;
1980
1981         while (n--)
1982                 p[n] = pi->rss[n];
1983         return 0;
1984 }
1985
1986 static int set_rss_table(struct net_device *dev, const u32 *p)
1987 {
1988         unsigned int i;
1989         struct port_info *pi = netdev_priv(dev);
1990
1991         for (i = 0; i < pi->rss_size; i++)
1992                 pi->rss[i] = p[i];
1993         if (pi->adapter->flags & FULL_INIT_DONE)
1994                 return write_rss(pi, pi->rss);
1995         return 0;
1996 }
1997
1998 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1999                      u32 *rules)
2000 {
2001         const struct port_info *pi = netdev_priv(dev);
2002
2003         switch (info->cmd) {
2004         case ETHTOOL_GRXFH: {
2005                 unsigned int v = pi->rss_mode;
2006
2007                 info->data = 0;
2008                 switch (info->flow_type) {
2009                 case TCP_V4_FLOW:
2010                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2011                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2012                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2013                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2014                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2015                         break;
2016                 case UDP_V4_FLOW:
2017                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2018                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2019                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2020                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2021                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2022                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2023                         break;
2024                 case SCTP_V4_FLOW:
2025                 case AH_ESP_V4_FLOW:
2026                 case IPV4_FLOW:
2027                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2028                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2029                         break;
2030                 case TCP_V6_FLOW:
2031                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2032                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2033                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2034                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2035                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2036                         break;
2037                 case UDP_V6_FLOW:
2038                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2039                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2040                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2041                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2042                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2043                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2044                         break;
2045                 case SCTP_V6_FLOW:
2046                 case AH_ESP_V6_FLOW:
2047                 case IPV6_FLOW:
2048                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2049                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2050                         break;
2051                 }
2052                 return 0;
2053         }
2054         case ETHTOOL_GRXRINGS:
2055                 info->data = pi->nqsets;
2056                 return 0;
2057         }
2058         return -EOPNOTSUPP;
2059 }
2060
2061 static const struct ethtool_ops cxgb_ethtool_ops = {
2062         .get_settings      = get_settings,
2063         .set_settings      = set_settings,
2064         .get_drvinfo       = get_drvinfo,
2065         .get_msglevel      = get_msglevel,
2066         .set_msglevel      = set_msglevel,
2067         .get_ringparam     = get_sge_param,
2068         .set_ringparam     = set_sge_param,
2069         .get_coalesce      = get_coalesce,
2070         .set_coalesce      = set_coalesce,
2071         .get_eeprom_len    = get_eeprom_len,
2072         .get_eeprom        = get_eeprom,
2073         .set_eeprom        = set_eeprom,
2074         .get_pauseparam    = get_pauseparam,
2075         .set_pauseparam    = set_pauseparam,
2076         .get_link          = ethtool_op_get_link,
2077         .get_strings       = get_strings,
2078         .set_phys_id       = identify_port,
2079         .nway_reset        = restart_autoneg,
2080         .get_sset_count    = get_sset_count,
2081         .get_ethtool_stats = get_stats,
2082         .get_regs_len      = get_regs_len,
2083         .get_regs          = get_regs,
2084         .get_wol           = get_wol,
2085         .set_wol           = set_wol,
2086         .get_rxnfc         = get_rxnfc,
2087         .get_rxfh_indir_size = get_rss_table_size,
2088         .get_rxfh_indir    = get_rss_table,
2089         .set_rxfh_indir    = set_rss_table,
2090         .flash_device      = set_flash,
2091 };
2092
2093 /*
2094  * debugfs support
2095  */
2096 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2097                         loff_t *ppos)
2098 {
2099         loff_t pos = *ppos;
2100         loff_t avail = file->f_path.dentry->d_inode->i_size;
2101         unsigned int mem = (uintptr_t)file->private_data & 3;
2102         struct adapter *adap = file->private_data - mem;
2103
2104         if (pos < 0)
2105                 return -EINVAL;
2106         if (pos >= avail)
2107                 return 0;
2108         if (count > avail - pos)
2109                 count = avail - pos;
2110
2111         while (count) {
2112                 size_t len;
2113                 int ret, ofst;
2114                 __be32 data[16];
2115
2116                 if (mem == MEM_MC)
2117                         ret = t4_mc_read(adap, pos, data, NULL);
2118                 else
2119                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2120                 if (ret)
2121                         return ret;
2122
2123                 ofst = pos % sizeof(data);
2124                 len = min(count, sizeof(data) - ofst);
2125                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2126                         return -EFAULT;
2127
2128                 buf += len;
2129                 pos += len;
2130                 count -= len;
2131         }
2132         count = pos - *ppos;
2133         *ppos = pos;
2134         return count;
2135 }
2136
2137 static const struct file_operations mem_debugfs_fops = {
2138         .owner   = THIS_MODULE,
2139         .open    = simple_open,
2140         .read    = mem_read,
2141         .llseek  = default_llseek,
2142 };
2143
2144 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2145                                       unsigned int idx, unsigned int size_mb)
2146 {
2147         struct dentry *de;
2148
2149         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2150                                  (void *)adap + idx, &mem_debugfs_fops);
2151         if (de && de->d_inode)
2152                 de->d_inode->i_size = size_mb << 20;
2153 }
2154
2155 static int __devinit setup_debugfs(struct adapter *adap)
2156 {
2157         int i;
2158
2159         if (IS_ERR_OR_NULL(adap->debugfs_root))
2160                 return -1;
2161
2162         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2163         if (i & EDRAM0_ENABLE)
2164                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2165         if (i & EDRAM1_ENABLE)
2166                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2167         if (i & EXT_MEM_ENABLE)
2168                 add_debugfs_mem(adap, "mc", MEM_MC,
2169                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2170         if (adap->l2t)
2171                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2172                                     &t4_l2t_fops);
2173         return 0;
2174 }
2175
2176 /*
2177  * upper-layer driver support
2178  */
2179
2180 /*
2181  * Allocate an active-open TID and set it to the supplied value.
2182  */
2183 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2184 {
2185         int atid = -1;
2186
2187         spin_lock_bh(&t->atid_lock);
2188         if (t->afree) {
2189                 union aopen_entry *p = t->afree;
2190
2191                 atid = p - t->atid_tab;
2192                 t->afree = p->next;
2193                 p->data = data;
2194                 t->atids_in_use++;
2195         }
2196         spin_unlock_bh(&t->atid_lock);
2197         return atid;
2198 }
2199 EXPORT_SYMBOL(cxgb4_alloc_atid);
2200
2201 /*
2202  * Release an active-open TID.
2203  */
2204 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2205 {
2206         union aopen_entry *p = &t->atid_tab[atid];
2207
2208         spin_lock_bh(&t->atid_lock);
2209         p->next = t->afree;
2210         t->afree = p;
2211         t->atids_in_use--;
2212         spin_unlock_bh(&t->atid_lock);
2213 }
2214 EXPORT_SYMBOL(cxgb4_free_atid);
2215
2216 /*
2217  * Allocate a server TID and set it to the supplied value.
2218  */
2219 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2220 {
2221         int stid;
2222
2223         spin_lock_bh(&t->stid_lock);
2224         if (family == PF_INET) {
2225                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2226                 if (stid < t->nstids)
2227                         __set_bit(stid, t->stid_bmap);
2228                 else
2229                         stid = -1;
2230         } else {
2231                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2232                 if (stid < 0)
2233                         stid = -1;
2234         }
2235         if (stid >= 0) {
2236                 t->stid_tab[stid].data = data;
2237                 stid += t->stid_base;
2238                 t->stids_in_use++;
2239         }
2240         spin_unlock_bh(&t->stid_lock);
2241         return stid;
2242 }
2243 EXPORT_SYMBOL(cxgb4_alloc_stid);
2244
2245 /*
2246  * Release a server TID.
2247  */
2248 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2249 {
2250         stid -= t->stid_base;
2251         spin_lock_bh(&t->stid_lock);
2252         if (family == PF_INET)
2253                 __clear_bit(stid, t->stid_bmap);
2254         else
2255                 bitmap_release_region(t->stid_bmap, stid, 2);
2256         t->stid_tab[stid].data = NULL;
2257         t->stids_in_use--;
2258         spin_unlock_bh(&t->stid_lock);
2259 }
2260 EXPORT_SYMBOL(cxgb4_free_stid);
2261
2262 /*
2263  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2264  */
2265 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2266                            unsigned int tid)
2267 {
2268         struct cpl_tid_release *req;
2269
2270         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2271         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2272         INIT_TP_WR(req, tid);
2273         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2274 }
2275
2276 /*
2277  * Queue a TID release request and if necessary schedule a work queue to
2278  * process it.
2279  */
2280 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2281                                     unsigned int tid)
2282 {
2283         void **p = &t->tid_tab[tid];
2284         struct adapter *adap = container_of(t, struct adapter, tids);
2285
2286         spin_lock_bh(&adap->tid_release_lock);
2287         *p = adap->tid_release_head;
2288         /* Low 2 bits encode the Tx channel number */
2289         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2290         if (!adap->tid_release_task_busy) {
2291                 adap->tid_release_task_busy = true;
2292                 queue_work(workq, &adap->tid_release_task);
2293         }
2294         spin_unlock_bh(&adap->tid_release_lock);
2295 }
2296
2297 /*
2298  * Process the list of pending TID release requests.
2299  */
2300 static void process_tid_release_list(struct work_struct *work)
2301 {
2302         struct sk_buff *skb;
2303         struct adapter *adap;
2304
2305         adap = container_of(work, struct adapter, tid_release_task);
2306
2307         spin_lock_bh(&adap->tid_release_lock);
2308         while (adap->tid_release_head) {
2309                 void **p = adap->tid_release_head;
2310                 unsigned int chan = (uintptr_t)p & 3;
2311                 p = (void *)p - chan;
2312
2313                 adap->tid_release_head = *p;
2314                 *p = NULL;
2315                 spin_unlock_bh(&adap->tid_release_lock);
2316
2317                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2318                                          GFP_KERNEL)))
2319                         schedule_timeout_uninterruptible(1);
2320
2321                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2322                 t4_ofld_send(adap, skb);
2323                 spin_lock_bh(&adap->tid_release_lock);
2324         }
2325         adap->tid_release_task_busy = false;
2326         spin_unlock_bh(&adap->tid_release_lock);
2327 }
2328
2329 /*
2330  * Release a TID and inform HW.  If we are unable to allocate the release
2331  * message we defer to a work queue.
2332  */
2333 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2334 {
2335         void *old;
2336         struct sk_buff *skb;
2337         struct adapter *adap = container_of(t, struct adapter, tids);
2338
2339         old = t->tid_tab[tid];
2340         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2341         if (likely(skb)) {
2342                 t->tid_tab[tid] = NULL;
2343                 mk_tid_release(skb, chan, tid);
2344                 t4_ofld_send(adap, skb);
2345         } else
2346                 cxgb4_queue_tid_release(t, chan, tid);
2347         if (old)
2348                 atomic_dec(&t->tids_in_use);
2349 }
2350 EXPORT_SYMBOL(cxgb4_remove_tid);
2351
2352 /*
2353  * Allocate and initialize the TID tables.  Returns 0 on success.
2354  */
2355 static int tid_init(struct tid_info *t)
2356 {
2357         size_t size;
2358         unsigned int natids = t->natids;
2359
2360         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2361                t->nstids * sizeof(*t->stid_tab) +
2362                BITS_TO_LONGS(t->nstids) * sizeof(long);
2363         t->tid_tab = t4_alloc_mem(size);
2364         if (!t->tid_tab)
2365                 return -ENOMEM;
2366
2367         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2368         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2369         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2370         spin_lock_init(&t->stid_lock);
2371         spin_lock_init(&t->atid_lock);
2372
2373         t->stids_in_use = 0;
2374         t->afree = NULL;
2375         t->atids_in_use = 0;
2376         atomic_set(&t->tids_in_use, 0);
2377
2378         /* Setup the free list for atid_tab and clear the stid bitmap. */
2379         if (natids) {
2380                 while (--natids)
2381                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2382                 t->afree = t->atid_tab;
2383         }
2384         bitmap_zero(t->stid_bmap, t->nstids);
2385         return 0;
2386 }
2387
2388 /**
2389  *      cxgb4_create_server - create an IP server
2390  *      @dev: the device
2391  *      @stid: the server TID
2392  *      @sip: local IP address to bind server to
2393  *      @sport: the server's TCP port
2394  *      @queue: queue to direct messages from this server to
2395  *
2396  *      Create an IP server for the given port and address.
2397  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2398  */
2399 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2400                         __be32 sip, __be16 sport, unsigned int queue)
2401 {
2402         unsigned int chan;
2403         struct sk_buff *skb;
2404         struct adapter *adap;
2405         struct cpl_pass_open_req *req;
2406
2407         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2408         if (!skb)
2409                 return -ENOMEM;
2410
2411         adap = netdev2adap(dev);
2412         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2413         INIT_TP_WR(req, 0);
2414         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2415         req->local_port = sport;
2416         req->peer_port = htons(0);
2417         req->local_ip = sip;
2418         req->peer_ip = htonl(0);
2419         chan = rxq_to_chan(&adap->sge, queue);
2420         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2421         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2422                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2423         return t4_mgmt_tx(adap, skb);
2424 }
2425 EXPORT_SYMBOL(cxgb4_create_server);
2426
2427 /**
2428  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2429  *      @mtus: the HW MTU table
2430  *      @mtu: the target MTU
2431  *      @idx: index of selected entry in the MTU table
2432  *
2433  *      Returns the index and the value in the HW MTU table that is closest to
2434  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2435  *      table, in which case that smallest available value is selected.
2436  */
2437 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2438                             unsigned int *idx)
2439 {
2440         unsigned int i = 0;
2441
2442         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2443                 ++i;
2444         if (idx)
2445                 *idx = i;
2446         return mtus[i];
2447 }
2448 EXPORT_SYMBOL(cxgb4_best_mtu);
2449
2450 /**
2451  *      cxgb4_port_chan - get the HW channel of a port
2452  *      @dev: the net device for the port
2453  *
2454  *      Return the HW Tx channel of the given port.
2455  */
2456 unsigned int cxgb4_port_chan(const struct net_device *dev)
2457 {
2458         return netdev2pinfo(dev)->tx_chan;
2459 }
2460 EXPORT_SYMBOL(cxgb4_port_chan);
2461
2462 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
2463 {
2464         struct adapter *adap = netdev2adap(dev);
2465         u32 v;
2466
2467         v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2468         return lpfifo ? G_LP_COUNT(v) : G_HP_COUNT(v);
2469 }
2470 EXPORT_SYMBOL(cxgb4_dbfifo_count);
2471
2472 /**
2473  *      cxgb4_port_viid - get the VI id of a port
2474  *      @dev: the net device for the port
2475  *
2476  *      Return the VI id of the given port.
2477  */
2478 unsigned int cxgb4_port_viid(const struct net_device *dev)
2479 {
2480         return netdev2pinfo(dev)->viid;
2481 }
2482 EXPORT_SYMBOL(cxgb4_port_viid);
2483
2484 /**
2485  *      cxgb4_port_idx - get the index of a port
2486  *      @dev: the net device for the port
2487  *
2488  *      Return the index of the given port.
2489  */
2490 unsigned int cxgb4_port_idx(const struct net_device *dev)
2491 {
2492         return netdev2pinfo(dev)->port_id;
2493 }
2494 EXPORT_SYMBOL(cxgb4_port_idx);
2495
2496 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2497                          struct tp_tcp_stats *v6)
2498 {
2499         struct adapter *adap = pci_get_drvdata(pdev);
2500
2501         spin_lock(&adap->stats_lock);
2502         t4_tp_get_tcp_stats(adap, v4, v6);
2503         spin_unlock(&adap->stats_lock);
2504 }
2505 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2506
2507 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2508                       const unsigned int *pgsz_order)
2509 {
2510         struct adapter *adap = netdev2adap(dev);
2511
2512         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2513         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2514                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2515                      HPZ3(pgsz_order[3]));
2516 }
2517 EXPORT_SYMBOL(cxgb4_iscsi_init);
2518
2519 int cxgb4_flush_eq_cache(struct net_device *dev)
2520 {
2521         struct adapter *adap = netdev2adap(dev);
2522         int ret;
2523
2524         ret = t4_fwaddrspace_write(adap, adap->mbox,
2525                                    0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
2526         return ret;
2527 }
2528 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
2529
2530 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
2531 {
2532         u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
2533         __be64 indices;
2534         int ret;
2535
2536         ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
2537         if (!ret) {
2538                 indices = be64_to_cpu(indices);
2539                 *cidx = (indices >> 25) & 0xffff;
2540                 *pidx = (indices >> 9) & 0xffff;
2541         }
2542         return ret;
2543 }
2544
2545 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
2546                         u16 size)
2547 {
2548         struct adapter *adap = netdev2adap(dev);
2549         u16 hw_pidx, hw_cidx;
2550         int ret;
2551
2552         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
2553         if (ret)
2554                 goto out;
2555
2556         if (pidx != hw_pidx) {
2557                 u16 delta;
2558
2559                 if (pidx >= hw_pidx)
2560                         delta = pidx - hw_pidx;
2561                 else
2562                         delta = size - hw_pidx + pidx;
2563                 wmb();
2564                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2565                              QID(qid) | PIDX(delta));
2566         }
2567 out:
2568         return ret;
2569 }
2570 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
2571
2572 static struct pci_driver cxgb4_driver;
2573
2574 static void check_neigh_update(struct neighbour *neigh)
2575 {
2576         const struct device *parent;
2577         const struct net_device *netdev = neigh->dev;
2578
2579         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2580                 netdev = vlan_dev_real_dev(netdev);
2581         parent = netdev->dev.parent;
2582         if (parent && parent->driver == &cxgb4_driver.driver)
2583                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2584 }
2585
2586 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2587                        void *data)
2588 {
2589         switch (event) {
2590         case NETEVENT_NEIGH_UPDATE:
2591                 check_neigh_update(data);
2592                 break;
2593         case NETEVENT_REDIRECT:
2594         default:
2595                 break;
2596         }
2597         return 0;
2598 }
2599
2600 static bool netevent_registered;
2601 static struct notifier_block cxgb4_netevent_nb = {
2602         .notifier_call = netevent_cb
2603 };
2604
2605 static void drain_db_fifo(struct adapter *adap, int usecs)
2606 {
2607         u32 v;
2608
2609         do {
2610                 set_current_state(TASK_UNINTERRUPTIBLE);
2611                 schedule_timeout(usecs_to_jiffies(usecs));
2612                 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2613                 if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0)
2614                         break;
2615         } while (1);
2616 }
2617
2618 static void disable_txq_db(struct sge_txq *q)
2619 {
2620         spin_lock_irq(&q->db_lock);
2621         q->db_disabled = 1;
2622         spin_unlock_irq(&q->db_lock);
2623 }
2624
2625 static void enable_txq_db(struct sge_txq *q)
2626 {
2627         spin_lock_irq(&q->db_lock);
2628         q->db_disabled = 0;
2629         spin_unlock_irq(&q->db_lock);
2630 }
2631
2632 static void disable_dbs(struct adapter *adap)
2633 {
2634         int i;
2635
2636         for_each_ethrxq(&adap->sge, i)
2637                 disable_txq_db(&adap->sge.ethtxq[i].q);
2638         for_each_ofldrxq(&adap->sge, i)
2639                 disable_txq_db(&adap->sge.ofldtxq[i].q);
2640         for_each_port(adap, i)
2641                 disable_txq_db(&adap->sge.ctrlq[i].q);
2642 }
2643
2644 static void enable_dbs(struct adapter *adap)
2645 {
2646         int i;
2647
2648         for_each_ethrxq(&adap->sge, i)
2649                 enable_txq_db(&adap->sge.ethtxq[i].q);
2650         for_each_ofldrxq(&adap->sge, i)
2651                 enable_txq_db(&adap->sge.ofldtxq[i].q);
2652         for_each_port(adap, i)
2653                 enable_txq_db(&adap->sge.ctrlq[i].q);
2654 }
2655
2656 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
2657 {
2658         u16 hw_pidx, hw_cidx;
2659         int ret;
2660
2661         spin_lock_bh(&q->db_lock);
2662         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
2663         if (ret)
2664                 goto out;
2665         if (q->db_pidx != hw_pidx) {
2666                 u16 delta;
2667
2668                 if (q->db_pidx >= hw_pidx)
2669                         delta = q->db_pidx - hw_pidx;
2670                 else
2671                         delta = q->size - hw_pidx + q->db_pidx;
2672                 wmb();
2673                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2674                              QID(q->cntxt_id) | PIDX(delta));
2675         }
2676 out:
2677         q->db_disabled = 0;
2678         spin_unlock_bh(&q->db_lock);
2679         if (ret)
2680                 CH_WARN(adap, "DB drop recovery failed.\n");
2681 }
2682 static void recover_all_queues(struct adapter *adap)
2683 {
2684         int i;
2685
2686         for_each_ethrxq(&adap->sge, i)
2687                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2688         for_each_ofldrxq(&adap->sge, i)
2689                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
2690         for_each_port(adap, i)
2691                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
2692 }
2693
2694 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
2695 {
2696         mutex_lock(&uld_mutex);
2697         if (adap->uld_handle[CXGB4_ULD_RDMA])
2698                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
2699                                 cmd);
2700         mutex_unlock(&uld_mutex);
2701 }
2702
2703 static void process_db_full(struct work_struct *work)
2704 {
2705         struct adapter *adap;
2706
2707         adap = container_of(work, struct adapter, db_full_task);
2708
2709         notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2710         drain_db_fifo(adap, dbfifo_drain_delay);
2711         t4_set_reg_field(adap, SGE_INT_ENABLE3,
2712                          DBFIFO_HP_INT | DBFIFO_LP_INT,
2713                          DBFIFO_HP_INT | DBFIFO_LP_INT);
2714         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2715 }
2716
2717 static void process_db_drop(struct work_struct *work)
2718 {
2719         struct adapter *adap;
2720
2721         adap = container_of(work, struct adapter, db_drop_task);
2722
2723         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
2724         disable_dbs(adap);
2725         notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
2726         drain_db_fifo(adap, 1);
2727         recover_all_queues(adap);
2728         enable_dbs(adap);
2729 }
2730
2731 void t4_db_full(struct adapter *adap)
2732 {
2733         t4_set_reg_field(adap, SGE_INT_ENABLE3,
2734                          DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
2735         queue_work(workq, &adap->db_full_task);
2736 }
2737
2738 void t4_db_dropped(struct adapter *adap)
2739 {
2740         queue_work(workq, &adap->db_drop_task);
2741 }
2742
2743 static void uld_attach(struct adapter *adap, unsigned int uld)
2744 {
2745         void *handle;
2746         struct cxgb4_lld_info lli;
2747
2748         lli.pdev = adap->pdev;
2749         lli.l2t = adap->l2t;
2750         lli.tids = &adap->tids;
2751         lli.ports = adap->port;
2752         lli.vr = &adap->vres;
2753         lli.mtus = adap->params.mtus;
2754         if (uld == CXGB4_ULD_RDMA) {
2755                 lli.rxq_ids = adap->sge.rdma_rxq;
2756                 lli.nrxq = adap->sge.rdmaqs;
2757         } else if (uld == CXGB4_ULD_ISCSI) {
2758                 lli.rxq_ids = adap->sge.ofld_rxq;
2759                 lli.nrxq = adap->sge.ofldqsets;
2760         }
2761         lli.ntxq = adap->sge.ofldqsets;
2762         lli.nchan = adap->params.nports;
2763         lli.nports = adap->params.nports;
2764         lli.wr_cred = adap->params.ofldq_wr_cred;
2765         lli.adapter_type = adap->params.rev;
2766         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2767         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2768                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2769                         (adap->fn * 4));
2770         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2771                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2772                         (adap->fn * 4));
2773         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2774         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2775         lli.fw_vers = adap->params.fw_vers;
2776         lli.dbfifo_int_thresh = dbfifo_int_thresh;
2777
2778         handle = ulds[uld].add(&lli);
2779         if (IS_ERR(handle)) {
2780                 dev_warn(adap->pdev_dev,
2781                          "could not attach to the %s driver, error %ld\n",
2782                          uld_str[uld], PTR_ERR(handle));
2783                 return;
2784         }
2785
2786         adap->uld_handle[uld] = handle;
2787
2788         if (!netevent_registered) {
2789                 register_netevent_notifier(&cxgb4_netevent_nb);
2790                 netevent_registered = true;
2791         }
2792
2793         if (adap->flags & FULL_INIT_DONE)
2794                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2795 }
2796
2797 static void attach_ulds(struct adapter *adap)
2798 {
2799         unsigned int i;
2800
2801         mutex_lock(&uld_mutex);
2802         list_add_tail(&adap->list_node, &adapter_list);
2803         for (i = 0; i < CXGB4_ULD_MAX; i++)
2804                 if (ulds[i].add)
2805                         uld_attach(adap, i);
2806         mutex_unlock(&uld_mutex);
2807 }
2808
2809 static void detach_ulds(struct adapter *adap)
2810 {
2811         unsigned int i;
2812
2813         mutex_lock(&uld_mutex);
2814         list_del(&adap->list_node);
2815         for (i = 0; i < CXGB4_ULD_MAX; i++)
2816                 if (adap->uld_handle[i]) {
2817                         ulds[i].state_change(adap->uld_handle[i],
2818                                              CXGB4_STATE_DETACH);
2819                         adap->uld_handle[i] = NULL;
2820                 }
2821         if (netevent_registered && list_empty(&adapter_list)) {
2822                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2823                 netevent_registered = false;
2824         }
2825         mutex_unlock(&uld_mutex);
2826 }
2827
2828 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2829 {
2830         unsigned int i;
2831
2832         mutex_lock(&uld_mutex);
2833         for (i = 0; i < CXGB4_ULD_MAX; i++)
2834                 if (adap->uld_handle[i])
2835                         ulds[i].state_change(adap->uld_handle[i], new_state);
2836         mutex_unlock(&uld_mutex);
2837 }
2838
2839 /**
2840  *      cxgb4_register_uld - register an upper-layer driver
2841  *      @type: the ULD type
2842  *      @p: the ULD methods
2843  *
2844  *      Registers an upper-layer driver with this driver and notifies the ULD
2845  *      about any presently available devices that support its type.  Returns
2846  *      %-EBUSY if a ULD of the same type is already registered.
2847  */
2848 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2849 {
2850         int ret = 0;
2851         struct adapter *adap;
2852
2853         if (type >= CXGB4_ULD_MAX)
2854                 return -EINVAL;
2855         mutex_lock(&uld_mutex);
2856         if (ulds[type].add) {
2857                 ret = -EBUSY;
2858                 goto out;
2859         }
2860         ulds[type] = *p;
2861         list_for_each_entry(adap, &adapter_list, list_node)
2862                 uld_attach(adap, type);
2863 out:    mutex_unlock(&uld_mutex);
2864         return ret;
2865 }
2866 EXPORT_SYMBOL(cxgb4_register_uld);
2867
2868 /**
2869  *      cxgb4_unregister_uld - unregister an upper-layer driver
2870  *      @type: the ULD type
2871  *
2872  *      Unregisters an existing upper-layer driver.
2873  */
2874 int cxgb4_unregister_uld(enum cxgb4_uld type)
2875 {
2876         struct adapter *adap;
2877
2878         if (type >= CXGB4_ULD_MAX)
2879                 return -EINVAL;
2880         mutex_lock(&uld_mutex);
2881         list_for_each_entry(adap, &adapter_list, list_node)
2882                 adap->uld_handle[type] = NULL;
2883         ulds[type].add = NULL;
2884         mutex_unlock(&uld_mutex);
2885         return 0;
2886 }
2887 EXPORT_SYMBOL(cxgb4_unregister_uld);
2888
2889 /**
2890  *      cxgb_up - enable the adapter
2891  *      @adap: adapter being enabled
2892  *
2893  *      Called when the first port is enabled, this function performs the
2894  *      actions necessary to make an adapter operational, such as completing
2895  *      the initialization of HW modules, and enabling interrupts.
2896  *
2897  *      Must be called with the rtnl lock held.
2898  */
2899 static int cxgb_up(struct adapter *adap)
2900 {
2901         int err;
2902
2903         err = setup_sge_queues(adap);
2904         if (err)
2905                 goto out;
2906         err = setup_rss(adap);
2907         if (err)
2908                 goto freeq;
2909
2910         if (adap->flags & USING_MSIX) {
2911                 name_msix_vecs(adap);
2912                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2913                                   adap->msix_info[0].desc, adap);
2914                 if (err)
2915                         goto irq_err;
2916
2917                 err = request_msix_queue_irqs(adap);
2918                 if (err) {
2919                         free_irq(adap->msix_info[0].vec, adap);
2920                         goto irq_err;
2921                 }
2922         } else {
2923                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2924                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2925                                   adap->port[0]->name, adap);
2926                 if (err)
2927                         goto irq_err;
2928         }
2929         enable_rx(adap);
2930         t4_sge_start(adap);
2931         t4_intr_enable(adap);
2932         adap->flags |= FULL_INIT_DONE;
2933         notify_ulds(adap, CXGB4_STATE_UP);
2934  out:
2935         return err;
2936  irq_err:
2937         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2938  freeq:
2939         t4_free_sge_resources(adap);
2940         goto out;
2941 }
2942
2943 static void cxgb_down(struct adapter *adapter)
2944 {
2945         t4_intr_disable(adapter);
2946         cancel_work_sync(&adapter->tid_release_task);
2947         cancel_work_sync(&adapter->db_full_task);
2948         cancel_work_sync(&adapter->db_drop_task);
2949         adapter->tid_release_task_busy = false;
2950         adapter->tid_release_head = NULL;
2951
2952         if (adapter->flags & USING_MSIX) {
2953                 free_msix_queue_irqs(adapter);
2954                 free_irq(adapter->msix_info[0].vec, adapter);
2955         } else
2956                 free_irq(adapter->pdev->irq, adapter);
2957         quiesce_rx(adapter);
2958         t4_sge_stop(adapter);
2959         t4_free_sge_resources(adapter);
2960         adapter->flags &= ~FULL_INIT_DONE;
2961 }
2962
2963 /*
2964  * net_device operations
2965  */
2966 static int cxgb_open(struct net_device *dev)
2967 {
2968         int err;
2969         struct port_info *pi = netdev_priv(dev);
2970         struct adapter *adapter = pi->adapter;
2971
2972         netif_carrier_off(dev);
2973
2974         if (!(adapter->flags & FULL_INIT_DONE)) {
2975                 err = cxgb_up(adapter);
2976                 if (err < 0)
2977                         return err;
2978         }
2979
2980         err = link_start(dev);
2981         if (!err)
2982                 netif_tx_start_all_queues(dev);
2983         return err;
2984 }
2985
2986 static int cxgb_close(struct net_device *dev)
2987 {
2988         struct port_info *pi = netdev_priv(dev);
2989         struct adapter *adapter = pi->adapter;
2990
2991         netif_tx_stop_all_queues(dev);
2992         netif_carrier_off(dev);
2993         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2994 }
2995
2996 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2997                                                 struct rtnl_link_stats64 *ns)
2998 {
2999         struct port_stats stats;
3000         struct port_info *p = netdev_priv(dev);
3001         struct adapter *adapter = p->adapter;
3002
3003         spin_lock(&adapter->stats_lock);
3004         t4_get_port_stats(adapter, p->tx_chan, &stats);
3005         spin_unlock(&adapter->stats_lock);
3006
3007         ns->tx_bytes   = stats.tx_octets;
3008         ns->tx_packets = stats.tx_frames;
3009         ns->rx_bytes   = stats.rx_octets;
3010         ns->rx_packets = stats.rx_frames;
3011         ns->multicast  = stats.rx_mcast_frames;
3012
3013         /* detailed rx_errors */
3014         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
3015                                stats.rx_runt;
3016         ns->rx_over_errors   = 0;
3017         ns->rx_crc_errors    = stats.rx_fcs_err;
3018         ns->rx_frame_errors  = stats.rx_symbol_err;
3019         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
3020                                stats.rx_ovflow2 + stats.rx_ovflow3 +
3021                                stats.rx_trunc0 + stats.rx_trunc1 +
3022                                stats.rx_trunc2 + stats.rx_trunc3;
3023         ns->rx_missed_errors = 0;
3024
3025         /* detailed tx_errors */
3026         ns->tx_aborted_errors   = 0;
3027         ns->tx_carrier_errors   = 0;
3028         ns->tx_fifo_errors      = 0;
3029         ns->tx_heartbeat_errors = 0;
3030         ns->tx_window_errors    = 0;
3031
3032         ns->tx_errors = stats.tx_error_frames;
3033         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
3034                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
3035         return ns;
3036 }
3037
3038 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
3039 {
3040         unsigned int mbox;
3041         int ret = 0, prtad, devad;
3042         struct port_info *pi = netdev_priv(dev);
3043         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
3044
3045         switch (cmd) {
3046         case SIOCGMIIPHY:
3047                 if (pi->mdio_addr < 0)
3048                         return -EOPNOTSUPP;
3049                 data->phy_id = pi->mdio_addr;
3050                 break;
3051         case SIOCGMIIREG:
3052         case SIOCSMIIREG:
3053                 if (mdio_phy_id_is_c45(data->phy_id)) {
3054                         prtad = mdio_phy_id_prtad(data->phy_id);
3055                         devad = mdio_phy_id_devad(data->phy_id);
3056                 } else if (data->phy_id < 32) {
3057                         prtad = data->phy_id;
3058                         devad = 0;
3059                         data->reg_num &= 0x1f;
3060                 } else
3061                         return -EINVAL;
3062
3063                 mbox = pi->adapter->fn;
3064                 if (cmd == SIOCGMIIREG)
3065                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
3066                                          data->reg_num, &data->val_out);
3067                 else
3068                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
3069                                          data->reg_num, data->val_in);
3070                 break;
3071         default:
3072                 return -EOPNOTSUPP;
3073         }
3074         return ret;
3075 }
3076
3077 static void cxgb_set_rxmode(struct net_device *dev)
3078 {
3079         /* unfortunately we can't return errors to the stack */
3080         set_rxmode(dev, -1, false);
3081 }
3082
3083 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
3084 {
3085         int ret;
3086         struct port_info *pi = netdev_priv(dev);
3087
3088         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
3089                 return -EINVAL;
3090         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
3091                             -1, -1, -1, true);
3092         if (!ret)
3093                 dev->mtu = new_mtu;
3094         return ret;
3095 }
3096
3097 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
3098 {
3099         int ret;
3100         struct sockaddr *addr = p;
3101         struct port_info *pi = netdev_priv(dev);
3102
3103         if (!is_valid_ether_addr(addr->sa_data))
3104                 return -EADDRNOTAVAIL;
3105
3106         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
3107                             pi->xact_addr_filt, addr->sa_data, true, true);
3108         if (ret < 0)
3109                 return ret;
3110
3111         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3112         pi->xact_addr_filt = ret;
3113         return 0;
3114 }
3115
3116 #ifdef CONFIG_NET_POLL_CONTROLLER
3117 static void cxgb_netpoll(struct net_device *dev)
3118 {
3119         struct port_info *pi = netdev_priv(dev);
3120         struct adapter *adap = pi->adapter;
3121
3122         if (adap->flags & USING_MSIX) {
3123                 int i;
3124                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
3125
3126                 for (i = pi->nqsets; i; i--, rx++)
3127                         t4_sge_intr_msix(0, &rx->rspq);
3128         } else
3129                 t4_intr_handler(adap)(0, adap);
3130 }
3131 #endif
3132
3133 static const struct net_device_ops cxgb4_netdev_ops = {
3134         .ndo_open             = cxgb_open,
3135         .ndo_stop             = cxgb_close,
3136         .ndo_start_xmit       = t4_eth_xmit,
3137         .ndo_get_stats64      = cxgb_get_stats,
3138         .ndo_set_rx_mode      = cxgb_set_rxmode,
3139         .ndo_set_mac_address  = cxgb_set_mac_addr,
3140         .ndo_set_features     = cxgb_set_features,
3141         .ndo_validate_addr    = eth_validate_addr,
3142         .ndo_do_ioctl         = cxgb_ioctl,
3143         .ndo_change_mtu       = cxgb_change_mtu,
3144 #ifdef CONFIG_NET_POLL_CONTROLLER
3145         .ndo_poll_controller  = cxgb_netpoll,
3146 #endif
3147 };
3148
3149 void t4_fatal_err(struct adapter *adap)
3150 {
3151         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
3152         t4_intr_disable(adap);
3153         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
3154 }
3155
3156 static void setup_memwin(struct adapter *adap)
3157 {
3158         u32 bar0;
3159
3160         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
3161         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
3162                      (bar0 + MEMWIN0_BASE) | BIR(0) |
3163                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
3164         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
3165                      (bar0 + MEMWIN1_BASE) | BIR(0) |
3166                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
3167         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
3168                      (bar0 + MEMWIN2_BASE) | BIR(0) |
3169                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
3170 }
3171
3172 static void setup_memwin_rdma(struct adapter *adap)
3173 {
3174         if (adap->vres.ocq.size) {
3175                 unsigned int start, sz_kb;
3176
3177                 start = pci_resource_start(adap->pdev, 2) +
3178                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
3179                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
3180                 t4_write_reg(adap,
3181                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
3182                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
3183                 t4_write_reg(adap,
3184                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
3185                              adap->vres.ocq.start);
3186                 t4_read_reg(adap,
3187                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
3188         }
3189 }
3190
3191 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
3192 {
3193         u32 v;
3194         int ret;
3195
3196         /* get device capabilities */
3197         memset(c, 0, sizeof(*c));
3198         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3199                                FW_CMD_REQUEST | FW_CMD_READ);
3200         c->retval_len16 = htonl(FW_LEN16(*c));
3201         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
3202         if (ret < 0)
3203                 return ret;
3204
3205         /* select capabilities we'll be using */
3206         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3207                 if (!vf_acls)
3208                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3209                 else
3210                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3211         } else if (vf_acls) {
3212                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
3213                 return ret;
3214         }
3215         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3216                                FW_CMD_REQUEST | FW_CMD_WRITE);
3217         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
3218         if (ret < 0)
3219                 return ret;
3220
3221         ret = t4_config_glbl_rss(adap, adap->fn,
3222                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3223                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3224                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
3225         if (ret < 0)
3226                 return ret;
3227
3228         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
3229                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
3230         if (ret < 0)
3231                 return ret;
3232
3233         t4_sge_init(adap);
3234
3235         /* tweak some settings */
3236         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
3237         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
3238         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
3239         v = t4_read_reg(adap, TP_PIO_DATA);
3240         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
3241
3242         /* get basic stuff going */
3243         return t4_early_init(adap, adap->fn);
3244 }
3245
3246 /*
3247  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
3248  */
3249 #define MAX_ATIDS 8192U
3250
3251 /*
3252  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3253  *
3254  * If the firmware we're dealing with has Configuration File support, then
3255  * we use that to perform all configuration
3256  */
3257
3258 /*
3259  * Tweak configuration based on module parameters, etc.  Most of these have
3260  * defaults assigned to them by Firmware Configuration Files (if we're using
3261  * them) but need to be explicitly set if we're using hard-coded
3262  * initialization.  But even in the case of using Firmware Configuration
3263  * Files, we'd like to expose the ability to change these via module
3264  * parameters so these are essentially common tweaks/settings for
3265  * Configuration Files and hard-coded initialization ...
3266  */
3267 static int adap_init0_tweaks(struct adapter *adapter)
3268 {
3269         /*
3270          * Fix up various Host-Dependent Parameters like Page Size, Cache
3271          * Line Size, etc.  The firmware default is for a 4KB Page Size and
3272          * 64B Cache Line Size ...
3273          */
3274         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
3275
3276         /*
3277          * Process module parameters which affect early initialization.
3278          */
3279         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
3280                 dev_err(&adapter->pdev->dev,
3281                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
3282                         rx_dma_offset);
3283                 rx_dma_offset = 2;
3284         }
3285         t4_set_reg_field(adapter, SGE_CONTROL,
3286                          PKTSHIFT_MASK,
3287                          PKTSHIFT(rx_dma_offset));
3288
3289         /*
3290          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
3291          * adds the pseudo header itself.
3292          */
3293         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
3294                                CSUM_HAS_PSEUDO_HDR, 0);
3295
3296         return 0;
3297 }
3298
3299 /*
3300  * Attempt to initialize the adapter via a Firmware Configuration File.
3301  */
3302 static int adap_init0_config(struct adapter *adapter, int reset)
3303 {
3304         struct fw_caps_config_cmd caps_cmd;
3305         const struct firmware *cf;
3306         unsigned long mtype = 0, maddr = 0;
3307         u32 finiver, finicsum, cfcsum;
3308         int ret, using_flash;
3309
3310         /*
3311          * Reset device if necessary.
3312          */
3313         if (reset) {
3314                 ret = t4_fw_reset(adapter, adapter->mbox,
3315                                   PIORSTMODE | PIORST);
3316                 if (ret < 0)
3317                         goto bye;
3318         }
3319
3320         /*
3321          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
3322          * then use that.  Otherwise, use the configuration file stored
3323          * in the adapter flash ...
3324          */
3325         ret = request_firmware(&cf, FW_CFNAME, adapter->pdev_dev);
3326         if (ret < 0) {
3327                 using_flash = 1;
3328                 mtype = FW_MEMTYPE_CF_FLASH;
3329                 maddr = t4_flash_cfg_addr(adapter);
3330         } else {
3331                 u32 params[7], val[7];
3332
3333                 using_flash = 0;
3334                 if (cf->size >= FLASH_CFG_MAX_SIZE)
3335                         ret = -ENOMEM;
3336                 else {
3337                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3338                              FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
3339                         ret = t4_query_params(adapter, adapter->mbox,
3340                                               adapter->fn, 0, 1, params, val);
3341                         if (ret == 0) {
3342                                 /*
3343                                  * For t4_memory_write() below addresses and
3344                                  * sizes have to be in terms of multiples of 4
3345                                  * bytes.  So, if the Configuration File isn't
3346                                  * a multiple of 4 bytes in length we'll have
3347                                  * to write that out separately since we can't
3348                                  * guarantee that the bytes following the
3349                                  * residual byte in the buffer returned by
3350                                  * request_firmware() are zeroed out ...
3351                                  */
3352                                 size_t resid = cf->size & 0x3;
3353                                 size_t size = cf->size & ~0x3;
3354                                 __be32 *data = (__be32 *)cf->data;
3355
3356                                 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
3357                                 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
3358
3359                                 ret = t4_memory_write(adapter, mtype, maddr,
3360                                                       size, data);
3361                                 if (ret == 0 && resid != 0) {
3362                                         union {
3363                                                 __be32 word;
3364                                                 char buf[4];
3365                                         } last;
3366                                         int i;
3367
3368                                         last.word = data[size >> 2];
3369                                         for (i = resid; i < 4; i++)
3370                                                 last.buf[i] = 0;
3371                                         ret = t4_memory_write(adapter, mtype,
3372                                                               maddr + size,
3373                                                               4, &last.word);
3374                                 }
3375                         }
3376                 }
3377
3378                 release_firmware(cf);
3379                 if (ret)
3380                         goto bye;
3381         }
3382
3383         /*
3384          * Issue a Capability Configuration command to the firmware to get it
3385          * to parse the Configuration File.  We don't use t4_fw_config_file()
3386          * because we want the ability to modify various features after we've
3387          * processed the configuration file ...
3388          */
3389         memset(&caps_cmd, 0, sizeof(caps_cmd));
3390         caps_cmd.op_to_write =
3391                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3392                       FW_CMD_REQUEST |
3393                       FW_CMD_READ);
3394         caps_cmd.retval_len16 =
3395                 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
3396                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3397                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
3398                       FW_LEN16(caps_cmd));
3399         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3400                          &caps_cmd);
3401         if (ret < 0)
3402                 goto bye;
3403
3404         finiver = ntohl(caps_cmd.finiver);
3405         finicsum = ntohl(caps_cmd.finicsum);
3406         cfcsum = ntohl(caps_cmd.cfcsum);
3407         if (finicsum != cfcsum)
3408                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
3409                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
3410                          finicsum, cfcsum);
3411
3412         /*
3413          * If we're a pure NIC driver then disable all offloading facilities.
3414          * This will allow the firmware to optimize aspects of the hardware
3415          * configuration which will result in improved performance.
3416          */
3417         caps_cmd.ofldcaps = 0;
3418         caps_cmd.iscsicaps = 0;
3419         caps_cmd.rdmacaps = 0;
3420         caps_cmd.fcoecaps = 0;
3421
3422         /*
3423          * And now tell the firmware to use the configuration we just loaded.
3424          */
3425         caps_cmd.op_to_write =
3426                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3427                       FW_CMD_REQUEST |
3428                       FW_CMD_WRITE);
3429         caps_cmd.retval_len16 = htonl(FW_LEN16(caps_cmd));
3430         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3431                          NULL);
3432         if (ret < 0)
3433                 goto bye;
3434
3435         /*
3436          * Tweak configuration based on system architecture, module
3437          * parameters, etc.
3438          */
3439         ret = adap_init0_tweaks(adapter);
3440         if (ret < 0)
3441                 goto bye;
3442
3443         /*
3444          * And finally tell the firmware to initialize itself using the
3445          * parameters from the Configuration File.
3446          */
3447         ret = t4_fw_initialize(adapter, adapter->mbox);
3448         if (ret < 0)
3449                 goto bye;
3450
3451         /*
3452          * Return successfully and note that we're operating with parameters
3453          * not supplied by the driver, rather than from hard-wired
3454          * initialization constants burried in the driver.
3455          */
3456         adapter->flags |= USING_SOFT_PARAMS;
3457         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
3458                  "Configuration File %s, version %#x, computed checksum %#x\n",
3459                  (using_flash
3460                   ? "in device FLASH"
3461                   : "/lib/firmware/" FW_CFNAME),
3462                  finiver, cfcsum);
3463         return 0;
3464
3465         /*
3466          * Something bad happened.  Return the error ...  (If the "error"
3467          * is that there's no Configuration File on the adapter we don't
3468          * want to issue a warning since this is fairly common.)
3469          */
3470 bye:
3471         if (ret != -ENOENT)
3472                 dev_warn(adapter->pdev_dev, "Configuration file error %d\n",
3473                          -ret);
3474         return ret;
3475 }
3476
3477 /*
3478  * Attempt to initialize the adapter via hard-coded, driver supplied
3479  * parameters ...
3480  */
3481 static int adap_init0_no_config(struct adapter *adapter, int reset)
3482 {
3483         struct sge *s = &adapter->sge;
3484         struct fw_caps_config_cmd caps_cmd;
3485         u32 v;
3486         int i, ret;
3487
3488         /*
3489          * Reset device if necessary
3490          */
3491         if (reset) {
3492                 ret = t4_fw_reset(adapter, adapter->mbox,
3493                                   PIORSTMODE | PIORST);
3494                 if (ret < 0)
3495                         goto bye;
3496         }
3497
3498         /*
3499          * Get device capabilities and select which we'll be using.
3500          */
3501         memset(&caps_cmd, 0, sizeof(caps_cmd));
3502         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3503                                      FW_CMD_REQUEST | FW_CMD_READ);
3504         caps_cmd.retval_len16 = htonl(FW_LEN16(caps_cmd));
3505         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3506                          &caps_cmd);
3507         if (ret < 0)
3508                 goto bye;
3509
3510 #ifndef CONFIG_CHELSIO_T4_OFFLOAD
3511         /*
3512          * If we're a pure NIC driver then disable all offloading facilities.
3513          * This will allow the firmware to optimize aspects of the hardware
3514          * configuration which will result in improved performance.
3515          */
3516         caps_cmd.ofldcaps = 0;
3517         caps_cmd.iscsicaps = 0;
3518         caps_cmd.rdmacaps = 0;
3519         caps_cmd.fcoecaps = 0;
3520 #endif
3521
3522         if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3523                 if (!vf_acls)
3524                         caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3525                 else
3526                         caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3527         } else if (vf_acls) {
3528                 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
3529                 goto bye;
3530         }
3531         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3532                               FW_CMD_REQUEST | FW_CMD_WRITE);
3533         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3534                          NULL);
3535         if (ret < 0)
3536                 goto bye;
3537
3538         /*
3539          * Tweak configuration based on system architecture, module
3540          * parameters, etc.
3541          */
3542         ret = adap_init0_tweaks(adapter);
3543         if (ret < 0)
3544                 goto bye;
3545
3546         /*
3547          * Select RSS Global Mode we want to use.  We use "Basic Virtual"
3548          * mode which maps each Virtual Interface to its own section of
3549          * the RSS Table and we turn on all map and hash enables ...
3550          */
3551         adapter->flags |= RSS_TNLALLLOOKUP;
3552         ret = t4_config_glbl_rss(adapter, adapter->mbox,
3553                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3554                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3555                                  FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
3556                                  ((adapter->flags & RSS_TNLALLLOOKUP) ?
3557                                         FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
3558         if (ret < 0)
3559                 goto bye;
3560
3561         /*
3562          * Set up our own fundamental resource provisioning ...
3563          */
3564         ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
3565                           PFRES_NEQ, PFRES_NETHCTRL,
3566                           PFRES_NIQFLINT, PFRES_NIQ,
3567                           PFRES_TC, PFRES_NVI,
3568                           FW_PFVF_CMD_CMASK_MASK,
3569                           pfvfres_pmask(adapter, adapter->fn, 0),
3570                           PFRES_NEXACTF,
3571                           PFRES_R_CAPS, PFRES_WX_CAPS);
3572         if (ret < 0)
3573                 goto bye;
3574
3575         /*
3576          * Perform low level SGE initialization.  We need to do this before we
3577          * send the firmware the INITIALIZE command because that will cause
3578          * any other PF Drivers which are waiting for the Master
3579          * Initialization to proceed forward.
3580          */
3581         for (i = 0; i < SGE_NTIMERS - 1; i++)
3582                 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
3583         s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3584         s->counter_val[0] = 1;
3585         for (i = 1; i < SGE_NCOUNTERS; i++)
3586                 s->counter_val[i] = min(intr_cnt[i - 1],
3587                                         THRESHOLD_0_GET(THRESHOLD_0_MASK));
3588         t4_sge_init(adapter);
3589
3590 #ifdef CONFIG_PCI_IOV
3591         /*
3592          * Provision resource limits for Virtual Functions.  We currently
3593          * grant them all the same static resource limits except for the Port
3594          * Access Rights Mask which we're assigning based on the PF.  All of
3595          * the static provisioning stuff for both the PF and VF really needs
3596          * to be managed in a persistent manner for each device which the
3597          * firmware controls.
3598          */
3599         {
3600                 int pf, vf;
3601
3602                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3603                         if (num_vf[pf] <= 0)
3604                                 continue;
3605
3606                         /* VF numbering starts at 1! */
3607                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3608                                 ret = t4_cfg_pfvf(adapter, adapter->mbox,
3609                                                   pf, vf,
3610                                                   VFRES_NEQ, VFRES_NETHCTRL,
3611                                                   VFRES_NIQFLINT, VFRES_NIQ,
3612                                                   VFRES_TC, VFRES_NVI,
3613                                                   FW_PFVF_CMD_CMASK_GET(
3614                                                   FW_PFVF_CMD_CMASK_MASK),
3615                                                   pfvfres_pmask(
3616                                                   adapter, pf, vf),
3617                                                   VFRES_NEXACTF,
3618                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3619                                 if (ret < 0)
3620                                         dev_warn(adapter->pdev_dev,
3621                                                  "failed to "\
3622                                                  "provision pf/vf=%d/%d; "
3623                                                  "err=%d\n", pf, vf, ret);
3624                         }
3625                 }
3626         }
3627 #endif
3628
3629         /*
3630          * Set up the default filter mode.  Later we'll want to implement this
3631          * via a firmware command, etc. ...  This needs to be done before the
3632          * firmare initialization command ...  If the selected set of fields
3633          * isn't equal to the default value, we'll need to make sure that the
3634          * field selections will fit in the 36-bit budget.
3635          */
3636         if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
3637                 int i, bits = 0;
3638
3639                 for (i = TP_VLAN_PRI_MAP_FIRST; i <= TP_VLAN_PRI_MAP_LAST; i++)
3640                         switch (tp_vlan_pri_map & (1 << i)) {
3641                         case 0:
3642                                 /* compressed filter field not enabled */
3643                                 break;
3644                         case FCOE_MASK:
3645                                 bits +=  1;
3646                                 break;
3647                         case PORT_MASK:
3648                                 bits +=  3;
3649                                 break;
3650                         case VNIC_ID_MASK:
3651                                 bits += 17;
3652                                 break;
3653                         case VLAN_MASK:
3654                                 bits += 17;
3655                                 break;
3656                         case TOS_MASK:
3657                                 bits +=  8;
3658                                 break;
3659                         case PROTOCOL_MASK:
3660                                 bits +=  8;
3661                                 break;
3662                         case ETHERTYPE_MASK:
3663                                 bits += 16;
3664                                 break;
3665                         case MACMATCH_MASK:
3666                                 bits +=  9;
3667                                 break;
3668                         case MPSHITTYPE_MASK:
3669                                 bits +=  3;
3670                                 break;
3671                         case FRAGMENTATION_MASK:
3672                                 bits +=  1;
3673                                 break;
3674                         }
3675
3676                 if (bits > 36) {
3677                         dev_err(adapter->pdev_dev,
3678                                 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
3679                                 " using %#x\n", tp_vlan_pri_map, bits,
3680                                 TP_VLAN_PRI_MAP_DEFAULT);
3681                         tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
3682                 }
3683         }
3684         v = tp_vlan_pri_map;
3685         t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
3686                           &v, 1, TP_VLAN_PRI_MAP);
3687
3688         /*
3689          * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
3690          * to support any of the compressed filter fields above.  Newer
3691          * versions of the firmware do this automatically but it doesn't hurt
3692          * to set it here.  Meanwhile, we do _not_ need to set Lookup Every
3693          * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
3694          * since the firmware automatically turns this on and off when we have
3695          * a non-zero number of filters active (since it does have a
3696          * performance impact).
3697          */
3698         if (tp_vlan_pri_map)
3699                 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
3700                                  FIVETUPLELOOKUP_MASK,
3701                                  FIVETUPLELOOKUP_MASK);
3702
3703         /*
3704          * Tweak some settings.
3705          */
3706         t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
3707                      RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
3708                      PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
3709                      KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
3710
3711         /*
3712          * Get basic stuff going by issuing the Firmware Initialize command.
3713          * Note that this _must_ be after all PFVF commands ...
3714          */
3715         ret = t4_fw_initialize(adapter, adapter->mbox);
3716         if (ret < 0)
3717                 goto bye;
3718
3719         /*
3720          * Return successfully!
3721          */
3722         dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
3723                  "driver parameters\n");
3724         return 0;
3725
3726         /*
3727          * Something bad happened.  Return the error ...
3728          */
3729 bye:
3730         return ret;
3731 }
3732
3733 /*
3734  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3735  */
3736 static int adap_init0(struct adapter *adap)
3737 {
3738         int ret;
3739         u32 v, port_vec;
3740         enum dev_state state;
3741         u32 params[7], val[7];
3742         int reset = 1, j;
3743
3744         /*
3745          * Contact FW, advertising Master capability (and potentially forcing
3746          * ourselves as the Master PF if our module parameter force_init is
3747          * set).
3748          */
3749         ret = t4_fw_hello(adap, adap->mbox, adap->fn,
3750                           force_init ? MASTER_MUST : MASTER_MAY,
3751                           &state);
3752         if (ret < 0) {
3753                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3754                         ret);
3755                 return ret;
3756         }
3757         if (ret == adap->mbox)
3758                 adap->flags |= MASTER_PF;
3759         if (force_init && state == DEV_STATE_INIT)
3760                 state = DEV_STATE_UNINIT;
3761
3762         /*
3763          * If we're the Master PF Driver and the device is uninitialized,
3764          * then let's consider upgrading the firmware ...  (We always want
3765          * to check the firmware version number in order to A. get it for
3766          * later reporting and B. to warn if the currently loaded firmware
3767          * is excessively mismatched relative to the driver.)
3768          */
3769         ret = t4_check_fw_version(adap);
3770         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
3771                 if (ret == -EINVAL || ret > 0) {
3772                         if (upgrade_fw(adap) >= 0) {
3773                                 /*
3774                                  * Note that the chip was reset as part of the
3775                                  * firmware upgrade so we don't reset it again
3776                                  * below and grab the new firmware version.
3777                                  */
3778                                 reset = 0;
3779                                 ret = t4_check_fw_version(adap);
3780                         }
3781                 }
3782                 if (ret < 0)
3783                         return ret;
3784         }
3785
3786         /*
3787          * Grab VPD parameters.  This should be done after we establish a
3788          * connection to the firmware since some of the VPD parameters
3789          * (notably the Core Clock frequency) are retrieved via requests to
3790          * the firmware.  On the other hand, we need these fairly early on
3791          * so we do this right after getting ahold of the firmware.
3792          */
3793         ret = get_vpd_params(adap, &adap->params.vpd);
3794         if (ret < 0)
3795                 goto bye;
3796
3797         /*
3798          * Find out what ports are available to us.  Note that we need to do
3799          * this before calling adap_init0_no_config() since it needs nports
3800          * and portvec ...
3801          */
3802         v =
3803             FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3804             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
3805         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
3806         if (ret < 0)
3807                 goto bye;
3808
3809         adap->params.nports = hweight32(port_vec);
3810         adap->params.portvec = port_vec;
3811
3812         /*
3813          * If the firmware is initialized already (and we're not forcing a
3814          * master initialization), note that we're living with existing
3815          * adapter parameters.  Otherwise, it's time to try initializing the
3816          * adapter ...
3817          */
3818         if (state == DEV_STATE_INIT) {
3819                 dev_info(adap->pdev_dev, "Coming up as %s: "\
3820                          "Adapter already initialized\n",
3821                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
3822                 adap->flags |= USING_SOFT_PARAMS;
3823         } else {
3824                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
3825                          "Initializing adapter\n");
3826
3827                 /*
3828                  * If the firmware doesn't support Configuration
3829                  * Files warn user and exit,
3830                  */
3831                 if (ret < 0)
3832                         dev_warn(adap->pdev_dev, "Firmware doesn't support "
3833                                  "configuration file.\n");
3834                 if (force_old_init)
3835                         ret = adap_init0_no_config(adap, reset);
3836                 else {
3837                         /*
3838                          * Find out whether we're dealing with a version of
3839                          * the firmware which has configuration file support.
3840                          */
3841                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3842                                      FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
3843                         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
3844                                               params, val);
3845
3846                         /*
3847                          * If the firmware doesn't support Configuration
3848                          * Files, use the old Driver-based, hard-wired
3849                          * initialization.  Otherwise, try using the
3850                          * Configuration File support and fall back to the
3851                          * Driver-based initialization if there's no
3852                          * Configuration File found.
3853                          */
3854                         if (ret < 0)
3855                                 ret = adap_init0_no_config(adap, reset);
3856                         else {
3857                                 /*
3858                                  * The firmware provides us with a memory
3859                                  * buffer where we can load a Configuration
3860                                  * File from the host if we want to override
3861                                  * the Configuration File in flash.
3862                                  */
3863
3864                                 ret = adap_init0_config(adap, reset);
3865                                 if (ret == -ENOENT) {
3866                                         dev_info(adap->pdev_dev,
3867                                             "No Configuration File present "
3868                                             "on adapter.  Using hard-wired "
3869                                             "configuration parameters.\n");
3870                                         ret = adap_init0_no_config(adap, reset);
3871                                 }
3872                         }
3873                 }
3874                 if (ret < 0) {
3875                         dev_err(adap->pdev_dev,
3876                                 "could not initialize adapter, error %d\n",
3877                                 -ret);
3878                         goto bye;
3879                 }
3880         }
3881
3882         /*
3883          * If we're living with non-hard-coded parameters (either from a
3884          * Firmware Configuration File or values programmed by a different PF
3885          * Driver), give the SGE code a chance to pull in anything that it
3886          * needs ...  Note that this must be called after we retrieve our VPD
3887          * parameters in order to know how to convert core ticks to seconds.
3888          */
3889         if (adap->flags & USING_SOFT_PARAMS) {
3890                 ret = t4_sge_init(adap);
3891                 if (ret < 0)
3892                         goto bye;
3893         }
3894
3895         /*
3896          * Grab some of our basic fundamental operating parameters.
3897          */
3898 #define FW_PARAM_DEV(param) \
3899         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3900         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3901
3902 #define FW_PARAM_PFVF(param) \
3903         FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3904         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)|  \
3905         FW_PARAMS_PARAM_Y(0) | \
3906         FW_PARAMS_PARAM_Z(0)
3907
3908         params[0] = FW_PARAM_PFVF(EQ_START);
3909         params[1] = FW_PARAM_PFVF(L2T_START);
3910         params[2] = FW_PARAM_PFVF(L2T_END);
3911         params[3] = FW_PARAM_PFVF(FILTER_START);
3912         params[4] = FW_PARAM_PFVF(FILTER_END);
3913         params[5] = FW_PARAM_PFVF(IQFLINT_START);
3914         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
3915         if (ret < 0)
3916                 goto bye;
3917         adap->sge.egr_start = val[0];
3918         adap->l2t_start = val[1];
3919         adap->l2t_end = val[2];
3920         adap->tids.ftid_base = val[3];
3921         adap->tids.nftids = val[4] - val[3] + 1;
3922         adap->sge.ingr_start = val[5];
3923
3924         /* query params related to active filter region */
3925         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
3926         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
3927         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
3928         /* If Active filter size is set we enable establishing
3929          * offload connection through firmware work request
3930          */
3931         if ((val[0] != val[1]) && (ret >= 0)) {
3932                 adap->flags |= FW_OFLD_CONN;
3933                 adap->tids.aftid_base = val[0];
3934                 adap->tids.aftid_end = val[1];
3935         }
3936
3937 #ifdef CONFIG_CHELSIO_T4_OFFLOAD
3938         /*
3939          * Get device capabilities so we can determine what resources we need
3940          * to manage.
3941          */
3942         memset(&caps_cmd, 0, sizeof(caps_cmd));
3943         caps_cmd.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3944                                      FW_CMD_REQUEST | FW_CMD_READ);
3945         caps_cmd.retval_len16 = htonl(FW_LEN16(caps_cmd));
3946         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
3947                          &caps_cmd);
3948         if (ret < 0)
3949                 goto bye;
3950
3951         if (caps_cmd.ofldcaps) {
3952                 /* query offload-related parameters */
3953                 params[0] = FW_PARAM_DEV(NTID);
3954                 params[1] = FW_PARAM_PFVF(SERVER_START);
3955                 params[2] = FW_PARAM_PFVF(SERVER_END);
3956                 params[3] = FW_PARAM_PFVF(TDDP_START);
3957                 params[4] = FW_PARAM_PFVF(TDDP_END);
3958                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3959                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
3960                                       params, val);
3961                 if (ret < 0)
3962                         goto bye;
3963                 adap->tids.ntids = val[0];
3964                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3965                 adap->tids.stid_base = val[1];
3966                 adap->tids.nstids = val[2] - val[1] + 1;
3967                 /*
3968                  * Setup server filter region. Divide the availble filter
3969                  * region into two parts. Regular filters get 1/3rd and server
3970                  * filters get 2/3rd part. This is only enabled if workarond
3971                  * path is enabled.
3972                  * 1. For regular filters.
3973                  * 2. Server filter: This are special filters which are used
3974                  * to redirect SYN packets to offload queue.
3975                  */
3976                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
3977                         adap->tids.sftid_base = adap->tids.ftid_base +
3978                                         DIV_ROUND_UP(adap->tids.nftids, 3);
3979                         adap->tids.nsftids = adap->tids.nftids -
3980                                          DIV_ROUND_UP(adap->tids.nftids, 3);
3981                         adap->tids.nftids = adap->tids.sftid_base -
3982                                                 adap->tids.ftid_base;
3983                 }
3984                 adap->vres.ddp.start = val[3];
3985                 adap->vres.ddp.size = val[4] - val[3] + 1;
3986                 adap->params.ofldq_wr_cred = val[5];
3987
3988                 params[0] = FW_PARAM_PFVF(ETHOFLD_START);
3989                 params[1] = FW_PARAM_PFVF(ETHOFLD_END);
3990                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
3991                                       params, val);
3992                 if ((val[0] != val[1]) && (ret >= 0)) {
3993                         adap->tids.uotid_base = val[0];
3994                         adap->tids.nuotids = val[1] - val[0] + 1;
3995                 }
3996
3997                 adap->params.offload = 1;
3998         }
3999         if (caps_cmd.rdmacaps) {
4000                 params[0] = FW_PARAM_PFVF(STAG_START);
4001                 params[1] = FW_PARAM_PFVF(STAG_END);
4002                 params[2] = FW_PARAM_PFVF(RQ_START);
4003                 params[3] = FW_PARAM_PFVF(RQ_END);
4004                 params[4] = FW_PARAM_PFVF(PBL_START);
4005                 params[5] = FW_PARAM_PFVF(PBL_END);
4006                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
4007                                       params, val);
4008                 if (ret < 0)
4009                         goto bye;
4010                 adap->vres.stag.start = val[0];
4011                 adap->vres.stag.size = val[1] - val[0] + 1;
4012                 adap->vres.rq.start = val[2];
4013                 adap->vres.rq.size = val[3] - val[2] + 1;
4014                 adap->vres.pbl.start = val[4];
4015                 adap->vres.pbl.size = val[5] - val[4] + 1;
4016
4017                 params[0] = FW_PARAM_PFVF(SQRQ_START);
4018                 params[1] = FW_PARAM_PFVF(SQRQ_END);
4019                 params[2] = FW_PARAM_PFVF(CQ_START);
4020                 params[3] = FW_PARAM_PFVF(CQ_END);
4021                 params[4] = FW_PARAM_PFVF(OCQ_START);
4022                 params[5] = FW_PARAM_PFVF(OCQ_END);
4023                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
4024                 if (ret < 0)
4025                         goto bye;
4026                 adap->vres.qp.start = val[0];
4027                 adap->vres.qp.size = val[1] - val[0] + 1;
4028                 adap->vres.cq.start = val[2];
4029                 adap->vres.cq.size = val[3] - val[2] + 1;
4030                 adap->vres.ocq.start = val[4];
4031                 adap->vres.ocq.size = val[5] - val[4] + 1;
4032         }
4033         if (caps_cmd.iscsicaps) {
4034                 params[0] = FW_PARAM_PFVF(ISCSI_START);
4035                 params[1] = FW_PARAM_PFVF(ISCSI_END);
4036                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
4037                                       params, val);
4038                 if (ret < 0)
4039                         goto bye;
4040                 adap->vres.iscsi.start = val[0];
4041                 adap->vres.iscsi.size = val[1] - val[0] + 1;
4042         }
4043 #undef FW_PARAM_PFVF
4044 #undef FW_PARAM_DEV
4045 #endif /* CONFIG_CHELSIO_T4_OFFLOAD */
4046
4047         /*
4048          * These are finalized by FW initialization, load their values now.
4049          */
4050         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
4051         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
4052         adap->params.tp.dack_re = DELAYEDACKRESOLUTION_GET(v);
4053         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
4054         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
4055                      adap->params.b_wnd);
4056
4057         /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
4058         for (j = 0; j < NCHAN; j++)
4059                 adap->params.tp.tx_modq[j] = j;
4060
4061         adap->flags |= FW_OK;
4062         return 0;
4063
4064         /*
4065          * Something bad happened.  If a command timed out or failed with EIO
4066          * FW does not operate within its spec or something catastrophic
4067          * happened to HW/FW, stop issuing commands.
4068          */
4069 bye:
4070         if (ret != -ETIMEDOUT && ret != -EIO)
4071                 t4_fw_bye(adap, adap->mbox);
4072         return ret;
4073 }
4074
4075 /* EEH callbacks */
4076
4077 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
4078                                          pci_channel_state_t state)
4079 {
4080         int i;
4081         struct adapter *adap = pci_get_drvdata(pdev);
4082
4083         if (!adap)
4084                 goto out;
4085
4086         rtnl_lock();
4087         adap->flags &= ~FW_OK;
4088         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
4089         for_each_port(adap, i) {
4090                 struct net_device *dev = adap->port[i];
4091
4092                 netif_device_detach(dev);
4093                 netif_carrier_off(dev);
4094         }
4095         if (adap->flags & FULL_INIT_DONE)
4096                 cxgb_down(adap);
4097         rtnl_unlock();
4098         pci_disable_device(pdev);
4099 out:    return state == pci_channel_io_perm_failure ?
4100                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
4101 }
4102
4103 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
4104 {
4105         int i, ret;
4106         struct fw_caps_config_cmd c;
4107         struct adapter *adap = pci_get_drvdata(pdev);
4108
4109         if (!adap) {
4110                 pci_restore_state(pdev);
4111                 pci_save_state(pdev);
4112                 return PCI_ERS_RESULT_RECOVERED;
4113         }
4114
4115         if (pci_enable_device(pdev)) {
4116                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
4117                 return PCI_ERS_RESULT_DISCONNECT;
4118         }
4119
4120         pci_set_master(pdev);
4121         pci_restore_state(pdev);
4122         pci_save_state(pdev);
4123         pci_cleanup_aer_uncorrect_error_status(pdev);
4124
4125         if (t4_wait_dev_ready(adap) < 0)
4126                 return PCI_ERS_RESULT_DISCONNECT;
4127         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
4128                 return PCI_ERS_RESULT_DISCONNECT;
4129         adap->flags |= FW_OK;
4130         if (adap_init1(adap, &c))
4131                 return PCI_ERS_RESULT_DISCONNECT;
4132
4133         for_each_port(adap, i) {
4134                 struct port_info *p = adap2pinfo(adap, i);
4135
4136                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
4137                                   NULL, NULL);
4138                 if (ret < 0)
4139                         return PCI_ERS_RESULT_DISCONNECT;
4140                 p->viid = ret;
4141                 p->xact_addr_filt = -1;
4142         }
4143
4144         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
4145                      adap->params.b_wnd);
4146         setup_memwin(adap);
4147         if (cxgb_up(adap))
4148                 return PCI_ERS_RESULT_DISCONNECT;
4149         return PCI_ERS_RESULT_RECOVERED;
4150 }
4151
4152 static void eeh_resume(struct pci_dev *pdev)
4153 {
4154         int i;
4155         struct adapter *adap = pci_get_drvdata(pdev);
4156
4157         if (!adap)
4158                 return;
4159
4160         rtnl_lock();
4161         for_each_port(adap, i) {
4162                 struct net_device *dev = adap->port[i];
4163
4164                 if (netif_running(dev)) {
4165                         link_start(dev);
4166                         cxgb_set_rxmode(dev);
4167                 }
4168                 netif_device_attach(dev);
4169         }
4170         rtnl_unlock();
4171 }
4172
4173 static const struct pci_error_handlers cxgb4_eeh = {
4174         .error_detected = eeh_err_detected,
4175         .slot_reset     = eeh_slot_reset,
4176         .resume         = eeh_resume,
4177 };
4178
4179 static inline bool is_10g_port(const struct link_config *lc)
4180 {
4181         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
4182 }
4183
4184 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
4185                              unsigned int size, unsigned int iqe_size)
4186 {
4187         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
4188                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
4189         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
4190         q->iqe_len = iqe_size;
4191         q->size = size;
4192 }
4193
4194 /*
4195  * Perform default configuration of DMA queues depending on the number and type
4196  * of ports we found and the number of available CPUs.  Most settings can be
4197  * modified by the admin prior to actual use.
4198  */
4199 static void __devinit cfg_queues(struct adapter *adap)
4200 {
4201         struct sge *s = &adap->sge;
4202         int i, q10g = 0, n10g = 0, qidx = 0;
4203
4204         for_each_port(adap, i)
4205                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
4206
4207         /*
4208          * We default to 1 queue per non-10G port and up to # of cores queues
4209          * per 10G port.
4210          */
4211         if (n10g)
4212                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
4213         if (q10g > netif_get_num_default_rss_queues())
4214                 q10g = netif_get_num_default_rss_queues();
4215
4216         for_each_port(adap, i) {
4217                 struct port_info *pi = adap2pinfo(adap, i);
4218
4219                 pi->first_qset = qidx;
4220                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
4221                 qidx += pi->nqsets;
4222         }
4223
4224         s->ethqsets = qidx;
4225         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
4226
4227         if (is_offload(adap)) {
4228                 /*
4229                  * For offload we use 1 queue/channel if all ports are up to 1G,
4230                  * otherwise we divide all available queues amongst the channels
4231                  * capped by the number of available cores.
4232                  */
4233                 if (n10g) {
4234                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
4235                                   num_online_cpus());
4236                         s->ofldqsets = roundup(i, adap->params.nports);
4237                 } else
4238                         s->ofldqsets = adap->params.nports;
4239                 /* For RDMA one Rx queue per channel suffices */
4240                 s->rdmaqs = adap->params.nports;
4241         }
4242
4243         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
4244                 struct sge_eth_rxq *r = &s->ethrxq[i];
4245
4246                 init_rspq(&r->rspq, 0, 0, 1024, 64);
4247                 r->fl.size = 72;
4248         }
4249
4250         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
4251                 s->ethtxq[i].q.size = 1024;
4252
4253         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
4254                 s->ctrlq[i].q.size = 512;
4255
4256         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
4257                 s->ofldtxq[i].q.size = 1024;
4258
4259         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
4260                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
4261
4262                 init_rspq(&r->rspq, 0, 0, 1024, 64);
4263                 r->rspq.uld = CXGB4_ULD_ISCSI;
4264                 r->fl.size = 72;
4265         }
4266
4267         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
4268                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
4269
4270                 init_rspq(&r->rspq, 0, 0, 511, 64);
4271                 r->rspq.uld = CXGB4_ULD_RDMA;
4272                 r->fl.size = 72;
4273         }
4274
4275         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
4276         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
4277 }
4278
4279 /*
4280  * Reduce the number of Ethernet queues across all ports to at most n.
4281  * n provides at least one queue per port.
4282  */
4283 static void __devinit reduce_ethqs(struct adapter *adap, int n)
4284 {
4285         int i;
4286         struct port_info *pi;
4287
4288         while (n < adap->sge.ethqsets)
4289                 for_each_port(adap, i) {
4290                         pi = adap2pinfo(adap, i);
4291                         if (pi->nqsets > 1) {
4292                                 pi->nqsets--;
4293                                 adap->sge.ethqsets--;
4294                                 if (adap->sge.ethqsets <= n)
4295                                         break;
4296                         }
4297                 }
4298
4299         n = 0;
4300         for_each_port(adap, i) {
4301                 pi = adap2pinfo(adap, i);
4302                 pi->first_qset = n;
4303                 n += pi->nqsets;
4304         }
4305 }
4306
4307 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
4308 #define EXTRA_VECS 2
4309
4310 static int __devinit enable_msix(struct adapter *adap)
4311 {
4312         int ofld_need = 0;
4313         int i, err, want, need;
4314         struct sge *s = &adap->sge;
4315         unsigned int nchan = adap->params.nports;
4316         struct msix_entry entries[MAX_INGQ + 1];
4317
4318         for (i = 0; i < ARRAY_SIZE(entries); ++i)
4319                 entries[i].entry = i;
4320
4321         want = s->max_ethqsets + EXTRA_VECS;
4322         if (is_offload(adap)) {
4323                 want += s->rdmaqs + s->ofldqsets;
4324                 /* need nchan for each possible ULD */
4325                 ofld_need = 2 * nchan;
4326         }
4327         need = adap->params.nports + EXTRA_VECS + ofld_need;
4328
4329         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
4330                 want = err;
4331
4332         if (!err) {
4333                 /*
4334                  * Distribute available vectors to the various queue groups.
4335                  * Every group gets its minimum requirement and NIC gets top
4336                  * priority for leftovers.
4337                  */
4338                 i = want - EXTRA_VECS - ofld_need;
4339                 if (i < s->max_ethqsets) {
4340                         s->max_ethqsets = i;
4341                         if (i < s->ethqsets)
4342                                 reduce_ethqs(adap, i);
4343                 }
4344                 if (is_offload(adap)) {
4345                         i = want - EXTRA_VECS - s->max_ethqsets;
4346                         i -= ofld_need - nchan;
4347                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
4348                 }
4349                 for (i = 0; i < want; ++i)
4350                         adap->msix_info[i].vec = entries[i].vector;
4351         } else if (err > 0)
4352                 dev_info(adap->pdev_dev,
4353                          "only %d MSI-X vectors left, not using MSI-X\n", err);
4354         return err;
4355 }
4356
4357 #undef EXTRA_VECS
4358
4359 static int __devinit init_rss(struct adapter *adap)
4360 {
4361         unsigned int i, j;
4362
4363         for_each_port(adap, i) {
4364                 struct port_info *pi = adap2pinfo(adap, i);
4365
4366                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
4367                 if (!pi->rss)
4368                         return -ENOMEM;
4369                 for (j = 0; j < pi->rss_size; j++)
4370                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
4371         }
4372         return 0;
4373 }
4374
4375 static void __devinit print_port_info(const struct net_device *dev)
4376 {
4377         static const char *base[] = {
4378                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
4379                 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
4380         };
4381
4382         char buf[80];
4383         char *bufp = buf;
4384         const char *spd = "";
4385         const struct port_info *pi = netdev_priv(dev);
4386         const struct adapter *adap = pi->adapter;
4387
4388         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
4389                 spd = " 2.5 GT/s";
4390         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
4391                 spd = " 5 GT/s";
4392
4393         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
4394                 bufp += sprintf(bufp, "100/");
4395         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
4396                 bufp += sprintf(bufp, "1000/");
4397         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
4398                 bufp += sprintf(bufp, "10G/");
4399         if (bufp != buf)
4400                 --bufp;
4401         sprintf(bufp, "BASE-%s", base[pi->port_type]);
4402
4403         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
4404                     adap->params.vpd.id, adap->params.rev, buf,
4405                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
4406                     (adap->flags & USING_MSIX) ? " MSI-X" :
4407                     (adap->flags & USING_MSI) ? " MSI" : "");
4408         netdev_info(dev, "S/N: %s, E/C: %s\n",
4409                     adap->params.vpd.sn, adap->params.vpd.ec);
4410 }
4411
4412 static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
4413 {
4414         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
4415 }
4416
4417 /*
4418  * Free the following resources:
4419  * - memory used for tables
4420  * - MSI/MSI-X
4421  * - net devices
4422  * - resources FW is holding for us
4423  */
4424 static void free_some_resources(struct adapter *adapter)
4425 {
4426         unsigned int i;
4427
4428         t4_free_mem(adapter->l2t);
4429         t4_free_mem(adapter->tids.tid_tab);
4430         disable_msi(adapter);
4431
4432         for_each_port(adapter, i)
4433                 if (adapter->port[i]) {
4434                         kfree(adap2pinfo(adapter, i)->rss);
4435                         free_netdev(adapter->port[i]);
4436                 }
4437         if (adapter->flags & FW_OK)
4438                 t4_fw_bye(adapter, adapter->fn);
4439 }
4440
4441 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
4442 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
4443                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
4444
4445 static int __devinit init_one(struct pci_dev *pdev,
4446                               const struct pci_device_id *ent)
4447 {
4448         int func, i, err;
4449         struct port_info *pi;
4450         bool highdma = false;
4451         struct adapter *adapter = NULL;
4452
4453         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
4454
4455         err = pci_request_regions(pdev, KBUILD_MODNAME);
4456         if (err) {
4457                 /* Just info, some other driver may have claimed the device. */
4458                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
4459                 return err;
4460         }
4461
4462         /* We control everything through one PF */
4463         func = PCI_FUNC(pdev->devfn);
4464         if (func != ent->driver_data) {
4465                 pci_save_state(pdev);        /* to restore SR-IOV later */
4466                 goto sriov;
4467         }
4468
4469         err = pci_enable_device(pdev);
4470         if (err) {
4471                 dev_err(&pdev->dev, "cannot enable PCI device\n");
4472                 goto out_release_regions;
4473         }
4474
4475         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
4476                 highdma = true;
4477                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4478                 if (err) {
4479                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
4480                                 "coherent allocations\n");
4481                         goto out_disable_device;
4482                 }
4483         } else {
4484                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4485                 if (err) {
4486                         dev_err(&pdev->dev, "no usable DMA configuration\n");
4487                         goto out_disable_device;
4488                 }
4489         }
4490
4491         pci_enable_pcie_error_reporting(pdev);
4492         enable_pcie_relaxed_ordering(pdev);
4493         pci_set_master(pdev);
4494         pci_save_state(pdev);
4495
4496         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
4497         if (!adapter) {
4498                 err = -ENOMEM;
4499                 goto out_disable_device;
4500         }
4501
4502         adapter->regs = pci_ioremap_bar(pdev, 0);
4503         if (!adapter->regs) {
4504                 dev_err(&pdev->dev, "cannot map device registers\n");
4505                 err = -ENOMEM;
4506                 goto out_free_adapter;
4507         }
4508
4509         adapter->pdev = pdev;
4510         adapter->pdev_dev = &pdev->dev;
4511         adapter->mbox = func;
4512         adapter->fn = func;
4513         adapter->msg_enable = dflt_msg_enable;
4514         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
4515
4516         spin_lock_init(&adapter->stats_lock);
4517         spin_lock_init(&adapter->tid_release_lock);
4518
4519         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
4520         INIT_WORK(&adapter->db_full_task, process_db_full);
4521         INIT_WORK(&adapter->db_drop_task, process_db_drop);
4522
4523         err = t4_prep_adapter(adapter);
4524         if (err)
4525                 goto out_unmap_bar;
4526         setup_memwin(adapter);
4527         err = adap_init0(adapter);
4528         setup_memwin_rdma(adapter);
4529         if (err)
4530                 goto out_unmap_bar;
4531
4532         for_each_port(adapter, i) {
4533                 struct net_device *netdev;
4534
4535                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
4536                                            MAX_ETH_QSETS);
4537                 if (!netdev) {
4538                         err = -ENOMEM;
4539                         goto out_free_dev;
4540                 }
4541
4542                 SET_NETDEV_DEV(netdev, &pdev->dev);
4543
4544                 adapter->port[i] = netdev;
4545                 pi = netdev_priv(netdev);
4546                 pi->adapter = adapter;
4547                 pi->xact_addr_filt = -1;
4548                 pi->port_id = i;
4549                 netdev->irq = pdev->irq;
4550
4551                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
4552                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4553                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
4554                         NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
4555                 if (highdma)
4556                         netdev->hw_features |= NETIF_F_HIGHDMA;
4557                 netdev->features |= netdev->hw_features;
4558                 netdev->vlan_features = netdev->features & VLAN_FEAT;
4559
4560                 netdev->priv_flags |= IFF_UNICAST_FLT;
4561
4562                 netdev->netdev_ops = &cxgb4_netdev_ops;
4563                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
4564         }
4565
4566         pci_set_drvdata(pdev, adapter);
4567
4568         if (adapter->flags & FW_OK) {
4569                 err = t4_port_init(adapter, func, func, 0);
4570                 if (err)
4571                         goto out_free_dev;
4572         }
4573
4574         /*
4575          * Configure queues and allocate tables now, they can be needed as
4576          * soon as the first register_netdev completes.
4577          */
4578         cfg_queues(adapter);
4579
4580         adapter->l2t = t4_init_l2t();
4581         if (!adapter->l2t) {
4582                 /* We tolerate a lack of L2T, giving up some functionality */
4583                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
4584                 adapter->params.offload = 0;
4585         }
4586
4587         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
4588                 dev_warn(&pdev->dev, "could not allocate TID table, "
4589                          "continuing\n");
4590                 adapter->params.offload = 0;
4591         }
4592
4593         /* See what interrupts we'll be using */
4594         if (msi > 1 && enable_msix(adapter) == 0)
4595                 adapter->flags |= USING_MSIX;
4596         else if (msi > 0 && pci_enable_msi(pdev) == 0)
4597                 adapter->flags |= USING_MSI;
4598
4599         err = init_rss(adapter);
4600         if (err)
4601                 goto out_free_dev;
4602
4603         /*
4604          * The card is now ready to go.  If any errors occur during device
4605          * registration we do not fail the whole card but rather proceed only
4606          * with the ports we manage to register successfully.  However we must
4607          * register at least one net device.
4608          */
4609         for_each_port(adapter, i) {
4610                 pi = adap2pinfo(adapter, i);
4611                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
4612                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
4613
4614                 err = register_netdev(adapter->port[i]);
4615                 if (err)
4616                         break;
4617                 adapter->chan_map[pi->tx_chan] = i;
4618                 print_port_info(adapter->port[i]);
4619         }
4620         if (i == 0) {
4621                 dev_err(&pdev->dev, "could not register any net devices\n");
4622                 goto out_free_dev;
4623         }
4624         if (err) {
4625                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
4626                 err = 0;
4627         }
4628
4629         if (cxgb4_debugfs_root) {
4630                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
4631                                                            cxgb4_debugfs_root);
4632                 setup_debugfs(adapter);
4633         }
4634
4635         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
4636         pdev->needs_freset = 1;
4637
4638         if (is_offload(adapter))
4639                 attach_ulds(adapter);
4640
4641 sriov:
4642 #ifdef CONFIG_PCI_IOV
4643         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
4644                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
4645                         dev_info(&pdev->dev,
4646                                  "instantiated %u virtual functions\n",
4647                                  num_vf[func]);
4648 #endif
4649         return 0;
4650
4651  out_free_dev:
4652         free_some_resources(adapter);
4653  out_unmap_bar:
4654         iounmap(adapter->regs);
4655  out_free_adapter:
4656         kfree(adapter);
4657  out_disable_device:
4658         pci_disable_pcie_error_reporting(pdev);
4659         pci_disable_device(pdev);
4660  out_release_regions:
4661         pci_release_regions(pdev);
4662         pci_set_drvdata(pdev, NULL);
4663         return err;
4664 }
4665
4666 static void __devexit remove_one(struct pci_dev *pdev)
4667 {
4668         struct adapter *adapter = pci_get_drvdata(pdev);
4669
4670 #ifdef CONFIG_PCI_IOV
4671         pci_disable_sriov(pdev);
4672
4673 #endif
4674
4675         if (adapter) {
4676                 int i;
4677
4678                 if (is_offload(adapter))
4679                         detach_ulds(adapter);
4680
4681                 for_each_port(adapter, i)
4682                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
4683                                 unregister_netdev(adapter->port[i]);
4684
4685                 if (adapter->debugfs_root)
4686                         debugfs_remove_recursive(adapter->debugfs_root);
4687
4688                 if (adapter->flags & FULL_INIT_DONE)
4689                         cxgb_down(adapter);
4690
4691                 free_some_resources(adapter);
4692                 iounmap(adapter->regs);
4693                 kfree(adapter);
4694                 pci_disable_pcie_error_reporting(pdev);
4695                 pci_disable_device(pdev);
4696                 pci_release_regions(pdev);
4697                 pci_set_drvdata(pdev, NULL);
4698         } else
4699                 pci_release_regions(pdev);
4700 }
4701
4702 static struct pci_driver cxgb4_driver = {
4703         .name     = KBUILD_MODNAME,
4704         .id_table = cxgb4_pci_tbl,
4705         .probe    = init_one,
4706         .remove   = __devexit_p(remove_one),
4707         .err_handler = &cxgb4_eeh,
4708 };
4709
4710 static int __init cxgb4_init_module(void)
4711 {
4712         int ret;
4713
4714         workq = create_singlethread_workqueue("cxgb4");
4715         if (!workq)
4716                 return -ENOMEM;
4717
4718         /* Debugfs support is optional, just warn if this fails */
4719         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
4720         if (!cxgb4_debugfs_root)
4721                 pr_warning("could not create debugfs entry, continuing\n");
4722
4723         ret = pci_register_driver(&cxgb4_driver);
4724         if (ret < 0)
4725                 debugfs_remove(cxgb4_debugfs_root);
4726         return ret;
4727 }
4728
4729 static void __exit cxgb4_cleanup_module(void)
4730 {
4731         pci_unregister_driver(&cxgb4_driver);
4732         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
4733         flush_workqueue(workq);
4734         destroy_workqueue(workq);
4735 }
4736
4737 module_init(cxgb4_init_module);
4738 module_exit(cxgb4_cleanup_module);