drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/pci.h>
  24 #include <linux/firmware.h>
  25 #include <linux/ptp_clock_kernel.h>
  26 #include <net/vxlan.h>
  27 #include "liquidio_common.h"
  28 #include "octeon_droq.h"
  29 #include "octeon_iq.h"
  30 #include "response_manager.h"
  31 #include "octeon_device.h"
  32 #include "octeon_nic.h"
  33 #include "octeon_main.h"
  34 #include "octeon_network.h"
  35 #include "cn66xx_regs.h"
  36 #include "cn66xx_device.h"
  37 #include "cn68xx_device.h"
  38 #include "cn23xx_pf_device.h"
  39 #include "liquidio_image.h"
  40
  41 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  42 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  43 MODULE_LICENSE("GPL");
  44 MODULE_VERSION(LIQUIDIO_VERSION);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
  46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
  47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
  48
  49 static int ddr_timeout = 10000;
  50 module_param(ddr_timeout, int, 0644);
  51 MODULE_PARM_DESC(ddr_timeout,
  52                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  53
  54 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  55
  56 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
  57         (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
  58
  59 static int debug = -1;
  60 module_param(debug, int, 0644);
  61 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  62
  63 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  64 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  65 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  66
  67 static int conf_type;
  68 module_param(conf_type, int, 0);
  69 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
  70
  71 static int ptp_enable = 1;
  72
  73 /* Bit mask values for lio->ifstate */
  74 #define   LIO_IFSTATE_DROQ_OPS             0x01
  75 #define   LIO_IFSTATE_REGISTERED           0x02
  76 #define   LIO_IFSTATE_RUNNING              0x04
  77 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
  78
  79 /* Polling interval for determining when NIC application is alive */
  80 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  81
  82 /* runtime link query interval */
  83 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  84
  85 struct liquidio_if_cfg_context {
  86         int octeon_id;
  87
  88         wait_queue_head_t wc;
  89
  90         int cond;
  91 };
  92
  93 struct liquidio_if_cfg_resp {
  94         u64 rh;
  95         struct liquidio_if_cfg_info cfg_info;
  96         u64 status;
  97 };
  98
  99 struct oct_link_status_resp {
 100         u64 rh;
 101         struct oct_link_info link_info;
 102         u64 status;
 103 };
 104
 105 struct oct_timestamp_resp {
 106         u64 rh;
 107         u64 timestamp;
 108         u64 status;
 109 };
 110
 111 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 112
 113 union tx_info {
 114         u64 u64;
 115         struct {
 116 #ifdef __BIG_ENDIAN_BITFIELD
 117                 u16 gso_size;
 118                 u16 gso_segs;
 119                 u32 reserved;
 120 #else
 121                 u32 reserved;
 122                 u16 gso_segs;
 123                 u16 gso_size;
 124 #endif
 125         } s;
 126 };
 127
 128 /** Octeon device properties to be used by the NIC module.
 129  * Each octeon device in the system will be represented
 130  * by this structure in the NIC module.
 131  */
 132
 133 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 134
 135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 136 #define OCTNIC_GSO_MAX_SIZE                                                    \
 137         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 138
 139 /** Structure of a node in list of gather components maintained by
 140  * NIC driver for each network device.
 141  */
 142 struct octnic_gather {
 143         /** List manipulation. Next and prev pointers. */
 144         struct list_head list;
 145
 146         /** Size of the gather component at sg in bytes. */
 147         int sg_size;
 148
 149         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 150         int adjust;
 151
 152         /** Gather component that can accommodate max sized fragment list
 153          *  received from the IP layer.
 154          */
 155         struct octeon_sg_entry *sg;
 156
 157         u64 sg_dma_ptr;
 158 };
 159
 160 struct handshake {
 161         struct completion init;
 162         struct completion started;
 163         struct pci_dev *pci_dev;
 164         int init_ok;
 165         int started_ok;
 166 };
 167
 168 struct octeon_device_priv {
 169         /** Tasklet structures for this device. */
 170         struct tasklet_struct droq_tasklet;
 171         unsigned long napi_mask;
 172 };
 173
 174 static int octeon_device_init(struct octeon_device *);
 175 static int liquidio_stop(struct net_device *netdev);
 176 static void liquidio_remove(struct pci_dev *pdev);
 177 static int liquidio_probe(struct pci_dev *pdev,
 178                           const struct pci_device_id *ent);
 179
 180 static struct handshake handshake[MAX_OCTEON_DEVICES];
 181 static struct completion first_stage;
 182
 183 static void octeon_droq_bh(unsigned long pdev)
 184 {
 185         int q_no;
 186         int reschedule = 0;
 187         struct octeon_device *oct = (struct octeon_device *)pdev;
 188         struct octeon_device_priv *oct_priv =
 189                 (struct octeon_device_priv *)oct->priv;
 190
 191         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
 192         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 193                 if (!(oct->io_qmask.oq & (1ULL << q_no)))
 194                         continue;
 195                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 196                                                           MAX_PACKET_BUDGET);
 197                 lio_enable_irq(oct->droq[q_no], NULL);
 198         }
 199
 200         if (reschedule)
 201                 tasklet_schedule(&oct_priv->droq_tasklet);
 202 }
 203
 204 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 205 {
 206         struct octeon_device_priv *oct_priv =
 207                 (struct octeon_device_priv *)oct->priv;
 208         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 209         int i;
 210
 211         do {
 212                 pending_pkts = 0;
 213
 214                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 215                         if (!(oct->io_qmask.oq & (1ULL << i)))
 216                                 continue;
 217                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 218                 }
 219                 if (pkt_cnt > 0) {
 220                         pending_pkts += pkt_cnt;
 221                         tasklet_schedule(&oct_priv->droq_tasklet);
 222                 }
 223                 pkt_cnt = 0;
 224                 schedule_timeout_uninterruptible(1);
 225
 226         } while (retry-- && pending_pkts);
 227
 228         return pkt_cnt;
 229 }
 230
 231 /**
 232  * \brief Forces all IO queues off on a given device
 233  * @param oct Pointer to Octeon device
 234  */
 235 static void force_io_queues_off(struct octeon_device *oct)
 236 {
 237         if ((oct->chip_id == OCTEON_CN66XX) ||
 238             (oct->chip_id == OCTEON_CN68XX)) {
 239                 /* Reset the Enable bits for Input Queues. */
 240                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 241
 242                 /* Reset the Enable bits for Output Queues. */
 243                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 244         }
 245 }
 246
 247 /**
 248  * \brief wait for all pending requests to complete
 249  * @param oct Pointer to Octeon device
 250  *
 251  * Called during shutdown sequence
 252  */
 253 static int wait_for_pending_requests(struct octeon_device *oct)
 254 {
 255         int i, pcount = 0;
 256
 257         for (i = 0; i < 100; i++) {
 258                 pcount =
 259                         atomic_read(&oct->response_list
 260                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 261                 if (pcount)
 262                         schedule_timeout_uninterruptible(HZ / 10);
 263                 else
 264                         break;
 265         }
 266
 267         if (pcount)
 268                 return 1;
 269
 270         return 0;
 271 }
 272
 273 /**
 274  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 275  * @param oct Pointer to Octeon device
 276  */
 277 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 278 {
 279         int i;
 280
 281         /* Disable the input and output queues now. No more packets will
 282          * arrive from Octeon, but we should wait for all packet processing
 283          * to finish.
 284          */
 285         force_io_queues_off(oct);
 286
 287         /* To allow for in-flight requests */
 288         schedule_timeout_uninterruptible(100);
 289
 290         if (wait_for_pending_requests(oct))
 291                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 292
 293         /* Force all requests waiting to be fetched by OCTEON to complete. */
 294         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 295                 struct octeon_instr_queue *iq;
 296
 297                 if (!(oct->io_qmask.iq & (1ULL << i)))
 298                         continue;
 299                 iq = oct->instr_queue[i];
 300
 301                 if (atomic_read(&iq->instr_pending)) {
 302                         spin_lock_bh(&iq->lock);
 303                         iq->fill_cnt = 0;
 304                         iq->octeon_read_index = iq->host_write_index;
 305                         iq->stats.instr_processed +=
 306                                 atomic_read(&iq->instr_pending);
 307                         lio_process_iq_request_list(oct, iq, 0);
 308                         spin_unlock_bh(&iq->lock);
 309                 }
 310         }
 311
 312         /* Force all pending ordered list requests to time out. */
 313         lio_process_ordered_list(oct, 1);
 314
 315         /* We do not need to wait for output queue packets to be processed. */
 316 }
 317
 318 /**
 319  * \brief Cleanup PCI AER uncorrectable error status
 320  * @param dev Pointer to PCI device
 321  */
 322 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 323 {
 324         int pos = 0x100;
 325         u32 status, mask;
 326
 327         pr_info("%s :\n", __func__);
 328
 329         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 330         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 331         if (dev->error_state == pci_channel_io_normal)
 332                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 333         else
 334                 status &= mask;         /* Clear corresponding fatal bits */
 335         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 336 }
 337
 338 /**
 339  * \brief Stop all PCI IO to a given device
 340  * @param dev Pointer to Octeon device
 341  */
 342 static void stop_pci_io(struct octeon_device *oct)
 343 {
 344         /* No more instructions will be forwarded. */
 345         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 346
 347         pci_disable_device(oct->pci_dev);
 348
 349         /* Disable interrupts  */
 350         oct->fn_list.disable_interrupt(oct->chip);
 351
 352         pcierror_quiesce_device(oct);
 353
 354         /* Release the interrupt line */
 355         free_irq(oct->pci_dev->irq, oct);
 356
 357         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 358                 pci_disable_msi(oct->pci_dev);
 359
 360         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 361                 lio_get_state_string(&oct->status));
 362
 363         /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
 364         /* making it a common function for all OCTEON models */
 365         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 366 }
 367
 368 /**
 369  * \brief called when PCI error is detected
 370  * @param pdev Pointer to PCI device
 371  * @param state The current pci connection state
 372  *
 373  * This function is called after a PCI bus error affecting
 374  * this device has been detected.
 375  */
 376 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 377                                                      pci_channel_state_t state)
 378 {
 379         struct octeon_device *oct = pci_get_drvdata(pdev);
 380
 381         /* Non-correctable Non-fatal errors */
 382         if (state == pci_channel_io_normal) {
 383                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 384                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 385                 return PCI_ERS_RESULT_CAN_RECOVER;
 386         }
 387
 388         /* Non-correctable Fatal errors */
 389         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 390         stop_pci_io(oct);
 391
 392         /* Always return a DISCONNECT. There is no support for recovery but only
 393          * for a clean shutdown.
 394          */
 395         return PCI_ERS_RESULT_DISCONNECT;
 396 }
 397
 398 /**
 399  * \brief mmio handler
 400  * @param pdev Pointer to PCI device
 401  */
 402 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 403                                 struct pci_dev *pdev __attribute__((unused)))
 404 {
 405         /* We should never hit this since we never ask for a reset for a Fatal
 406          * Error. We always return DISCONNECT in io_error above.
 407          * But play safe and return RECOVERED for now.
 408          */
 409         return PCI_ERS_RESULT_RECOVERED;
 410 }
 411
 412 /**
 413  * \brief called after the pci bus has been reset.
 414  * @param pdev Pointer to PCI device
 415  *
 416  * Restart the card from scratch, as if from a cold-boot. Implementation
 417  * resembles the first-half of the octeon_resume routine.
 418  */
 419 static pci_ers_result_t liquidio_pcie_slot_reset(
 420                                 struct pci_dev *pdev __attribute__((unused)))
 421 {
 422         /* We should never hit this since we never ask for a reset for a Fatal
 423          * Error. We always return DISCONNECT in io_error above.
 424          * But play safe and return RECOVERED for now.
 425          */
 426         return PCI_ERS_RESULT_RECOVERED;
 427 }
 428
 429 /**
 430  * \brief called when traffic can start flowing again.
 431  * @param pdev Pointer to PCI device
 432  *
 433  * This callback is called when the error recovery driver tells us that
 434  * its OK to resume normal operation. Implementation resembles the
 435  * second-half of the octeon_resume routine.
 436  */
 437 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 438 {
 439         /* Nothing to be done here. */
 440 }
 441
 442 #ifdef CONFIG_PM
 443 /**
 444  * \brief called when suspending
 445  * @param pdev Pointer to PCI device
 446  * @param state state to suspend to
 447  */
 448 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 449                             pm_message_t state __attribute__((unused)))
 450 {
 451         return 0;
 452 }
 453
 454 /**
 455  * \brief called when resuming
 456  * @param pdev Pointer to PCI device
 457  */
 458 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 459 {
 460         return 0;
 461 }
 462 #endif
 463
 464 /* For PCI-E Advanced Error Recovery (AER) Interface */
 465 static const struct pci_error_handlers liquidio_err_handler = {
 466         .error_detected = liquidio_pcie_error_detected,
 467         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 468         .slot_reset     = liquidio_pcie_slot_reset,
 469         .resume         = liquidio_pcie_resume,
 470 };
 471
 472 static const struct pci_device_id liquidio_pci_tbl[] = {
 473         {       /* 68xx */
 474                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 475         },
 476         {       /* 66xx */
 477                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 478         },
 479         {       /* 23xx pf */
 480                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 481         },
 482         {
 483                 0, 0, 0, 0, 0, 0, 0
 484         }
 485 };
 486 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 487
 488 static struct pci_driver liquidio_pci_driver = {
 489         .name           = "LiquidIO",
 490         .id_table       = liquidio_pci_tbl,
 491         .probe          = liquidio_probe,
 492         .remove         = liquidio_remove,
 493         .err_handler    = &liquidio_err_handler,    /* For AER */
 494
 495 #ifdef CONFIG_PM
 496         .suspend        = liquidio_suspend,
 497         .resume         = liquidio_resume,
 498 #endif
 499 };
 500
 501 /**
 502  * \brief register PCI driver
 503  */
 504 static int liquidio_init_pci(void)
 505 {
 506         return pci_register_driver(&liquidio_pci_driver);
 507 }
 508
 509 /**
 510  * \brief unregister PCI driver
 511  */
 512 static void liquidio_deinit_pci(void)
 513 {
 514         pci_unregister_driver(&liquidio_pci_driver);
 515 }
 516
 517 /**
 518  * \brief check interface state
 519  * @param lio per-network private data
 520  * @param state_flag flag state to check
 521  */
 522 static inline int ifstate_check(struct lio *lio, int state_flag)
 523 {
 524         return atomic_read(&lio->ifstate) & state_flag;
 525 }
 526
 527 /**
 528  * \brief set interface state
 529  * @param lio per-network private data
 530  * @param state_flag flag state to set
 531  */
 532 static inline void ifstate_set(struct lio *lio, int state_flag)
 533 {
 534         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
 535 }
 536
 537 /**
 538  * \brief clear interface state
 539  * @param lio per-network private data
 540  * @param state_flag flag state to clear
 541  */
 542 static inline void ifstate_reset(struct lio *lio, int state_flag)
 543 {
 544         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 545 }
 546
 547 /**
 548  * \brief Stop Tx queues
 549  * @param netdev network device
 550  */
 551 static inline void txqs_stop(struct net_device *netdev)
 552 {
 553         if (netif_is_multiqueue(netdev)) {
 554                 int i;
 555
 556                 for (i = 0; i < netdev->num_tx_queues; i++)
 557                         netif_stop_subqueue(netdev, i);
 558         } else {
 559                 netif_stop_queue(netdev);
 560         }
 561 }
 562
 563 /**
 564  * \brief Start Tx queues
 565  * @param netdev network device
 566  */
 567 static inline void txqs_start(struct net_device *netdev)
 568 {
 569         if (netif_is_multiqueue(netdev)) {
 570                 int i;
 571
 572                 for (i = 0; i < netdev->num_tx_queues; i++)
 573                         netif_start_subqueue(netdev, i);
 574         } else {
 575                 netif_start_queue(netdev);
 576         }
 577 }
 578
 579 /**
 580  * \brief Wake Tx queues
 581  * @param netdev network device
 582  */
 583 static inline void txqs_wake(struct net_device *netdev)
 584 {
 585         struct lio *lio = GET_LIO(netdev);
 586
 587         if (netif_is_multiqueue(netdev)) {
 588                 int i;
 589
 590                 for (i = 0; i < netdev->num_tx_queues; i++) {
 591                         int qno = lio->linfo.txpciq[i %
 592                                 (lio->linfo.num_txpciq)].s.q_no;
 593
 594                         if (__netif_subqueue_stopped(netdev, i)) {
 595                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
 596                                                           tx_restart, 1);
 597                                 netif_wake_subqueue(netdev, i);
 598                         }
 599                 }
 600         } else {
 601                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 602                                           tx_restart, 1);
 603                 netif_wake_queue(netdev);
 604         }
 605 }
 606
 607 /**
 608  * \brief Stop Tx queue
 609  * @param netdev network device
 610  */
 611 static void stop_txq(struct net_device *netdev)
 612 {
 613         txqs_stop(netdev);
 614 }
 615
 616 /**
 617  * \brief Start Tx queue
 618  * @param netdev network device
 619  */
 620 static void start_txq(struct net_device *netdev)
 621 {
 622         struct lio *lio = GET_LIO(netdev);
 623
 624         if (lio->linfo.link.s.link_up) {
 625                 txqs_start(netdev);
 626                 return;
 627         }
 628 }
 629
 630 /**
 631  * \brief Wake a queue
 632  * @param netdev network device
 633  * @param q which queue to wake
 634  */
 635 static inline void wake_q(struct net_device *netdev, int q)
 636 {
 637         if (netif_is_multiqueue(netdev))
 638                 netif_wake_subqueue(netdev, q);
 639         else
 640                 netif_wake_queue(netdev);
 641 }
 642
 643 /**
 644  * \brief Stop a queue
 645  * @param netdev network device
 646  * @param q which queue to stop
 647  */
 648 static inline void stop_q(struct net_device *netdev, int q)
 649 {
 650         if (netif_is_multiqueue(netdev))
 651                 netif_stop_subqueue(netdev, q);
 652         else
 653                 netif_stop_queue(netdev);
 654 }
 655
 656 /**
 657  * \brief Check Tx queue status, and take appropriate action
 658  * @param lio per-network private data
 659  * @returns 0 if full, number of queues woken up otherwise
 660  */
 661 static inline int check_txq_status(struct lio *lio)
 662 {
 663         int ret_val = 0;
 664
 665         if (netif_is_multiqueue(lio->netdev)) {
 666                 int numqs = lio->netdev->num_tx_queues;
 667                 int q, iq = 0;
 668
 669                 /* check each sub-queue state */
 670                 for (q = 0; q < numqs; q++) {
 671                         iq = lio->linfo.txpciq[q %
 672                                 (lio->linfo.num_txpciq)].s.q_no;
 673                         if (octnet_iq_is_full(lio->oct_dev, iq))
 674                                 continue;
 675                         if (__netif_subqueue_stopped(lio->netdev, q)) {
 676                                 wake_q(lio->netdev, q);
 677                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 678                                                           tx_restart, 1);
 679                                 ret_val++;
 680                         }
 681                 }
 682         } else {
 683                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 684                         return 0;
 685                 wake_q(lio->netdev, lio->txq);
 686                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 687                                           tx_restart, 1);
 688                 ret_val = 1;
 689         }
 690         return ret_val;
 691 }
 692
 693 /**
 694  * Remove the node at the head of the list. The list would be empty at
 695  * the end of this call if there are no more nodes in the list.
 696  */
 697 static inline struct list_head *list_delete_head(struct list_head *root)
 698 {
 699         struct list_head *node;
 700
 701         if ((root->prev == root) && (root->next == root))
 702                 node = NULL;
 703         else
 704                 node = root->next;
 705
 706         if (node)
 707                 list_del(node);
 708
 709         return node;
 710 }
 711
 712 /**
 713  * \brief Delete gather lists
 714  * @param lio per-network private data
 715  */
 716 static void delete_glists(struct lio *lio)
 717 {
 718         struct octnic_gather *g;
 719         int i;
 720
 721         if (!lio->glist)
 722                 return;
 723
 724         for (i = 0; i < lio->linfo.num_txpciq; i++) {
 725                 do {
 726                         g = (struct octnic_gather *)
 727                                 list_delete_head(&lio->glist[i]);
 728                         if (g) {
 729                                 if (g->sg) {
 730                                         dma_unmap_single(&lio->oct_dev->
 731                                                          pci_dev->dev,
 732                                                          g->sg_dma_ptr,
 733                                                          g->sg_size,
 734                                                          DMA_TO_DEVICE);
 735                                         kfree((void *)((unsigned long)g->sg -
 736                                                        g->adjust));
 737                                 }
 738                                 kfree(g);
 739                         }
 740                 } while (g);
 741         }
 742
 743         kfree((void *)lio->glist);
 744 }
 745
 746 /**
 747  * \brief Setup gather lists
 748  * @param lio per-network private data
 749  */
 750 static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 751 {
 752         int i, j;
 753         struct octnic_gather *g;
 754
 755         lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
 756                                   GFP_KERNEL);
 757         if (!lio->glist_lock)
 758                 return 1;
 759
 760         lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
 761                              GFP_KERNEL);
 762         if (!lio->glist) {
 763                 kfree((void *)lio->glist_lock);
 764                 return 1;
 765         }
 766
 767         for (i = 0; i < num_iqs; i++) {
 768                 int numa_node = cpu_to_node(i % num_online_cpus());
 769
 770                 spin_lock_init(&lio->glist_lock[i]);
 771
 772                 INIT_LIST_HEAD(&lio->glist[i]);
 773
 774                 for (j = 0; j < lio->tx_qsize; j++) {
 775                         g = kzalloc_node(sizeof(*g), GFP_KERNEL,
 776                                          numa_node);
 777                         if (!g)
 778                                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 779                         if (!g)
 780                                 break;
 781
 782                         g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) *
 783                                       OCT_SG_ENTRY_SIZE);
 784
 785                         g->sg = kmalloc_node(g->sg_size + 8,
 786                                              GFP_KERNEL, numa_node);
 787                         if (!g->sg)
 788                                 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
 789                         if (!g->sg) {
 790                                 kfree(g);
 791                                 break;
 792                         }
 793
 794                         /* The gather component should be aligned on 64-bit
 795                          * boundary
 796                          */
 797                         if (((unsigned long)g->sg) & 7) {
 798                                 g->adjust = 8 - (((unsigned long)g->sg) & 7);
 799                                 g->sg = (struct octeon_sg_entry *)
 800                                         ((unsigned long)g->sg + g->adjust);
 801                         }
 802                         g->sg_dma_ptr = dma_map_single(&oct->pci_dev->dev,
 803                                                        g->sg, g->sg_size,
 804                                                        DMA_TO_DEVICE);
 805                         if (dma_mapping_error(&oct->pci_dev->dev,
 806                                               g->sg_dma_ptr)) {
 807                                 kfree((void *)((unsigned long)g->sg -
 808                                                g->adjust));
 809                                 kfree(g);
 810                                 break;
 811                         }
 812
 813                         list_add_tail(&g->list, &lio->glist[i]);
 814                 }
 815
 816                 if (j != lio->tx_qsize) {
 817                         delete_glists(lio);
 818                         return 1;
 819                 }
 820         }
 821
 822         return 0;
 823 }
 824
 825 /**
 826  * \brief Print link information
 827  * @param netdev network device
 828  */
 829 static void print_link_info(struct net_device *netdev)
 830 {
 831         struct lio *lio = GET_LIO(netdev);
 832
 833         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 834                 struct oct_link_info *linfo = &lio->linfo;
 835
 836                 if (linfo->link.s.link_up) {
 837                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 838                                    linfo->link.s.speed,
 839                                    (linfo->link.s.duplex) ? "Full" : "Half");
 840                 } else {
 841                         netif_info(lio, link, lio->netdev, "Link Down\n");
 842                 }
 843         }
 844 }
 845
 846 /**
 847  * \brief Update link status
 848  * @param netdev network device
 849  * @param ls link status structure
 850  *
 851  * Called on receipt of a link status response from the core application to
 852  * update each interface's link status.
 853  */
 854 static inline void update_link_status(struct net_device *netdev,
 855                                       union oct_link_status *ls)
 856 {
 857         struct lio *lio = GET_LIO(netdev);
 858         int changed = (lio->linfo.link.u64 != ls->u64);
 859
 860         lio->linfo.link.u64 = ls->u64;
 861
 862         if ((lio->intf_open) && (changed)) {
 863                 print_link_info(netdev);
 864                 lio->link_changes++;
 865
 866                 if (lio->linfo.link.s.link_up) {
 867                         netif_carrier_on(netdev);
 868                         /* start_txq(netdev); */
 869                         txqs_wake(netdev);
 870                 } else {
 871                         netif_carrier_off(netdev);
 872                         stop_txq(netdev);
 873                 }
 874         }
 875 }
 876
 877 /* Runs in interrupt context. */
 878 static void update_txq_status(struct octeon_device *oct, int iq_num)
 879 {
 880         struct net_device *netdev;
 881         struct lio *lio;
 882         struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 883
 884         /*octeon_update_iq_read_idx(oct, iq);*/
 885
 886         netdev = oct->props[iq->ifidx].netdev;
 887
 888         /* This is needed because the first IQ does not have
 889          * a netdev associated with it.
 890          */
 891         if (!netdev)
 892                 return;
 893
 894         lio = GET_LIO(netdev);
 895         if (netif_is_multiqueue(netdev)) {
 896                 if (__netif_subqueue_stopped(netdev, iq->q_index) &&
 897                     lio->linfo.link.s.link_up &&
 898                     (!octnet_iq_is_full(oct, iq_num))) {
 899                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
 900                                                   tx_restart, 1);
 901                         netif_wake_subqueue(netdev, iq->q_index);
 902                 } else {
 903                         if (!octnet_iq_is_full(oct, lio->txq)) {
 904                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
 905                                                           lio->txq,
 906                                                           tx_restart, 1);
 907                                 wake_q(netdev, lio->txq);
 908                         }
 909                 }
 910         }
 911 }
 912
 913 /**
 914  * \brief Droq packet processor sceduler
 915  * @param oct octeon device
 916  */
 917 static
 918 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 919 {
 920         struct octeon_device_priv *oct_priv =
 921                 (struct octeon_device_priv *)oct->priv;
 922         u64 oq_no;
 923         struct octeon_droq *droq;
 924
 925         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 926                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
 927                      oq_no++) {
 928                         if (!(oct->droq_intr & (1ULL << oq_no)))
 929                                 continue;
 930
 931                         droq = oct->droq[oq_no];
 932
 933                         if (droq->ops.poll_mode) {
 934                                 droq->ops.napi_fn(droq);
 935                                 oct_priv->napi_mask |= (1 << oq_no);
 936                         } else {
 937                                 tasklet_schedule(&oct_priv->droq_tasklet);
 938                         }
 939                 }
 940         }
 941 }
 942
 943 /**
 944  * \brief Interrupt handler for octeon
 945  * @param irq unused
 946  * @param dev octeon device
 947  */
 948 static
 949 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 950 {
 951         struct octeon_device *oct = (struct octeon_device *)dev;
 952         irqreturn_t ret;
 953
 954         /* Disable our interrupts for the duration of ISR */
 955         oct->fn_list.disable_interrupt(oct->chip);
 956
 957         ret = oct->fn_list.process_interrupt_regs(oct);
 958
 959         if (ret == IRQ_HANDLED)
 960                 liquidio_schedule_droq_pkt_handlers(oct);
 961
 962         /* Re-enable our interrupts  */
 963         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
 964                 oct->fn_list.enable_interrupt(oct->chip);
 965
 966         return ret;
 967 }
 968
 969 /**
 970  * \brief Setup interrupt for octeon device
 971  * @param oct octeon device
 972  *
 973  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
 974  */
 975 static int octeon_setup_interrupt(struct octeon_device *oct)
 976 {
 977         int irqret, err;
 978
 979         err = pci_enable_msi(oct->pci_dev);
 980         if (err)
 981                 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
 982                          err);
 983         else
 984                 oct->flags |= LIO_FLAG_MSI_ENABLED;
 985
 986         irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
 987                              IRQF_SHARED, "octeon", oct);
 988         if (irqret) {
 989                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
 990                         pci_disable_msi(oct->pci_dev);
 991                 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
 992                         irqret);
 993                 return 1;
 994         }
 995
 996         return 0;
 997 }
 998
 999 /**
1000  * \brief PCI probe handler
1001  * @param pdev PCI device structure
1002  * @param ent unused
1003  */
1004 static int
1005 liquidio_probe(struct pci_dev *pdev,
1006                const struct pci_device_id *ent __attribute__((unused)))
1007 {
1008         struct octeon_device *oct_dev = NULL;
1009         struct handshake *hs;
1010
1011         oct_dev = octeon_allocate_device(pdev->device,
1012                                          sizeof(struct octeon_device_priv));
1013         if (!oct_dev) {
1014                 dev_err(&pdev->dev, "Unable to allocate device\n");
1015                 return -ENOMEM;
1016         }
1017
1018         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1019                  (u32)pdev->vendor, (u32)pdev->device);
1020
1021         /* Assign octeon_device for this device to the private data area. */
1022         pci_set_drvdata(pdev, oct_dev);
1023
1024         /* set linux specific device pointer */
1025         oct_dev->pci_dev = (void *)pdev;
1026
1027         hs = &handshake[oct_dev->octeon_id];
1028         init_completion(&hs->init);
1029         init_completion(&hs->started);
1030         hs->pci_dev = pdev;
1031
1032         if (oct_dev->octeon_id == 0)
1033                 /* first LiquidIO NIC is detected */
1034                 complete(&first_stage);
1035
1036         if (octeon_device_init(oct_dev)) {
1037                 liquidio_remove(pdev);
1038                 return -ENOMEM;
1039         }
1040
1041         oct_dev->rx_pause = 1;
1042         oct_dev->tx_pause = 1;
1043
1044         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1045
1046         return 0;
1047 }
1048
1049 /**
1050  *\brief Destroy resources associated with octeon device
1051  * @param pdev PCI device structure
1052  * @param ent unused
1053  */
1054 static void octeon_destroy_resources(struct octeon_device *oct)
1055 {
1056         int i;
1057         struct octeon_device_priv *oct_priv =
1058                 (struct octeon_device_priv *)oct->priv;
1059
1060         struct handshake *hs;
1061
1062         switch (atomic_read(&oct->status)) {
1063         case OCT_DEV_RUNNING:
1064         case OCT_DEV_CORE_OK:
1065
1066                 /* No more instructions will be forwarded. */
1067                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1068
1069                 oct->app_mode = CVM_DRV_INVALID_APP;
1070                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1071                         lio_get_state_string(&oct->status));
1072
1073                 schedule_timeout_uninterruptible(HZ / 10);
1074
1075                 /* fallthrough */
1076         case OCT_DEV_HOST_OK:
1077
1078                 /* fallthrough */
1079         case OCT_DEV_CONSOLE_INIT_DONE:
1080                 /* Remove any consoles */
1081                 octeon_remove_consoles(oct);
1082
1083                 /* fallthrough */
1084         case OCT_DEV_IO_QUEUES_DONE:
1085                 if (wait_for_pending_requests(oct))
1086                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1087
1088                 if (lio_wait_for_instr_fetch(oct))
1089                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1090
1091                 /* Disable the input and output queues now. No more packets will
1092                  * arrive from Octeon, but we should wait for all packet
1093                  * processing to finish.
1094                  */
1095                 oct->fn_list.disable_io_queues(oct);
1096
1097                 if (lio_wait_for_oq_pkts(oct))
1098                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1099
1100                 /* Disable interrupts  */
1101                 oct->fn_list.disable_interrupt(oct->chip);
1102
1103                 /* Release the interrupt line */
1104                 free_irq(oct->pci_dev->irq, oct);
1105
1106                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1107                         pci_disable_msi(oct->pci_dev);
1108
1109                 /* fallthrough */
1110         case OCT_DEV_IN_RESET:
1111         case OCT_DEV_DROQ_INIT_DONE:
1112                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1113                 mdelay(100);
1114                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1115                         if (!(oct->io_qmask.oq & (1ULL << i)))
1116                                 continue;
1117                         octeon_delete_droq(oct, i);
1118                 }
1119
1120                 /* Force any pending handshakes to complete */
1121                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1122                         hs = &handshake[i];
1123
1124                         if (hs->pci_dev) {
1125                                 handshake[oct->octeon_id].init_ok = 0;
1126                                 complete(&handshake[oct->octeon_id].init);
1127                                 handshake[oct->octeon_id].started_ok = 0;
1128                                 complete(&handshake[oct->octeon_id].started);
1129                         }
1130                 }
1131
1132                 /* fallthrough */
1133         case OCT_DEV_RESP_LIST_INIT_DONE:
1134                 octeon_delete_response_list(oct);
1135
1136                 /* fallthrough */
1137         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1138                 octeon_free_sc_buffer_pool(oct);
1139
1140                 /* fallthrough */
1141         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1142                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1143                         if (!(oct->io_qmask.iq & (1ULL << i)))
1144                                 continue;
1145                         octeon_delete_instr_queue(oct, i);
1146                 }
1147
1148                 /* fallthrough */
1149         case OCT_DEV_DISPATCH_INIT_DONE:
1150                 octeon_delete_dispatch_list(oct);
1151                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1152
1153                 /* fallthrough */
1154         case OCT_DEV_PCI_MAP_DONE:
1155
1156                 /* Soft reset the octeon device before exiting */
1157                 oct->fn_list.soft_reset(oct);
1158
1159                 octeon_unmap_pci_barx(oct, 0);
1160                 octeon_unmap_pci_barx(oct, 1);
1161
1162                 /* fallthrough */
1163         case OCT_DEV_BEGIN_STATE:
1164                 /* Disable the device, releasing the PCI INT */
1165                 pci_disable_device(oct->pci_dev);
1166
1167                 /* Nothing to be done here either */
1168                 break;
1169         }                       /* end switch (oct->status) */
1170
1171         tasklet_kill(&oct_priv->droq_tasklet);
1172 }
1173
1174 /**
1175  * \brief Send Rx control command
1176  * @param lio per-network private data
1177  * @param start_stop whether to start or stop
1178  */
1179 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1180 {
1181         struct octnic_ctrl_pkt nctrl;
1182
1183         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1184
1185         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
1186         nctrl.ncmd.s.param1 = start_stop;
1187         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1188         nctrl.netpndev = (u64)lio->netdev;
1189
1190         if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl) < 0)
1191                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1192 }
1193
1194 /**
1195  * \brief Destroy NIC device interface
1196  * @param oct octeon device
1197  * @param ifidx which interface to destroy
1198  *
1199  * Cleanup associated with each interface for an Octeon device  when NIC
1200  * module is being unloaded or if initialization fails during load.
1201  */
1202 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1203 {
1204         struct net_device *netdev = oct->props[ifidx].netdev;
1205         struct lio *lio;
1206         struct napi_struct *napi, *n;
1207
1208         if (!netdev) {
1209                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1210                         __func__, ifidx);
1211                 return;
1212         }
1213
1214         lio = GET_LIO(netdev);
1215
1216         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1217
1218         send_rx_ctrl_cmd(lio, 0);
1219
1220         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1221                 txqs_stop(netdev);
1222
1223         if (oct->props[lio->ifidx].napi_enabled == 1) {
1224                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1225                         napi_disable(napi);
1226
1227                 oct->props[lio->ifidx].napi_enabled = 0;
1228         }
1229
1230         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1231                 unregister_netdev(netdev);
1232
1233         delete_glists(lio);
1234
1235         free_netdev(netdev);
1236
1237         oct->props[ifidx].gmxport = -1;
1238
1239         oct->props[ifidx].netdev = NULL;
1240 }
1241
1242 /**
1243  * \brief Stop complete NIC functionality
1244  * @param oct octeon device
1245  */
1246 static int liquidio_stop_nic_module(struct octeon_device *oct)
1247 {
1248         int i, j;
1249         struct lio *lio;
1250
1251         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1252         if (!oct->ifcount) {
1253                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1254                 return 1;
1255         }
1256
1257         spin_lock_bh(&oct->cmd_resp_wqlock);
1258         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1259         spin_unlock_bh(&oct->cmd_resp_wqlock);
1260
1261         for (i = 0; i < oct->ifcount; i++) {
1262                 lio = GET_LIO(oct->props[i].netdev);
1263                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1264                         octeon_unregister_droq_ops(oct,
1265                                                    lio->linfo.rxpciq[j].s.q_no);
1266         }
1267
1268         for (i = 0; i < oct->ifcount; i++)
1269                 liquidio_destroy_nic_device(oct, i);
1270
1271         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1272         return 0;
1273 }
1274
1275 /**
1276  * \brief Cleans up resources at unload time
1277  * @param pdev PCI device structure
1278  */
1279 static void liquidio_remove(struct pci_dev *pdev)
1280 {
1281         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1282
1283         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1284
1285         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1286                 liquidio_stop_nic_module(oct_dev);
1287
1288         /* Reset the octeon device and cleanup all memory allocated for
1289          * the octeon device by driver.
1290          */
1291         octeon_destroy_resources(oct_dev);
1292
1293         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1294
1295         /* This octeon device has been removed. Update the global
1296          * data structure to reflect this. Free the device structure.
1297          */
1298         octeon_free_device_mem(oct_dev);
1299 }
1300
1301 /**
1302  * \brief Identify the Octeon device and to map the BAR address space
1303  * @param oct octeon device
1304  */
1305 static int octeon_chip_specific_setup(struct octeon_device *oct)
1306 {
1307         u32 dev_id, rev_id;
1308         int ret = 1;
1309         char *s;
1310
1311         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1312         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1313         oct->rev_id = rev_id & 0xff;
1314
1315         switch (dev_id) {
1316         case OCTEON_CN68XX_PCIID:
1317                 oct->chip_id = OCTEON_CN68XX;
1318                 ret = lio_setup_cn68xx_octeon_device(oct);
1319                 s = "CN68XX";
1320                 break;
1321
1322         case OCTEON_CN66XX_PCIID:
1323                 oct->chip_id = OCTEON_CN66XX;
1324                 ret = lio_setup_cn66xx_octeon_device(oct);
1325                 s = "CN66XX";
1326                 break;
1327
1328         case OCTEON_CN23XX_PCIID_PF:
1329                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1330                 ret = setup_cn23xx_octeon_pf_device(oct);
1331                 s = "CN23XX";
1332                 break;
1333
1334         default:
1335                 s = "?";
1336                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1337                         dev_id);
1338         }
1339
1340         if (!ret)
1341                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1342                          OCTEON_MAJOR_REV(oct),
1343                          OCTEON_MINOR_REV(oct),
1344                          octeon_get_conf(oct)->card_name,
1345                          LIQUIDIO_VERSION);
1346
1347         return ret;
1348 }
1349
1350 /**
1351  * \brief PCI initialization for each Octeon device.
1352  * @param oct octeon device
1353  */
1354 static int octeon_pci_os_setup(struct octeon_device *oct)
1355 {
1356         /* setup PCI stuff first */
1357         if (pci_enable_device(oct->pci_dev)) {
1358                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1359                 return 1;
1360         }
1361
1362         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1363                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1364                 return 1;
1365         }
1366
1367         /* Enable PCI DMA Master. */
1368         pci_set_master(oct->pci_dev);
1369
1370         return 0;
1371 }
1372
1373 static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1374 {
1375         int q = 0;
1376
1377         if (netif_is_multiqueue(lio->netdev))
1378                 q = skb->queue_mapping % lio->linfo.num_txpciq;
1379
1380         return q;
1381 }
1382
1383 /**
1384  * \brief Check Tx queue state for a given network buffer
1385  * @param lio per-network private data
1386  * @param skb network buffer
1387  */
1388 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1389 {
1390         int q = 0, iq = 0;
1391
1392         if (netif_is_multiqueue(lio->netdev)) {
1393                 q = skb->queue_mapping;
1394                 iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
1395         } else {
1396                 iq = lio->txq;
1397                 q = iq;
1398         }
1399
1400         if (octnet_iq_is_full(lio->oct_dev, iq))
1401                 return 0;
1402
1403         if (__netif_subqueue_stopped(lio->netdev, q)) {
1404                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
1405                 wake_q(lio->netdev, q);
1406         }
1407         return 1;
1408 }
1409
1410 /**
1411  * \brief Unmap and free network buffer
1412  * @param buf buffer
1413  */
1414 static void free_netbuf(void *buf)
1415 {
1416         struct sk_buff *skb;
1417         struct octnet_buf_free_info *finfo;
1418         struct lio *lio;
1419
1420         finfo = (struct octnet_buf_free_info *)buf;
1421         skb = finfo->skb;
1422         lio = finfo->lio;
1423
1424         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1425                          DMA_TO_DEVICE);
1426
1427         check_txq_state(lio, skb);
1428
1429         tx_buffer_free(skb);
1430 }
1431
1432 /**
1433  * \brief Unmap and free gather buffer
1434  * @param buf buffer
1435  */
1436 static void free_netsgbuf(void *buf)
1437 {
1438         struct octnet_buf_free_info *finfo;
1439         struct sk_buff *skb;
1440         struct lio *lio;
1441         struct octnic_gather *g;
1442         int i, frags, iq;
1443
1444         finfo = (struct octnet_buf_free_info *)buf;
1445         skb = finfo->skb;
1446         lio = finfo->lio;
1447         g = finfo->g;
1448         frags = skb_shinfo(skb)->nr_frags;
1449
1450         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1451                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1452                          DMA_TO_DEVICE);
1453
1454         i = 1;
1455         while (frags--) {
1456                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1457
1458                 pci_unmap_page((lio->oct_dev)->pci_dev,
1459                                g->sg[(i >> 2)].ptr[(i & 3)],
1460                                frag->size, DMA_TO_DEVICE);
1461                 i++;
1462         }
1463
1464         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1465                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1466
1467         iq = skb_iq(lio, skb);
1468         spin_lock(&lio->glist_lock[iq]);
1469         list_add_tail(&g->list, &lio->glist[iq]);
1470         spin_unlock(&lio->glist_lock[iq]);
1471
1472         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1473
1474         tx_buffer_free(skb);
1475 }
1476
1477 /**
1478  * \brief Unmap and free gather buffer with response
1479  * @param buf buffer
1480  */
1481 static void free_netsgbuf_with_resp(void *buf)
1482 {
1483         struct octeon_soft_command *sc;
1484         struct octnet_buf_free_info *finfo;
1485         struct sk_buff *skb;
1486         struct lio *lio;
1487         struct octnic_gather *g;
1488         int i, frags, iq;
1489
1490         sc = (struct octeon_soft_command *)buf;
1491         skb = (struct sk_buff *)sc->callback_arg;
1492         finfo = (struct octnet_buf_free_info *)&skb->cb;
1493
1494         lio = finfo->lio;
1495         g = finfo->g;
1496         frags = skb_shinfo(skb)->nr_frags;
1497
1498         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1499                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1500                          DMA_TO_DEVICE);
1501
1502         i = 1;
1503         while (frags--) {
1504                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1505
1506                 pci_unmap_page((lio->oct_dev)->pci_dev,
1507                                g->sg[(i >> 2)].ptr[(i & 3)],
1508                                frag->size, DMA_TO_DEVICE);
1509                 i++;
1510         }
1511
1512         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1513                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1514
1515         iq = skb_iq(lio, skb);
1516
1517         spin_lock(&lio->glist_lock[iq]);
1518         list_add_tail(&g->list, &lio->glist[iq]);
1519         spin_unlock(&lio->glist_lock[iq]);
1520
1521         /* Don't free the skb yet */
1522
1523         check_txq_state(lio, skb);
1524 }
1525
1526 /**
1527  * \brief Adjust ptp frequency
1528  * @param ptp PTP clock info
1529  * @param ppb how much to adjust by, in parts-per-billion
1530  */
1531 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1532 {
1533         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1534         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1535         u64 comp, delta;
1536         unsigned long flags;
1537         bool neg_adj = false;
1538
1539         if (ppb < 0) {
1540                 neg_adj = true;
1541                 ppb = -ppb;
1542         }
1543
1544         /* The hardware adds the clock compensation value to the
1545          * PTP clock on every coprocessor clock cycle, so we
1546          * compute the delta in terms of coprocessor clocks.
1547          */
1548         delta = (u64)ppb << 32;
1549         do_div(delta, oct->coproc_clock_rate);
1550
1551         spin_lock_irqsave(&lio->ptp_lock, flags);
1552         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1553         if (neg_adj)
1554                 comp -= delta;
1555         else
1556                 comp += delta;
1557         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1558         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1559
1560         return 0;
1561 }
1562
1563 /**
1564  * \brief Adjust ptp time
1565  * @param ptp PTP clock info
1566  * @param delta how much to adjust by, in nanosecs
1567  */
1568 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1569 {
1570         unsigned long flags;
1571         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1572
1573         spin_lock_irqsave(&lio->ptp_lock, flags);
1574         lio->ptp_adjust += delta;
1575         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1576
1577         return 0;
1578 }
1579
1580 /**
1581  * \brief Get hardware clock time, including any adjustment
1582  * @param ptp PTP clock info
1583  * @param ts timespec
1584  */
1585 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1586                                 struct timespec64 *ts)
1587 {
1588         u64 ns;
1589         unsigned long flags;
1590         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1591         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1592
1593         spin_lock_irqsave(&lio->ptp_lock, flags);
1594         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1595         ns += lio->ptp_adjust;
1596         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1597
1598         *ts = ns_to_timespec64(ns);
1599
1600         return 0;
1601 }
1602
1603 /**
1604  * \brief Set hardware clock time. Reset adjustment
1605  * @param ptp PTP clock info
1606  * @param ts timespec
1607  */
1608 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1609                                 const struct timespec64 *ts)
1610 {
1611         u64 ns;
1612         unsigned long flags;
1613         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1614         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1615
1616         ns = timespec_to_ns(ts);
1617
1618         spin_lock_irqsave(&lio->ptp_lock, flags);
1619         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1620         lio->ptp_adjust = 0;
1621         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1622
1623         return 0;
1624 }
1625
1626 /**
1627  * \brief Check if PTP is enabled
1628  * @param ptp PTP clock info
1629  * @param rq request
1630  * @param on is it on
1631  */
1632 static int
1633 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1634                     struct ptp_clock_request *rq __attribute__((unused)),
1635                     int on __attribute__((unused)))
1636 {
1637         return -EOPNOTSUPP;
1638 }
1639
1640 /**
1641  * \brief Open PTP clock source
1642  * @param netdev network device
1643  */
1644 static void oct_ptp_open(struct net_device *netdev)
1645 {
1646         struct lio *lio = GET_LIO(netdev);
1647         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1648
1649         spin_lock_init(&lio->ptp_lock);
1650
1651         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1652         lio->ptp_info.owner = THIS_MODULE;
1653         lio->ptp_info.max_adj = 250000000;
1654         lio->ptp_info.n_alarm = 0;
1655         lio->ptp_info.n_ext_ts = 0;
1656         lio->ptp_info.n_per_out = 0;
1657         lio->ptp_info.pps = 0;
1658         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1659         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1660         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1661         lio->ptp_info.settime64 = liquidio_ptp_settime;
1662         lio->ptp_info.enable = liquidio_ptp_enable;
1663
1664         lio->ptp_adjust = 0;
1665
1666         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1667                                              &oct->pci_dev->dev);
1668
1669         if (IS_ERR(lio->ptp_clock))
1670                 lio->ptp_clock = NULL;
1671 }
1672
1673 /**
1674  * \brief Init PTP clock
1675  * @param oct octeon device
1676  */
1677 static void liquidio_ptp_init(struct octeon_device *oct)
1678 {
1679         u64 clock_comp, cfg;
1680
1681         clock_comp = (u64)NSEC_PER_SEC << 32;
1682         do_div(clock_comp, oct->coproc_clock_rate);
1683         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1684
1685         /* Enable */
1686         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1687         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1688 }
1689
1690 /**
1691  * \brief Load firmware to device
1692  * @param oct octeon device
1693  *
1694  * Maps device to firmware filename, requests firmware, and downloads it
1695  */
1696 static int load_firmware(struct octeon_device *oct)
1697 {
1698         int ret = 0;
1699         const struct firmware *fw;
1700         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1701         char *tmp_fw_type;
1702
1703         if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1704                     sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
1705                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
1706                 return ret;
1707         }
1708
1709         if (fw_type[0] == '\0')
1710                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1711         else
1712                 tmp_fw_type = fw_type;
1713
1714         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1715                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1716                 LIO_FW_NAME_SUFFIX);
1717
1718         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1719         if (ret) {
1720                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
1721                         fw_name);
1722                 release_firmware(fw);
1723                 return ret;
1724         }
1725
1726         ret = octeon_download_firmware(oct, fw->data, fw->size);
1727
1728         release_firmware(fw);
1729
1730         return ret;
1731 }
1732
1733 /**
1734  * \brief Setup output queue
1735  * @param oct octeon device
1736  * @param q_no which queue
1737  * @param num_descs how many descriptors
1738  * @param desc_size size of each descriptor
1739  * @param app_ctx application context
1740  */
1741 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
1742                              int desc_size, void *app_ctx)
1743 {
1744         int ret_val = 0;
1745
1746         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
1747         /* droq creation and local register settings. */
1748         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
1749         if (ret_val < 0)
1750                 return ret_val;
1751
1752         if (ret_val == 1) {
1753                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
1754                 return 0;
1755         }
1756         /* tasklet creation for the droq */
1757
1758         /* Enable the droq queues */
1759         octeon_set_droq_pkt_op(oct, q_no, 1);
1760
1761         /* Send Credit for Octeon Output queues. Credits are always
1762          * sent after the output queue is enabled.
1763          */
1764         writel(oct->droq[q_no]->max_count,
1765                oct->droq[q_no]->pkts_credit_reg);
1766
1767         return ret_val;
1768 }
1769
1770 /**
1771  * \brief Callback for getting interface configuration
1772  * @param status status of request
1773  * @param buf pointer to resp structure
1774  */
1775 static void if_cfg_callback(struct octeon_device *oct,
1776                             u32 status __attribute__((unused)),
1777                             void *buf)
1778 {
1779         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1780         struct liquidio_if_cfg_resp *resp;
1781         struct liquidio_if_cfg_context *ctx;
1782
1783         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1784         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1785
1786         oct = lio_get_device(ctx->octeon_id);
1787         if (resp->status)
1788                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
1789                         CVM_CAST64(resp->status));
1790         WRITE_ONCE(ctx->cond, 1);
1791
1792         snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
1793                  resp->cfg_info.liquidio_firmware_version);
1794
1795         /* This barrier is required to be sure that the response has been
1796          * written fully before waking up the handler
1797          */
1798         wmb();
1799
1800         wake_up_interruptible(&ctx->wc);
1801 }
1802
1803 /**
1804  * \brief Select queue based on hash
1805  * @param dev Net device
1806  * @param skb sk_buff structure
1807  * @returns selected queue number
1808  */
1809 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
1810                     void *accel_priv __attribute__((unused)),
1811                     select_queue_fallback_t fallback __attribute__((unused)))
1812 {
1813         u32 qindex = 0;
1814         struct lio *lio;
1815
1816         lio = GET_LIO(dev);
1817         qindex = skb_tx_hash(dev, skb);
1818
1819         return (u16)(qindex % (lio->linfo.num_txpciq));
1820 }
1821
1822 /** Routine to push packets arriving on Octeon interface upto network layer.
1823  * @param oct_id   - octeon device id.
1824  * @param skbuff   - skbuff struct to be passed to network layer.
1825  * @param len      - size of total data received.
1826  * @param rh       - Control header associated with the packet
1827  * @param param    - additional control data with the packet
1828  * @param arg      - farg registered in droq_ops
1829  */
1830 static void
1831 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
1832                      void *skbuff,
1833                      u32 len,
1834                      union octeon_rh *rh,
1835                      void *param,
1836                      void *arg)
1837 {
1838         struct napi_struct *napi = param;
1839         struct sk_buff *skb = (struct sk_buff *)skbuff;
1840         struct skb_shared_hwtstamps *shhwtstamps;
1841         u64 ns;
1842         u16 vtag = 0;
1843         struct net_device *netdev = (struct net_device *)arg;
1844         struct octeon_droq *droq = container_of(param, struct octeon_droq,
1845                                                 napi);
1846         if (netdev) {
1847                 int packet_was_received;
1848                 struct lio *lio = GET_LIO(netdev);
1849                 struct octeon_device *oct = lio->oct_dev;
1850
1851                 /* Do not proceed if the interface is not in RUNNING state. */
1852                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1853                         recv_buffer_free(skb);
1854                         droq->stats.rx_dropped++;
1855                         return;
1856                 }
1857
1858                 skb->dev = netdev;
1859
1860                 skb_record_rx_queue(skb, droq->q_no);
1861                 if (likely(len > MIN_SKB_SIZE)) {
1862                         struct octeon_skb_page_info *pg_info;
1863                         unsigned char *va;
1864
1865                         pg_info = ((struct octeon_skb_page_info *)(skb->cb));
1866                         if (pg_info->page) {
1867                                 /* For Paged allocation use the frags */
1868                                 va = page_address(pg_info->page) +
1869                                         pg_info->page_offset;
1870                                 memcpy(skb->data, va, MIN_SKB_SIZE);
1871                                 skb_put(skb, MIN_SKB_SIZE);
1872                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1873                                                 pg_info->page,
1874                                                 pg_info->page_offset +
1875                                                 MIN_SKB_SIZE,
1876                                                 len - MIN_SKB_SIZE,
1877                                                 LIO_RXBUFFER_SZ);
1878                         }
1879                 } else {
1880                         struct octeon_skb_page_info *pg_info =
1881                                 ((struct octeon_skb_page_info *)(skb->cb));
1882                         skb_copy_to_linear_data(skb, page_address(pg_info->page)
1883                                                 + pg_info->page_offset, len);
1884                         skb_put(skb, len);
1885                         put_page(pg_info->page);
1886                 }
1887
1888                 if (((oct->chip_id == OCTEON_CN66XX) ||
1889                      (oct->chip_id == OCTEON_CN68XX)) &&
1890                     ptp_enable) {
1891                         if (rh->r_dh.has_hwtstamp) {
1892                                 /* timestamp is included from the hardware at
1893                                  * the beginning of the packet.
1894                                  */
1895                                 if (ifstate_check
1896                                     (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
1897                                         /* Nanoseconds are in the first 64-bits
1898                                          * of the packet.
1899                                          */
1900                                         memcpy(&ns, (skb->data), sizeof(ns));
1901                                         shhwtstamps = skb_hwtstamps(skb);
1902                                         shhwtstamps->hwtstamp =
1903                                                 ns_to_ktime(ns +
1904                                                             lio->ptp_adjust);
1905                                 }
1906                                 skb_pull(skb, sizeof(ns));
1907                         }
1908                 }
1909
1910                 skb->protocol = eth_type_trans(skb, skb->dev);
1911                 if ((netdev->features & NETIF_F_RXCSUM) &&
1912                     (((rh->r_dh.encap_on) &&
1913                       (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
1914                      (!(rh->r_dh.encap_on) &&
1915                       (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
1916                         /* checksum has already been verified */
1917                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1918                 else
1919                         skb->ip_summed = CHECKSUM_NONE;
1920
1921                 /* Setting Encapsulation field on basis of status received
1922                  * from the firmware
1923                  */
1924                 if (rh->r_dh.encap_on) {
1925                         skb->encapsulation = 1;
1926                         skb->csum_level = 1;
1927                         droq->stats.rx_vxlan++;
1928                 }
1929
1930                 /* inbound VLAN tag */
1931                 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1932                     (rh->r_dh.vlan != 0)) {
1933                         u16 vid = rh->r_dh.vlan;
1934                         u16 priority = rh->r_dh.priority;
1935
1936                         vtag = priority << 13 | vid;
1937                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
1938                 }
1939
1940                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
1941
1942                 if (packet_was_received) {
1943                         droq->stats.rx_bytes_received += len;
1944                         droq->stats.rx_pkts_received++;
1945                         netdev->last_rx = jiffies;
1946                 } else {
1947                         droq->stats.rx_dropped++;
1948                         netif_info(lio, rx_err, lio->netdev,
1949                                    "droq:%d  error rx_dropped:%llu\n",
1950                                    droq->q_no, droq->stats.rx_dropped);
1951                 }
1952
1953         } else {
1954                 recv_buffer_free(skb);
1955         }
1956 }
1957
1958 /**
1959  * \brief wrapper for calling napi_schedule
1960  * @param param parameters to pass to napi_schedule
1961  *
1962  * Used when scheduling on different CPUs
1963  */
1964 static void napi_schedule_wrapper(void *param)
1965 {
1966         struct napi_struct *napi = param;
1967
1968         napi_schedule(napi);
1969 }
1970
1971 /**
1972  * \brief callback when receive interrupt occurs and we are in NAPI mode
1973  * @param arg pointer to octeon output queue
1974  */
1975 static void liquidio_napi_drv_callback(void *arg)
1976 {
1977         struct octeon_droq *droq = arg;
1978         int this_cpu = smp_processor_id();
1979
1980         if (droq->cpu_id == this_cpu) {
1981                 napi_schedule(&droq->napi);
1982         } else {
1983                 struct call_single_data *csd = &droq->csd;
1984
1985                 csd->func = napi_schedule_wrapper;
1986                 csd->info = &droq->napi;
1987                 csd->flags = 0;
1988
1989                 smp_call_function_single_async(droq->cpu_id, csd);
1990         }
1991 }
1992
1993 /**
1994  * \brief Entry point for NAPI polling
1995  * @param napi NAPI structure
1996  * @param budget maximum number of items to process
1997  */
1998 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
1999 {
2000         struct octeon_droq *droq;
2001         int work_done;
2002         int tx_done = 0, iq_no;
2003         struct octeon_instr_queue *iq;
2004         struct octeon_device *oct;
2005
2006         droq = container_of(napi, struct octeon_droq, napi);
2007         oct = droq->oct_dev;
2008         iq_no = droq->q_no;
2009         /* Handle Droq descriptors */
2010         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
2011                                                  POLL_EVENT_PROCESS_PKTS,
2012                                                  budget);
2013
2014         /* Flush the instruction queue */
2015         iq = oct->instr_queue[iq_no];
2016         if (iq) {
2017                 /* Process iq buffers with in the budget limits */
2018                 tx_done = octeon_flush_iq(oct, iq, 1, budget);
2019                 /* Update iq read-index rather than waiting for next interrupt.
2020                  * Return back if tx_done is false.
2021                  */
2022                 update_txq_status(oct, iq_no);
2023                 /*tx_done = (iq->flush_index == iq->octeon_read_index);*/
2024         } else {
2025                 dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
2026                         __func__, iq_no);
2027         }
2028
2029         if ((work_done < budget) && (tx_done)) {
2030                 napi_complete(napi);
2031                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
2032                                              POLL_EVENT_ENABLE_INTR, 0);
2033                 return 0;
2034         }
2035
2036         return (!tx_done) ? (budget) : (work_done);
2037 }
2038
2039 /**
2040  * \brief Setup input and output queues
2041  * @param octeon_dev octeon device
2042  * @param ifidx  Interface Index
2043  *
2044  * Note: Queues are with respect to the octeon device. Thus
2045  * an input queue is for egress packets, and output queues
2046  * are for ingress packets.
2047  */
2048 static inline int setup_io_queues(struct octeon_device *octeon_dev,
2049                                   int ifidx)
2050 {
2051         struct octeon_droq_ops droq_ops;
2052         struct net_device *netdev;
2053         static int cpu_id;
2054         static int cpu_id_modulus;
2055         struct octeon_droq *droq;
2056         struct napi_struct *napi;
2057         int q, q_no, retval = 0;
2058         struct lio *lio;
2059         int num_tx_descs;
2060
2061         netdev = octeon_dev->props[ifidx].netdev;
2062
2063         lio = GET_LIO(netdev);
2064
2065         memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
2066
2067         droq_ops.fptr = liquidio_push_packet;
2068         droq_ops.farg = (void *)netdev;
2069
2070         droq_ops.poll_mode = 1;
2071         droq_ops.napi_fn = liquidio_napi_drv_callback;
2072         cpu_id = 0;
2073         cpu_id_modulus = num_present_cpus();
2074
2075         /* set up DROQs. */
2076         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
2077                 q_no = lio->linfo.rxpciq[q].s.q_no;
2078                 dev_dbg(&octeon_dev->pci_dev->dev,
2079                         "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2080                         q, q_no);
2081                 retval = octeon_setup_droq(octeon_dev, q_no,
2082                                            CFG_GET_NUM_RX_DESCS_NIC_IF
2083                                                    (octeon_get_conf(octeon_dev),
2084                                                    lio->ifidx),
2085                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2086                                                    (octeon_get_conf(octeon_dev),
2087                                                    lio->ifidx), NULL);
2088                 if (retval) {
2089                         dev_err(&octeon_dev->pci_dev->dev,
2090                                 "%s : Runtime DROQ(RxQ) creation failed.\n",
2091                                 __func__);
2092                         return 1;
2093                 }
2094
2095                 droq = octeon_dev->droq[q_no];
2096                 napi = &droq->napi;
2097                 dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n",
2098                         (u64)netdev, (u64)octeon_dev, octeon_dev->pf_num);
2099                 netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
2100
2101                 /* designate a CPU for this droq */
2102                 droq->cpu_id = cpu_id;
2103                 cpu_id++;
2104                 if (cpu_id >= cpu_id_modulus)
2105                         cpu_id = 0;
2106
2107                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
2108         }
2109
2110         /* set up IQs. */
2111         for (q = 0; q < lio->linfo.num_txpciq; q++) {
2112                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2113                                                            (octeon_dev),
2114                                                            lio->ifidx);
2115                 retval = octeon_setup_iq(octeon_dev, ifidx, q,
2116                                          lio->linfo.txpciq[q], num_tx_descs,
2117                                          netdev_get_tx_queue(netdev, q));
2118                 if (retval) {
2119                         dev_err(&octeon_dev->pci_dev->dev,
2120                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2121                                 __func__);
2122                         return 1;
2123                 }
2124         }
2125
2126         return 0;
2127 }
2128
2129 /**
2130  * \brief Poll routine for checking transmit queue status
2131  * @param work work_struct data structure
2132  */
2133 static void octnet_poll_check_txq_status(struct work_struct *work)
2134 {
2135         struct cavium_wk *wk = (struct cavium_wk *)work;
2136         struct lio *lio = (struct lio *)wk->ctxptr;
2137
2138         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2139                 return;
2140
2141         check_txq_status(lio);
2142         queue_delayed_work(lio->txq_status_wq.wq,
2143                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2144 }
2145
2146 /**
2147  * \brief Sets up the txq poll check
2148  * @param netdev network device
2149  */
2150 static inline void setup_tx_poll_fn(struct net_device *netdev)
2151 {
2152         struct lio *lio = GET_LIO(netdev);
2153         struct octeon_device *oct = lio->oct_dev;
2154
2155         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2156                                                 WQ_MEM_RECLAIM, 0);
2157         if (!lio->txq_status_wq.wq) {
2158                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2159                 return;
2160         }
2161         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2162                           octnet_poll_check_txq_status);
2163         lio->txq_status_wq.wk.ctxptr = lio;
2164         queue_delayed_work(lio->txq_status_wq.wq,
2165                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2166 }
2167
2168 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2169 {
2170         struct lio *lio = GET_LIO(netdev);
2171
2172         cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2173         destroy_workqueue(lio->txq_status_wq.wq);
2174 }
2175
2176 /**
2177  * \brief Net device open for LiquidIO
2178  * @param netdev network device
2179  */
2180 static int liquidio_open(struct net_device *netdev)
2181 {
2182         struct lio *lio = GET_LIO(netdev);
2183         struct octeon_device *oct = lio->oct_dev;
2184         struct napi_struct *napi, *n;
2185
2186         if (oct->props[lio->ifidx].napi_enabled == 0) {
2187                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2188                         napi_enable(napi);
2189
2190                 oct->props[lio->ifidx].napi_enabled = 1;
2191         }
2192
2193         oct_ptp_open(netdev);
2194
2195         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2196
2197         setup_tx_poll_fn(netdev);
2198
2199         start_txq(netdev);
2200
2201         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2202
2203         /* tell Octeon to start forwarding packets to host */
2204         send_rx_ctrl_cmd(lio, 1);
2205
2206         /* Ready for link status updates */
2207         lio->intf_open = 1;
2208
2209         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2210                  netdev->name);
2211
2212         return 0;
2213 }
2214
2215 /**
2216  * \brief Net device stop for LiquidIO
2217  * @param netdev network device
2218  */
2219 static int liquidio_stop(struct net_device *netdev)
2220 {
2221         struct lio *lio = GET_LIO(netdev);
2222         struct octeon_device *oct = lio->oct_dev;
2223
2224         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2225
2226         netif_tx_disable(netdev);
2227
2228         /* Inform that netif carrier is down */
2229         netif_carrier_off(netdev);
2230         lio->intf_open = 0;
2231         lio->linfo.link.s.link_up = 0;
2232         lio->link_changes++;
2233
2234         /* Pause for a moment and wait for Octeon to flush out (to the wire) any
2235          * egress packets that are in-flight.
2236          */
2237         set_current_state(TASK_INTERRUPTIBLE);
2238         schedule_timeout(msecs_to_jiffies(100));
2239
2240         /* Now it should be safe to tell Octeon that nic interface is down. */
2241         send_rx_ctrl_cmd(lio, 0);
2242
2243         cleanup_tx_poll_fn(netdev);
2244
2245         if (lio->ptp_clock) {
2246                 ptp_clock_unregister(lio->ptp_clock);
2247                 lio->ptp_clock = NULL;
2248         }
2249
2250         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2251
2252         return 0;
2253 }
2254
2255 /**
2256  * \brief Converts a mask based on net device flags
2257  * @param netdev network device
2258  *
2259  * This routine generates a octnet_ifflags mask from the net device flags
2260  * received from the OS.
2261  */
2262 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2263 {
2264         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2265
2266         if (netdev->flags & IFF_PROMISC)
2267                 f |= OCTNET_IFFLAG_PROMISC;
2268
2269         if (netdev->flags & IFF_ALLMULTI)
2270                 f |= OCTNET_IFFLAG_ALLMULTI;
2271
2272         if (netdev->flags & IFF_MULTICAST) {
2273                 f |= OCTNET_IFFLAG_MULTICAST;
2274
2275                 /* Accept all multicast addresses if there are more than we
2276                  * can handle
2277                  */
2278                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2279                         f |= OCTNET_IFFLAG_ALLMULTI;
2280         }
2281
2282         if (netdev->flags & IFF_BROADCAST)
2283                 f |= OCTNET_IFFLAG_BROADCAST;
2284
2285         return f;
2286 }
2287
2288 /**
2289  * \brief Net device set_multicast_list
2290  * @param netdev network device
2291  */
2292 static void liquidio_set_mcast_list(struct net_device *netdev)
2293 {
2294         struct lio *lio = GET_LIO(netdev);
2295         struct octeon_device *oct = lio->oct_dev;
2296         struct octnic_ctrl_pkt nctrl;
2297         struct netdev_hw_addr *ha;
2298         u64 *mc;
2299         int ret;
2300         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2301
2302         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2303
2304         /* Create a ctrl pkt command to be sent to core app. */
2305         nctrl.ncmd.u64 = 0;
2306         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2307         nctrl.ncmd.s.param1 = get_new_flags(netdev);
2308         nctrl.ncmd.s.param2 = mc_count;
2309         nctrl.ncmd.s.more = mc_count;
2310         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2311         nctrl.netpndev = (u64)netdev;
2312         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2313
2314         /* copy all the addresses into the udd */
2315         mc = &nctrl.udd[0];
2316         netdev_for_each_mc_addr(ha, netdev) {
2317                 *mc = 0;
2318                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2319                 /* no need to swap bytes */
2320
2321                 if (++mc > &nctrl.udd[mc_count])
2322                         break;
2323         }
2324
2325         /* Apparently, any activity in this call from the kernel has to
2326          * be atomic. So we won't wait for response.
2327          */
2328         nctrl.wait_time = 0;
2329
2330         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2331         if (ret < 0) {
2332                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2333                         ret);
2334         }
2335 }
2336
2337 /**
2338  * \brief Net device set_mac_address
2339  * @param netdev network device
2340  */
2341 static int liquidio_set_mac(struct net_device *netdev, void *p)
2342 {
2343         int ret = 0;
2344         struct lio *lio = GET_LIO(netdev);
2345         struct octeon_device *oct = lio->oct_dev;
2346         struct sockaddr *addr = (struct sockaddr *)p;
2347         struct octnic_ctrl_pkt nctrl;
2348
2349         if (!is_valid_ether_addr(addr->sa_data))
2350                 return -EADDRNOTAVAIL;
2351
2352         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2353
2354         nctrl.ncmd.u64 = 0;
2355         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2356         nctrl.ncmd.s.param1 = 0;
2357         nctrl.ncmd.s.more = 1;
2358         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2359         nctrl.netpndev = (u64)netdev;
2360         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2361         nctrl.wait_time = 100;
2362
2363         nctrl.udd[0] = 0;
2364         /* The MAC Address is presented in network byte order. */
2365         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2366
2367         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2368         if (ret < 0) {
2369                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2370                 return -ENOMEM;
2371         }
2372         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2373         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2374
2375         return 0;
2376 }
2377
2378 /**
2379  * \brief Net device get_stats
2380  * @param netdev network device
2381  */
2382 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2383 {
2384         struct lio *lio = GET_LIO(netdev);
2385         struct net_device_stats *stats = &netdev->stats;
2386         struct octeon_device *oct;
2387         u64 pkts = 0, drop = 0, bytes = 0;
2388         struct oct_droq_stats *oq_stats;
2389         struct oct_iq_stats *iq_stats;
2390         int i, iq_no, oq_no;
2391
2392         oct = lio->oct_dev;
2393
2394         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2395                 iq_no = lio->linfo.txpciq[i].s.q_no;
2396                 iq_stats = &oct->instr_queue[iq_no]->stats;
2397                 pkts += iq_stats->tx_done;
2398                 drop += iq_stats->tx_dropped;
2399                 bytes += iq_stats->tx_tot_bytes;
2400         }
2401
2402         stats->tx_packets = pkts;
2403         stats->tx_bytes = bytes;
2404         stats->tx_dropped = drop;
2405
2406         pkts = 0;
2407         drop = 0;
2408         bytes = 0;
2409
2410         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2411                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2412                 oq_stats = &oct->droq[oq_no]->stats;
2413                 pkts += oq_stats->rx_pkts_received;
2414                 drop += (oq_stats->rx_dropped +
2415                          oq_stats->dropped_nodispatch +
2416                          oq_stats->dropped_toomany +
2417                          oq_stats->dropped_nomem);
2418                 bytes += oq_stats->rx_bytes_received;
2419         }
2420
2421         stats->rx_bytes = bytes;
2422         stats->rx_packets = pkts;
2423         stats->rx_dropped = drop;
2424
2425         return stats;
2426 }
2427
2428 /**
2429  * \brief Net device change_mtu
2430  * @param netdev network device
2431  */
2432 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2433 {
2434         struct lio *lio = GET_LIO(netdev);
2435         struct octeon_device *oct = lio->oct_dev;
2436         struct octnic_ctrl_pkt nctrl;
2437         int ret = 0;
2438
2439         /* Limit the MTU to make sure the ethernet packets are between 68 bytes
2440          * and 16000 bytes
2441          */
2442         if ((new_mtu < LIO_MIN_MTU_SIZE) ||
2443             (new_mtu > LIO_MAX_MTU_SIZE)) {
2444                 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2445                 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2446                         LIO_MIN_MTU_SIZE, LIO_MAX_MTU_SIZE);
2447                 return -EINVAL;
2448         }
2449
2450         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2451
2452         nctrl.ncmd.u64 = 0;
2453         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2454         nctrl.ncmd.s.param1 = new_mtu;
2455         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2456         nctrl.wait_time = 100;
2457         nctrl.netpndev = (u64)netdev;
2458         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2459
2460         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2461         if (ret < 0) {
2462                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2463                 return -1;
2464         }
2465
2466         lio->mtu = new_mtu;
2467
2468         return 0;
2469 }
2470
2471 /**
2472  * \brief Handler for SIOCSHWTSTAMP ioctl
2473  * @param netdev network device
2474  * @param ifr interface request
2475  * @param cmd command
2476  */
2477 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2478 {
2479         struct hwtstamp_config conf;
2480         struct lio *lio = GET_LIO(netdev);
2481
2482         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2483                 return -EFAULT;
2484
2485         if (conf.flags)
2486                 return -EINVAL;
2487
2488         switch (conf.tx_type) {
2489         case HWTSTAMP_TX_ON:
2490         case HWTSTAMP_TX_OFF:
2491                 break;
2492         default:
2493                 return -ERANGE;
2494         }
2495
2496         switch (conf.rx_filter) {
2497         case HWTSTAMP_FILTER_NONE:
2498                 break;
2499         case HWTSTAMP_FILTER_ALL:
2500         case HWTSTAMP_FILTER_SOME:
2501         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2502         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2503         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2504         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2505         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2506         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2507         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2508         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2509         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2510         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2511         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2512         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2513                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2514                 break;
2515         default:
2516                 return -ERANGE;
2517         }
2518
2519         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2520                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2521
2522         else
2523                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2524
2525         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2526 }
2527
2528 /**
2529  * \brief ioctl handler
2530  * @param netdev network device
2531  * @param ifr interface request
2532  * @param cmd command
2533  */
2534 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2535 {
2536         switch (cmd) {
2537         case SIOCSHWTSTAMP:
2538                 return hwtstamp_ioctl(netdev, ifr);
2539         default:
2540                 return -EOPNOTSUPP;
2541         }
2542 }
2543
2544 /**
2545  * \brief handle a Tx timestamp response
2546  * @param status response status
2547  * @param buf pointer to skb
2548  */
2549 static void handle_timestamp(struct octeon_device *oct,
2550                              u32 status,
2551                              void *buf)
2552 {
2553         struct octnet_buf_free_info *finfo;
2554         struct octeon_soft_command *sc;
2555         struct oct_timestamp_resp *resp;
2556         struct lio *lio;
2557         struct sk_buff *skb = (struct sk_buff *)buf;
2558
2559         finfo = (struct octnet_buf_free_info *)skb->cb;
2560         lio = finfo->lio;
2561         sc = finfo->sc;
2562         oct = lio->oct_dev;
2563         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2564
2565         if (status != OCTEON_REQUEST_DONE) {
2566                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2567                         CVM_CAST64(status));
2568                 resp->timestamp = 0;
2569         }
2570
2571         octeon_swap_8B_data(&resp->timestamp, 1);
2572
2573         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2574                 struct skb_shared_hwtstamps ts;
2575                 u64 ns = resp->timestamp;
2576
2577                 netif_info(lio, tx_done, lio->netdev,
2578                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2579                            skb, (unsigned long long)ns);
2580                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2581                 skb_tstamp_tx(skb, &ts);
2582         }
2583
2584         octeon_free_soft_command(oct, sc);
2585         tx_buffer_free(skb);
2586 }
2587
2588 /* \brief Send a data packet that will be timestamped
2589  * @param oct octeon device
2590  * @param ndata pointer to network data
2591  * @param finfo pointer to private network data
2592  */
2593 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2594                                          struct octnic_data_pkt *ndata,
2595                                          struct octnet_buf_free_info *finfo)
2596 {
2597         int retval;
2598         struct octeon_soft_command *sc;
2599         struct lio *lio;
2600         int ring_doorbell;
2601         u32 len;
2602
2603         lio = finfo->lio;
2604
2605         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2606                                             sizeof(struct oct_timestamp_resp));
2607         finfo->sc = sc;
2608
2609         if (!sc) {
2610                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2611                 return IQ_SEND_FAILED;
2612         }
2613
2614         if (ndata->reqtype == REQTYPE_NORESP_NET)
2615                 ndata->reqtype = REQTYPE_RESP_NET;
2616         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2617                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2618
2619         sc->callback = handle_timestamp;
2620         sc->callback_arg = finfo->skb;
2621         sc->iq_no = ndata->q_no;
2622
2623         len = (u32)((struct octeon_instr_ih2 *)(&sc->cmd.cmd2.ih2))->dlengsz;
2624
2625         ring_doorbell = 1;
2626         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2627                                      sc, len, ndata->reqtype);
2628
2629         if (retval == IQ_SEND_FAILED) {
2630                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2631                         retval);
2632                 octeon_free_soft_command(oct, sc);
2633         } else {
2634                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2635         }
2636
2637         return retval;
2638 }
2639
2640 /** \brief Transmit networks packets to the Octeon interface
2641  * @param skbuff   skbuff struct to be passed to network layer.
2642  * @param netdev    pointer to network device
2643  * @returns whether the packet was transmitted to the device okay or not
2644  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2645  */
2646 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2647 {
2648         struct lio *lio;
2649         struct octnet_buf_free_info *finfo;
2650         union octnic_cmd_setup cmdsetup;
2651         struct octnic_data_pkt ndata;
2652         struct octeon_device *oct;
2653         struct oct_iq_stats *stats;
2654         struct octeon_instr_irh *irh;
2655         union tx_info *tx_info;
2656         int status = 0;
2657         int q_idx = 0, iq_no = 0;
2658         int j;
2659         u64 dptr = 0;
2660         u32 tag = 0;
2661
2662         lio = GET_LIO(netdev);
2663         oct = lio->oct_dev;
2664
2665         if (netif_is_multiqueue(netdev)) {
2666                 q_idx = skb->queue_mapping;
2667                 q_idx = (q_idx % (lio->linfo.num_txpciq));
2668                 tag = q_idx;
2669                 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2670         } else {
2671                 iq_no = lio->txq;
2672         }
2673
2674         stats = &oct->instr_queue[iq_no]->stats;
2675
2676         /* Check for all conditions in which the current packet cannot be
2677          * transmitted.
2678          */
2679         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2680             (!lio->linfo.link.s.link_up) ||
2681             (skb->len <= 0)) {
2682                 netif_info(lio, tx_err, lio->netdev,
2683                            "Transmit failed link_status : %d\n",
2684                            lio->linfo.link.s.link_up);
2685                 goto lio_xmit_failed;
2686         }
2687
2688         /* Use space in skb->cb to store info used to unmap and
2689          * free the buffers.
2690          */
2691         finfo = (struct octnet_buf_free_info *)skb->cb;
2692         finfo->lio = lio;
2693         finfo->skb = skb;
2694         finfo->sc = NULL;
2695
2696         /* Prepare the attributes for the data to be passed to OSI. */
2697         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2698
2699         ndata.buf = (void *)finfo;
2700
2701         ndata.q_no = iq_no;
2702
2703         if (netif_is_multiqueue(netdev)) {
2704                 if (octnet_iq_is_full(oct, ndata.q_no)) {
2705                         /* defer sending if queue is full */
2706                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2707                                    ndata.q_no);
2708                         stats->tx_iq_busy++;
2709                         return NETDEV_TX_BUSY;
2710                 }
2711         } else {
2712                 if (octnet_iq_is_full(oct, lio->txq)) {
2713                         /* defer sending if queue is full */
2714                         stats->tx_iq_busy++;
2715                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2716                                    lio->txq);
2717                         return NETDEV_TX_BUSY;
2718                 }
2719         }
2720         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2721          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2722          */
2723
2724         ndata.datasize = skb->len;
2725
2726         cmdsetup.u64 = 0;
2727         cmdsetup.s.iq_no = iq_no;
2728
2729         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2730                 if (skb->encapsulation) {
2731                         cmdsetup.s.tnl_csum = 1;
2732                         stats->tx_vxlan++;
2733                 } else {
2734                         cmdsetup.s.transport_csum = 1;
2735                 }
2736         }
2737         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2738                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2739                 cmdsetup.s.timestamp = 1;
2740         }
2741
2742         if (skb_shinfo(skb)->nr_frags == 0) {
2743                 cmdsetup.s.u.datasize = skb->len;
2744                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2745
2746                 /* Offload checksum calculation for TCP/UDP packets */
2747                 dptr = dma_map_single(&oct->pci_dev->dev,
2748                                       skb->data,
2749                                       skb->len,
2750                                       DMA_TO_DEVICE);
2751                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2752                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2753                                 __func__);
2754                         return NETDEV_TX_BUSY;
2755                 }
2756
2757                 ndata.cmd.cmd2.dptr = dptr;
2758                 finfo->dptr = dptr;
2759                 ndata.reqtype = REQTYPE_NORESP_NET;
2760
2761         } else {
2762                 int i, frags;
2763                 struct skb_frag_struct *frag;
2764                 struct octnic_gather *g;
2765
2766                 spin_lock(&lio->glist_lock[q_idx]);
2767                 g = (struct octnic_gather *)
2768                         list_delete_head(&lio->glist[q_idx]);
2769                 spin_unlock(&lio->glist_lock[q_idx]);
2770
2771                 if (!g) {
2772                         netif_info(lio, tx_err, lio->netdev,
2773                                    "Transmit scatter gather: glist null!\n");
2774                         goto lio_xmit_failed;
2775                 }
2776
2777                 cmdsetup.s.gather = 1;
2778                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2779                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2780
2781                 memset(g->sg, 0, g->sg_size);
2782
2783                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2784                                                  skb->data,
2785                                                  (skb->len - skb->data_len),
2786                                                  DMA_TO_DEVICE);
2787                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2788                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2789                                 __func__);
2790                         return NETDEV_TX_BUSY;
2791                 }
2792                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2793
2794                 frags = skb_shinfo(skb)->nr_frags;
2795                 i = 1;
2796                 while (frags--) {
2797                         frag = &skb_shinfo(skb)->frags[i - 1];
2798
2799                         g->sg[(i >> 2)].ptr[(i & 3)] =
2800                                 dma_map_page(&oct->pci_dev->dev,
2801                                              frag->page.p,
2802                                              frag->page_offset,
2803                                              frag->size,
2804                                              DMA_TO_DEVICE);
2805
2806                         if (dma_mapping_error(&oct->pci_dev->dev,
2807                                               g->sg[i >> 2].ptr[i & 3])) {
2808                                 dma_unmap_single(&oct->pci_dev->dev,
2809                                                  g->sg[0].ptr[0],
2810                                                  skb->len - skb->data_len,
2811                                                  DMA_TO_DEVICE);
2812                                 for (j = 1; j < i; j++) {
2813                                         frag = &skb_shinfo(skb)->frags[j - 1];
2814                                         dma_unmap_page(&oct->pci_dev->dev,
2815                                                        g->sg[j >> 2].ptr[j & 3],
2816                                                        frag->size,
2817                                                        DMA_TO_DEVICE);
2818                                 }
2819                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2820                                         __func__);
2821                                 return NETDEV_TX_BUSY;
2822                         }
2823
2824                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2825                         i++;
2826                 }
2827
2828                 dma_sync_single_for_device(&oct->pci_dev->dev, g->sg_dma_ptr,
2829                                            g->sg_size, DMA_TO_DEVICE);
2830                 dptr = g->sg_dma_ptr;
2831
2832                 ndata.cmd.cmd2.dptr = dptr;
2833                 finfo->dptr = dptr;
2834                 finfo->g = g;
2835
2836                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2837         }
2838
2839         irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2840         tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2841
2842         if (skb_shinfo(skb)->gso_size) {
2843                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2844                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2845                 stats->tx_gso++;
2846         }
2847
2848         /* HW insert VLAN tag */
2849         if (skb_vlan_tag_present(skb)) {
2850                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2851                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2852         }
2853
2854         if (unlikely(cmdsetup.s.timestamp))
2855                 status = send_nic_timestamp_pkt(oct, &ndata, finfo);
2856         else
2857                 status = octnet_send_nic_data_pkt(oct, &ndata);
2858         if (status == IQ_SEND_FAILED)
2859                 goto lio_xmit_failed;
2860
2861         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2862
2863         if (status == IQ_SEND_STOP)
2864                 stop_q(lio->netdev, q_idx);
2865
2866         netif_trans_update(netdev);
2867
2868         if (skb_shinfo(skb)->gso_size)
2869                 stats->tx_done += skb_shinfo(skb)->gso_segs;
2870         else
2871                 stats->tx_done++;
2872         stats->tx_tot_bytes += skb->len;
2873
2874         return NETDEV_TX_OK;
2875
2876 lio_xmit_failed:
2877         stats->tx_dropped++;
2878         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2879                    iq_no, stats->tx_dropped);
2880         if (dptr)
2881                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2882                                  ndata.datasize, DMA_TO_DEVICE);
2883         tx_buffer_free(skb);
2884         return NETDEV_TX_OK;
2885 }
2886
2887 /** \brief Network device Tx timeout
2888  * @param netdev    pointer to network device
2889  */
2890 static void liquidio_tx_timeout(struct net_device *netdev)
2891 {
2892         struct lio *lio;
2893
2894         lio = GET_LIO(netdev);
2895
2896         netif_info(lio, tx_err, lio->netdev,
2897                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2898                    netdev->stats.tx_dropped);
2899         netif_trans_update(netdev);
2900         txqs_wake(netdev);
2901 }
2902
2903 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2904                                     __be16 proto __attribute__((unused)),
2905                                     u16 vid)
2906 {
2907         struct lio *lio = GET_LIO(netdev);
2908         struct octeon_device *oct = lio->oct_dev;
2909         struct octnic_ctrl_pkt nctrl;
2910         int ret = 0;
2911
2912         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2913
2914         nctrl.ncmd.u64 = 0;
2915         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2916         nctrl.ncmd.s.param1 = vid;
2917         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2918         nctrl.wait_time = 100;
2919         nctrl.netpndev = (u64)netdev;
2920         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2921
2922         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2923         if (ret < 0) {
2924                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2925                         ret);
2926         }
2927
2928         return ret;
2929 }
2930
2931 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2932                                      __be16 proto __attribute__((unused)),
2933                                      u16 vid)
2934 {
2935         struct lio *lio = GET_LIO(netdev);
2936         struct octeon_device *oct = lio->oct_dev;
2937         struct octnic_ctrl_pkt nctrl;
2938         int ret = 0;
2939
2940         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2941
2942         nctrl.ncmd.u64 = 0;
2943         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2944         nctrl.ncmd.s.param1 = vid;
2945         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2946         nctrl.wait_time = 100;
2947         nctrl.netpndev = (u64)netdev;
2948         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2949
2950         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2951         if (ret < 0) {
2952                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2953                         ret);
2954         }
2955         return ret;
2956 }
2957
2958 /** Sending command to enable/disable RX checksum offload
2959  * @param netdev                pointer to network device
2960  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2961  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2962  *                              OCTNET_CMD_RXCSUM_DISABLE
2963  * @returns                     SUCCESS or FAILURE
2964  */
2965 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2966                                        u8 rx_cmd)
2967 {
2968         struct lio *lio = GET_LIO(netdev);
2969         struct octeon_device *oct = lio->oct_dev;
2970         struct octnic_ctrl_pkt nctrl;
2971         int ret = 0;
2972
2973         nctrl.ncmd.u64 = 0;
2974         nctrl.ncmd.s.cmd = command;
2975         nctrl.ncmd.s.param1 = rx_cmd;
2976         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2977         nctrl.wait_time = 100;
2978         nctrl.netpndev = (u64)netdev;
2979         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2980
2981         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2982         if (ret < 0) {
2983                 dev_err(&oct->pci_dev->dev,
2984                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2985                         ret);
2986         }
2987         return ret;
2988 }
2989
2990 /** Sending command to add/delete VxLAN UDP port to firmware
2991  * @param netdev                pointer to network device
2992  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2993  * @param vxlan_port            VxLAN port to be added or deleted
2994  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2995  *                              OCTNET_CMD_VXLAN_PORT_DEL
2996  * @returns                     SUCCESS or FAILURE
2997  */
2998 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2999                                        u16 vxlan_port, u8 vxlan_cmd_bit)
3000 {
3001         struct lio *lio = GET_LIO(netdev);
3002         struct octeon_device *oct = lio->oct_dev;
3003         struct octnic_ctrl_pkt nctrl;
3004         int ret = 0;
3005
3006         nctrl.ncmd.u64 = 0;
3007         nctrl.ncmd.s.cmd = command;
3008         nctrl.ncmd.s.more = vxlan_cmd_bit;
3009         nctrl.ncmd.s.param1 = vxlan_port;
3010         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3011         nctrl.wait_time = 100;
3012         nctrl.netpndev = (u64)netdev;
3013         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3014
3015         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3016         if (ret < 0) {
3017                 dev_err(&oct->pci_dev->dev,
3018                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
3019                         ret);
3020         }
3021         return ret;
3022 }
3023
3024 /** \brief Net device fix features
3025  * @param netdev  pointer to network device
3026  * @param request features requested
3027  * @returns updated features list
3028  */
3029 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3030                                                netdev_features_t request)
3031 {
3032         struct lio *lio = netdev_priv(netdev);
3033
3034         if ((request & NETIF_F_RXCSUM) &&
3035             !(lio->dev_capability & NETIF_F_RXCSUM))
3036                 request &= ~NETIF_F_RXCSUM;
3037
3038         if ((request & NETIF_F_HW_CSUM) &&
3039             !(lio->dev_capability & NETIF_F_HW_CSUM))
3040                 request &= ~NETIF_F_HW_CSUM;
3041
3042         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3043                 request &= ~NETIF_F_TSO;
3044
3045         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3046                 request &= ~NETIF_F_TSO6;
3047
3048         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3049                 request &= ~NETIF_F_LRO;
3050
3051         /*Disable LRO if RXCSUM is off */
3052         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3053             (lio->dev_capability & NETIF_F_LRO))
3054                 request &= ~NETIF_F_LRO;
3055
3056         return request;
3057 }
3058
3059 /** \brief Net device set features
3060  * @param netdev  pointer to network device
3061  * @param features features to enable/disable
3062  */
3063 static int liquidio_set_features(struct net_device *netdev,
3064                                  netdev_features_t features)
3065 {
3066         struct lio *lio = netdev_priv(netdev);
3067
3068         if (!((netdev->features ^ features) & NETIF_F_LRO))
3069                 return 0;
3070
3071         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3072                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3073                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3074         else if (!(features & NETIF_F_LRO) &&
3075                  (lio->dev_capability & NETIF_F_LRO))
3076                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3077                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3078
3079         /* Sending command to firmware to enable/disable RX checksum
3080          * offload settings using ethtool
3081          */
3082         if (!(netdev->features & NETIF_F_RXCSUM) &&
3083             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3084             (features & NETIF_F_RXCSUM))
3085                 liquidio_set_rxcsum_command(netdev,
3086                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
3087                                             OCTNET_CMD_RXCSUM_ENABLE);
3088         else if ((netdev->features & NETIF_F_RXCSUM) &&
3089                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3090                  !(features & NETIF_F_RXCSUM))
3091                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3092                                             OCTNET_CMD_RXCSUM_DISABLE);
3093
3094         return 0;
3095 }
3096
3097 static void liquidio_add_vxlan_port(struct net_device *netdev,
3098                                     struct udp_tunnel_info *ti)
3099 {
3100         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3101                 return;
3102
3103         liquidio_vxlan_port_command(netdev,
3104                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3105                                     htons(ti->port),
3106                                     OCTNET_CMD_VXLAN_PORT_ADD);
3107 }
3108
3109 static void liquidio_del_vxlan_port(struct net_device *netdev,
3110                                     struct udp_tunnel_info *ti)
3111 {
3112         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3113                 return;
3114
3115         liquidio_vxlan_port_command(netdev,
3116                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3117                                     htons(ti->port),
3118                                     OCTNET_CMD_VXLAN_PORT_DEL);
3119 }
3120
3121 static struct net_device_ops lionetdevops = {
3122         .ndo_open               = liquidio_open,
3123         .ndo_stop               = liquidio_stop,
3124         .ndo_start_xmit         = liquidio_xmit,
3125         .ndo_get_stats          = liquidio_get_stats,
3126         .ndo_set_mac_address    = liquidio_set_mac,
3127         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3128         .ndo_tx_timeout         = liquidio_tx_timeout,
3129
3130         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3131         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3132         .ndo_change_mtu         = liquidio_change_mtu,
3133         .ndo_do_ioctl           = liquidio_ioctl,
3134         .ndo_fix_features       = liquidio_fix_features,
3135         .ndo_set_features       = liquidio_set_features,
3136         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3137         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3138 };
3139
3140 /** \brief Entry point for the liquidio module
3141  */
3142 static int __init liquidio_init(void)
3143 {
3144         int i;
3145         struct handshake *hs;
3146
3147         init_completion(&first_stage);
3148
3149         octeon_init_device_list(conf_type);
3150
3151         if (liquidio_init_pci())
3152                 return -EINVAL;
3153
3154         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3155
3156         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3157                 hs = &handshake[i];
3158                 if (hs->pci_dev) {
3159                         wait_for_completion(&hs->init);
3160                         if (!hs->init_ok) {
3161                                 /* init handshake failed */
3162                                 dev_err(&hs->pci_dev->dev,
3163                                         "Failed to init device\n");
3164                                 liquidio_deinit_pci();
3165                                 return -EIO;
3166                         }
3167                 }
3168         }
3169
3170         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3171                 hs = &handshake[i];
3172                 if (hs->pci_dev) {
3173                         wait_for_completion_timeout(&hs->started,
3174                                                     msecs_to_jiffies(30000));
3175                         if (!hs->started_ok) {
3176                                 /* starter handshake failed */
3177                                 dev_err(&hs->pci_dev->dev,
3178                                         "Firmware failed to start\n");
3179                                 liquidio_deinit_pci();
3180                                 return -EIO;
3181                         }
3182                 }
3183         }
3184
3185         return 0;
3186 }
3187
3188 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3189 {
3190         struct octeon_device *oct = (struct octeon_device *)buf;
3191         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3192         int gmxport = 0;
3193         union oct_link_status *ls;
3194         int i;
3195
3196         if (recv_pkt->buffer_size[0] != sizeof(*ls)) {
3197                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3198                         recv_pkt->buffer_size[0],
3199                         recv_pkt->rh.r_nic_info.gmxport);
3200                 goto nic_info_err;
3201         }
3202
3203         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3204         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3205
3206         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3207         for (i = 0; i < oct->ifcount; i++) {
3208                 if (oct->props[i].gmxport == gmxport) {
3209                         update_link_status(oct->props[i].netdev, ls);
3210                         break;
3211                 }
3212         }
3213
3214 nic_info_err:
3215         for (i = 0; i < recv_pkt->buffer_count; i++)
3216                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3217         octeon_free_recv_info(recv_info);
3218         return 0;
3219 }
3220
3221 /**
3222  * \brief Setup network interfaces
3223  * @param octeon_dev  octeon device
3224  *
3225  * Called during init time for each device. It assumes the NIC
3226  * is already up and running.  The link information for each
3227  * interface is passed in link_info.
3228  */
3229 static int setup_nic_devices(struct octeon_device *octeon_dev)
3230 {
3231         struct lio *lio = NULL;
3232         struct net_device *netdev;
3233         u8 mac[6], i, j;
3234         struct octeon_soft_command *sc;
3235         struct liquidio_if_cfg_context *ctx;
3236         struct liquidio_if_cfg_resp *resp;
3237         struct octdev_props *props;
3238         int retval, num_iqueues, num_oqueues;
3239         union oct_nic_if_cfg if_cfg;
3240         unsigned int base_queue;
3241         unsigned int gmx_port_id;
3242         u32 resp_size, ctx_size, data_size;
3243         u32 ifidx_or_pfnum;
3244         struct lio_version *vdata;
3245
3246         /* This is to handle link status changes */
3247         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3248                                     OPCODE_NIC_INFO,
3249                                     lio_nic_info, octeon_dev);
3250
3251         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3252          * They are handled directly.
3253          */
3254         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3255                                         free_netbuf);
3256
3257         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3258                                         free_netsgbuf);
3259
3260         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3261                                         free_netsgbuf_with_resp);
3262
3263         for (i = 0; i < octeon_dev->ifcount; i++) {
3264                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3265                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3266                 data_size = sizeof(struct lio_version);
3267                 sc = (struct octeon_soft_command *)
3268                         octeon_alloc_soft_command(octeon_dev, data_size,
3269                                                   resp_size, ctx_size);
3270                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3271                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3272                 vdata = (struct lio_version *)sc->virtdptr;
3273
3274                 *((u64 *)vdata) = 0;
3275                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3276                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3277                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3278
3279                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3280                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3281                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3282                         base_queue = octeon_dev->sriov_info.pf_srn;
3283
3284                         gmx_port_id = octeon_dev->pf_num;
3285                         ifidx_or_pfnum = octeon_dev->pf_num;
3286                 } else {
3287                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3288                                                 octeon_get_conf(octeon_dev), i);
3289                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3290                                                 octeon_get_conf(octeon_dev), i);
3291                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3292                                                 octeon_get_conf(octeon_dev), i);
3293                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
3294                                                 octeon_get_conf(octeon_dev), i);
3295                         ifidx_or_pfnum = i;
3296                 }
3297
3298                 dev_dbg(&octeon_dev->pci_dev->dev,
3299                         "requesting config for interface %d, iqs %d, oqs %d\n",
3300                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3301                 WRITE_ONCE(ctx->cond, 0);
3302                 ctx->octeon_id = lio_get_device_id(octeon_dev);
3303                 init_waitqueue_head(&ctx->wc);
3304
3305                 if_cfg.u64 = 0;
3306                 if_cfg.s.num_iqueues = num_iqueues;
3307                 if_cfg.s.num_oqueues = num_oqueues;
3308                 if_cfg.s.base_queue = base_queue;
3309                 if_cfg.s.gmx_port_id = gmx_port_id;
3310
3311                 sc->iq_no = 0;
3312
3313                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3314                                             OPCODE_NIC_IF_CFG, 0,
3315                                             if_cfg.u64, 0);
3316
3317                 sc->callback = if_cfg_callback;
3318                 sc->callback_arg = sc;
3319                 sc->wait_time = 3000;
3320
3321                 retval = octeon_send_soft_command(octeon_dev, sc);
3322                 if (retval == IQ_SEND_FAILED) {
3323                         dev_err(&octeon_dev->pci_dev->dev,
3324                                 "iq/oq config failed status: %x\n",
3325                                 retval);
3326                         /* Soft instr is freed by driver in case of failure. */
3327                         goto setup_nic_dev_fail;
3328                 }
3329
3330                 /* Sleep on a wait queue till the cond flag indicates that the
3331                  * response arrived or timed-out.
3332                  */
3333                 sleep_cond(&ctx->wc, &ctx->cond);
3334                 retval = resp->status;
3335                 if (retval) {
3336                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3337                         goto setup_nic_dev_fail;
3338                 }
3339
3340                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3341                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3342
3343                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3344                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3345
3346                 if (!(num_iqueues) || !(num_oqueues)) {
3347                         dev_err(&octeon_dev->pci_dev->dev,
3348                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3349                                 resp->cfg_info.iqmask,
3350                                 resp->cfg_info.oqmask);
3351                         goto setup_nic_dev_fail;
3352                 }
3353                 dev_dbg(&octeon_dev->pci_dev->dev,
3354                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3355                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3356                         num_iqueues, num_oqueues);
3357                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3358
3359                 if (!netdev) {
3360                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3361                         goto setup_nic_dev_fail;
3362                 }
3363
3364                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3365
3366                 if (num_iqueues > 1)
3367                         lionetdevops.ndo_select_queue = select_q;
3368
3369                 /* Associate the routines that will handle different
3370                  * netdev tasks.
3371                  */
3372                 netdev->netdev_ops = &lionetdevops;
3373
3374                 lio = GET_LIO(netdev);
3375
3376                 memset(lio, 0, sizeof(struct lio));
3377
3378                 lio->ifidx = ifidx_or_pfnum;
3379
3380                 props = &octeon_dev->props[i];
3381                 props->gmxport = resp->cfg_info.linfo.gmxport;
3382                 props->netdev = netdev;
3383
3384                 lio->linfo.num_rxpciq = num_oqueues;
3385                 lio->linfo.num_txpciq = num_iqueues;
3386                 for (j = 0; j < num_oqueues; j++) {
3387                         lio->linfo.rxpciq[j].u64 =
3388                                 resp->cfg_info.linfo.rxpciq[j].u64;
3389                 }
3390                 for (j = 0; j < num_iqueues; j++) {
3391                         lio->linfo.txpciq[j].u64 =
3392                                 resp->cfg_info.linfo.txpciq[j].u64;
3393                 }
3394                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3395                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3396                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3397
3398                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3399
3400                 if (OCTEON_CN23XX_PF(octeon_dev) ||
3401                     OCTEON_CN6XXX(octeon_dev)) {
3402                         lio->dev_capability = NETIF_F_HIGHDMA
3403                                               | NETIF_F_IP_CSUM
3404                                               | NETIF_F_IPV6_CSUM
3405                                               | NETIF_F_SG | NETIF_F_RXCSUM
3406                                               | NETIF_F_GRO
3407                                               | NETIF_F_TSO | NETIF_F_TSO6
3408                                               | NETIF_F_LRO;
3409                 }
3410                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3411
3412                 /*  Copy of transmit encapsulation capabilities:
3413                  *  TSO, TSO6, Checksums for this device
3414                  */
3415                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3416                                           | NETIF_F_IPV6_CSUM
3417                                           | NETIF_F_GSO_UDP_TUNNEL
3418                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3419                                           | NETIF_F_RXCSUM
3420                                           | NETIF_F_TSO | NETIF_F_TSO6
3421                                           | NETIF_F_LRO;
3422
3423                 netdev->hw_enc_features = (lio->enc_dev_capability &
3424                                            ~NETIF_F_LRO);
3425
3426                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3427
3428                 netdev->vlan_features = lio->dev_capability;
3429                 /* Add any unchangeable hw features */
3430                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3431                                         NETIF_F_HW_VLAN_CTAG_RX |
3432                                         NETIF_F_HW_VLAN_CTAG_TX;
3433
3434                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3435
3436                 netdev->hw_features = lio->dev_capability;
3437                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3438                 netdev->hw_features = netdev->hw_features &
3439                         ~NETIF_F_HW_VLAN_CTAG_RX;
3440
3441                 /* Point to the  properties for octeon device to which this
3442                  * interface belongs.
3443                  */
3444                 lio->oct_dev = octeon_dev;
3445                 lio->octprops = props;
3446                 lio->netdev = netdev;
3447
3448                 dev_dbg(&octeon_dev->pci_dev->dev,
3449                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3450                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3451
3452                 /* 64-bit swap required on LE machines */
3453                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3454                 for (j = 0; j < 6; j++)
3455                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3456
3457                 /* Copy MAC Address to OS network device structure */
3458
3459                 ether_addr_copy(netdev->dev_addr, mac);
3460
3461                 /* By default all interfaces on a single Octeon uses the same
3462                  * tx and rx queues
3463                  */
3464                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3465                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3466                 if (setup_io_queues(octeon_dev, i)) {
3467                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3468                         goto setup_nic_dev_fail;
3469                 }
3470
3471                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3472
3473                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3474                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3475
3476                 if (setup_glists(octeon_dev, lio, num_iqueues)) {
3477                         dev_err(&octeon_dev->pci_dev->dev,
3478                                 "Gather list allocation failed\n");
3479                         goto setup_nic_dev_fail;
3480                 }
3481
3482                 /* Register ethtool support */
3483                 liquidio_set_ethtool_ops(netdev);
3484                 octeon_dev->priv_flags = 0x0;
3485
3486                 if (netdev->features & NETIF_F_LRO)
3487                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3488                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3489
3490                 liquidio_set_feature(netdev, OCTNET_CMD_ENABLE_VLAN_FILTER, 0);
3491
3492                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3493                         liquidio_set_feature(netdev,
3494                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3495
3496                 /* Register the network device with the OS */
3497                 if (register_netdev(netdev)) {
3498                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3499                         goto setup_nic_dev_fail;
3500                 }
3501
3502                 dev_dbg(&octeon_dev->pci_dev->dev,
3503                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3504                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3505                 netif_carrier_off(netdev);
3506                 lio->link_changes++;
3507
3508                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3509
3510                 /* Sending command to firmware to enable Rx checksum offload
3511                  * by default at the time of setup of Liquidio driver for
3512                  * this device
3513                  */
3514                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3515                                             OCTNET_CMD_RXCSUM_ENABLE);
3516                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3517                                      OCTNET_CMD_TXCSUM_ENABLE);
3518
3519                 dev_dbg(&octeon_dev->pci_dev->dev,
3520                         "NIC ifidx:%d Setup successful\n", i);
3521
3522                 octeon_free_soft_command(octeon_dev, sc);
3523         }
3524
3525         return 0;
3526
3527 setup_nic_dev_fail:
3528
3529         octeon_free_soft_command(octeon_dev, sc);
3530
3531         while (i--) {
3532                 dev_err(&octeon_dev->pci_dev->dev,
3533                         "NIC ifidx:%d Setup failed\n", i);
3534                 liquidio_destroy_nic_device(octeon_dev, i);
3535         }
3536         return -ENODEV;
3537 }
3538
3539 /**
3540  * \brief initialize the NIC
3541  * @param oct octeon device
3542  *
3543  * This initialization routine is called once the Octeon device application is
3544  * up and running
3545  */
3546 static int liquidio_init_nic_module(struct octeon_device *oct)
3547 {
3548         struct oct_intrmod_cfg *intrmod_cfg;
3549         int i, retval = 0;
3550         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3551
3552         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3553
3554         /* only default iq and oq were initialized
3555          * initialize the rest as well
3556          */
3557         /* run port_config command for each port */
3558         oct->ifcount = num_nic_ports;
3559
3560         memset(oct->props, 0,
3561                sizeof(struct octdev_props) * num_nic_ports);
3562
3563         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3564                 oct->props[i].gmxport = -1;
3565
3566         retval = setup_nic_devices(oct);
3567         if (retval) {
3568                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3569                 goto octnet_init_failure;
3570         }
3571
3572         liquidio_ptp_init(oct);
3573
3574         /* Initialize interrupt moderation params */
3575         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
3576         intrmod_cfg->rx_enable = 1;
3577         intrmod_cfg->check_intrvl =   LIO_INTRMOD_CHECK_INTERVAL;
3578         intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
3579         intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
3580         intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
3581         intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER;
3582         intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER;
3583         intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER;
3584         intrmod_cfg->tx_enable = 1;
3585         intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER;
3586         intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
3587         intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
3588         intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
3589         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3590
3591         return retval;
3592
3593 octnet_init_failure:
3594
3595         oct->ifcount = 0;
3596
3597         return retval;
3598 }
3599
3600 /**
3601  * \brief starter callback that invokes the remaining initialization work after
3602  * the NIC is up and running.
3603  * @param octptr  work struct work_struct
3604  */
3605 static void nic_starter(struct work_struct *work)
3606 {
3607         struct octeon_device *oct;
3608         struct cavium_wk *wk = (struct cavium_wk *)work;
3609
3610         oct = (struct octeon_device *)wk->ctxptr;
3611
3612         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3613                 return;
3614
3615         /* If the status of the device is CORE_OK, the core
3616          * application has reported its application type. Call
3617          * any registered handlers now and move to the RUNNING
3618          * state.
3619          */
3620         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3621                 schedule_delayed_work(&oct->nic_poll_work.work,
3622                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3623                 return;
3624         }
3625
3626         atomic_set(&oct->status, OCT_DEV_RUNNING);
3627
3628         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3629                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3630
3631                 if (liquidio_init_nic_module(oct))
3632                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3633                 else
3634                         handshake[oct->octeon_id].started_ok = 1;
3635         } else {
3636                 dev_err(&oct->pci_dev->dev,
3637                         "Unexpected application running on NIC (%d). Check firmware.\n",
3638                         oct->app_mode);
3639         }
3640
3641         complete(&handshake[oct->octeon_id].started);
3642 }
3643
3644 /**
3645  * \brief Device initialization for each Octeon device that is probed
3646  * @param octeon_dev  octeon device
3647  */
3648 static int octeon_device_init(struct octeon_device *octeon_dev)
3649 {
3650         int j, ret;
3651         char bootcmd[] = "\n";
3652         struct octeon_device_priv *oct_priv =
3653                 (struct octeon_device_priv *)octeon_dev->priv;
3654         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
3655
3656         /* Enable access to the octeon device and make its DMA capability
3657          * known to the OS.
3658          */
3659         if (octeon_pci_os_setup(octeon_dev))
3660                 return 1;
3661
3662         /* Identify the Octeon type and map the BAR address space. */
3663         if (octeon_chip_specific_setup(octeon_dev)) {
3664                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
3665                 return 1;
3666         }
3667
3668         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
3669
3670         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
3671
3672         /* Do a soft reset of the Octeon device. */
3673         if (octeon_dev->fn_list.soft_reset(octeon_dev))
3674                 return 1;
3675
3676         /* Initialize the dispatch mechanism used to push packets arriving on
3677          * Octeon Output queues.
3678          */
3679         if (octeon_init_dispatch_list(octeon_dev))
3680                 return 1;
3681
3682         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3683                                     OPCODE_NIC_CORE_DRV_ACTIVE,
3684                                     octeon_core_drv_init,
3685                                     octeon_dev);
3686
3687         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
3688         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
3689         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
3690                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3691
3692         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
3693
3694         octeon_set_io_queues_off(octeon_dev);
3695
3696         if (OCTEON_CN23XX_PF(octeon_dev)) {
3697                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3698                 if (ret) {
3699                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
3700                         return ret;
3701                 }
3702         }
3703
3704         /* Initialize soft command buffer pool
3705          */
3706         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
3707                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
3708                 return 1;
3709         }
3710         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
3711
3712         /*  Setup the data structures that manage this Octeon's Input queues. */
3713         if (octeon_setup_instr_queues(octeon_dev)) {
3714                 dev_err(&octeon_dev->pci_dev->dev,
3715                         "instruction queue initialization failed\n");
3716                 /* On error, release any previously allocated queues */
3717                 for (j = 0; j < octeon_dev->num_iqs; j++)
3718                         octeon_delete_instr_queue(octeon_dev, j);
3719                 return 1;
3720         }
3721         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
3722
3723         /* Initialize lists to manage the requests of different types that
3724          * arrive from user & kernel applications for this octeon device.
3725          */
3726         if (octeon_setup_response_list(octeon_dev)) {
3727                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
3728                 return 1;
3729         }
3730         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
3731
3732         if (octeon_setup_output_queues(octeon_dev)) {
3733                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
3734                 /* Release any previously allocated queues */
3735                 for (j = 0; j < octeon_dev->num_oqs; j++)
3736                         octeon_delete_droq(octeon_dev, j);
3737                 return 1;
3738         }
3739
3740         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
3741
3742         /* The input and output queue registers were setup earlier (the queues
3743          * were not enabled). Any additional registers that need to be
3744          * programmed should be done now.
3745          */
3746         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3747         if (ret) {
3748                 dev_err(&octeon_dev->pci_dev->dev,
3749                         "Failed to configure device registers\n");
3750                 return ret;
3751         }
3752
3753         /* Initialize the tasklet that handles output queue packet processing.*/
3754         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
3755         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
3756                      (unsigned long)octeon_dev);
3757
3758         /* Setup the interrupt handler and record the INT SUM register address
3759          */
3760         if (octeon_setup_interrupt(octeon_dev))
3761                 return 1;
3762
3763         /* Enable Octeon device interrupts */
3764         octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
3765
3766         /* Enable the input and output queues for this Octeon device */
3767         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
3768         if (ret) {
3769                 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
3770                 return ret;
3771         }
3772
3773         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
3774
3775         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
3776
3777         if (ddr_timeout == 0)
3778                 dev_info(&octeon_dev->pci_dev->dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3779
3780         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
3781
3782         /* Wait for the octeon to initialize DDR after the soft-reset. */
3783         while (ddr_timeout == 0) {
3784                 set_current_state(TASK_INTERRUPTIBLE);
3785                 if (schedule_timeout(HZ / 10)) {
3786                         /* user probably pressed Control-C */
3787                         return 1;
3788                 }
3789         }
3790         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
3791         if (ret) {
3792                 dev_err(&octeon_dev->pci_dev->dev,
3793                         "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3794                         ret);
3795                 return 1;
3796         }
3797
3798         if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
3799                 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
3800                 return 1;
3801         }
3802
3803         /* Divert uboot to take commands from host instead. */
3804         ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
3805
3806         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
3807         ret = octeon_init_consoles(octeon_dev);
3808         if (ret) {
3809                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
3810                 return 1;
3811         }
3812         ret = octeon_add_console(octeon_dev, 0);
3813         if (ret) {
3814                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
3815                 return 1;
3816         }
3817
3818         atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
3819
3820         dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
3821         ret = load_firmware(octeon_dev);
3822         if (ret) {
3823                 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
3824                 return 1;
3825         }
3826
3827         handshake[octeon_dev->octeon_id].init_ok = 1;
3828         complete(&handshake[octeon_dev->octeon_id].init);
3829
3830         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
3831
3832         /* Send Credit for Octeon Output queues. Credits are always sent after
3833          * the output queue is enabled.
3834          */
3835         for (j = 0; j < octeon_dev->num_oqs; j++)
3836                 writel(octeon_dev->droq[j]->max_count,
3837                        octeon_dev->droq[j]->pkts_credit_reg);
3838
3839         /* Packets can start arriving on the output queues from this point. */
3840
3841         return 0;
3842 }
3843
3844 /**
3845  * \brief Exits the module
3846  */
3847 static void __exit liquidio_exit(void)
3848 {
3849         liquidio_deinit_pci();
3850
3851         pr_info("LiquidIO network module is now unloaded\n");
3852 }
3853
3854 module_init(liquidio_init);
3855 module_exit(liquidio_exit);