drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/pci.h>
  24 #include <linux/firmware.h>
  25 #include <linux/ptp_clock_kernel.h>
  26 #include <net/vxlan.h>
  27 #include "liquidio_common.h"
  28 #include "octeon_droq.h"
  29 #include "octeon_iq.h"
  30 #include "response_manager.h"
  31 #include "octeon_device.h"
  32 #include "octeon_nic.h"
  33 #include "octeon_main.h"
  34 #include "octeon_network.h"
  35 #include "cn66xx_regs.h"
  36 #include "cn66xx_device.h"
  37 #include "cn68xx_device.h"
  38 #include "liquidio_image.h"
  39
  40 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  41 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  42 MODULE_LICENSE("GPL");
  43 MODULE_VERSION(LIQUIDIO_VERSION);
  44 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
  46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
  47
  48 static int ddr_timeout = 10000;
  49 module_param(ddr_timeout, int, 0644);
  50 MODULE_PARM_DESC(ddr_timeout,
  51                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  52
  53 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  54
  55 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
  56         (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
  57
  58 static int debug = -1;
  59 module_param(debug, int, 0644);
  60 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  61
  62 static char fw_type[LIO_MAX_FW_TYPE_LEN];
  63 module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
  64 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
  65
  66 static int conf_type;
  67 module_param(conf_type, int, 0);
  68 MODULE_PARM_DESC(conf_type, "select octeon configuration 0 default 1 ovs");
  69
  70 static int ptp_enable = 1;
  71
  72 /* Bit mask values for lio->ifstate */
  73 #define   LIO_IFSTATE_DROQ_OPS             0x01
  74 #define   LIO_IFSTATE_REGISTERED           0x02
  75 #define   LIO_IFSTATE_RUNNING              0x04
  76 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
  77
  78 /* Polling interval for determining when NIC application is alive */
  79 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  80
  81 /* runtime link query interval */
  82 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  83
  84 struct liquidio_if_cfg_context {
  85         int octeon_id;
  86
  87         wait_queue_head_t wc;
  88
  89         int cond;
  90 };
  91
  92 struct liquidio_if_cfg_resp {
  93         u64 rh;
  94         struct liquidio_if_cfg_info cfg_info;
  95         u64 status;
  96 };
  97
  98 struct oct_link_status_resp {
  99         u64 rh;
 100         struct oct_link_info link_info;
 101         u64 status;
 102 };
 103
 104 struct oct_timestamp_resp {
 105         u64 rh;
 106         u64 timestamp;
 107         u64 status;
 108 };
 109
 110 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 111
 112 union tx_info {
 113         u64 u64;
 114         struct {
 115 #ifdef __BIG_ENDIAN_BITFIELD
 116                 u16 gso_size;
 117                 u16 gso_segs;
 118                 u32 reserved;
 119 #else
 120                 u32 reserved;
 121                 u16 gso_segs;
 122                 u16 gso_size;
 123 #endif
 124         } s;
 125 };
 126
 127 /** Octeon device properties to be used by the NIC module.
 128  * Each octeon device in the system will be represented
 129  * by this structure in the NIC module.
 130  */
 131
 132 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 133
 134 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 135 #define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE)
 136
 137 /** Structure of a node in list of gather components maintained by
 138  * NIC driver for each network device.
 139  */
 140 struct octnic_gather {
 141         /** List manipulation. Next and prev pointers. */
 142         struct list_head list;
 143
 144         /** Size of the gather component at sg in bytes. */
 145         int sg_size;
 146
 147         /** Number of bytes that sg was adjusted to make it 8B-aligned. */
 148         int adjust;
 149
 150         /** Gather component that can accommodate max sized fragment list
 151          *  received from the IP layer.
 152          */
 153         struct octeon_sg_entry *sg;
 154
 155         u64 sg_dma_ptr;
 156 };
 157
 158 struct handshake {
 159         struct completion init;
 160         struct completion started;
 161         struct pci_dev *pci_dev;
 162         int init_ok;
 163         int started_ok;
 164 };
 165
 166 struct octeon_device_priv {
 167         /** Tasklet structures for this device. */
 168         struct tasklet_struct droq_tasklet;
 169         unsigned long napi_mask;
 170 };
 171
 172 static int octeon_device_init(struct octeon_device *);
 173 static int liquidio_stop(struct net_device *netdev);
 174 static void liquidio_remove(struct pci_dev *pdev);
 175 static int liquidio_probe(struct pci_dev *pdev,
 176                           const struct pci_device_id *ent);
 177
 178 static struct handshake handshake[MAX_OCTEON_DEVICES];
 179 static struct completion first_stage;
 180
 181 static void octeon_droq_bh(unsigned long pdev)
 182 {
 183         int q_no;
 184         int reschedule = 0;
 185         struct octeon_device *oct = (struct octeon_device *)pdev;
 186         struct octeon_device_priv *oct_priv =
 187                 (struct octeon_device_priv *)oct->priv;
 188
 189         /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
 190         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 191                 if (!(oct->io_qmask.oq & (1ULL << q_no)))
 192                         continue;
 193                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 194                                                           MAX_PACKET_BUDGET);
 195         }
 196
 197         if (reschedule)
 198                 tasklet_schedule(&oct_priv->droq_tasklet);
 199 }
 200
 201 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 202 {
 203         struct octeon_device_priv *oct_priv =
 204                 (struct octeon_device_priv *)oct->priv;
 205         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 206         int i;
 207
 208         do {
 209                 pending_pkts = 0;
 210
 211                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 212                         if (!(oct->io_qmask.oq & (1ULL << i)))
 213                                 continue;
 214                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 215                 }
 216                 if (pkt_cnt > 0) {
 217                         pending_pkts += pkt_cnt;
 218                         tasklet_schedule(&oct_priv->droq_tasklet);
 219                 }
 220                 pkt_cnt = 0;
 221                 schedule_timeout_uninterruptible(1);
 222
 223         } while (retry-- && pending_pkts);
 224
 225         return pkt_cnt;
 226 }
 227
 228 /**
 229  * \brief Forces all IO queues off on a given device
 230  * @param oct Pointer to Octeon device
 231  */
 232 static void force_io_queues_off(struct octeon_device *oct)
 233 {
 234         if ((oct->chip_id == OCTEON_CN66XX) ||
 235             (oct->chip_id == OCTEON_CN68XX)) {
 236                 /* Reset the Enable bits for Input Queues. */
 237                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 238
 239                 /* Reset the Enable bits for Output Queues. */
 240                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 241         }
 242 }
 243
 244 /**
 245  * \brief wait for all pending requests to complete
 246  * @param oct Pointer to Octeon device
 247  *
 248  * Called during shutdown sequence
 249  */
 250 static int wait_for_pending_requests(struct octeon_device *oct)
 251 {
 252         int i, pcount = 0;
 253
 254         for (i = 0; i < 100; i++) {
 255                 pcount =
 256                         atomic_read(&oct->response_list
 257                                 [OCTEON_ORDERED_SC_LIST].pending_req_count);
 258                 if (pcount)
 259                         schedule_timeout_uninterruptible(HZ / 10);
 260                 else
 261                         break;
 262         }
 263
 264         if (pcount)
 265                 return 1;
 266
 267         return 0;
 268 }
 269
 270 /**
 271  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 272  * @param oct Pointer to Octeon device
 273  */
 274 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 275 {
 276         int i;
 277
 278         /* Disable the input and output queues now. No more packets will
 279          * arrive from Octeon, but we should wait for all packet processing
 280          * to finish.
 281          */
 282         force_io_queues_off(oct);
 283
 284         /* To allow for in-flight requests */
 285         schedule_timeout_uninterruptible(100);
 286
 287         if (wait_for_pending_requests(oct))
 288                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 289
 290         /* Force all requests waiting to be fetched by OCTEON to complete. */
 291         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 292                 struct octeon_instr_queue *iq;
 293
 294                 if (!(oct->io_qmask.iq & (1ULL << i)))
 295                         continue;
 296                 iq = oct->instr_queue[i];
 297
 298                 if (atomic_read(&iq->instr_pending)) {
 299                         spin_lock_bh(&iq->lock);
 300                         iq->fill_cnt = 0;
 301                         iq->octeon_read_index = iq->host_write_index;
 302                         iq->stats.instr_processed +=
 303                                 atomic_read(&iq->instr_pending);
 304                         lio_process_iq_request_list(oct, iq, 0);
 305                         spin_unlock_bh(&iq->lock);
 306                 }
 307         }
 308
 309         /* Force all pending ordered list requests to time out. */
 310         lio_process_ordered_list(oct, 1);
 311
 312         /* We do not need to wait for output queue packets to be processed. */
 313 }
 314
 315 /**
 316  * \brief Cleanup PCI AER uncorrectable error status
 317  * @param dev Pointer to PCI device
 318  */
 319 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 320 {
 321         int pos = 0x100;
 322         u32 status, mask;
 323
 324         pr_info("%s :\n", __func__);
 325
 326         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 327         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 328         if (dev->error_state == pci_channel_io_normal)
 329                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 330         else
 331                 status &= mask;         /* Clear corresponding fatal bits */
 332         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 333 }
 334
 335 /**
 336  * \brief Stop all PCI IO to a given device
 337  * @param dev Pointer to Octeon device
 338  */
 339 static void stop_pci_io(struct octeon_device *oct)
 340 {
 341         /* No more instructions will be forwarded. */
 342         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 343
 344         pci_disable_device(oct->pci_dev);
 345
 346         /* Disable interrupts  */
 347         oct->fn_list.disable_interrupt(oct->chip);
 348
 349         pcierror_quiesce_device(oct);
 350
 351         /* Release the interrupt line */
 352         free_irq(oct->pci_dev->irq, oct);
 353
 354         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 355                 pci_disable_msi(oct->pci_dev);
 356
 357         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 358                 lio_get_state_string(&oct->status));
 359
 360         /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
 361         /* making it a common function for all OCTEON models */
 362         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 363 }
 364
 365 /**
 366  * \brief called when PCI error is detected
 367  * @param pdev Pointer to PCI device
 368  * @param state The current pci connection state
 369  *
 370  * This function is called after a PCI bus error affecting
 371  * this device has been detected.
 372  */
 373 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 374                                                      pci_channel_state_t state)
 375 {
 376         struct octeon_device *oct = pci_get_drvdata(pdev);
 377
 378         /* Non-correctable Non-fatal errors */
 379         if (state == pci_channel_io_normal) {
 380                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 381                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 382                 return PCI_ERS_RESULT_CAN_RECOVER;
 383         }
 384
 385         /* Non-correctable Fatal errors */
 386         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 387         stop_pci_io(oct);
 388
 389         /* Always return a DISCONNECT. There is no support for recovery but only
 390          * for a clean shutdown.
 391          */
 392         return PCI_ERS_RESULT_DISCONNECT;
 393 }
 394
 395 /**
 396  * \brief mmio handler
 397  * @param pdev Pointer to PCI device
 398  */
 399 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 400                                 struct pci_dev *pdev __attribute__((unused)))
 401 {
 402         /* We should never hit this since we never ask for a reset for a Fatal
 403          * Error. We always return DISCONNECT in io_error above.
 404          * But play safe and return RECOVERED for now.
 405          */
 406         return PCI_ERS_RESULT_RECOVERED;
 407 }
 408
 409 /**
 410  * \brief called after the pci bus has been reset.
 411  * @param pdev Pointer to PCI device
 412  *
 413  * Restart the card from scratch, as if from a cold-boot. Implementation
 414  * resembles the first-half of the octeon_resume routine.
 415  */
 416 static pci_ers_result_t liquidio_pcie_slot_reset(
 417                                 struct pci_dev *pdev __attribute__((unused)))
 418 {
 419         /* We should never hit this since we never ask for a reset for a Fatal
 420          * Error. We always return DISCONNECT in io_error above.
 421          * But play safe and return RECOVERED for now.
 422          */
 423         return PCI_ERS_RESULT_RECOVERED;
 424 }
 425
 426 /**
 427  * \brief called when traffic can start flowing again.
 428  * @param pdev Pointer to PCI device
 429  *
 430  * This callback is called when the error recovery driver tells us that
 431  * its OK to resume normal operation. Implementation resembles the
 432  * second-half of the octeon_resume routine.
 433  */
 434 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 435 {
 436         /* Nothing to be done here. */
 437 }
 438
 439 #ifdef CONFIG_PM
 440 /**
 441  * \brief called when suspending
 442  * @param pdev Pointer to PCI device
 443  * @param state state to suspend to
 444  */
 445 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 446                             pm_message_t state __attribute__((unused)))
 447 {
 448         return 0;
 449 }
 450
 451 /**
 452  * \brief called when resuming
 453  * @param pdev Pointer to PCI device
 454  */
 455 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 456 {
 457         return 0;
 458 }
 459 #endif
 460
 461 /* For PCI-E Advanced Error Recovery (AER) Interface */
 462 static const struct pci_error_handlers liquidio_err_handler = {
 463         .error_detected = liquidio_pcie_error_detected,
 464         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 465         .slot_reset     = liquidio_pcie_slot_reset,
 466         .resume         = liquidio_pcie_resume,
 467 };
 468
 469 static const struct pci_device_id liquidio_pci_tbl[] = {
 470         {       /* 68xx */
 471                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 472         },
 473         {       /* 66xx */
 474                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 475         },
 476         {
 477                 0, 0, 0, 0, 0, 0, 0
 478         }
 479 };
 480 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 481
 482 static struct pci_driver liquidio_pci_driver = {
 483         .name           = "LiquidIO",
 484         .id_table       = liquidio_pci_tbl,
 485         .probe          = liquidio_probe,
 486         .remove         = liquidio_remove,
 487         .err_handler    = &liquidio_err_handler,    /* For AER */
 488
 489 #ifdef CONFIG_PM
 490         .suspend        = liquidio_suspend,
 491         .resume         = liquidio_resume,
 492 #endif
 493
 494 };
 495
 496 /**
 497  * \brief register PCI driver
 498  */
 499 static int liquidio_init_pci(void)
 500 {
 501         return pci_register_driver(&liquidio_pci_driver);
 502 }
 503
 504 /**
 505  * \brief unregister PCI driver
 506  */
 507 static void liquidio_deinit_pci(void)
 508 {
 509         pci_unregister_driver(&liquidio_pci_driver);
 510 }
 511
 512 /**
 513  * \brief check interface state
 514  * @param lio per-network private data
 515  * @param state_flag flag state to check
 516  */
 517 static inline int ifstate_check(struct lio *lio, int state_flag)
 518 {
 519         return atomic_read(&lio->ifstate) & state_flag;
 520 }
 521
 522 /**
 523  * \brief set interface state
 524  * @param lio per-network private data
 525  * @param state_flag flag state to set
 526  */
 527 static inline void ifstate_set(struct lio *lio, int state_flag)
 528 {
 529         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
 530 }
 531
 532 /**
 533  * \brief clear interface state
 534  * @param lio per-network private data
 535  * @param state_flag flag state to clear
 536  */
 537 static inline void ifstate_reset(struct lio *lio, int state_flag)
 538 {
 539         atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 540 }
 541
 542 /**
 543  * \brief Stop Tx queues
 544  * @param netdev network device
 545  */
 546 static inline void txqs_stop(struct net_device *netdev)
 547 {
 548         if (netif_is_multiqueue(netdev)) {
 549                 int i;
 550
 551                 for (i = 0; i < netdev->num_tx_queues; i++)
 552                         netif_stop_subqueue(netdev, i);
 553         } else {
 554                 netif_stop_queue(netdev);
 555         }
 556 }
 557
 558 /**
 559  * \brief Start Tx queues
 560  * @param netdev network device
 561  */
 562 static inline void txqs_start(struct net_device *netdev)
 563 {
 564         if (netif_is_multiqueue(netdev)) {
 565                 int i;
 566
 567                 for (i = 0; i < netdev->num_tx_queues; i++)
 568                         netif_start_subqueue(netdev, i);
 569         } else {
 570                 netif_start_queue(netdev);
 571         }
 572 }
 573
 574 /**
 575  * \brief Wake Tx queues
 576  * @param netdev network device
 577  */
 578 static inline void txqs_wake(struct net_device *netdev)
 579 {
 580         struct lio *lio = GET_LIO(netdev);
 581
 582         if (netif_is_multiqueue(netdev)) {
 583                 int i;
 584
 585                 for (i = 0; i < netdev->num_tx_queues; i++) {
 586                         int qno = lio->linfo.txpciq[i %
 587                                 (lio->linfo.num_txpciq)].s.q_no;
 588
 589                         if (__netif_subqueue_stopped(netdev, i)) {
 590                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
 591                                                           tx_restart, 1);
 592                                 netif_wake_subqueue(netdev, i);
 593                         }
 594                 }
 595         } else {
 596                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 597                                           tx_restart, 1);
 598                 netif_wake_queue(netdev);
 599         }
 600 }
 601
 602 /**
 603  * \brief Stop Tx queue
 604  * @param netdev network device
 605  */
 606 static void stop_txq(struct net_device *netdev)
 607 {
 608         txqs_stop(netdev);
 609 }
 610
 611 /**
 612  * \brief Start Tx queue
 613  * @param netdev network device
 614  */
 615 static void start_txq(struct net_device *netdev)
 616 {
 617         struct lio *lio = GET_LIO(netdev);
 618
 619         if (lio->linfo.link.s.link_up) {
 620                 txqs_start(netdev);
 621                 return;
 622         }
 623 }
 624
 625 /**
 626  * \brief Wake a queue
 627  * @param netdev network device
 628  * @param q which queue to wake
 629  */
 630 static inline void wake_q(struct net_device *netdev, int q)
 631 {
 632         if (netif_is_multiqueue(netdev))
 633                 netif_wake_subqueue(netdev, q);
 634         else
 635                 netif_wake_queue(netdev);
 636 }
 637
 638 /**
 639  * \brief Stop a queue
 640  * @param netdev network device
 641  * @param q which queue to stop
 642  */
 643 static inline void stop_q(struct net_device *netdev, int q)
 644 {
 645         if (netif_is_multiqueue(netdev))
 646                 netif_stop_subqueue(netdev, q);
 647         else
 648                 netif_stop_queue(netdev);
 649 }
 650
 651 /**
 652  * \brief Check Tx queue status, and take appropriate action
 653  * @param lio per-network private data
 654  * @returns 0 if full, number of queues woken up otherwise
 655  */
 656 static inline int check_txq_status(struct lio *lio)
 657 {
 658         int ret_val = 0;
 659
 660         if (netif_is_multiqueue(lio->netdev)) {
 661                 int numqs = lio->netdev->num_tx_queues;
 662                 int q, iq = 0;
 663
 664                 /* check each sub-queue state */
 665                 for (q = 0; q < numqs; q++) {
 666                         iq = lio->linfo.txpciq[q %
 667                                 (lio->linfo.num_txpciq)].s.q_no;
 668                         if (octnet_iq_is_full(lio->oct_dev, iq))
 669                                 continue;
 670                         if (__netif_subqueue_stopped(lio->netdev, q)) {
 671                                 wake_q(lio->netdev, q);
 672                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 673                                                           tx_restart, 1);
 674                                 ret_val++;
 675                         }
 676                 }
 677         } else {
 678                 if (octnet_iq_is_full(lio->oct_dev, lio->txq))
 679                         return 0;
 680                 wake_q(lio->netdev, lio->txq);
 681                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
 682                                           tx_restart, 1);
 683                 ret_val = 1;
 684         }
 685         return ret_val;
 686 }
 687
 688 /**
 689  * Remove the node at the head of the list. The list would be empty at
 690  * the end of this call if there are no more nodes in the list.
 691  */
 692 static inline struct list_head *list_delete_head(struct list_head *root)
 693 {
 694         struct list_head *node;
 695
 696         if ((root->prev == root) && (root->next == root))
 697                 node = NULL;
 698         else
 699                 node = root->next;
 700
 701         if (node)
 702                 list_del(node);
 703
 704         return node;
 705 }
 706
 707 /**
 708  * \brief Delete gather lists
 709  * @param lio per-network private data
 710  */
 711 static void delete_glists(struct lio *lio)
 712 {
 713         struct octnic_gather *g;
 714         int i;
 715
 716         if (!lio->glist)
 717                 return;
 718
 719         for (i = 0; i < lio->linfo.num_txpciq; i++) {
 720                 do {
 721                         g = (struct octnic_gather *)
 722                                 list_delete_head(&lio->glist[i]);
 723                         if (g) {
 724                                 if (g->sg) {
 725                                         dma_unmap_single(&lio->oct_dev->
 726                                                          pci_dev->dev,
 727                                                          g->sg_dma_ptr,
 728                                                          g->sg_size,
 729                                                          DMA_TO_DEVICE);
 730                                         kfree((void *)((unsigned long)g->sg -
 731                                                        g->adjust));
 732                                 }
 733                                 kfree(g);
 734                         }
 735                 } while (g);
 736         }
 737
 738         kfree((void *)lio->glist);
 739 }
 740
 741 /**
 742  * \brief Setup gather lists
 743  * @param lio per-network private data
 744  */
 745 static int setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
 746 {
 747         int i, j;
 748         struct octnic_gather *g;
 749
 750         lio->glist_lock = kcalloc(num_iqs, sizeof(*lio->glist_lock),
 751                                   GFP_KERNEL);
 752         if (!lio->glist_lock)
 753                 return 1;
 754
 755         lio->glist = kcalloc(num_iqs, sizeof(*lio->glist),
 756                              GFP_KERNEL);
 757         if (!lio->glist) {
 758                 kfree((void *)lio->glist_lock);
 759                 return 1;
 760         }
 761
 762         for (i = 0; i < num_iqs; i++) {
 763                 int numa_node = cpu_to_node(i % num_online_cpus());
 764
 765                 spin_lock_init(&lio->glist_lock[i]);
 766
 767                 INIT_LIST_HEAD(&lio->glist[i]);
 768
 769                 for (j = 0; j < lio->tx_qsize; j++) {
 770                         g = kzalloc_node(sizeof(*g), GFP_KERNEL,
 771                                          numa_node);
 772                         if (!g)
 773                                 g = kzalloc(sizeof(*g), GFP_KERNEL);
 774                         if (!g)
 775                                 break;
 776
 777                         g->sg_size = ((ROUNDUP4(OCTNIC_MAX_SG) >> 2) *
 778                                       OCT_SG_ENTRY_SIZE);
 779
 780                         g->sg = kmalloc_node(g->sg_size + 8,
 781                                              GFP_KERNEL, numa_node);
 782                         if (!g->sg)
 783                                 g->sg = kmalloc(g->sg_size + 8, GFP_KERNEL);
 784                         if (!g->sg) {
 785                                 kfree(g);
 786                                 break;
 787                         }
 788
 789                         /* The gather component should be aligned on 64-bit
 790                          * boundary
 791                          */
 792                         if (((unsigned long)g->sg) & 7) {
 793                                 g->adjust = 8 - (((unsigned long)g->sg) & 7);
 794                                 g->sg = (struct octeon_sg_entry *)
 795                                         ((unsigned long)g->sg + g->adjust);
 796                         }
 797                         g->sg_dma_ptr = dma_map_single(&oct->pci_dev->dev,
 798                                                        g->sg, g->sg_size,
 799                                                        DMA_TO_DEVICE);
 800                         if (dma_mapping_error(&oct->pci_dev->dev,
 801                                               g->sg_dma_ptr)) {
 802                                 kfree((void *)((unsigned long)g->sg -
 803                                                g->adjust));
 804                                 kfree(g);
 805                                 break;
 806                         }
 807
 808                         list_add_tail(&g->list, &lio->glist[i]);
 809                 }
 810
 811                 if (j != lio->tx_qsize) {
 812                         delete_glists(lio);
 813                         return 1;
 814                 }
 815         }
 816
 817         return 0;
 818 }
 819
 820 /**
 821  * \brief Print link information
 822  * @param netdev network device
 823  */
 824 static void print_link_info(struct net_device *netdev)
 825 {
 826         struct lio *lio = GET_LIO(netdev);
 827
 828         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
 829                 struct oct_link_info *linfo = &lio->linfo;
 830
 831                 if (linfo->link.s.link_up) {
 832                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 833                                    linfo->link.s.speed,
 834                                    (linfo->link.s.duplex) ? "Full" : "Half");
 835                 } else {
 836                         netif_info(lio, link, lio->netdev, "Link Down\n");
 837                 }
 838         }
 839 }
 840
 841 /**
 842  * \brief Update link status
 843  * @param netdev network device
 844  * @param ls link status structure
 845  *
 846  * Called on receipt of a link status response from the core application to
 847  * update each interface's link status.
 848  */
 849 static inline void update_link_status(struct net_device *netdev,
 850                                       union oct_link_status *ls)
 851 {
 852         struct lio *lio = GET_LIO(netdev);
 853         int changed = (lio->linfo.link.u64 != ls->u64);
 854
 855         lio->linfo.link.u64 = ls->u64;
 856
 857         if ((lio->intf_open) && (changed)) {
 858                 print_link_info(netdev);
 859                 lio->link_changes++;
 860
 861                 if (lio->linfo.link.s.link_up) {
 862                         netif_carrier_on(netdev);
 863                         /* start_txq(netdev); */
 864                         txqs_wake(netdev);
 865                 } else {
 866                         netif_carrier_off(netdev);
 867                         stop_txq(netdev);
 868                 }
 869         }
 870 }
 871
 872 /* Runs in interrupt context. */
 873 static void update_txq_status(struct octeon_device *oct, int iq_num)
 874 {
 875         struct net_device *netdev;
 876         struct lio *lio;
 877         struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 878
 879         /*octeon_update_iq_read_idx(oct, iq);*/
 880
 881         netdev = oct->props[iq->ifidx].netdev;
 882
 883         /* This is needed because the first IQ does not have
 884          * a netdev associated with it.
 885          */
 886         if (!netdev)
 887                 return;
 888
 889         lio = GET_LIO(netdev);
 890         if (netif_is_multiqueue(netdev)) {
 891                 if (__netif_subqueue_stopped(netdev, iq->q_index) &&
 892                     lio->linfo.link.s.link_up &&
 893                     (!octnet_iq_is_full(oct, iq_num))) {
 894                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
 895                                                   tx_restart, 1);
 896                         netif_wake_subqueue(netdev, iq->q_index);
 897                 } else {
 898                         if (!octnet_iq_is_full(oct, lio->txq)) {
 899                                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
 900                                                           lio->txq,
 901                                                           tx_restart, 1);
 902                                 wake_q(netdev, lio->txq);
 903                         }
 904                 }
 905         }
 906 }
 907
 908 /**
 909  * \brief Droq packet processor sceduler
 910  * @param oct octeon device
 911  */
 912 static
 913 void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 914 {
 915         struct octeon_device_priv *oct_priv =
 916                 (struct octeon_device_priv *)oct->priv;
 917         u64 oq_no;
 918         struct octeon_droq *droq;
 919
 920         if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
 921                 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
 922                      oq_no++) {
 923                         if (!(oct->droq_intr & (1ULL << oq_no)))
 924                                 continue;
 925
 926                         droq = oct->droq[oq_no];
 927
 928                         if (droq->ops.poll_mode) {
 929                                 droq->ops.napi_fn(droq);
 930                                 oct_priv->napi_mask |= (1 << oq_no);
 931                         } else {
 932                                 tasklet_schedule(&oct_priv->droq_tasklet);
 933                         }
 934                 }
 935         }
 936 }
 937
 938 /**
 939  * \brief Interrupt handler for octeon
 940  * @param irq unused
 941  * @param dev octeon device
 942  */
 943 static
 944 irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 945 {
 946         struct octeon_device *oct = (struct octeon_device *)dev;
 947         irqreturn_t ret;
 948
 949         /* Disable our interrupts for the duration of ISR */
 950         oct->fn_list.disable_interrupt(oct->chip);
 951
 952         ret = oct->fn_list.process_interrupt_regs(oct);
 953
 954         if (ret == IRQ_HANDLED)
 955                 liquidio_schedule_droq_pkt_handlers(oct);
 956
 957         /* Re-enable our interrupts  */
 958         if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
 959                 oct->fn_list.enable_interrupt(oct->chip);
 960
 961         return ret;
 962 }
 963
 964 /**
 965  * \brief Setup interrupt for octeon device
 966  * @param oct octeon device
 967  *
 968  *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
 969  */
 970 static int octeon_setup_interrupt(struct octeon_device *oct)
 971 {
 972         int irqret, err;
 973
 974         err = pci_enable_msi(oct->pci_dev);
 975         if (err)
 976                 dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
 977                          err);
 978         else
 979                 oct->flags |= LIO_FLAG_MSI_ENABLED;
 980
 981         irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
 982                              IRQF_SHARED, "octeon", oct);
 983         if (irqret) {
 984                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
 985                         pci_disable_msi(oct->pci_dev);
 986                 dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
 987                         irqret);
 988                 return 1;
 989         }
 990
 991         return 0;
 992 }
 993
 994 /**
 995  * \brief PCI probe handler
 996  * @param pdev PCI device structure
 997  * @param ent unused
 998  */
 999 static int
1000 liquidio_probe(struct pci_dev *pdev,
1001                const struct pci_device_id *ent __attribute__((unused)))
1002 {
1003         struct octeon_device *oct_dev = NULL;
1004         struct handshake *hs;
1005
1006         oct_dev = octeon_allocate_device(pdev->device,
1007                                          sizeof(struct octeon_device_priv));
1008         if (!oct_dev) {
1009                 dev_err(&pdev->dev, "Unable to allocate device\n");
1010                 return -ENOMEM;
1011         }
1012
1013         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
1014                  (u32)pdev->vendor, (u32)pdev->device);
1015
1016         /* Assign octeon_device for this device to the private data area. */
1017         pci_set_drvdata(pdev, oct_dev);
1018
1019         /* set linux specific device pointer */
1020         oct_dev->pci_dev = (void *)pdev;
1021
1022         hs = &handshake[oct_dev->octeon_id];
1023         init_completion(&hs->init);
1024         init_completion(&hs->started);
1025         hs->pci_dev = pdev;
1026
1027         if (oct_dev->octeon_id == 0)
1028                 /* first LiquidIO NIC is detected */
1029                 complete(&first_stage);
1030
1031         if (octeon_device_init(oct_dev)) {
1032                 liquidio_remove(pdev);
1033                 return -ENOMEM;
1034         }
1035
1036         oct_dev->rx_pause = 1;
1037         oct_dev->tx_pause = 1;
1038
1039         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
1040
1041         return 0;
1042 }
1043
1044 /**
1045  *\brief Destroy resources associated with octeon device
1046  * @param pdev PCI device structure
1047  * @param ent unused
1048  */
1049 static void octeon_destroy_resources(struct octeon_device *oct)
1050 {
1051         int i;
1052         struct octeon_device_priv *oct_priv =
1053                 (struct octeon_device_priv *)oct->priv;
1054
1055         struct handshake *hs;
1056
1057         switch (atomic_read(&oct->status)) {
1058         case OCT_DEV_RUNNING:
1059         case OCT_DEV_CORE_OK:
1060
1061                 /* No more instructions will be forwarded. */
1062                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
1063
1064                 oct->app_mode = CVM_DRV_INVALID_APP;
1065                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
1066                         lio_get_state_string(&oct->status));
1067
1068                 schedule_timeout_uninterruptible(HZ / 10);
1069
1070                 /* fallthrough */
1071         case OCT_DEV_HOST_OK:
1072
1073                 /* fallthrough */
1074         case OCT_DEV_CONSOLE_INIT_DONE:
1075                 /* Remove any consoles */
1076                 octeon_remove_consoles(oct);
1077
1078                 /* fallthrough */
1079         case OCT_DEV_IO_QUEUES_DONE:
1080                 if (wait_for_pending_requests(oct))
1081                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1082
1083                 if (lio_wait_for_instr_fetch(oct))
1084                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1085
1086                 /* Disable the input and output queues now. No more packets will
1087                  * arrive from Octeon, but we should wait for all packet
1088                  * processing to finish.
1089                  */
1090                 oct->fn_list.disable_io_queues(oct);
1091
1092                 if (lio_wait_for_oq_pkts(oct))
1093                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1094
1095                 /* Disable interrupts  */
1096                 oct->fn_list.disable_interrupt(oct->chip);
1097
1098                 /* Release the interrupt line */
1099                 free_irq(oct->pci_dev->irq, oct);
1100
1101                 if (oct->flags & LIO_FLAG_MSI_ENABLED)
1102                         pci_disable_msi(oct->pci_dev);
1103
1104                 /* fallthrough */
1105         case OCT_DEV_IN_RESET:
1106         case OCT_DEV_DROQ_INIT_DONE:
1107                 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1108                 mdelay(100);
1109                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1110                         if (!(oct->io_qmask.oq & (1ULL << i)))
1111                                 continue;
1112                         octeon_delete_droq(oct, i);
1113                 }
1114
1115                 /* Force any pending handshakes to complete */
1116                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1117                         hs = &handshake[i];
1118
1119                         if (hs->pci_dev) {
1120                                 handshake[oct->octeon_id].init_ok = 0;
1121                                 complete(&handshake[oct->octeon_id].init);
1122                                 handshake[oct->octeon_id].started_ok = 0;
1123                                 complete(&handshake[oct->octeon_id].started);
1124                         }
1125                 }
1126
1127                 /* fallthrough */
1128         case OCT_DEV_RESP_LIST_INIT_DONE:
1129                 octeon_delete_response_list(oct);
1130
1131                 /* fallthrough */
1132         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1133                 octeon_free_sc_buffer_pool(oct);
1134
1135                 /* fallthrough */
1136         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1137                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1138                         if (!(oct->io_qmask.iq & (1ULL << i)))
1139                                 continue;
1140                         octeon_delete_instr_queue(oct, i);
1141                 }
1142
1143                 /* fallthrough */
1144         case OCT_DEV_DISPATCH_INIT_DONE:
1145                 octeon_delete_dispatch_list(oct);
1146                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1147
1148                 /* fallthrough */
1149         case OCT_DEV_PCI_MAP_DONE:
1150
1151                 /* Soft reset the octeon device before exiting */
1152                 oct->fn_list.soft_reset(oct);
1153
1154                 octeon_unmap_pci_barx(oct, 0);
1155                 octeon_unmap_pci_barx(oct, 1);
1156
1157                 /* fallthrough */
1158         case OCT_DEV_BEGIN_STATE:
1159                 /* Disable the device, releasing the PCI INT */
1160                 pci_disable_device(oct->pci_dev);
1161
1162                 /* Nothing to be done here either */
1163                 break;
1164         }                       /* end switch (oct->status) */
1165
1166         tasklet_kill(&oct_priv->droq_tasklet);
1167 }
1168
1169 /**
1170  * \brief Send Rx control command
1171  * @param lio per-network private data
1172  * @param start_stop whether to start or stop
1173  */
1174 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1175 {
1176         struct octnic_ctrl_pkt nctrl;
1177
1178         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1179
1180         nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
1181         nctrl.ncmd.s.param1 = start_stop;
1182         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1183         nctrl.netpndev = (u64)lio->netdev;
1184
1185         if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl) < 0)
1186                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1187 }
1188
1189 /**
1190  * \brief Destroy NIC device interface
1191  * @param oct octeon device
1192  * @param ifidx which interface to destroy
1193  *
1194  * Cleanup associated with each interface for an Octeon device  when NIC
1195  * module is being unloaded or if initialization fails during load.
1196  */
1197 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1198 {
1199         struct net_device *netdev = oct->props[ifidx].netdev;
1200         struct lio *lio;
1201         struct napi_struct *napi, *n;
1202
1203         if (!netdev) {
1204                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1205                         __func__, ifidx);
1206                 return;
1207         }
1208
1209         lio = GET_LIO(netdev);
1210
1211         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1212
1213         send_rx_ctrl_cmd(lio, 0);
1214
1215         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1216                 txqs_stop(netdev);
1217
1218         if (oct->props[lio->ifidx].napi_enabled == 1) {
1219                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1220                         napi_disable(napi);
1221
1222                 oct->props[lio->ifidx].napi_enabled = 0;
1223         }
1224
1225         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1226                 unregister_netdev(netdev);
1227
1228         delete_glists(lio);
1229
1230         free_netdev(netdev);
1231
1232         oct->props[ifidx].gmxport = -1;
1233
1234         oct->props[ifidx].netdev = NULL;
1235 }
1236
1237 /**
1238  * \brief Stop complete NIC functionality
1239  * @param oct octeon device
1240  */
1241 static int liquidio_stop_nic_module(struct octeon_device *oct)
1242 {
1243         int i, j;
1244         struct lio *lio;
1245
1246         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1247         if (!oct->ifcount) {
1248                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1249                 return 1;
1250         }
1251
1252         spin_lock_bh(&oct->cmd_resp_wqlock);
1253         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1254         spin_unlock_bh(&oct->cmd_resp_wqlock);
1255
1256         for (i = 0; i < oct->ifcount; i++) {
1257                 lio = GET_LIO(oct->props[i].netdev);
1258                 for (j = 0; j < lio->linfo.num_rxpciq; j++)
1259                         octeon_unregister_droq_ops(oct,
1260                                                    lio->linfo.rxpciq[j].s.q_no);
1261         }
1262
1263         for (i = 0; i < oct->ifcount; i++)
1264                 liquidio_destroy_nic_device(oct, i);
1265
1266         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1267         return 0;
1268 }
1269
1270 /**
1271  * \brief Cleans up resources at unload time
1272  * @param pdev PCI device structure
1273  */
1274 static void liquidio_remove(struct pci_dev *pdev)
1275 {
1276         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1277
1278         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1279
1280         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1281                 liquidio_stop_nic_module(oct_dev);
1282
1283         /* Reset the octeon device and cleanup all memory allocated for
1284          * the octeon device by driver.
1285          */
1286         octeon_destroy_resources(oct_dev);
1287
1288         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1289
1290         /* This octeon device has been removed. Update the global
1291          * data structure to reflect this. Free the device structure.
1292          */
1293         octeon_free_device_mem(oct_dev);
1294 }
1295
1296 /**
1297  * \brief Identify the Octeon device and to map the BAR address space
1298  * @param oct octeon device
1299  */
1300 static int octeon_chip_specific_setup(struct octeon_device *oct)
1301 {
1302         u32 dev_id, rev_id;
1303         int ret = 1;
1304         char *s;
1305
1306         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1307         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1308         oct->rev_id = rev_id & 0xff;
1309
1310         switch (dev_id) {
1311         case OCTEON_CN68XX_PCIID:
1312                 oct->chip_id = OCTEON_CN68XX;
1313                 ret = lio_setup_cn68xx_octeon_device(oct);
1314                 s = "CN68XX";
1315                 break;
1316
1317         case OCTEON_CN66XX_PCIID:
1318                 oct->chip_id = OCTEON_CN66XX;
1319                 ret = lio_setup_cn66xx_octeon_device(oct);
1320                 s = "CN66XX";
1321                 break;
1322
1323         default:
1324                 s = "?";
1325                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1326                         dev_id);
1327         }
1328
1329         if (!ret)
1330                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1331                          OCTEON_MAJOR_REV(oct),
1332                          OCTEON_MINOR_REV(oct),
1333                          octeon_get_conf(oct)->card_name,
1334                          LIQUIDIO_VERSION);
1335
1336         return ret;
1337 }
1338
1339 /**
1340  * \brief PCI initialization for each Octeon device.
1341  * @param oct octeon device
1342  */
1343 static int octeon_pci_os_setup(struct octeon_device *oct)
1344 {
1345         /* setup PCI stuff first */
1346         if (pci_enable_device(oct->pci_dev)) {
1347                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1348                 return 1;
1349         }
1350
1351         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1352                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1353                 return 1;
1354         }
1355
1356         /* Enable PCI DMA Master. */
1357         pci_set_master(oct->pci_dev);
1358
1359         return 0;
1360 }
1361
1362 static inline int skb_iq(struct lio *lio, struct sk_buff *skb)
1363 {
1364         int q = 0;
1365
1366         if (netif_is_multiqueue(lio->netdev))
1367                 q = skb->queue_mapping % lio->linfo.num_txpciq;
1368
1369         return q;
1370 }
1371
1372 /**
1373  * \brief Check Tx queue state for a given network buffer
1374  * @param lio per-network private data
1375  * @param skb network buffer
1376  */
1377 static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
1378 {
1379         int q = 0, iq = 0;
1380
1381         if (netif_is_multiqueue(lio->netdev)) {
1382                 q = skb->queue_mapping;
1383                 iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
1384         } else {
1385                 iq = lio->txq;
1386                 q = iq;
1387         }
1388
1389         if (octnet_iq_is_full(lio->oct_dev, iq))
1390                 return 0;
1391
1392         if (__netif_subqueue_stopped(lio->netdev, q)) {
1393                 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq, tx_restart, 1);
1394                 wake_q(lio->netdev, q);
1395         }
1396         return 1;
1397 }
1398
1399 /**
1400  * \brief Unmap and free network buffer
1401  * @param buf buffer
1402  */
1403 static void free_netbuf(void *buf)
1404 {
1405         struct sk_buff *skb;
1406         struct octnet_buf_free_info *finfo;
1407         struct lio *lio;
1408
1409         finfo = (struct octnet_buf_free_info *)buf;
1410         skb = finfo->skb;
1411         lio = finfo->lio;
1412
1413         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1414                          DMA_TO_DEVICE);
1415
1416         check_txq_state(lio, skb);
1417
1418         tx_buffer_free(skb);
1419 }
1420
1421 /**
1422  * \brief Unmap and free gather buffer
1423  * @param buf buffer
1424  */
1425 static void free_netsgbuf(void *buf)
1426 {
1427         struct octnet_buf_free_info *finfo;
1428         struct sk_buff *skb;
1429         struct lio *lio;
1430         struct octnic_gather *g;
1431         int i, frags, iq;
1432
1433         finfo = (struct octnet_buf_free_info *)buf;
1434         skb = finfo->skb;
1435         lio = finfo->lio;
1436         g = finfo->g;
1437         frags = skb_shinfo(skb)->nr_frags;
1438
1439         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1440                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1441                          DMA_TO_DEVICE);
1442
1443         i = 1;
1444         while (frags--) {
1445                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1446
1447                 pci_unmap_page((lio->oct_dev)->pci_dev,
1448                                g->sg[(i >> 2)].ptr[(i & 3)],
1449                                frag->size, DMA_TO_DEVICE);
1450                 i++;
1451         }
1452
1453         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1454                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1455
1456         iq = skb_iq(lio, skb);
1457         spin_lock(&lio->glist_lock[iq]);
1458         list_add_tail(&g->list, &lio->glist[iq]);
1459         spin_unlock(&lio->glist_lock[iq]);
1460
1461         check_txq_state(lio, skb);     /* mq support: sub-queue state check */
1462
1463         tx_buffer_free(skb);
1464 }
1465
1466 /**
1467  * \brief Unmap and free gather buffer with response
1468  * @param buf buffer
1469  */
1470 static void free_netsgbuf_with_resp(void *buf)
1471 {
1472         struct octeon_soft_command *sc;
1473         struct octnet_buf_free_info *finfo;
1474         struct sk_buff *skb;
1475         struct lio *lio;
1476         struct octnic_gather *g;
1477         int i, frags, iq;
1478
1479         sc = (struct octeon_soft_command *)buf;
1480         skb = (struct sk_buff *)sc->callback_arg;
1481         finfo = (struct octnet_buf_free_info *)&skb->cb;
1482
1483         lio = finfo->lio;
1484         g = finfo->g;
1485         frags = skb_shinfo(skb)->nr_frags;
1486
1487         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1488                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1489                          DMA_TO_DEVICE);
1490
1491         i = 1;
1492         while (frags--) {
1493                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1494
1495                 pci_unmap_page((lio->oct_dev)->pci_dev,
1496                                g->sg[(i >> 2)].ptr[(i & 3)],
1497                                frag->size, DMA_TO_DEVICE);
1498                 i++;
1499         }
1500
1501         dma_sync_single_for_cpu(&lio->oct_dev->pci_dev->dev,
1502                                 g->sg_dma_ptr, g->sg_size, DMA_TO_DEVICE);
1503
1504         iq = skb_iq(lio, skb);
1505
1506         spin_lock(&lio->glist_lock[iq]);
1507         list_add_tail(&g->list, &lio->glist[iq]);
1508         spin_unlock(&lio->glist_lock[iq]);
1509
1510         /* Don't free the skb yet */
1511
1512         check_txq_state(lio, skb);
1513 }
1514
1515 /**
1516  * \brief Adjust ptp frequency
1517  * @param ptp PTP clock info
1518  * @param ppb how much to adjust by, in parts-per-billion
1519  */
1520 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1521 {
1522         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1523         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1524         u64 comp, delta;
1525         unsigned long flags;
1526         bool neg_adj = false;
1527
1528         if (ppb < 0) {
1529                 neg_adj = true;
1530                 ppb = -ppb;
1531         }
1532
1533         /* The hardware adds the clock compensation value to the
1534          * PTP clock on every coprocessor clock cycle, so we
1535          * compute the delta in terms of coprocessor clocks.
1536          */
1537         delta = (u64)ppb << 32;
1538         do_div(delta, oct->coproc_clock_rate);
1539
1540         spin_lock_irqsave(&lio->ptp_lock, flags);
1541         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1542         if (neg_adj)
1543                 comp -= delta;
1544         else
1545                 comp += delta;
1546         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1547         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1548
1549         return 0;
1550 }
1551
1552 /**
1553  * \brief Adjust ptp time
1554  * @param ptp PTP clock info
1555  * @param delta how much to adjust by, in nanosecs
1556  */
1557 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1558 {
1559         unsigned long flags;
1560         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1561
1562         spin_lock_irqsave(&lio->ptp_lock, flags);
1563         lio->ptp_adjust += delta;
1564         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1565
1566         return 0;
1567 }
1568
1569 /**
1570  * \brief Get hardware clock time, including any adjustment
1571  * @param ptp PTP clock info
1572  * @param ts timespec
1573  */
1574 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1575                                 struct timespec64 *ts)
1576 {
1577         u64 ns;
1578         unsigned long flags;
1579         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1580         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1581
1582         spin_lock_irqsave(&lio->ptp_lock, flags);
1583         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1584         ns += lio->ptp_adjust;
1585         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1586
1587         *ts = ns_to_timespec64(ns);
1588
1589         return 0;
1590 }
1591
1592 /**
1593  * \brief Set hardware clock time. Reset adjustment
1594  * @param ptp PTP clock info
1595  * @param ts timespec
1596  */
1597 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1598                                 const struct timespec64 *ts)
1599 {
1600         u64 ns;
1601         unsigned long flags;
1602         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1603         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1604
1605         ns = timespec_to_ns(ts);
1606
1607         spin_lock_irqsave(&lio->ptp_lock, flags);
1608         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1609         lio->ptp_adjust = 0;
1610         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1611
1612         return 0;
1613 }
1614
1615 /**
1616  * \brief Check if PTP is enabled
1617  * @param ptp PTP clock info
1618  * @param rq request
1619  * @param on is it on
1620  */
1621 static int
1622 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1623                     struct ptp_clock_request *rq __attribute__((unused)),
1624                     int on __attribute__((unused)))
1625 {
1626         return -EOPNOTSUPP;
1627 }
1628
1629 /**
1630  * \brief Open PTP clock source
1631  * @param netdev network device
1632  */
1633 static void oct_ptp_open(struct net_device *netdev)
1634 {
1635         struct lio *lio = GET_LIO(netdev);
1636         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1637
1638         spin_lock_init(&lio->ptp_lock);
1639
1640         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1641         lio->ptp_info.owner = THIS_MODULE;
1642         lio->ptp_info.max_adj = 250000000;
1643         lio->ptp_info.n_alarm = 0;
1644         lio->ptp_info.n_ext_ts = 0;
1645         lio->ptp_info.n_per_out = 0;
1646         lio->ptp_info.pps = 0;
1647         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1648         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1649         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1650         lio->ptp_info.settime64 = liquidio_ptp_settime;
1651         lio->ptp_info.enable = liquidio_ptp_enable;
1652
1653         lio->ptp_adjust = 0;
1654
1655         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1656                                              &oct->pci_dev->dev);
1657
1658         if (IS_ERR(lio->ptp_clock))
1659                 lio->ptp_clock = NULL;
1660 }
1661
1662 /**
1663  * \brief Init PTP clock
1664  * @param oct octeon device
1665  */
1666 static void liquidio_ptp_init(struct octeon_device *oct)
1667 {
1668         u64 clock_comp, cfg;
1669
1670         clock_comp = (u64)NSEC_PER_SEC << 32;
1671         do_div(clock_comp, oct->coproc_clock_rate);
1672         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1673
1674         /* Enable */
1675         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1676         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1677 }
1678
1679 /**
1680  * \brief Load firmware to device
1681  * @param oct octeon device
1682  *
1683  * Maps device to firmware filename, requests firmware, and downloads it
1684  */
1685 static int load_firmware(struct octeon_device *oct)
1686 {
1687         int ret = 0;
1688         const struct firmware *fw;
1689         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1690         char *tmp_fw_type;
1691
1692         if (strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
1693                     sizeof(LIO_FW_NAME_TYPE_NONE)) == 0) {
1694                 dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
1695                 return ret;
1696         }
1697
1698         if (fw_type[0] == '\0')
1699                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1700         else
1701                 tmp_fw_type = fw_type;
1702
1703         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1704                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1705                 LIO_FW_NAME_SUFFIX);
1706
1707         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1708         if (ret) {
1709                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n.",
1710                         fw_name);
1711                 release_firmware(fw);
1712                 return ret;
1713         }
1714
1715         ret = octeon_download_firmware(oct, fw->data, fw->size);
1716
1717         release_firmware(fw);
1718
1719         return ret;
1720 }
1721
1722 /**
1723  * \brief Setup output queue
1724  * @param oct octeon device
1725  * @param q_no which queue
1726  * @param num_descs how many descriptors
1727  * @param desc_size size of each descriptor
1728  * @param app_ctx application context
1729  */
1730 static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
1731                              int desc_size, void *app_ctx)
1732 {
1733         int ret_val = 0;
1734
1735         dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
1736         /* droq creation and local register settings. */
1737         ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
1738         if (ret_val < 0)
1739                 return ret_val;
1740
1741         if (ret_val == 1) {
1742                 dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
1743                 return 0;
1744         }
1745         /* tasklet creation for the droq */
1746
1747         /* Enable the droq queues */
1748         octeon_set_droq_pkt_op(oct, q_no, 1);
1749
1750         /* Send Credit for Octeon Output queues. Credits are always
1751          * sent after the output queue is enabled.
1752          */
1753         writel(oct->droq[q_no]->max_count,
1754                oct->droq[q_no]->pkts_credit_reg);
1755
1756         return ret_val;
1757 }
1758
1759 /**
1760  * \brief Callback for getting interface configuration
1761  * @param status status of request
1762  * @param buf pointer to resp structure
1763  */
1764 static void if_cfg_callback(struct octeon_device *oct,
1765                             u32 status __attribute__((unused)),
1766                             void *buf)
1767 {
1768         struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
1769         struct liquidio_if_cfg_resp *resp;
1770         struct liquidio_if_cfg_context *ctx;
1771
1772         resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
1773         ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
1774
1775         oct = lio_get_device(ctx->octeon_id);
1776         if (resp->status)
1777                 dev_err(&oct->pci_dev->dev, "nic if cfg instruction failed. Status: %llx\n",
1778                         CVM_CAST64(resp->status));
1779         WRITE_ONCE(ctx->cond, 1);
1780
1781         snprintf(oct->fw_info.liquidio_firmware_version, 32, "%s",
1782                  resp->cfg_info.liquidio_firmware_version);
1783
1784         /* This barrier is required to be sure that the response has been
1785          * written fully before waking up the handler
1786          */
1787         wmb();
1788
1789         wake_up_interruptible(&ctx->wc);
1790 }
1791
1792 /**
1793  * \brief Select queue based on hash
1794  * @param dev Net device
1795  * @param skb sk_buff structure
1796  * @returns selected queue number
1797  */
1798 static u16 select_q(struct net_device *dev, struct sk_buff *skb,
1799                     void *accel_priv __attribute__((unused)),
1800                     select_queue_fallback_t fallback __attribute__((unused)))
1801 {
1802         u32 qindex = 0;
1803         struct lio *lio;
1804
1805         lio = GET_LIO(dev);
1806         qindex = skb_tx_hash(dev, skb);
1807
1808         return (u16)(qindex % (lio->linfo.num_txpciq));
1809 }
1810
1811 /** Routine to push packets arriving on Octeon interface upto network layer.
1812  * @param oct_id   - octeon device id.
1813  * @param skbuff   - skbuff struct to be passed to network layer.
1814  * @param len      - size of total data received.
1815  * @param rh       - Control header associated with the packet
1816  * @param param    - additional control data with the packet
1817  * @param arg      - farg registered in droq_ops
1818  */
1819 static void
1820 liquidio_push_packet(u32 octeon_id __attribute__((unused)),
1821                      void *skbuff,
1822                      u32 len,
1823                      union octeon_rh *rh,
1824                      void *param,
1825                      void *arg)
1826 {
1827         struct napi_struct *napi = param;
1828         struct sk_buff *skb = (struct sk_buff *)skbuff;
1829         struct skb_shared_hwtstamps *shhwtstamps;
1830         u64 ns;
1831         u16 vtag = 0;
1832         struct net_device *netdev = (struct net_device *)arg;
1833         struct octeon_droq *droq = container_of(param, struct octeon_droq,
1834                                                 napi);
1835         if (netdev) {
1836                 int packet_was_received;
1837                 struct lio *lio = GET_LIO(netdev);
1838                 struct octeon_device *oct = lio->oct_dev;
1839
1840                 /* Do not proceed if the interface is not in RUNNING state. */
1841                 if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1842                         recv_buffer_free(skb);
1843                         droq->stats.rx_dropped++;
1844                         return;
1845                 }
1846
1847                 skb->dev = netdev;
1848
1849                 skb_record_rx_queue(skb, droq->q_no);
1850                 if (likely(len > MIN_SKB_SIZE)) {
1851                         struct octeon_skb_page_info *pg_info;
1852                         unsigned char *va;
1853
1854                         pg_info = ((struct octeon_skb_page_info *)(skb->cb));
1855                         if (pg_info->page) {
1856                                 /* For Paged allocation use the frags */
1857                                 va = page_address(pg_info->page) +
1858                                         pg_info->page_offset;
1859                                 memcpy(skb->data, va, MIN_SKB_SIZE);
1860                                 skb_put(skb, MIN_SKB_SIZE);
1861                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1862                                                 pg_info->page,
1863                                                 pg_info->page_offset +
1864                                                 MIN_SKB_SIZE,
1865                                                 len - MIN_SKB_SIZE,
1866                                                 LIO_RXBUFFER_SZ);
1867                         }
1868                 } else {
1869                         struct octeon_skb_page_info *pg_info =
1870                                 ((struct octeon_skb_page_info *)(skb->cb));
1871                         skb_copy_to_linear_data(skb, page_address(pg_info->page)
1872                                                 + pg_info->page_offset, len);
1873                         skb_put(skb, len);
1874                         put_page(pg_info->page);
1875                 }
1876
1877                 if (((oct->chip_id == OCTEON_CN66XX) ||
1878                      (oct->chip_id == OCTEON_CN68XX)) &&
1879                     ptp_enable) {
1880                         if (rh->r_dh.has_hwtstamp) {
1881                                 /* timestamp is included from the hardware at
1882                                  * the beginning of the packet.
1883                                  */
1884                                 if (ifstate_check
1885                                     (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
1886                                         /* Nanoseconds are in the first 64-bits
1887                                          * of the packet.
1888                                          */
1889                                         memcpy(&ns, (skb->data), sizeof(ns));
1890                                         shhwtstamps = skb_hwtstamps(skb);
1891                                         shhwtstamps->hwtstamp =
1892                                                 ns_to_ktime(ns +
1893                                                             lio->ptp_adjust);
1894                                 }
1895                                 skb_pull(skb, sizeof(ns));
1896                         }
1897                 }
1898
1899                 skb->protocol = eth_type_trans(skb, skb->dev);
1900                 if ((netdev->features & NETIF_F_RXCSUM) &&
1901                     (((rh->r_dh.encap_on) &&
1902                       (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
1903                      (!(rh->r_dh.encap_on) &&
1904                       (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
1905                         /* checksum has already been verified */
1906                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1907                 else
1908                         skb->ip_summed = CHECKSUM_NONE;
1909
1910                 /* Setting Encapsulation field on basis of status received
1911                  * from the firmware
1912                  */
1913                 if (rh->r_dh.encap_on) {
1914                         skb->encapsulation = 1;
1915                         skb->csum_level = 1;
1916                         droq->stats.rx_vxlan++;
1917                 }
1918
1919                 /* inbound VLAN tag */
1920                 if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1921                     (rh->r_dh.vlan != 0)) {
1922                         u16 vid = rh->r_dh.vlan;
1923                         u16 priority = rh->r_dh.priority;
1924
1925                         vtag = priority << 13 | vid;
1926                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
1927                 }
1928
1929                 packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
1930
1931                 if (packet_was_received) {
1932                         droq->stats.rx_bytes_received += len;
1933                         droq->stats.rx_pkts_received++;
1934                         netdev->last_rx = jiffies;
1935                 } else {
1936                         droq->stats.rx_dropped++;
1937                         netif_info(lio, rx_err, lio->netdev,
1938                                    "droq:%d  error rx_dropped:%llu\n",
1939                                    droq->q_no, droq->stats.rx_dropped);
1940                 }
1941
1942         } else {
1943                 recv_buffer_free(skb);
1944         }
1945 }
1946
1947 /**
1948  * \brief wrapper for calling napi_schedule
1949  * @param param parameters to pass to napi_schedule
1950  *
1951  * Used when scheduling on different CPUs
1952  */
1953 static void napi_schedule_wrapper(void *param)
1954 {
1955         struct napi_struct *napi = param;
1956
1957         napi_schedule(napi);
1958 }
1959
1960 /**
1961  * \brief callback when receive interrupt occurs and we are in NAPI mode
1962  * @param arg pointer to octeon output queue
1963  */
1964 static void liquidio_napi_drv_callback(void *arg)
1965 {
1966         struct octeon_droq *droq = arg;
1967         int this_cpu = smp_processor_id();
1968
1969         if (droq->cpu_id == this_cpu) {
1970                 napi_schedule(&droq->napi);
1971         } else {
1972                 struct call_single_data *csd = &droq->csd;
1973
1974                 csd->func = napi_schedule_wrapper;
1975                 csd->info = &droq->napi;
1976                 csd->flags = 0;
1977
1978                 smp_call_function_single_async(droq->cpu_id, csd);
1979         }
1980 }
1981
1982 /**
1983  * \brief Entry point for NAPI polling
1984  * @param napi NAPI structure
1985  * @param budget maximum number of items to process
1986  */
1987 static int liquidio_napi_poll(struct napi_struct *napi, int budget)
1988 {
1989         struct octeon_droq *droq;
1990         int work_done;
1991         int tx_done = 0, iq_no;
1992         struct octeon_instr_queue *iq;
1993         struct octeon_device *oct;
1994
1995         droq = container_of(napi, struct octeon_droq, napi);
1996         oct = droq->oct_dev;
1997         iq_no = droq->q_no;
1998         /* Handle Droq descriptors */
1999         work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
2000                                                  POLL_EVENT_PROCESS_PKTS,
2001                                                  budget);
2002
2003         /* Flush the instruction queue */
2004         iq = oct->instr_queue[iq_no];
2005         if (iq) {
2006                 /* Process iq buffers with in the budget limits */
2007                 tx_done = octeon_flush_iq(oct, iq, 1, budget);
2008                 /* Update iq read-index rather than waiting for next interrupt.
2009                  * Return back if tx_done is false.
2010                  */
2011                 update_txq_status(oct, iq_no);
2012                 /*tx_done = (iq->flush_index == iq->octeon_read_index);*/
2013         } else {
2014                 dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
2015                         __func__, iq_no);
2016         }
2017
2018         if ((work_done < budget) && (tx_done)) {
2019                 napi_complete(napi);
2020                 octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
2021                                              POLL_EVENT_ENABLE_INTR, 0);
2022                 return 0;
2023         }
2024
2025         return (!tx_done) ? (budget) : (work_done);
2026 }
2027
2028 /**
2029  * \brief Setup input and output queues
2030  * @param octeon_dev octeon device
2031  * @param ifidx  Interface Index
2032  *
2033  * Note: Queues are with respect to the octeon device. Thus
2034  * an input queue is for egress packets, and output queues
2035  * are for ingress packets.
2036  */
2037 static inline int setup_io_queues(struct octeon_device *octeon_dev,
2038                                   int ifidx)
2039 {
2040         struct octeon_droq_ops droq_ops;
2041         struct net_device *netdev;
2042         static int cpu_id;
2043         static int cpu_id_modulus;
2044         struct octeon_droq *droq;
2045         struct napi_struct *napi;
2046         int q, q_no, retval = 0;
2047         struct lio *lio;
2048         int num_tx_descs;
2049
2050         netdev = octeon_dev->props[ifidx].netdev;
2051
2052         lio = GET_LIO(netdev);
2053
2054         memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
2055
2056         droq_ops.fptr = liquidio_push_packet;
2057         droq_ops.farg = (void *)netdev;
2058
2059         droq_ops.poll_mode = 1;
2060         droq_ops.napi_fn = liquidio_napi_drv_callback;
2061         cpu_id = 0;
2062         cpu_id_modulus = num_present_cpus();
2063
2064         /* set up DROQs. */
2065         for (q = 0; q < lio->linfo.num_rxpciq; q++) {
2066                 q_no = lio->linfo.rxpciq[q].s.q_no;
2067                 dev_dbg(&octeon_dev->pci_dev->dev,
2068                         "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2069                         q, q_no);
2070                 retval = octeon_setup_droq(octeon_dev, q_no,
2071                                            CFG_GET_NUM_RX_DESCS_NIC_IF
2072                                                    (octeon_get_conf(octeon_dev),
2073                                                    lio->ifidx),
2074                                            CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2075                                                    (octeon_get_conf(octeon_dev),
2076                                                    lio->ifidx), NULL);
2077                 if (retval) {
2078                         dev_err(&octeon_dev->pci_dev->dev,
2079                                 "%s : Runtime DROQ(RxQ) creation failed.\n",
2080                                 __func__);
2081                         return 1;
2082                 }
2083
2084                 droq = octeon_dev->droq[q_no];
2085                 napi = &droq->napi;
2086                 dev_dbg(&octeon_dev->pci_dev->dev,
2087                         "netif_napi_add netdev:%llx oct:%llx\n",
2088                         (u64)netdev,
2089                         (u64)octeon_dev);
2090                 netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
2091
2092                 /* designate a CPU for this droq */
2093                 droq->cpu_id = cpu_id;
2094                 cpu_id++;
2095                 if (cpu_id >= cpu_id_modulus)
2096                         cpu_id = 0;
2097
2098                 octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
2099         }
2100
2101         /* set up IQs. */
2102         for (q = 0; q < lio->linfo.num_txpciq; q++) {
2103                 num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2104                                                            (octeon_dev),
2105                                                            lio->ifidx);
2106                 retval = octeon_setup_iq(octeon_dev, ifidx, q,
2107                                          lio->linfo.txpciq[q], num_tx_descs,
2108                                          netdev_get_tx_queue(netdev, q));
2109                 if (retval) {
2110                         dev_err(&octeon_dev->pci_dev->dev,
2111                                 " %s : Runtime IQ(TxQ) creation failed.\n",
2112                                 __func__);
2113                         return 1;
2114                 }
2115         }
2116
2117         return 0;
2118 }
2119
2120 /**
2121  * \brief Poll routine for checking transmit queue status
2122  * @param work work_struct data structure
2123  */
2124 static void octnet_poll_check_txq_status(struct work_struct *work)
2125 {
2126         struct cavium_wk *wk = (struct cavium_wk *)work;
2127         struct lio *lio = (struct lio *)wk->ctxptr;
2128
2129         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
2130                 return;
2131
2132         check_txq_status(lio);
2133         queue_delayed_work(lio->txq_status_wq.wq,
2134                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2135 }
2136
2137 /**
2138  * \brief Sets up the txq poll check
2139  * @param netdev network device
2140  */
2141 static inline void setup_tx_poll_fn(struct net_device *netdev)
2142 {
2143         struct lio *lio = GET_LIO(netdev);
2144         struct octeon_device *oct = lio->oct_dev;
2145
2146         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
2147                                                 WQ_MEM_RECLAIM, 0);
2148         if (!lio->txq_status_wq.wq) {
2149                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
2150                 return;
2151         }
2152         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
2153                           octnet_poll_check_txq_status);
2154         lio->txq_status_wq.wk.ctxptr = lio;
2155         queue_delayed_work(lio->txq_status_wq.wq,
2156                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
2157 }
2158
2159 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
2160 {
2161         struct lio *lio = GET_LIO(netdev);
2162
2163         cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
2164         destroy_workqueue(lio->txq_status_wq.wq);
2165 }
2166
2167 /**
2168  * \brief Net device open for LiquidIO
2169  * @param netdev network device
2170  */
2171 static int liquidio_open(struct net_device *netdev)
2172 {
2173         struct lio *lio = GET_LIO(netdev);
2174         struct octeon_device *oct = lio->oct_dev;
2175         struct napi_struct *napi, *n;
2176
2177         if (oct->props[lio->ifidx].napi_enabled == 0) {
2178                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
2179                         napi_enable(napi);
2180
2181                 oct->props[lio->ifidx].napi_enabled = 1;
2182         }
2183
2184         oct_ptp_open(netdev);
2185
2186         ifstate_set(lio, LIO_IFSTATE_RUNNING);
2187
2188         setup_tx_poll_fn(netdev);
2189
2190         start_txq(netdev);
2191
2192         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
2193
2194         /* tell Octeon to start forwarding packets to host */
2195         send_rx_ctrl_cmd(lio, 1);
2196
2197         /* Ready for link status updates */
2198         lio->intf_open = 1;
2199
2200         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
2201                  netdev->name);
2202
2203         return 0;
2204 }
2205
2206 /**
2207  * \brief Net device stop for LiquidIO
2208  * @param netdev network device
2209  */
2210 static int liquidio_stop(struct net_device *netdev)
2211 {
2212         struct lio *lio = GET_LIO(netdev);
2213         struct octeon_device *oct = lio->oct_dev;
2214
2215         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
2216
2217         netif_tx_disable(netdev);
2218
2219         /* Inform that netif carrier is down */
2220         netif_carrier_off(netdev);
2221         lio->intf_open = 0;
2222         lio->linfo.link.s.link_up = 0;
2223         lio->link_changes++;
2224
2225         /* Pause for a moment and wait for Octeon to flush out (to the wire) any
2226          * egress packets that are in-flight.
2227          */
2228         set_current_state(TASK_INTERRUPTIBLE);
2229         schedule_timeout(msecs_to_jiffies(100));
2230
2231         /* Now it should be safe to tell Octeon that nic interface is down. */
2232         send_rx_ctrl_cmd(lio, 0);
2233
2234         cleanup_tx_poll_fn(netdev);
2235
2236         if (lio->ptp_clock) {
2237                 ptp_clock_unregister(lio->ptp_clock);
2238                 lio->ptp_clock = NULL;
2239         }
2240
2241         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
2242
2243         return 0;
2244 }
2245
2246 /**
2247  * \brief Converts a mask based on net device flags
2248  * @param netdev network device
2249  *
2250  * This routine generates a octnet_ifflags mask from the net device flags
2251  * received from the OS.
2252  */
2253 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
2254 {
2255         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
2256
2257         if (netdev->flags & IFF_PROMISC)
2258                 f |= OCTNET_IFFLAG_PROMISC;
2259
2260         if (netdev->flags & IFF_ALLMULTI)
2261                 f |= OCTNET_IFFLAG_ALLMULTI;
2262
2263         if (netdev->flags & IFF_MULTICAST) {
2264                 f |= OCTNET_IFFLAG_MULTICAST;
2265
2266                 /* Accept all multicast addresses if there are more than we
2267                  * can handle
2268                  */
2269                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
2270                         f |= OCTNET_IFFLAG_ALLMULTI;
2271         }
2272
2273         if (netdev->flags & IFF_BROADCAST)
2274                 f |= OCTNET_IFFLAG_BROADCAST;
2275
2276         return f;
2277 }
2278
2279 /**
2280  * \brief Net device set_multicast_list
2281  * @param netdev network device
2282  */
2283 static void liquidio_set_mcast_list(struct net_device *netdev)
2284 {
2285         struct lio *lio = GET_LIO(netdev);
2286         struct octeon_device *oct = lio->oct_dev;
2287         struct octnic_ctrl_pkt nctrl;
2288         struct netdev_hw_addr *ha;
2289         u64 *mc;
2290         int ret;
2291         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
2292
2293         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2294
2295         /* Create a ctrl pkt command to be sent to core app. */
2296         nctrl.ncmd.u64 = 0;
2297         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
2298         nctrl.ncmd.s.param1 = get_new_flags(netdev);
2299         nctrl.ncmd.s.param2 = mc_count;
2300         nctrl.ncmd.s.more = mc_count;
2301         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2302         nctrl.netpndev = (u64)netdev;
2303         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2304
2305         /* copy all the addresses into the udd */
2306         mc = &nctrl.udd[0];
2307         netdev_for_each_mc_addr(ha, netdev) {
2308                 *mc = 0;
2309                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
2310                 /* no need to swap bytes */
2311
2312                 if (++mc > &nctrl.udd[mc_count])
2313                         break;
2314         }
2315
2316         /* Apparently, any activity in this call from the kernel has to
2317          * be atomic. So we won't wait for response.
2318          */
2319         nctrl.wait_time = 0;
2320
2321         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2322         if (ret < 0) {
2323                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2324                         ret);
2325         }
2326 }
2327
2328 /**
2329  * \brief Net device set_mac_address
2330  * @param netdev network device
2331  */
2332 static int liquidio_set_mac(struct net_device *netdev, void *p)
2333 {
2334         int ret = 0;
2335         struct lio *lio = GET_LIO(netdev);
2336         struct octeon_device *oct = lio->oct_dev;
2337         struct sockaddr *addr = (struct sockaddr *)p;
2338         struct octnic_ctrl_pkt nctrl;
2339
2340         if (!is_valid_ether_addr(addr->sa_data))
2341                 return -EADDRNOTAVAIL;
2342
2343         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2344
2345         nctrl.ncmd.u64 = 0;
2346         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2347         nctrl.ncmd.s.param1 = 0;
2348         nctrl.ncmd.s.more = 1;
2349         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2350         nctrl.netpndev = (u64)netdev;
2351         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2352         nctrl.wait_time = 100;
2353
2354         nctrl.udd[0] = 0;
2355         /* The MAC Address is presented in network byte order. */
2356         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2357
2358         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2359         if (ret < 0) {
2360                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2361                 return -ENOMEM;
2362         }
2363         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2364         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2365
2366         return 0;
2367 }
2368
2369 /**
2370  * \brief Net device get_stats
2371  * @param netdev network device
2372  */
2373 static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
2374 {
2375         struct lio *lio = GET_LIO(netdev);
2376         struct net_device_stats *stats = &netdev->stats;
2377         struct octeon_device *oct;
2378         u64 pkts = 0, drop = 0, bytes = 0;
2379         struct oct_droq_stats *oq_stats;
2380         struct oct_iq_stats *iq_stats;
2381         int i, iq_no, oq_no;
2382
2383         oct = lio->oct_dev;
2384
2385         for (i = 0; i < lio->linfo.num_txpciq; i++) {
2386                 iq_no = lio->linfo.txpciq[i].s.q_no;
2387                 iq_stats = &oct->instr_queue[iq_no]->stats;
2388                 pkts += iq_stats->tx_done;
2389                 drop += iq_stats->tx_dropped;
2390                 bytes += iq_stats->tx_tot_bytes;
2391         }
2392
2393         stats->tx_packets = pkts;
2394         stats->tx_bytes = bytes;
2395         stats->tx_dropped = drop;
2396
2397         pkts = 0;
2398         drop = 0;
2399         bytes = 0;
2400
2401         for (i = 0; i < lio->linfo.num_rxpciq; i++) {
2402                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2403                 oq_stats = &oct->droq[oq_no]->stats;
2404                 pkts += oq_stats->rx_pkts_received;
2405                 drop += (oq_stats->rx_dropped +
2406                          oq_stats->dropped_nodispatch +
2407                          oq_stats->dropped_toomany +
2408                          oq_stats->dropped_nomem);
2409                 bytes += oq_stats->rx_bytes_received;
2410         }
2411
2412         stats->rx_bytes = bytes;
2413         stats->rx_packets = pkts;
2414         stats->rx_dropped = drop;
2415
2416         return stats;
2417 }
2418
2419 /**
2420  * \brief Net device change_mtu
2421  * @param netdev network device
2422  */
2423 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
2424 {
2425         struct lio *lio = GET_LIO(netdev);
2426         struct octeon_device *oct = lio->oct_dev;
2427         struct octnic_ctrl_pkt nctrl;
2428         int ret = 0;
2429
2430         /* Limit the MTU to make sure the ethernet packets are between 68 bytes
2431          * and 16000 bytes
2432          */
2433         if ((new_mtu < LIO_MIN_MTU_SIZE) ||
2434             (new_mtu > LIO_MAX_MTU_SIZE)) {
2435                 dev_err(&oct->pci_dev->dev, "Invalid MTU: %d\n", new_mtu);
2436                 dev_err(&oct->pci_dev->dev, "Valid range %d and %d\n",
2437                         LIO_MIN_MTU_SIZE, LIO_MAX_MTU_SIZE);
2438                 return -EINVAL;
2439         }
2440
2441         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2442
2443         nctrl.ncmd.u64 = 0;
2444         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
2445         nctrl.ncmd.s.param1 = new_mtu;
2446         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2447         nctrl.wait_time = 100;
2448         nctrl.netpndev = (u64)netdev;
2449         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2450
2451         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2452         if (ret < 0) {
2453                 dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
2454                 return -1;
2455         }
2456
2457         lio->mtu = new_mtu;
2458
2459         return 0;
2460 }
2461
2462 /**
2463  * \brief Handler for SIOCSHWTSTAMP ioctl
2464  * @param netdev network device
2465  * @param ifr interface request
2466  * @param cmd command
2467  */
2468 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2469 {
2470         struct hwtstamp_config conf;
2471         struct lio *lio = GET_LIO(netdev);
2472
2473         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2474                 return -EFAULT;
2475
2476         if (conf.flags)
2477                 return -EINVAL;
2478
2479         switch (conf.tx_type) {
2480         case HWTSTAMP_TX_ON:
2481         case HWTSTAMP_TX_OFF:
2482                 break;
2483         default:
2484                 return -ERANGE;
2485         }
2486
2487         switch (conf.rx_filter) {
2488         case HWTSTAMP_FILTER_NONE:
2489                 break;
2490         case HWTSTAMP_FILTER_ALL:
2491         case HWTSTAMP_FILTER_SOME:
2492         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2493         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2494         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2495         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2496         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2497         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2498         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2499         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2500         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2501         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2502         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2503         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2504                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2505                 break;
2506         default:
2507                 return -ERANGE;
2508         }
2509
2510         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2511                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2512
2513         else
2514                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2515
2516         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2517 }
2518
2519 /**
2520  * \brief ioctl handler
2521  * @param netdev network device
2522  * @param ifr interface request
2523  * @param cmd command
2524  */
2525 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2526 {
2527         switch (cmd) {
2528         case SIOCSHWTSTAMP:
2529                 return hwtstamp_ioctl(netdev, ifr);
2530         default:
2531                 return -EOPNOTSUPP;
2532         }
2533 }
2534
2535 /**
2536  * \brief handle a Tx timestamp response
2537  * @param status response status
2538  * @param buf pointer to skb
2539  */
2540 static void handle_timestamp(struct octeon_device *oct,
2541                              u32 status,
2542                              void *buf)
2543 {
2544         struct octnet_buf_free_info *finfo;
2545         struct octeon_soft_command *sc;
2546         struct oct_timestamp_resp *resp;
2547         struct lio *lio;
2548         struct sk_buff *skb = (struct sk_buff *)buf;
2549
2550         finfo = (struct octnet_buf_free_info *)skb->cb;
2551         lio = finfo->lio;
2552         sc = finfo->sc;
2553         oct = lio->oct_dev;
2554         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2555
2556         if (status != OCTEON_REQUEST_DONE) {
2557                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2558                         CVM_CAST64(status));
2559                 resp->timestamp = 0;
2560         }
2561
2562         octeon_swap_8B_data(&resp->timestamp, 1);
2563
2564         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2565                 struct skb_shared_hwtstamps ts;
2566                 u64 ns = resp->timestamp;
2567
2568                 netif_info(lio, tx_done, lio->netdev,
2569                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2570                            skb, (unsigned long long)ns);
2571                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2572                 skb_tstamp_tx(skb, &ts);
2573         }
2574
2575         octeon_free_soft_command(oct, sc);
2576         tx_buffer_free(skb);
2577 }
2578
2579 /* \brief Send a data packet that will be timestamped
2580  * @param oct octeon device
2581  * @param ndata pointer to network data
2582  * @param finfo pointer to private network data
2583  */
2584 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2585                                          struct octnic_data_pkt *ndata,
2586                                          struct octnet_buf_free_info *finfo)
2587 {
2588         int retval;
2589         struct octeon_soft_command *sc;
2590         struct lio *lio;
2591         int ring_doorbell;
2592         u32 len;
2593
2594         lio = finfo->lio;
2595
2596         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2597                                             sizeof(struct oct_timestamp_resp));
2598         finfo->sc = sc;
2599
2600         if (!sc) {
2601                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2602                 return IQ_SEND_FAILED;
2603         }
2604
2605         if (ndata->reqtype == REQTYPE_NORESP_NET)
2606                 ndata->reqtype = REQTYPE_RESP_NET;
2607         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2608                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2609
2610         sc->callback = handle_timestamp;
2611         sc->callback_arg = finfo->skb;
2612         sc->iq_no = ndata->q_no;
2613
2614         len = (u32)((struct octeon_instr_ih2 *)(&sc->cmd.cmd2.ih2))->dlengsz;
2615
2616         ring_doorbell = 1;
2617         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2618                                      sc, len, ndata->reqtype);
2619
2620         if (retval == IQ_SEND_FAILED) {
2621                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2622                         retval);
2623                 octeon_free_soft_command(oct, sc);
2624         } else {
2625                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2626         }
2627
2628         return retval;
2629 }
2630
2631 /** \brief Transmit networks packets to the Octeon interface
2632  * @param skbuff   skbuff struct to be passed to network layer.
2633  * @param netdev    pointer to network device
2634  * @returns whether the packet was transmitted to the device okay or not
2635  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2636  */
2637 static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2638 {
2639         struct lio *lio;
2640         struct octnet_buf_free_info *finfo;
2641         union octnic_cmd_setup cmdsetup;
2642         struct octnic_data_pkt ndata;
2643         struct octeon_device *oct;
2644         struct oct_iq_stats *stats;
2645         struct octeon_instr_irh *irh;
2646         union tx_info *tx_info;
2647         int status = 0;
2648         int q_idx = 0, iq_no = 0;
2649         int j;
2650         u64 dptr = 0;
2651         u32 tag = 0;
2652
2653         lio = GET_LIO(netdev);
2654         oct = lio->oct_dev;
2655
2656         if (netif_is_multiqueue(netdev)) {
2657                 q_idx = skb->queue_mapping;
2658                 q_idx = (q_idx % (lio->linfo.num_txpciq));
2659                 tag = q_idx;
2660                 iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2661         } else {
2662                 iq_no = lio->txq;
2663         }
2664
2665         stats = &oct->instr_queue[iq_no]->stats;
2666
2667         /* Check for all conditions in which the current packet cannot be
2668          * transmitted.
2669          */
2670         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2671             (!lio->linfo.link.s.link_up) ||
2672             (skb->len <= 0)) {
2673                 netif_info(lio, tx_err, lio->netdev,
2674                            "Transmit failed link_status : %d\n",
2675                            lio->linfo.link.s.link_up);
2676                 goto lio_xmit_failed;
2677         }
2678
2679         /* Use space in skb->cb to store info used to unmap and
2680          * free the buffers.
2681          */
2682         finfo = (struct octnet_buf_free_info *)skb->cb;
2683         finfo->lio = lio;
2684         finfo->skb = skb;
2685         finfo->sc = NULL;
2686
2687         /* Prepare the attributes for the data to be passed to OSI. */
2688         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2689
2690         ndata.buf = (void *)finfo;
2691
2692         ndata.q_no = iq_no;
2693
2694         if (netif_is_multiqueue(netdev)) {
2695                 if (octnet_iq_is_full(oct, ndata.q_no)) {
2696                         /* defer sending if queue is full */
2697                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2698                                    ndata.q_no);
2699                         stats->tx_iq_busy++;
2700                         return NETDEV_TX_BUSY;
2701                 }
2702         } else {
2703                 if (octnet_iq_is_full(oct, lio->txq)) {
2704                         /* defer sending if queue is full */
2705                         stats->tx_iq_busy++;
2706                         netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2707                                    lio->txq);
2708                         return NETDEV_TX_BUSY;
2709                 }
2710         }
2711         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2712          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2713          */
2714
2715         ndata.datasize = skb->len;
2716
2717         cmdsetup.u64 = 0;
2718         cmdsetup.s.iq_no = iq_no;
2719
2720         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2721                 if (skb->encapsulation) {
2722                         cmdsetup.s.tnl_csum = 1;
2723                         stats->tx_vxlan++;
2724                 } else {
2725                         cmdsetup.s.transport_csum = 1;
2726                 }
2727         }
2728         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2729                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2730                 cmdsetup.s.timestamp = 1;
2731         }
2732
2733         if (skb_shinfo(skb)->nr_frags == 0) {
2734                 cmdsetup.s.u.datasize = skb->len;
2735                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2736
2737                 /* Offload checksum calculation for TCP/UDP packets */
2738                 dptr = dma_map_single(&oct->pci_dev->dev,
2739                                       skb->data,
2740                                       skb->len,
2741                                       DMA_TO_DEVICE);
2742                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2743                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2744                                 __func__);
2745                         return NETDEV_TX_BUSY;
2746                 }
2747
2748                 ndata.cmd.cmd2.dptr = dptr;
2749                 finfo->dptr = dptr;
2750                 ndata.reqtype = REQTYPE_NORESP_NET;
2751
2752         } else {
2753                 int i, frags;
2754                 struct skb_frag_struct *frag;
2755                 struct octnic_gather *g;
2756
2757                 spin_lock(&lio->glist_lock[q_idx]);
2758                 g = (struct octnic_gather *)
2759                         list_delete_head(&lio->glist[q_idx]);
2760                 spin_unlock(&lio->glist_lock[q_idx]);
2761
2762                 if (!g) {
2763                         netif_info(lio, tx_err, lio->netdev,
2764                                    "Transmit scatter gather: glist null!\n");
2765                         goto lio_xmit_failed;
2766                 }
2767
2768                 cmdsetup.s.gather = 1;
2769                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2770                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2771
2772                 memset(g->sg, 0, g->sg_size);
2773
2774                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2775                                                  skb->data,
2776                                                  (skb->len - skb->data_len),
2777                                                  DMA_TO_DEVICE);
2778                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2779                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2780                                 __func__);
2781                         return NETDEV_TX_BUSY;
2782                 }
2783                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2784
2785                 frags = skb_shinfo(skb)->nr_frags;
2786                 i = 1;
2787                 while (frags--) {
2788                         frag = &skb_shinfo(skb)->frags[i - 1];
2789
2790                         g->sg[(i >> 2)].ptr[(i & 3)] =
2791                                 dma_map_page(&oct->pci_dev->dev,
2792                                              frag->page.p,
2793                                              frag->page_offset,
2794                                              frag->size,
2795                                              DMA_TO_DEVICE);
2796
2797                         if (dma_mapping_error(&oct->pci_dev->dev,
2798                                               g->sg[i >> 2].ptr[i & 3])) {
2799                                 dma_unmap_single(&oct->pci_dev->dev,
2800                                                  g->sg[0].ptr[0],
2801                                                  skb->len - skb->data_len,
2802                                                  DMA_TO_DEVICE);
2803                                 for (j = 1; j < i; j++) {
2804                                         frag = &skb_shinfo(skb)->frags[j - 1];
2805                                         dma_unmap_page(&oct->pci_dev->dev,
2806                                                        g->sg[j >> 2].ptr[j & 3],
2807                                                        frag->size,
2808                                                        DMA_TO_DEVICE);
2809                                 }
2810                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2811                                         __func__);
2812                                 return NETDEV_TX_BUSY;
2813                         }
2814
2815                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2816                         i++;
2817                 }
2818
2819                 dma_sync_single_for_device(&oct->pci_dev->dev, g->sg_dma_ptr,
2820                                            g->sg_size, DMA_TO_DEVICE);
2821                 dptr = g->sg_dma_ptr;
2822
2823                 ndata.cmd.cmd2.dptr = dptr;
2824                 finfo->dptr = dptr;
2825                 finfo->g = g;
2826
2827                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2828         }
2829
2830         irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2831         tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2832
2833         if (skb_shinfo(skb)->gso_size) {
2834                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2835                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2836                 stats->tx_gso++;
2837         }
2838
2839         /* HW insert VLAN tag */
2840         if (skb_vlan_tag_present(skb)) {
2841                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2842                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2843         }
2844
2845         if (unlikely(cmdsetup.s.timestamp))
2846                 status = send_nic_timestamp_pkt(oct, &ndata, finfo);
2847         else
2848                 status = octnet_send_nic_data_pkt(oct, &ndata);
2849         if (status == IQ_SEND_FAILED)
2850                 goto lio_xmit_failed;
2851
2852         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2853
2854         if (status == IQ_SEND_STOP)
2855                 stop_q(lio->netdev, q_idx);
2856
2857         netif_trans_update(netdev);
2858
2859         if (skb_shinfo(skb)->gso_size)
2860                 stats->tx_done += skb_shinfo(skb)->gso_segs;
2861         else
2862                 stats->tx_done++;
2863         stats->tx_tot_bytes += skb->len;
2864
2865         return NETDEV_TX_OK;
2866
2867 lio_xmit_failed:
2868         stats->tx_dropped++;
2869         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2870                    iq_no, stats->tx_dropped);
2871         if (dptr)
2872                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2873                                  ndata.datasize, DMA_TO_DEVICE);
2874         tx_buffer_free(skb);
2875         return NETDEV_TX_OK;
2876 }
2877
2878 /** \brief Network device Tx timeout
2879  * @param netdev    pointer to network device
2880  */
2881 static void liquidio_tx_timeout(struct net_device *netdev)
2882 {
2883         struct lio *lio;
2884
2885         lio = GET_LIO(netdev);
2886
2887         netif_info(lio, tx_err, lio->netdev,
2888                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2889                    netdev->stats.tx_dropped);
2890         netif_trans_update(netdev);
2891         txqs_wake(netdev);
2892 }
2893
2894 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2895                                     __be16 proto __attribute__((unused)),
2896                                     u16 vid)
2897 {
2898         struct lio *lio = GET_LIO(netdev);
2899         struct octeon_device *oct = lio->oct_dev;
2900         struct octnic_ctrl_pkt nctrl;
2901         int ret = 0;
2902
2903         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2904
2905         nctrl.ncmd.u64 = 0;
2906         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2907         nctrl.ncmd.s.param1 = vid;
2908         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2909         nctrl.wait_time = 100;
2910         nctrl.netpndev = (u64)netdev;
2911         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2912
2913         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2914         if (ret < 0) {
2915                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2916                         ret);
2917         }
2918
2919         return ret;
2920 }
2921
2922 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2923                                      __be16 proto __attribute__((unused)),
2924                                      u16 vid)
2925 {
2926         struct lio *lio = GET_LIO(netdev);
2927         struct octeon_device *oct = lio->oct_dev;
2928         struct octnic_ctrl_pkt nctrl;
2929         int ret = 0;
2930
2931         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2932
2933         nctrl.ncmd.u64 = 0;
2934         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2935         nctrl.ncmd.s.param1 = vid;
2936         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2937         nctrl.wait_time = 100;
2938         nctrl.netpndev = (u64)netdev;
2939         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2940
2941         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2942         if (ret < 0) {
2943                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2944                         ret);
2945         }
2946         return ret;
2947 }
2948
2949 /** Sending command to enable/disable RX checksum offload
2950  * @param netdev                pointer to network device
2951  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2952  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2953  *                              OCTNET_CMD_RXCSUM_DISABLE
2954  * @returns                     SUCCESS or FAILURE
2955  */
2956 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2957                                        u8 rx_cmd)
2958 {
2959         struct lio *lio = GET_LIO(netdev);
2960         struct octeon_device *oct = lio->oct_dev;
2961         struct octnic_ctrl_pkt nctrl;
2962         int ret = 0;
2963
2964         nctrl.ncmd.u64 = 0;
2965         nctrl.ncmd.s.cmd = command;
2966         nctrl.ncmd.s.param1 = rx_cmd;
2967         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2968         nctrl.wait_time = 100;
2969         nctrl.netpndev = (u64)netdev;
2970         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2971
2972         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2973         if (ret < 0) {
2974                 dev_err(&oct->pci_dev->dev,
2975                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2976                         ret);
2977         }
2978         return ret;
2979 }
2980
2981 /** Sending command to add/delete VxLAN UDP port to firmware
2982  * @param netdev                pointer to network device
2983  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2984  * @param vxlan_port            VxLAN port to be added or deleted
2985  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2986  *                              OCTNET_CMD_VXLAN_PORT_DEL
2987  * @returns                     SUCCESS or FAILURE
2988  */
2989 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2990                                        u16 vxlan_port, u8 vxlan_cmd_bit)
2991 {
2992         struct lio *lio = GET_LIO(netdev);
2993         struct octeon_device *oct = lio->oct_dev;
2994         struct octnic_ctrl_pkt nctrl;
2995         int ret = 0;
2996
2997         nctrl.ncmd.u64 = 0;
2998         nctrl.ncmd.s.cmd = command;
2999         nctrl.ncmd.s.more = vxlan_cmd_bit;
3000         nctrl.ncmd.s.param1 = vxlan_port;
3001         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3002         nctrl.wait_time = 100;
3003         nctrl.netpndev = (u64)netdev;
3004         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
3005
3006         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
3007         if (ret < 0) {
3008                 dev_err(&oct->pci_dev->dev,
3009                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
3010                         ret);
3011         }
3012         return ret;
3013 }
3014
3015 /** \brief Net device fix features
3016  * @param netdev  pointer to network device
3017  * @param request features requested
3018  * @returns updated features list
3019  */
3020 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
3021                                                netdev_features_t request)
3022 {
3023         struct lio *lio = netdev_priv(netdev);
3024
3025         if ((request & NETIF_F_RXCSUM) &&
3026             !(lio->dev_capability & NETIF_F_RXCSUM))
3027                 request &= ~NETIF_F_RXCSUM;
3028
3029         if ((request & NETIF_F_HW_CSUM) &&
3030             !(lio->dev_capability & NETIF_F_HW_CSUM))
3031                 request &= ~NETIF_F_HW_CSUM;
3032
3033         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
3034                 request &= ~NETIF_F_TSO;
3035
3036         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
3037                 request &= ~NETIF_F_TSO6;
3038
3039         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
3040                 request &= ~NETIF_F_LRO;
3041
3042         /*Disable LRO if RXCSUM is off */
3043         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
3044             (lio->dev_capability & NETIF_F_LRO))
3045                 request &= ~NETIF_F_LRO;
3046
3047         return request;
3048 }
3049
3050 /** \brief Net device set features
3051  * @param netdev  pointer to network device
3052  * @param features features to enable/disable
3053  */
3054 static int liquidio_set_features(struct net_device *netdev,
3055                                  netdev_features_t features)
3056 {
3057         struct lio *lio = netdev_priv(netdev);
3058
3059         if (!((netdev->features ^ features) & NETIF_F_LRO))
3060                 return 0;
3061
3062         if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO))
3063                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3064                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3065         else if (!(features & NETIF_F_LRO) &&
3066                  (lio->dev_capability & NETIF_F_LRO))
3067                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
3068                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3069
3070         /* Sending command to firmware to enable/disable RX checksum
3071          * offload settings using ethtool
3072          */
3073         if (!(netdev->features & NETIF_F_RXCSUM) &&
3074             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3075             (features & NETIF_F_RXCSUM))
3076                 liquidio_set_rxcsum_command(netdev,
3077                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
3078                                             OCTNET_CMD_RXCSUM_ENABLE);
3079         else if ((netdev->features & NETIF_F_RXCSUM) &&
3080                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
3081                  !(features & NETIF_F_RXCSUM))
3082                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3083                                             OCTNET_CMD_RXCSUM_DISABLE);
3084
3085         return 0;
3086 }
3087
3088 static void liquidio_add_vxlan_port(struct net_device *netdev,
3089                                     struct udp_tunnel_info *ti)
3090 {
3091         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3092                 return;
3093
3094         liquidio_vxlan_port_command(netdev,
3095                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3096                                     htons(ti->port),
3097                                     OCTNET_CMD_VXLAN_PORT_ADD);
3098 }
3099
3100 static void liquidio_del_vxlan_port(struct net_device *netdev,
3101                                     struct udp_tunnel_info *ti)
3102 {
3103         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3104                 return;
3105
3106         liquidio_vxlan_port_command(netdev,
3107                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
3108                                     htons(ti->port),
3109                                     OCTNET_CMD_VXLAN_PORT_DEL);
3110 }
3111
3112 static struct net_device_ops lionetdevops = {
3113         .ndo_open               = liquidio_open,
3114         .ndo_stop               = liquidio_stop,
3115         .ndo_start_xmit         = liquidio_xmit,
3116         .ndo_get_stats          = liquidio_get_stats,
3117         .ndo_set_mac_address    = liquidio_set_mac,
3118         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3119         .ndo_tx_timeout         = liquidio_tx_timeout,
3120
3121         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3122         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3123         .ndo_change_mtu         = liquidio_change_mtu,
3124         .ndo_do_ioctl           = liquidio_ioctl,
3125         .ndo_fix_features       = liquidio_fix_features,
3126         .ndo_set_features       = liquidio_set_features,
3127         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3128         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3129 };
3130
3131 /** \brief Entry point for the liquidio module
3132  */
3133 static int __init liquidio_init(void)
3134 {
3135         int i;
3136         struct handshake *hs;
3137
3138         init_completion(&first_stage);
3139
3140         octeon_init_device_list(conf_type);
3141
3142         if (liquidio_init_pci())
3143                 return -EINVAL;
3144
3145         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3146
3147         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3148                 hs = &handshake[i];
3149                 if (hs->pci_dev) {
3150                         wait_for_completion(&hs->init);
3151                         if (!hs->init_ok) {
3152                                 /* init handshake failed */
3153                                 dev_err(&hs->pci_dev->dev,
3154                                         "Failed to init device\n");
3155                                 liquidio_deinit_pci();
3156                                 return -EIO;
3157                         }
3158                 }
3159         }
3160
3161         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3162                 hs = &handshake[i];
3163                 if (hs->pci_dev) {
3164                         wait_for_completion_timeout(&hs->started,
3165                                                     msecs_to_jiffies(30000));
3166                         if (!hs->started_ok) {
3167                                 /* starter handshake failed */
3168                                 dev_err(&hs->pci_dev->dev,
3169                                         "Firmware failed to start\n");
3170                                 liquidio_deinit_pci();
3171                                 return -EIO;
3172                         }
3173                 }
3174         }
3175
3176         return 0;
3177 }
3178
3179 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3180 {
3181         struct octeon_device *oct = (struct octeon_device *)buf;
3182         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3183         int gmxport = 0;
3184         union oct_link_status *ls;
3185         int i;
3186
3187         if (recv_pkt->buffer_size[0] != sizeof(*ls)) {
3188                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3189                         recv_pkt->buffer_size[0],
3190                         recv_pkt->rh.r_nic_info.gmxport);
3191                 goto nic_info_err;
3192         }
3193
3194         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3195         ls = (union oct_link_status *)get_rbd(recv_pkt->buffer_ptr[0]);
3196
3197         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3198         for (i = 0; i < oct->ifcount; i++) {
3199                 if (oct->props[i].gmxport == gmxport) {
3200                         update_link_status(oct->props[i].netdev, ls);
3201                         break;
3202                 }
3203         }
3204
3205 nic_info_err:
3206         for (i = 0; i < recv_pkt->buffer_count; i++)
3207                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3208         octeon_free_recv_info(recv_info);
3209         return 0;
3210 }
3211
3212 /**
3213  * \brief Setup network interfaces
3214  * @param octeon_dev  octeon device
3215  *
3216  * Called during init time for each device. It assumes the NIC
3217  * is already up and running.  The link information for each
3218  * interface is passed in link_info.
3219  */
3220 static int setup_nic_devices(struct octeon_device *octeon_dev)
3221 {
3222         struct lio *lio = NULL;
3223         struct net_device *netdev;
3224         u8 mac[6], i, j;
3225         struct octeon_soft_command *sc;
3226         struct liquidio_if_cfg_context *ctx;
3227         struct liquidio_if_cfg_resp *resp;
3228         struct octdev_props *props;
3229         int retval, num_iqueues, num_oqueues;
3230         union oct_nic_if_cfg if_cfg;
3231         unsigned int base_queue;
3232         unsigned int gmx_port_id;
3233         u32 resp_size, ctx_size, data_size;
3234         u32 ifidx_or_pfnum;
3235         struct lio_version *vdata;
3236
3237         /* This is to handle link status changes */
3238         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3239                                     OPCODE_NIC_INFO,
3240                                     lio_nic_info, octeon_dev);
3241
3242         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3243          * They are handled directly.
3244          */
3245         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3246                                         free_netbuf);
3247
3248         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3249                                         free_netsgbuf);
3250
3251         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3252                                         free_netsgbuf_with_resp);
3253
3254         for (i = 0; i < octeon_dev->ifcount; i++) {
3255                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3256                 ctx_size = sizeof(struct liquidio_if_cfg_context);
3257                 data_size = sizeof(struct lio_version);
3258                 sc = (struct octeon_soft_command *)
3259                         octeon_alloc_soft_command(octeon_dev, data_size,
3260                                                   resp_size, ctx_size);
3261                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3262                 ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
3263                 vdata = (struct lio_version *)sc->virtdptr;
3264
3265                 *((u64 *)vdata) = 0;
3266                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3267                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3268                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3269
3270                 num_iqueues =
3271                         CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3272                 num_oqueues =
3273                         CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
3274                 base_queue =
3275                         CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i);
3276                 gmx_port_id =
3277                         CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i);
3278                 ifidx_or_pfnum = i;
3279
3280                 dev_dbg(&octeon_dev->pci_dev->dev,
3281                         "requesting config for interface %d, iqs %d, oqs %d\n",
3282                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3283                 WRITE_ONCE(ctx->cond, 0);
3284                 ctx->octeon_id = lio_get_device_id(octeon_dev);
3285                 init_waitqueue_head(&ctx->wc);
3286
3287                 if_cfg.u64 = 0;
3288                 if_cfg.s.num_iqueues = num_iqueues;
3289                 if_cfg.s.num_oqueues = num_oqueues;
3290                 if_cfg.s.base_queue = base_queue;
3291                 if_cfg.s.gmx_port_id = gmx_port_id;
3292
3293                 sc->iq_no = 0;
3294
3295                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3296                                             OPCODE_NIC_IF_CFG, 0,
3297                                             if_cfg.u64, 0);
3298
3299                 sc->callback = if_cfg_callback;
3300                 sc->callback_arg = sc;
3301                 sc->wait_time = 3000;
3302
3303                 retval = octeon_send_soft_command(octeon_dev, sc);
3304                 if (retval == IQ_SEND_FAILED) {
3305                         dev_err(&octeon_dev->pci_dev->dev,
3306                                 "iq/oq config failed status: %x\n",
3307                                 retval);
3308                         /* Soft instr is freed by driver in case of failure. */
3309                         goto setup_nic_dev_fail;
3310                 }
3311
3312                 /* Sleep on a wait queue till the cond flag indicates that the
3313                  * response arrived or timed-out.
3314                  */
3315                 sleep_cond(&ctx->wc, &ctx->cond);
3316                 retval = resp->status;
3317                 if (retval) {
3318                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3319                         goto setup_nic_dev_fail;
3320                 }
3321
3322                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3323                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3324
3325                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3326                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3327
3328                 if (!(num_iqueues) || !(num_oqueues)) {
3329                         dev_err(&octeon_dev->pci_dev->dev,
3330                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3331                                 resp->cfg_info.iqmask,
3332                                 resp->cfg_info.oqmask);
3333                         goto setup_nic_dev_fail;
3334                 }
3335                 dev_dbg(&octeon_dev->pci_dev->dev,
3336                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3337                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3338                         num_iqueues, num_oqueues);
3339                 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues);
3340
3341                 if (!netdev) {
3342                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3343                         goto setup_nic_dev_fail;
3344                 }
3345
3346                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3347
3348                 if (num_iqueues > 1)
3349                         lionetdevops.ndo_select_queue = select_q;
3350
3351                 /* Associate the routines that will handle different
3352                  * netdev tasks.
3353                  */
3354                 netdev->netdev_ops = &lionetdevops;
3355
3356                 lio = GET_LIO(netdev);
3357
3358                 memset(lio, 0, sizeof(struct lio));
3359
3360                 lio->ifidx = ifidx_or_pfnum;
3361
3362                 props = &octeon_dev->props[i];
3363                 props->gmxport = resp->cfg_info.linfo.gmxport;
3364                 props->netdev = netdev;
3365
3366                 lio->linfo.num_rxpciq = num_oqueues;
3367                 lio->linfo.num_txpciq = num_iqueues;
3368                 for (j = 0; j < num_oqueues; j++) {
3369                         lio->linfo.rxpciq[j].u64 =
3370                                 resp->cfg_info.linfo.rxpciq[j].u64;
3371                 }
3372                 for (j = 0; j < num_iqueues; j++) {
3373                         lio->linfo.txpciq[j].u64 =
3374                                 resp->cfg_info.linfo.txpciq[j].u64;
3375                 }
3376                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3377                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3378                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3379
3380                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3381
3382                 lio->dev_capability = NETIF_F_HIGHDMA
3383                                 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
3384                                 | NETIF_F_SG | NETIF_F_RXCSUM
3385                                 | NETIF_F_GRO
3386                                 | NETIF_F_TSO | NETIF_F_TSO6
3387                                 | NETIF_F_LRO;
3388                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3389
3390                 /*  Copy of transmit encapsulation capabilities:
3391                  *  TSO, TSO6, Checksums for this device
3392                  */
3393                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3394                                           | NETIF_F_IPV6_CSUM
3395                                           | NETIF_F_GSO_UDP_TUNNEL
3396                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3397                                           | NETIF_F_RXCSUM
3398                                           | NETIF_F_TSO | NETIF_F_TSO6
3399                                           | NETIF_F_LRO;
3400
3401                 netdev->hw_enc_features = (lio->enc_dev_capability &
3402                                            ~NETIF_F_LRO);
3403
3404                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3405
3406                 netdev->vlan_features = lio->dev_capability;
3407                 /* Add any unchangeable hw features */
3408                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3409                                         NETIF_F_HW_VLAN_CTAG_RX |
3410                                         NETIF_F_HW_VLAN_CTAG_TX;
3411
3412                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3413
3414                 netdev->hw_features = lio->dev_capability;
3415                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3416                 netdev->hw_features = netdev->hw_features &
3417                         ~NETIF_F_HW_VLAN_CTAG_RX;
3418
3419                 /* Point to the  properties for octeon device to which this
3420                  * interface belongs.
3421                  */
3422                 lio->oct_dev = octeon_dev;
3423                 lio->octprops = props;
3424                 lio->netdev = netdev;
3425
3426                 dev_dbg(&octeon_dev->pci_dev->dev,
3427                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3428                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3429
3430                 /* 64-bit swap required on LE machines */
3431                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3432                 for (j = 0; j < 6; j++)
3433                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3434
3435                 /* Copy MAC Address to OS network device structure */
3436
3437                 ether_addr_copy(netdev->dev_addr, mac);
3438
3439                 /* By default all interfaces on a single Octeon uses the same
3440                  * tx and rx queues
3441                  */
3442                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3443                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3444                 if (setup_io_queues(octeon_dev, i)) {
3445                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3446                         goto setup_nic_dev_fail;
3447                 }
3448
3449                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3450
3451                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3452                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3453
3454                 if (setup_glists(octeon_dev, lio, num_iqueues)) {
3455                         dev_err(&octeon_dev->pci_dev->dev,
3456                                 "Gather list allocation failed\n");
3457                         goto setup_nic_dev_fail;
3458                 }
3459
3460                 /* Register ethtool support */
3461                 liquidio_set_ethtool_ops(netdev);
3462                 octeon_dev->priv_flags = 0x0;
3463
3464                 if (netdev->features & NETIF_F_LRO)
3465                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3466                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3467
3468                 liquidio_set_feature(netdev, OCTNET_CMD_ENABLE_VLAN_FILTER, 0);
3469
3470                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3471                         liquidio_set_feature(netdev,
3472                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3473
3474                 /* Register the network device with the OS */
3475                 if (register_netdev(netdev)) {
3476                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3477                         goto setup_nic_dev_fail;
3478                 }
3479
3480                 dev_dbg(&octeon_dev->pci_dev->dev,
3481                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3482                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3483                 netif_carrier_off(netdev);
3484                 lio->link_changes++;
3485
3486                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3487
3488                 /* Sending command to firmware to enable Rx checksum offload
3489                  * by default at the time of setup of Liquidio driver for
3490                  * this device
3491                  */
3492                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3493                                             OCTNET_CMD_RXCSUM_ENABLE);
3494                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3495                                      OCTNET_CMD_TXCSUM_ENABLE);
3496
3497                 dev_dbg(&octeon_dev->pci_dev->dev,
3498                         "NIC ifidx:%d Setup successful\n", i);
3499
3500                 octeon_free_soft_command(octeon_dev, sc);
3501         }
3502
3503         return 0;
3504
3505 setup_nic_dev_fail:
3506
3507         octeon_free_soft_command(octeon_dev, sc);
3508
3509         while (i--) {
3510                 dev_err(&octeon_dev->pci_dev->dev,
3511                         "NIC ifidx:%d Setup failed\n", i);
3512                 liquidio_destroy_nic_device(octeon_dev, i);
3513         }
3514         return -ENODEV;
3515 }
3516
3517 /**
3518  * \brief initialize the NIC
3519  * @param oct octeon device
3520  *
3521  * This initialization routine is called once the Octeon device application is
3522  * up and running
3523  */
3524 static int liquidio_init_nic_module(struct octeon_device *oct)
3525 {
3526         struct oct_intrmod_cfg *intrmod_cfg;
3527         int i, retval = 0;
3528         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3529
3530         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3531
3532         /* only default iq and oq were initialized
3533          * initialize the rest as well
3534          */
3535         /* run port_config command for each port */
3536         oct->ifcount = num_nic_ports;
3537
3538         memset(oct->props, 0,
3539                sizeof(struct octdev_props) * num_nic_ports);
3540
3541         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3542                 oct->props[i].gmxport = -1;
3543
3544         retval = setup_nic_devices(oct);
3545         if (retval) {
3546                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3547                 goto octnet_init_failure;
3548         }
3549
3550         liquidio_ptp_init(oct);
3551
3552         /* Initialize interrupt moderation params */
3553         intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
3554         intrmod_cfg->rx_enable = 1;
3555         intrmod_cfg->check_intrvl =   LIO_INTRMOD_CHECK_INTERVAL;
3556         intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
3557         intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
3558         intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
3559         intrmod_cfg->rx_maxtmr_trigger = LIO_INTRMOD_RXMAXTMR_TRIGGER;
3560         intrmod_cfg->rx_mintmr_trigger = LIO_INTRMOD_RXMINTMR_TRIGGER;
3561         intrmod_cfg->rx_mincnt_trigger = LIO_INTRMOD_RXMINCNT_TRIGGER;
3562         intrmod_cfg->tx_enable = 1;
3563         intrmod_cfg->tx_maxcnt_trigger = LIO_INTRMOD_TXMAXCNT_TRIGGER;
3564         intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
3565         intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
3566         intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
3567         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3568
3569         return retval;
3570
3571 octnet_init_failure:
3572
3573         oct->ifcount = 0;
3574
3575         return retval;
3576 }
3577
3578 /**
3579  * \brief starter callback that invokes the remaining initialization work after
3580  * the NIC is up and running.
3581  * @param octptr  work struct work_struct
3582  */
3583 static void nic_starter(struct work_struct *work)
3584 {
3585         struct octeon_device *oct;
3586         struct cavium_wk *wk = (struct cavium_wk *)work;
3587
3588         oct = (struct octeon_device *)wk->ctxptr;
3589
3590         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3591                 return;
3592
3593         /* If the status of the device is CORE_OK, the core
3594          * application has reported its application type. Call
3595          * any registered handlers now and move to the RUNNING
3596          * state.
3597          */
3598         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3599                 schedule_delayed_work(&oct->nic_poll_work.work,
3600                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3601                 return;
3602         }
3603
3604         atomic_set(&oct->status, OCT_DEV_RUNNING);
3605
3606         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3607                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3608
3609                 if (liquidio_init_nic_module(oct))
3610                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3611                 else
3612                         handshake[oct->octeon_id].started_ok = 1;
3613         } else {
3614                 dev_err(&oct->pci_dev->dev,
3615                         "Unexpected application running on NIC (%d). Check firmware.\n",
3616                         oct->app_mode);
3617         }
3618
3619         complete(&handshake[oct->octeon_id].started);
3620 }
3621
3622 /**
3623  * \brief Device initialization for each Octeon device that is probed
3624  * @param octeon_dev  octeon device
3625  */
3626 static int octeon_device_init(struct octeon_device *octeon_dev)
3627 {
3628         int j, ret;
3629         char bootcmd[] = "\n";
3630         struct octeon_device_priv *oct_priv =
3631                 (struct octeon_device_priv *)octeon_dev->priv;
3632         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
3633
3634         /* Enable access to the octeon device and make its DMA capability
3635          * known to the OS.
3636          */
3637         if (octeon_pci_os_setup(octeon_dev))
3638                 return 1;
3639
3640         /* Identify the Octeon type and map the BAR address space. */
3641         if (octeon_chip_specific_setup(octeon_dev)) {
3642                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
3643                 return 1;
3644         }
3645
3646         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
3647
3648         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
3649
3650         /* Do a soft reset of the Octeon device. */
3651         if (octeon_dev->fn_list.soft_reset(octeon_dev))
3652                 return 1;
3653
3654         /* Initialize the dispatch mechanism used to push packets arriving on
3655          * Octeon Output queues.
3656          */
3657         if (octeon_init_dispatch_list(octeon_dev))
3658                 return 1;
3659
3660         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3661                                     OPCODE_NIC_CORE_DRV_ACTIVE,
3662                                     octeon_core_drv_init,
3663                                     octeon_dev);
3664
3665         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
3666         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
3667         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
3668                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3669
3670         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
3671
3672         octeon_set_io_queues_off(octeon_dev);
3673
3674         /*  Setup the data structures that manage this Octeon's Input queues. */
3675         if (octeon_setup_instr_queues(octeon_dev)) {
3676                 dev_err(&octeon_dev->pci_dev->dev,
3677                         "instruction queue initialization failed\n");
3678                 /* On error, release any previously allocated queues */
3679                 for (j = 0; j < octeon_dev->num_iqs; j++)
3680                         octeon_delete_instr_queue(octeon_dev, j);
3681                 return 1;
3682         }
3683         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
3684
3685         /* Initialize soft command buffer pool
3686          */
3687         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
3688                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
3689                 return 1;
3690         }
3691         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
3692
3693         /* Initialize lists to manage the requests of different types that
3694          * arrive from user & kernel applications for this octeon device.
3695          */
3696         if (octeon_setup_response_list(octeon_dev)) {
3697                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
3698                 return 1;
3699         }
3700         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
3701
3702         if (octeon_setup_output_queues(octeon_dev)) {
3703                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
3704                 /* Release any previously allocated queues */
3705                 for (j = 0; j < octeon_dev->num_oqs; j++)
3706                         octeon_delete_droq(octeon_dev, j);
3707                 return 1;
3708         }
3709
3710         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
3711
3712         /* The input and output queue registers were setup earlier (the queues
3713          * were not enabled). Any additional registers that need to be
3714          * programmed should be done now.
3715          */
3716         ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
3717         if (ret) {
3718                 dev_err(&octeon_dev->pci_dev->dev,
3719                         "Failed to configure device registers\n");
3720                 return ret;
3721         }
3722
3723         /* Initialize the tasklet that handles output queue packet processing.*/
3724         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
3725         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
3726                      (unsigned long)octeon_dev);
3727
3728         /* Setup the interrupt handler and record the INT SUM register address
3729          */
3730         if (octeon_setup_interrupt(octeon_dev))
3731                 return 1;
3732
3733         /* Enable Octeon device interrupts */
3734         octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
3735
3736         /* Enable the input and output queues for this Octeon device */
3737         octeon_dev->fn_list.enable_io_queues(octeon_dev);
3738
3739         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
3740
3741         dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
3742
3743         if (ddr_timeout == 0)
3744                 dev_info(&octeon_dev->pci_dev->dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3745
3746         schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
3747
3748         /* Wait for the octeon to initialize DDR after the soft-reset. */
3749         while (ddr_timeout == 0) {
3750                 set_current_state(TASK_INTERRUPTIBLE);
3751                 if (schedule_timeout(HZ / 10)) {
3752                         /* user probably pressed Control-C */
3753                         return 1;
3754                 }
3755         }
3756         ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
3757         if (ret) {
3758                 dev_err(&octeon_dev->pci_dev->dev,
3759                         "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3760                         ret);
3761                 return 1;
3762         }
3763
3764         if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
3765                 dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
3766                 return 1;
3767         }
3768
3769         /* Divert uboot to take commands from host instead. */
3770         ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
3771
3772         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
3773         ret = octeon_init_consoles(octeon_dev);
3774         if (ret) {
3775                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
3776                 return 1;
3777         }
3778         ret = octeon_add_console(octeon_dev, 0);
3779         if (ret) {
3780                 dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
3781                 return 1;
3782         }
3783
3784         atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
3785
3786         dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
3787         ret = load_firmware(octeon_dev);
3788         if (ret) {
3789                 dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
3790                 return 1;
3791         }
3792
3793         handshake[octeon_dev->octeon_id].init_ok = 1;
3794         complete(&handshake[octeon_dev->octeon_id].init);
3795
3796         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
3797
3798         /* Send Credit for Octeon Output queues. Credits are always sent after
3799          * the output queue is enabled.
3800          */
3801         for (j = 0; j < octeon_dev->num_oqs; j++)
3802                 writel(octeon_dev->droq[j]->max_count,
3803                        octeon_dev->droq[j]->pkts_credit_reg);
3804
3805         /* Packets can start arriving on the output queues from this point. */
3806
3807         return 0;
3808 }
3809
3810 /**
3811  * \brief Exits the module
3812  */
3813 static void __exit liquidio_exit(void)
3814 {
3815         liquidio_deinit_pci();
3816
3817         pr_info("LiquidIO network module is now unloaded\n");
3818 }
3819
3820 module_init(liquidio_init);
3821 module_exit(liquidio_exit);