drivers/ntb/hw/intel/ntb_hw_intel.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *
   9  *   This program is free software; you can redistribute it and/or modify
  10  *   it under the terms of version 2 of the GNU General Public License as
  11  *   published by the Free Software Foundation.
  12  *
  13  *   BSD LICENSE
  14  *
  15  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  16  *
  17  *   Redistribution and use in source and binary forms, with or without
  18  *   modification, are permitted provided that the following conditions
  19  *   are met:
  20  *
  21  *     * Redistributions of source code must retain the above copyright
  22  *       notice, this list of conditions and the following disclaimer.
  23  *     * Redistributions in binary form must reproduce the above copy
  24  *       notice, this list of conditions and the following disclaimer in
  25  *       the documentation and/or other materials provided with the
  26  *       distribution.
  27  *     * Neither the name of Intel Corporation nor the names of its
  28  *       contributors may be used to endorse or promote products derived
  29  *       from this software without specific prior written permission.
  30  *
  31  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42  *
  43  * Intel PCIe NTB Linux driver
  44  *
  45  * Contact Information:
  46  * Jon Mason <jon.mason@intel.com>
  47  */
  48 #include <linux/debugfs.h>
  49 #include <linux/delay.h>
  50 #include <linux/init.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/module.h>
  53 #include <linux/pci.h>
  54 #include <linux/random.h>
  55 #include <linux/slab.h>
  56 #include "ntb_hw_intel.h"
  57
  58 #define NTB_NAME        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  59 #define NTB_VER         "1.0"
  60
  61 MODULE_DESCRIPTION(NTB_NAME);
  62 MODULE_VERSION(NTB_VER);
  63 MODULE_LICENSE("Dual BSD/GPL");
  64 MODULE_AUTHOR("Intel Corporation");
  65
  66 enum {
  67         NTB_CONN_TRANSPARENT = 0,
  68         NTB_CONN_B2B,
  69         NTB_CONN_RP,
  70 };
  71
  72 enum {
  73         NTB_DEV_USD = 0,
  74         NTB_DEV_DSD,
  75 };
  76
  77 enum {
  78         SNB_HW = 0,
  79         BWD_HW,
  80 };
  81
  82 static struct dentry *debugfs_dir;
  83
  84 #define BWD_LINK_RECOVERY_TIME  500
  85
  86 /* Translate memory window 0,1,2 to BAR 2,4,5 */
  87 #define MW_TO_BAR(mw)   (mw == 0 ? 2 : (mw == 1 ? 4 : 5))
  88
  89 static const struct pci_device_id ntb_pci_tbl[] = {
  90         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
  91         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
  92         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
  93         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
  94         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
  95         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
  96         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
  97         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
  98         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
  99         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
 100         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
 101         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
 102         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
 103         {0}
 104 };
 105 MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
 106
 107 static int is_ntb_xeon(struct ntb_device *ndev)
 108 {
 109         switch (ndev->pdev->device) {
 110         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 111         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 112         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 113         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 114         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 115         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 116         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 117         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 118         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 119         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 120         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 121         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 122                 return 1;
 123         default:
 124                 return 0;
 125         }
 126
 127         return 0;
 128 }
 129
 130 static int is_ntb_atom(struct ntb_device *ndev)
 131 {
 132         switch (ndev->pdev->device) {
 133         case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
 134                 return 1;
 135         default:
 136                 return 0;
 137         }
 138
 139         return 0;
 140 }
 141
 142 static void ntb_set_errata_flags(struct ntb_device *ndev)
 143 {
 144         switch (ndev->pdev->device) {
 145         /*
 146          * this workaround applies to all platform up to IvyBridge
 147          * Haswell has splitbar support and use a different workaround
 148          */
 149         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 150         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 151         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 152         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 153         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 154         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 155         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 156         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 157         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 158         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 159         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 160         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 161                 ndev->wa_flags |= WA_SNB_ERR;
 162                 break;
 163         }
 164 }
 165
 166 /**
 167  * ntb_register_event_callback() - register event callback
 168  * @ndev: pointer to ntb_device instance
 169  * @func: callback function to register
 170  *
 171  * This function registers a callback for any HW driver events such as link
 172  * up/down, power management notices and etc.
 173  *
 174  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 175  */
 176 int ntb_register_event_callback(struct ntb_device *ndev,
 177                                 void (*func)(void *handle,
 178                                              enum ntb_hw_event event))
 179 {
 180         if (ndev->event_cb)
 181                 return -EINVAL;
 182
 183         ndev->event_cb = func;
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * ntb_unregister_event_callback() - unregisters the event callback
 190  * @ndev: pointer to ntb_device instance
 191  *
 192  * This function unregisters the existing callback from transport
 193  */
 194 void ntb_unregister_event_callback(struct ntb_device *ndev)
 195 {
 196         ndev->event_cb = NULL;
 197 }
 198
 199 static void ntb_irq_work(unsigned long data)
 200 {
 201         struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;
 202         int rc;
 203
 204         rc = db_cb->callback(db_cb->data, db_cb->db_num);
 205         if (rc)
 206                 tasklet_schedule(&db_cb->irq_work);
 207         else {
 208                 struct ntb_device *ndev = db_cb->ndev;
 209                 unsigned long mask;
 210
 211                 mask = readw(ndev->reg_ofs.ldb_mask);
 212                 clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
 213                 writew(mask, ndev->reg_ofs.ldb_mask);
 214         }
 215 }
 216
 217 /**
 218  * ntb_register_db_callback() - register a callback for doorbell interrupt
 219  * @ndev: pointer to ntb_device instance
 220  * @idx: doorbell index to register callback, zero based
 221  * @data: pointer to be returned to caller with every callback
 222  * @func: callback function to register
 223  *
 224  * This function registers a callback function for the doorbell interrupt
 225  * on the primary side. The function will unmask the doorbell as well to
 226  * allow interrupt.
 227  *
 228  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 229  */
 230 int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
 231                              void *data, int (*func)(void *data, int db_num))
 232 {
 233         unsigned long mask;
 234
 235         if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
 236                 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
 237                 return -EINVAL;
 238         }
 239
 240         ndev->db_cb[idx].callback = func;
 241         ndev->db_cb[idx].data = data;
 242         ndev->db_cb[idx].ndev = ndev;
 243
 244         tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,
 245                      (unsigned long) &ndev->db_cb[idx]);
 246
 247         /* unmask interrupt */
 248         mask = readw(ndev->reg_ofs.ldb_mask);
 249         clear_bit(idx * ndev->bits_per_vector, &mask);
 250         writew(mask, ndev->reg_ofs.ldb_mask);
 251
 252         return 0;
 253 }
 254
 255 /**
 256  * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
 257  * @ndev: pointer to ntb_device instance
 258  * @idx: doorbell index to register callback, zero based
 259  *
 260  * This function unregisters a callback function for the doorbell interrupt
 261  * on the primary side. The function will also mask the said doorbell.
 262  */
 263 void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
 264 {
 265         unsigned long mask;
 266
 267         if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
 268                 return;
 269
 270         mask = readw(ndev->reg_ofs.ldb_mask);
 271         set_bit(idx * ndev->bits_per_vector, &mask);
 272         writew(mask, ndev->reg_ofs.ldb_mask);
 273
 274         tasklet_disable(&ndev->db_cb[idx].irq_work);
 275
 276         ndev->db_cb[idx].callback = NULL;
 277 }
 278
 279 /**
 280  * ntb_find_transport() - find the transport pointer
 281  * @transport: pointer to pci device
 282  *
 283  * Given the pci device pointer, return the transport pointer passed in when
 284  * the transport attached when it was inited.
 285  *
 286  * RETURNS: pointer to transport.
 287  */
 288 void *ntb_find_transport(struct pci_dev *pdev)
 289 {
 290         struct ntb_device *ndev = pci_get_drvdata(pdev);
 291         return ndev->ntb_transport;
 292 }
 293
 294 /**
 295  * ntb_register_transport() - Register NTB transport with NTB HW driver
 296  * @transport: transport identifier
 297  *
 298  * This function allows a transport to reserve the hardware driver for
 299  * NTB usage.
 300  *
 301  * RETURNS: pointer to ntb_device, NULL on error.
 302  */
 303 struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
 304 {
 305         struct ntb_device *ndev = pci_get_drvdata(pdev);
 306
 307         if (ndev->ntb_transport)
 308                 return NULL;
 309
 310         ndev->ntb_transport = transport;
 311         return ndev;
 312 }
 313
 314 /**
 315  * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
 316  * @ndev - ntb_device of the transport to be freed
 317  *
 318  * This function unregisters the transport from the HW driver and performs any
 319  * necessary cleanups.
 320  */
 321 void ntb_unregister_transport(struct ntb_device *ndev)
 322 {
 323         int i;
 324
 325         if (!ndev->ntb_transport)
 326                 return;
 327
 328         for (i = 0; i < ndev->max_cbs; i++)
 329                 ntb_unregister_db_callback(ndev, i);
 330
 331         ntb_unregister_event_callback(ndev);
 332         ndev->ntb_transport = NULL;
 333 }
 334
 335 /**
 336  * ntb_write_local_spad() - write to the secondary scratchpad register
 337  * @ndev: pointer to ntb_device instance
 338  * @idx: index to the scratchpad register, 0 based
 339  * @val: the data value to put into the register
 340  *
 341  * This function allows writing of a 32bit value to the indexed scratchpad
 342  * register. This writes over the data mirrored to the local scratchpad register
 343  * by the remote system.
 344  *
 345  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 346  */
 347 int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 348 {
 349         if (idx >= ndev->limits.max_spads)
 350                 return -EINVAL;
 351
 352         dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
 353                 val, idx);
 354         writel(val, ndev->reg_ofs.spad_read + idx * 4);
 355
 356         return 0;
 357 }
 358
 359 /**
 360  * ntb_read_local_spad() - read from the primary scratchpad register
 361  * @ndev: pointer to ntb_device instance
 362  * @idx: index to scratchpad register, 0 based
 363  * @val: pointer to 32bit integer for storing the register value
 364  *
 365  * This function allows reading of the 32bit scratchpad register on
 366  * the primary (internal) side.  This allows the local system to read data
 367  * written and mirrored to the scratchpad register by the remote system.
 368  *
 369  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 370  */
 371 int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 372 {
 373         if (idx >= ndev->limits.max_spads)
 374                 return -EINVAL;
 375
 376         *val = readl(ndev->reg_ofs.spad_write + idx * 4);
 377         dev_dbg(&ndev->pdev->dev,
 378                 "Reading %x from local scratch pad index %d\n", *val, idx);
 379
 380         return 0;
 381 }
 382
 383 /**
 384  * ntb_write_remote_spad() - write to the secondary scratchpad register
 385  * @ndev: pointer to ntb_device instance
 386  * @idx: index to the scratchpad register, 0 based
 387  * @val: the data value to put into the register
 388  *
 389  * This function allows writing of a 32bit value to the indexed scratchpad
 390  * register. The register resides on the secondary (external) side.  This allows
 391  * the local system to write data to be mirrored to the remote systems
 392  * scratchpad register.
 393  *
 394  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 395  */
 396 int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 397 {
 398         if (idx >= ndev->limits.max_spads)
 399                 return -EINVAL;
 400
 401         dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
 402                 val, idx);
 403         writel(val, ndev->reg_ofs.spad_write + idx * 4);
 404
 405         return 0;
 406 }
 407
 408 /**
 409  * ntb_read_remote_spad() - read from the primary scratchpad register
 410  * @ndev: pointer to ntb_device instance
 411  * @idx: index to scratchpad register, 0 based
 412  * @val: pointer to 32bit integer for storing the register value
 413  *
 414  * This function allows reading of the 32bit scratchpad register on
 415  * the primary (internal) side.  This alloows the local system to read the data
 416  * it wrote to be mirrored on the remote system.
 417  *
 418  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 419  */
 420 int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 421 {
 422         if (idx >= ndev->limits.max_spads)
 423                 return -EINVAL;
 424
 425         *val = readl(ndev->reg_ofs.spad_read + idx * 4);
 426         dev_dbg(&ndev->pdev->dev,
 427                 "Reading %x from remote scratch pad index %d\n", *val, idx);
 428
 429         return 0;
 430 }
 431
 432 /**
 433  * ntb_get_mw_base() - get addr for the NTB memory window
 434  * @ndev: pointer to ntb_device instance
 435  * @mw: memory window number
 436  *
 437  * This function provides the base address of the memory window specified.
 438  *
 439  * RETURNS: address, or NULL on error.
 440  */
 441 resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
 442 {
 443         if (mw >= ntb_max_mw(ndev))
 444                 return 0;
 445
 446         return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
 447 }
 448
 449 /**
 450  * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
 451  * @ndev: pointer to ntb_device instance
 452  * @mw: memory window number
 453  *
 454  * This function provides the base virtual address of the memory window
 455  * specified.
 456  *
 457  * RETURNS: pointer to virtual address, or NULL on error.
 458  */
 459 void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
 460 {
 461         if (mw >= ntb_max_mw(ndev))
 462                 return NULL;
 463
 464         return ndev->mw[mw].vbase;
 465 }
 466
 467 /**
 468  * ntb_get_mw_size() - return size of NTB memory window
 469  * @ndev: pointer to ntb_device instance
 470  * @mw: memory window number
 471  *
 472  * This function provides the physical size of the memory window specified
 473  *
 474  * RETURNS: the size of the memory window or zero on error
 475  */
 476 u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
 477 {
 478         if (mw >= ntb_max_mw(ndev))
 479                 return 0;
 480
 481         return ndev->mw[mw].bar_sz;
 482 }
 483
 484 /**
 485  * ntb_set_mw_addr - set the memory window address
 486  * @ndev: pointer to ntb_device instance
 487  * @mw: memory window number
 488  * @addr: base address for data
 489  *
 490  * This function sets the base physical address of the memory window.  This
 491  * memory address is where data from the remote system will be transfered into
 492  * or out of depending on how the transport is configured.
 493  */
 494 void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
 495 {
 496         if (mw >= ntb_max_mw(ndev))
 497                 return;
 498
 499         dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
 500                 MW_TO_BAR(mw));
 501
 502         ndev->mw[mw].phys_addr = addr;
 503
 504         switch (MW_TO_BAR(mw)) {
 505         case NTB_BAR_23:
 506                 writeq(addr, ndev->reg_ofs.bar2_xlat);
 507                 break;
 508         case NTB_BAR_4:
 509                 if (ndev->split_bar)
 510                         writel(addr, ndev->reg_ofs.bar4_xlat);
 511                 else
 512                         writeq(addr, ndev->reg_ofs.bar4_xlat);
 513                 break;
 514         case NTB_BAR_5:
 515                 writel(addr, ndev->reg_ofs.bar5_xlat);
 516                 break;
 517         }
 518 }
 519
 520 /**
 521  * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
 522  * @ndev: pointer to ntb_device instance
 523  * @db: doorbell to ring
 524  *
 525  * This function allows triggering of a doorbell on the secondary/external
 526  * side that will initiate an interrupt on the remote host
 527  *
 528  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 529  */
 530 void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
 531 {
 532         dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
 533
 534         if (ndev->hw_type == BWD_HW)
 535                 writeq((u64) 1 << db, ndev->reg_ofs.rdb);
 536         else
 537                 writew(((1 << ndev->bits_per_vector) - 1) <<
 538                        (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
 539 }
 540
 541 static void bwd_recover_link(struct ntb_device *ndev)
 542 {
 543         u32 status;
 544
 545         /* Driver resets the NTB ModPhy lanes - magic! */
 546         writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
 547         writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
 548         writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
 549         writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
 550
 551         /* Driver waits 100ms to allow the NTB ModPhy to settle */
 552         msleep(100);
 553
 554         /* Clear AER Errors, write to clear */
 555         status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 556         dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
 557         status &= PCI_ERR_COR_REP_ROLL;
 558         writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 559
 560         /* Clear unexpected electrical idle event in LTSSM, write to clear */
 561         status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 562         dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
 563         status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
 564         writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 565
 566         /* Clear DeSkew Buffer error, write to clear */
 567         status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 568         dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
 569         status |= BWD_DESKEWSTS_DBERR;
 570         writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 571
 572         status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 573         dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
 574         status &= BWD_IBIST_ERR_OFLOW;
 575         writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 576
 577         /* Releases the NTB state machine to allow the link to retrain */
 578         status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 579         dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
 580         status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
 581         writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 582 }
 583
 584 static void ntb_link_event(struct ntb_device *ndev, int link_state)
 585 {
 586         unsigned int event;
 587
 588         if (ndev->link_status == link_state)
 589                 return;
 590
 591         if (link_state == NTB_LINK_UP) {
 592                 u16 status;
 593
 594                 dev_info(&ndev->pdev->dev, "Link Up\n");
 595                 ndev->link_status = NTB_LINK_UP;
 596                 event = NTB_EVENT_HW_LINK_UP;
 597
 598                 if (is_ntb_atom(ndev) ||
 599                     ndev->conn_type == NTB_CONN_TRANSPARENT)
 600                         status = readw(ndev->reg_ofs.lnk_stat);
 601                 else {
 602                         int rc = pci_read_config_word(ndev->pdev,
 603                                                       SNB_LINK_STATUS_OFFSET,
 604                                                       &status);
 605                         if (rc)
 606                                 return;
 607                 }
 608
 609                 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
 610                 ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
 611                 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
 612                          ndev->link_width, ndev->link_speed);
 613         } else {
 614                 dev_info(&ndev->pdev->dev, "Link Down\n");
 615                 ndev->link_status = NTB_LINK_DOWN;
 616                 event = NTB_EVENT_HW_LINK_DOWN;
 617                 /* Don't modify link width/speed, we need it in link recovery */
 618         }
 619
 620         /* notify the upper layer if we have an event change */
 621         if (ndev->event_cb)
 622                 ndev->event_cb(ndev->ntb_transport, event);
 623 }
 624
 625 static int ntb_link_status(struct ntb_device *ndev)
 626 {
 627         int link_state;
 628
 629         if (is_ntb_atom(ndev)) {
 630                 u32 ntb_cntl;
 631
 632                 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
 633                 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
 634                         link_state = NTB_LINK_DOWN;
 635                 else
 636                         link_state = NTB_LINK_UP;
 637         } else {
 638                 u16 status;
 639                 int rc;
 640
 641                 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
 642                                           &status);
 643                 if (rc)
 644                         return rc;
 645
 646                 if (status & NTB_LINK_STATUS_ACTIVE)
 647                         link_state = NTB_LINK_UP;
 648                 else
 649                         link_state = NTB_LINK_DOWN;
 650         }
 651
 652         ntb_link_event(ndev, link_state);
 653
 654         return 0;
 655 }
 656
 657 static void bwd_link_recovery(struct work_struct *work)
 658 {
 659         struct ntb_device *ndev = container_of(work, struct ntb_device,
 660                                                lr_timer.work);
 661         u32 status32;
 662
 663         bwd_recover_link(ndev);
 664         /* There is a potential race between the 2 NTB devices recovering at the
 665          * same time.  If the times are the same, the link will not recover and
 666          * the driver will be stuck in this loop forever.  Add a random interval
 667          * to the recovery time to prevent this race.
 668          */
 669         msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
 670
 671         status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 672         if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
 673                 goto retry;
 674
 675         status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 676         if (status32 & BWD_IBIST_ERR_OFLOW)
 677                 goto retry;
 678
 679         status32 = readl(ndev->reg_ofs.lnk_cntl);
 680         if (!(status32 & BWD_CNTL_LINK_DOWN)) {
 681                 unsigned char speed, width;
 682                 u16 status16;
 683
 684                 status16 = readw(ndev->reg_ofs.lnk_stat);
 685                 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
 686                 speed = (status16 & NTB_LINK_SPEED_MASK);
 687                 if (ndev->link_width != width || ndev->link_speed != speed)
 688                         goto retry;
 689         }
 690
 691         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 692         return;
 693
 694 retry:
 695         schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
 696 }
 697
 698 /* BWD doesn't have link status interrupt, poll on that platform */
 699 static void bwd_link_poll(struct work_struct *work)
 700 {
 701         struct ntb_device *ndev = container_of(work, struct ntb_device,
 702                                                hb_timer.work);
 703         unsigned long ts = jiffies;
 704
 705         /* If we haven't gotten an interrupt in a while, check the BWD link
 706          * status bit
 707          */
 708         if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
 709                 int rc = ntb_link_status(ndev);
 710                 if (rc)
 711                         dev_err(&ndev->pdev->dev,
 712                                 "Error determining link status\n");
 713
 714                 /* Check to see if a link error is the cause of the link down */
 715                 if (ndev->link_status == NTB_LINK_DOWN) {
 716                         u32 status32 = readl(ndev->reg_base +
 717                                              BWD_LTSSMSTATEJMP_OFFSET);
 718                         if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
 719                                 schedule_delayed_work(&ndev->lr_timer, 0);
 720                                 return;
 721                         }
 722                 }
 723         }
 724
 725         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 726 }
 727
 728 static int ntb_xeon_setup(struct ntb_device *ndev)
 729 {
 730         switch (ndev->conn_type) {
 731         case NTB_CONN_B2B:
 732                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 733                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 734                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 735                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 736                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 737                 if (ndev->split_bar)
 738                         ndev->reg_ofs.bar5_xlat =
 739                                 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
 740                 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
 741
 742                 /* There is a Xeon hardware errata related to writes to
 743                  * SDOORBELL or B2BDOORBELL in conjunction with inbound access
 744                  * to NTB MMIO Space, which may hang the system.  To workaround
 745                  * this use the second memory window to access the interrupt and
 746                  * scratch pad registers on the remote system.
 747                  */
 748                 if (ndev->wa_flags & WA_SNB_ERR) {
 749                         if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz)
 750                                 return -EINVAL;
 751
 752                         ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 753                         ndev->reg_ofs.spad_write =
 754                                 ndev->mw[ndev->limits.max_mw - 1].vbase +
 755                                 SNB_SPAD_OFFSET;
 756                         ndev->reg_ofs.rdb =
 757                                 ndev->mw[ndev->limits.max_mw - 1].vbase +
 758                                 SNB_PDOORBELL_OFFSET;
 759
 760                         /* Set the Limit register to 4k, the minimum size, to
 761                          * prevent an illegal access
 762                          */
 763                         writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
 764                                SNB_PBAR4LMT_OFFSET);
 765                         /* HW errata on the Limit registers.  They can only be
 766                          * written when the base register is 4GB aligned and
 767                          * < 32bit.  This should already be the case based on
 768                          * the driver defaults, but write the Limit registers
 769                          * first just in case.
 770                          */
 771
 772                         ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
 773                 } else {
 774                         /* HW Errata on bit 14 of b2bdoorbell register.  Writes
 775                          * will not be mirrored to the remote system.  Shrink
 776                          * the number of bits by one, since bit 14 is the last
 777                          * bit.
 778                          */
 779                         ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
 780                         ndev->reg_ofs.spad_write = ndev->reg_base +
 781                                                    SNB_B2B_SPAD_OFFSET;
 782                         ndev->reg_ofs.rdb = ndev->reg_base +
 783                                             SNB_B2B_DOORBELL_OFFSET;
 784
 785                         /* Disable the Limit register, just incase it is set to
 786                          * something silly. A 64bit write should handle it
 787                          * regardless of whether it has a split BAR or not.
 788                          */
 789                         writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
 790                         /* HW errata on the Limit registers.  They can only be
 791                          * written when the base register is 4GB aligned and
 792                          * < 32bit.  This should already be the case based on
 793                          * the driver defaults, but write the Limit registers
 794                          * first just in case.
 795                          */
 796                         if (ndev->split_bar)
 797                                 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 798                         else
 799                                 ndev->limits.max_mw = SNB_MAX_MW;
 800                 }
 801
 802                 /* The Xeon errata workaround requires setting SBAR Base
 803                  * addresses to known values, so that the PBAR XLAT can be
 804                  * pointed at SBAR0 of the remote system.
 805                  */
 806                 if (ndev->dev_type == NTB_DEV_USD) {
 807                         writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 808                                SNB_PBAR2XLAT_OFFSET);
 809                         if (ndev->wa_flags & WA_SNB_ERR)
 810                                 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 811                                        SNB_PBAR4XLAT_OFFSET);
 812                         else {
 813                                 if (ndev->split_bar) {
 814                                         writel(SNB_MBAR4_DSD_ADDR,
 815                                                ndev->reg_base +
 816                                                SNB_PBAR4XLAT_OFFSET);
 817                                         writel(SNB_MBAR5_DSD_ADDR,
 818                                                ndev->reg_base +
 819                                                SNB_PBAR5XLAT_OFFSET);
 820                                 } else
 821                                         writeq(SNB_MBAR4_DSD_ADDR,
 822                                                ndev->reg_base +
 823                                                SNB_PBAR4XLAT_OFFSET);
 824
 825                                 /* B2B_XLAT_OFFSET is a 64bit register, but can
 826                                  * only take 32bit writes
 827                                  */
 828                                 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
 829                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 830                                 writel(SNB_MBAR01_DSD_ADDR >> 32,
 831                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 832                         }
 833
 834                         writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 835                                SNB_SBAR0BASE_OFFSET);
 836                         writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 837                                SNB_SBAR2BASE_OFFSET);
 838                         if (ndev->split_bar) {
 839                                 writel(SNB_MBAR4_USD_ADDR, ndev->reg_base +
 840                                        SNB_SBAR4BASE_OFFSET);
 841                                 writel(SNB_MBAR5_USD_ADDR, ndev->reg_base +
 842                                        SNB_SBAR5BASE_OFFSET);
 843                         } else
 844                                 writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base +
 845                                        SNB_SBAR4BASE_OFFSET);
 846                 } else {
 847                         writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 848                                SNB_PBAR2XLAT_OFFSET);
 849                         if (ndev->wa_flags & WA_SNB_ERR)
 850                                 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 851                                        SNB_PBAR4XLAT_OFFSET);
 852                         else {
 853                                 if (ndev->split_bar) {
 854                                         writel(SNB_MBAR4_USD_ADDR,
 855                                                ndev->reg_base +
 856                                                SNB_PBAR4XLAT_OFFSET);
 857                                         writel(SNB_MBAR5_USD_ADDR,
 858                                                ndev->reg_base +
 859                                                SNB_PBAR5XLAT_OFFSET);
 860                                 } else
 861                                         writeq(SNB_MBAR4_USD_ADDR,
 862                                                ndev->reg_base +
 863                                                SNB_PBAR4XLAT_OFFSET);
 864
 865                                 /*
 866                                  * B2B_XLAT_OFFSET is a 64bit register, but can
 867                                  * only take 32bit writes
 868                                  */
 869                                 writel(SNB_MBAR01_USD_ADDR & 0xffffffff,
 870                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 871                                 writel(SNB_MBAR01_USD_ADDR >> 32,
 872                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 873                         }
 874                         writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 875                                SNB_SBAR0BASE_OFFSET);
 876                         writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 877                                SNB_SBAR2BASE_OFFSET);
 878                         if (ndev->split_bar) {
 879                                 writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
 880                                        SNB_SBAR4BASE_OFFSET);
 881                                 writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base +
 882                                        SNB_SBAR5BASE_OFFSET);
 883                         } else
 884                                 writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
 885                                        SNB_SBAR4BASE_OFFSET);
 886
 887                 }
 888                 break;
 889         case NTB_CONN_RP:
 890                 if (ndev->wa_flags & WA_SNB_ERR) {
 891                         dev_err(&ndev->pdev->dev,
 892                                 "NTB-RP disabled due to hardware errata.\n");
 893                         return -EINVAL;
 894                 }
 895
 896                 /* Scratch pads need to have exclusive access from the primary
 897                  * or secondary side.  Halve the num spads so that each side can
 898                  * have an equal amount.
 899                  */
 900                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 901                 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 902                 /* Note: The SDOORBELL is the cause of the errata.  You REALLY
 903                  * don't want to touch it.
 904                  */
 905                 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 906                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 907                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 908                 /* Offset the start of the spads to correspond to whether it is
 909                  * primary or secondary
 910                  */
 911                 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
 912                                            ndev->limits.max_spads * 4;
 913                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 914                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 915                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 916                 if (ndev->split_bar) {
 917                         ndev->reg_ofs.bar5_xlat =
 918                                 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
 919                         ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 920                 } else
 921                         ndev->limits.max_mw = SNB_MAX_MW;
 922                 break;
 923         case NTB_CONN_TRANSPARENT:
 924                 if (ndev->wa_flags & WA_SNB_ERR) {
 925                         dev_err(&ndev->pdev->dev,
 926                                 "NTB-TRANSPARENT disabled due to hardware errata.\n");
 927                         return -EINVAL;
 928                 }
 929
 930                 /* Scratch pads need to have exclusive access from the primary
 931                  * or secondary side.  Halve the num spads so that each side can
 932                  * have an equal amount.
 933                  */
 934                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 935                 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 936                 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 937                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 938                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
 939                 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
 940                 /* Offset the start of the spads to correspond to whether it is
 941                  * primary or secondary
 942                  */
 943                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
 944                                           ndev->limits.max_spads * 4;
 945                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
 946                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
 947
 948                 if (ndev->split_bar) {
 949                         ndev->reg_ofs.bar5_xlat =
 950                                 ndev->reg_base + SNB_PBAR5XLAT_OFFSET;
 951                         ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 952                 } else
 953                         ndev->limits.max_mw = SNB_MAX_MW;
 954                 break;
 955         default:
 956                 /*
 957                  * we should never hit this. the detect function should've
 958                  * take cared of everything.
 959                  */
 960                 return -EINVAL;
 961         }
 962
 963         ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
 964         ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
 965         ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
 966
 967         ndev->limits.msix_cnt = SNB_MSIX_CNT;
 968         ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
 969
 970         return 0;
 971 }
 972
 973 static int ntb_bwd_setup(struct ntb_device *ndev)
 974 {
 975         int rc;
 976         u32 val;
 977
 978         ndev->hw_type = BWD_HW;
 979
 980         rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
 981         if (rc)
 982                 return rc;
 983
 984         switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
 985         case NTB_CONN_B2B:
 986                 ndev->conn_type = NTB_CONN_B2B;
 987                 break;
 988         case NTB_CONN_RP:
 989         default:
 990                 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
 991                 return -EINVAL;
 992         }
 993
 994         if (val & BWD_PPD_DEV_TYPE)
 995                 ndev->dev_type = NTB_DEV_DSD;
 996         else
 997                 ndev->dev_type = NTB_DEV_USD;
 998
 999         /* Initiate PCI-E link training */
1000         rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
1001                                     val | BWD_PPD_INIT_LINK);
1002         if (rc)
1003                 return rc;
1004
1005         ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
1006         ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
1007         ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
1008         ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
1009         ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
1010         ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
1011         ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
1012         ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
1013         ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
1014         ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
1015         ndev->limits.max_mw = BWD_MAX_MW;
1016         ndev->limits.max_spads = BWD_MAX_SPADS;
1017         ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
1018         ndev->limits.msix_cnt = BWD_MSIX_CNT;
1019         ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
1020
1021         /* Since bwd doesn't have a link interrupt, setup a poll timer */
1022         INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
1023         INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
1024         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
1025
1026         return 0;
1027 }
1028
1029 static int ntb_device_setup(struct ntb_device *ndev)
1030 {
1031         int rc;
1032
1033         if (is_ntb_xeon(ndev))
1034                 rc = ntb_xeon_setup(ndev);
1035         else if (is_ntb_atom(ndev))
1036                 rc = ntb_bwd_setup(ndev);
1037         else
1038                 rc = -ENODEV;
1039
1040         if (rc)
1041                 return rc;
1042
1043         if (ndev->conn_type == NTB_CONN_B2B)
1044                 /* Enable Bus Master and Memory Space on the secondary side */
1045                 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1046                        ndev->reg_ofs.spci_cmd);
1047
1048         return 0;
1049 }
1050
1051 static void ntb_device_free(struct ntb_device *ndev)
1052 {
1053         if (is_ntb_atom(ndev)) {
1054                 cancel_delayed_work_sync(&ndev->hb_timer);
1055                 cancel_delayed_work_sync(&ndev->lr_timer);
1056         }
1057 }
1058
1059 static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
1060 {
1061         struct ntb_db_cb *db_cb = data;
1062         struct ntb_device *ndev = db_cb->ndev;
1063         unsigned long mask;
1064
1065         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
1066                 db_cb->db_num);
1067
1068         mask = readw(ndev->reg_ofs.ldb_mask);
1069         set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1070         writew(mask, ndev->reg_ofs.ldb_mask);
1071
1072         tasklet_schedule(&db_cb->irq_work);
1073
1074         /* No need to check for the specific HB irq, any interrupt means
1075          * we're connected.
1076          */
1077         ndev->last_ts = jiffies;
1078
1079         writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
1080
1081         return IRQ_HANDLED;
1082 }
1083
1084 static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
1085 {
1086         struct ntb_db_cb *db_cb = data;
1087         struct ntb_device *ndev = db_cb->ndev;
1088         unsigned long mask;
1089
1090         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
1091                 db_cb->db_num);
1092
1093         mask = readw(ndev->reg_ofs.ldb_mask);
1094         set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1095         writew(mask, ndev->reg_ofs.ldb_mask);
1096
1097         tasklet_schedule(&db_cb->irq_work);
1098
1099         /* On Sandybridge, there are 16 bits in the interrupt register
1100          * but only 4 vectors.  So, 5 bits are assigned to the first 3
1101          * vectors, with the 4th having a single bit for link
1102          * interrupts.
1103          */
1104         writew(((1 << ndev->bits_per_vector) - 1) <<
1105                (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
1106
1107         return IRQ_HANDLED;
1108 }
1109
1110 /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
1111 static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
1112 {
1113         struct ntb_device *ndev = dev;
1114         int rc;
1115
1116         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
1117
1118         rc = ntb_link_status(ndev);
1119         if (rc)
1120                 dev_err(&ndev->pdev->dev, "Error determining link status\n");
1121
1122         /* bit 15 is always the link bit */
1123         writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);
1124
1125         return IRQ_HANDLED;
1126 }
1127
1128 static irqreturn_t ntb_interrupt(int irq, void *dev)
1129 {
1130         struct ntb_device *ndev = dev;
1131         unsigned int i = 0;
1132
1133         if (is_ntb_atom(ndev)) {
1134                 u64 ldb = readq(ndev->reg_ofs.ldb);
1135
1136                 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
1137
1138                 while (ldb) {
1139                         i = __ffs(ldb);
1140                         ldb &= ldb - 1;
1141                         bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1142                 }
1143         } else {
1144                 u16 ldb = readw(ndev->reg_ofs.ldb);
1145
1146                 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
1147
1148                 if (ldb & SNB_DB_HW_LINK) {
1149                         xeon_event_msix_irq(irq, dev);
1150                         ldb &= ~SNB_DB_HW_LINK;
1151                 }
1152
1153                 while (ldb) {
1154                         i = __ffs(ldb);
1155                         ldb &= ldb - 1;
1156                         xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1157                 }
1158         }
1159
1160         return IRQ_HANDLED;
1161 }
1162
1163 static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries)
1164 {
1165         struct pci_dev *pdev = ndev->pdev;
1166         struct msix_entry *msix;
1167         int rc, i;
1168
1169         if (msix_entries < ndev->limits.msix_cnt)
1170                 return -ENOSPC;
1171
1172         rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries);
1173         if (rc < 0)
1174                 return rc;
1175
1176         for (i = 0; i < msix_entries; i++) {
1177                 msix = &ndev->msix_entries[i];
1178                 WARN_ON(!msix->vector);
1179
1180                 if (i == msix_entries - 1) {
1181                         rc = request_irq(msix->vector,
1182                                          xeon_event_msix_irq, 0,
1183                                          "ntb-event-msix", ndev);
1184                         if (rc)
1185                                 goto err;
1186                 } else {
1187                         rc = request_irq(msix->vector,
1188                                          xeon_callback_msix_irq, 0,
1189                                          "ntb-callback-msix",
1190                                          &ndev->db_cb[i]);
1191                         if (rc)
1192                                 goto err;
1193                 }
1194         }
1195
1196         ndev->num_msix = msix_entries;
1197         ndev->max_cbs = msix_entries - 1;
1198
1199         return 0;
1200
1201 err:
1202         while (--i >= 0) {
1203                 /* Code never reaches here for entry nr 'ndev->num_msix - 1' */
1204                 msix = &ndev->msix_entries[i];
1205                 free_irq(msix->vector, &ndev->db_cb[i]);
1206         }
1207
1208         pci_disable_msix(pdev);
1209         ndev->num_msix = 0;
1210
1211         return rc;
1212 }
1213
1214 static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries)
1215 {
1216         struct pci_dev *pdev = ndev->pdev;
1217         struct msix_entry *msix;
1218         int rc, i;
1219
1220         msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries,
1221                                              1, msix_entries);
1222         if (msix_entries < 0)
1223                 return msix_entries;
1224
1225         for (i = 0; i < msix_entries; i++) {
1226                 msix = &ndev->msix_entries[i];
1227                 WARN_ON(!msix->vector);
1228
1229                 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1230                                  "ntb-callback-msix", &ndev->db_cb[i]);
1231                 if (rc)
1232                         goto err;
1233         }
1234
1235         ndev->num_msix = msix_entries;
1236         ndev->max_cbs = msix_entries;
1237
1238         return 0;
1239
1240 err:
1241         while (--i >= 0)
1242                 free_irq(msix->vector, &ndev->db_cb[i]);
1243
1244         pci_disable_msix(pdev);
1245         ndev->num_msix = 0;
1246
1247         return rc;
1248 }
1249
1250 static int ntb_setup_msix(struct ntb_device *ndev)
1251 {
1252         struct pci_dev *pdev = ndev->pdev;
1253         int msix_entries;
1254         int rc, i;
1255
1256         msix_entries = pci_msix_vec_count(pdev);
1257         if (msix_entries < 0) {
1258                 rc = msix_entries;
1259                 goto err;
1260         } else if (msix_entries > ndev->limits.msix_cnt) {
1261                 rc = -EINVAL;
1262                 goto err;
1263         }
1264
1265         ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1266                                      GFP_KERNEL);
1267         if (!ndev->msix_entries) {
1268                 rc = -ENOMEM;
1269                 goto err;
1270         }
1271
1272         for (i = 0; i < msix_entries; i++)
1273                 ndev->msix_entries[i].entry = i;
1274
1275         if (is_ntb_atom(ndev))
1276                 rc = ntb_setup_bwd_msix(ndev, msix_entries);
1277         else
1278                 rc = ntb_setup_snb_msix(ndev, msix_entries);
1279         if (rc)
1280                 goto err1;
1281
1282         return 0;
1283
1284 err1:
1285         kfree(ndev->msix_entries);
1286 err:
1287         dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1288         return rc;
1289 }
1290
1291 static int ntb_setup_msi(struct ntb_device *ndev)
1292 {
1293         struct pci_dev *pdev = ndev->pdev;
1294         int rc;
1295
1296         rc = pci_enable_msi(pdev);
1297         if (rc)
1298                 return rc;
1299
1300         rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1301         if (rc) {
1302                 pci_disable_msi(pdev);
1303                 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1304                 return rc;
1305         }
1306
1307         return 0;
1308 }
1309
1310 static int ntb_setup_intx(struct ntb_device *ndev)
1311 {
1312         struct pci_dev *pdev = ndev->pdev;
1313         int rc;
1314
1315         /* Verify intx is enabled */
1316         pci_intx(pdev, 1);
1317
1318         rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1319                          ndev);
1320         if (rc)
1321                 return rc;
1322
1323         return 0;
1324 }
1325
1326 static int ntb_setup_interrupts(struct ntb_device *ndev)
1327 {
1328         int rc;
1329
1330         /* On BWD, disable all interrupts.  On SNB, disable all but Link
1331          * Interrupt.  The rest will be unmasked as callbacks are registered.
1332          */
1333         if (is_ntb_atom(ndev))
1334                 writeq(~0, ndev->reg_ofs.ldb_mask);
1335         else {
1336                 u16 var = 1 << SNB_LINK_DB;
1337                 writew(~var, ndev->reg_ofs.ldb_mask);
1338         }
1339
1340         rc = ntb_setup_msix(ndev);
1341         if (!rc)
1342                 goto done;
1343
1344         ndev->bits_per_vector = 1;
1345         ndev->max_cbs = ndev->limits.max_db_bits;
1346
1347         rc = ntb_setup_msi(ndev);
1348         if (!rc)
1349                 goto done;
1350
1351         rc = ntb_setup_intx(ndev);
1352         if (rc) {
1353                 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1354                 return rc;
1355         }
1356
1357 done:
1358         return 0;
1359 }
1360
1361 static void ntb_free_interrupts(struct ntb_device *ndev)
1362 {
1363         struct pci_dev *pdev = ndev->pdev;
1364
1365         /* mask interrupts */
1366         if (is_ntb_atom(ndev))
1367                 writeq(~0, ndev->reg_ofs.ldb_mask);
1368         else
1369                 writew(~0, ndev->reg_ofs.ldb_mask);
1370
1371         if (ndev->num_msix) {
1372                 struct msix_entry *msix;
1373                 u32 i;
1374
1375                 for (i = 0; i < ndev->num_msix; i++) {
1376                         msix = &ndev->msix_entries[i];
1377                         if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1)
1378                                 free_irq(msix->vector, ndev);
1379                         else
1380                                 free_irq(msix->vector, &ndev->db_cb[i]);
1381                 }
1382                 pci_disable_msix(pdev);
1383                 kfree(ndev->msix_entries);
1384         } else {
1385                 free_irq(pdev->irq, ndev);
1386
1387                 if (pci_dev_msi_enabled(pdev))
1388                         pci_disable_msi(pdev);
1389         }
1390 }
1391
1392 static int ntb_create_callbacks(struct ntb_device *ndev)
1393 {
1394         int i;
1395
1396         /* Chicken-egg issue.  We won't know how many callbacks are necessary
1397          * until we see how many MSI-X vectors we get, but these pointers need
1398          * to be passed into the MSI-X register function.  So, we allocate the
1399          * max, knowing that they might not all be used, to work around this.
1400          */
1401         ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1402                               sizeof(struct ntb_db_cb),
1403                               GFP_KERNEL);
1404         if (!ndev->db_cb)
1405                 return -ENOMEM;
1406
1407         for (i = 0; i < ndev->limits.max_db_bits; i++) {
1408                 ndev->db_cb[i].db_num = i;
1409                 ndev->db_cb[i].ndev = ndev;
1410         }
1411
1412         return 0;
1413 }
1414
1415 static void ntb_free_callbacks(struct ntb_device *ndev)
1416 {
1417         int i;
1418
1419         for (i = 0; i < ndev->limits.max_db_bits; i++)
1420                 ntb_unregister_db_callback(ndev, i);
1421
1422         kfree(ndev->db_cb);
1423 }
1424
1425 static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf,
1426                                 size_t count, loff_t *offp)
1427 {
1428         struct ntb_device *ndev;
1429         char *buf;
1430         ssize_t ret, offset, out_count;
1431
1432         out_count = 500;
1433
1434         buf = kmalloc(out_count, GFP_KERNEL);
1435         if (!buf)
1436                 return -ENOMEM;
1437
1438         ndev = filp->private_data;
1439         offset = 0;
1440         offset += snprintf(buf + offset, out_count - offset,
1441                            "NTB Device Information:\n");
1442         offset += snprintf(buf + offset, out_count - offset,
1443                            "Connection Type - \t\t%s\n",
1444                            ndev->conn_type == NTB_CONN_TRANSPARENT ?
1445                            "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ?
1446                            "Back to back" : "Root Port");
1447         offset += snprintf(buf + offset, out_count - offset,
1448                            "Device Type - \t\t\t%s\n",
1449                            ndev->dev_type == NTB_DEV_USD ?
1450                            "DSD/USP" : "USD/DSP");
1451         offset += snprintf(buf + offset, out_count - offset,
1452                            "Max Number of Callbacks - \t%u\n",
1453                            ntb_max_cbs(ndev));
1454         offset += snprintf(buf + offset, out_count - offset,
1455                            "Link Status - \t\t\t%s\n",
1456                            ntb_hw_link_status(ndev) ? "Up" : "Down");
1457         if (ntb_hw_link_status(ndev)) {
1458                 offset += snprintf(buf + offset, out_count - offset,
1459                                    "Link Speed - \t\t\tPCI-E Gen %u\n",
1460                                    ndev->link_speed);
1461                 offset += snprintf(buf + offset, out_count - offset,
1462                                    "Link Width - \t\t\tx%u\n",
1463                                    ndev->link_width);
1464         }
1465
1466         if (is_ntb_xeon(ndev)) {
1467                 u32 status32;
1468                 u16 status16;
1469                 int rc;
1470
1471                 offset += snprintf(buf + offset, out_count - offset,
1472                                    "\nNTB Device Statistics:\n");
1473                 offset += snprintf(buf + offset, out_count - offset,
1474                                    "Upstream Memory Miss - \t%u\n",
1475                                    readw(ndev->reg_base +
1476                                          SNB_USMEMMISS_OFFSET));
1477
1478                 offset += snprintf(buf + offset, out_count - offset,
1479                                    "\nNTB Hardware Errors:\n");
1480
1481                 rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET,
1482                                           &status16);
1483                 if (!rc)
1484                         offset += snprintf(buf + offset, out_count - offset,
1485                                            "DEVSTS - \t%#06x\n", status16);
1486
1487                 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
1488                                           &status16);
1489                 if (!rc)
1490                         offset += snprintf(buf + offset, out_count - offset,
1491                                            "LNKSTS - \t%#06x\n", status16);
1492
1493                 rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET,
1494                                            &status32);
1495                 if (!rc)
1496                         offset += snprintf(buf + offset, out_count - offset,
1497                                            "UNCERRSTS - \t%#010x\n", status32);
1498
1499                 rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET,
1500                                            &status32);
1501                 if (!rc)
1502                         offset += snprintf(buf + offset, out_count - offset,
1503                                            "CORERRSTS - \t%#010x\n", status32);
1504         }
1505
1506         if (offset > out_count)
1507                 offset = out_count;
1508
1509         ret = simple_read_from_buffer(ubuf, count, offp, buf, offset);
1510         kfree(buf);
1511         return ret;
1512 }
1513
1514 static const struct file_operations ntb_debugfs_info = {
1515         .owner = THIS_MODULE,
1516         .open = simple_open,
1517         .read = ntb_debugfs_read,
1518 };
1519
1520 static void ntb_setup_debugfs(struct ntb_device *ndev)
1521 {
1522         if (!debugfs_initialized())
1523                 return;
1524
1525         if (!debugfs_dir)
1526                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1527
1528         ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1529                                                debugfs_dir);
1530         if (ndev->debugfs_dir)
1531                 ndev->debugfs_info = debugfs_create_file("info", S_IRUSR,
1532                                                          ndev->debugfs_dir,
1533                                                          ndev,
1534                                                          &ntb_debugfs_info);
1535 }
1536
1537 static void ntb_free_debugfs(struct ntb_device *ndev)
1538 {
1539         debugfs_remove_recursive(ndev->debugfs_dir);
1540
1541         if (debugfs_dir && simple_empty(debugfs_dir)) {
1542                 debugfs_remove_recursive(debugfs_dir);
1543                 debugfs_dir = NULL;
1544         }
1545 }
1546
1547 static void ntb_hw_link_up(struct ntb_device *ndev)
1548 {
1549         if (ndev->conn_type == NTB_CONN_TRANSPARENT)
1550                 ntb_link_event(ndev, NTB_LINK_UP);
1551         else {
1552                 u32 ntb_cntl;
1553
1554                 /* Let's bring the NTB link up */
1555                 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1556                 ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1557                 ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1558                 ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
1559                 if (ndev->split_bar)
1560                         ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP |
1561                                     NTB_CNTL_S2P_BAR5_SNOOP;
1562
1563                 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1564         }
1565 }
1566
1567 static void ntb_hw_link_down(struct ntb_device *ndev)
1568 {
1569         u32 ntb_cntl;
1570
1571         if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
1572                 ntb_link_event(ndev, NTB_LINK_DOWN);
1573                 return;
1574         }
1575
1576         /* Bring NTB link down */
1577         ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1578         ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1579         ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1580         if (ndev->split_bar)
1581                 ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP |
1582                               NTB_CNTL_S2P_BAR5_SNOOP);
1583         ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1584         writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1585 }
1586
1587 static void ntb_max_mw_detect(struct ntb_device *ndev)
1588 {
1589         if (ndev->split_bar)
1590                 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
1591         else
1592                 ndev->limits.max_mw = SNB_MAX_MW;
1593 }
1594
1595 static int ntb_xeon_detect(struct ntb_device *ndev)
1596 {
1597         int rc, bars_mask;
1598         u32 bars;
1599         u8 ppd;
1600
1601         ndev->hw_type = SNB_HW;
1602
1603         rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd);
1604         if (rc)
1605                 return -EIO;
1606
1607         if (ppd & SNB_PPD_DEV_TYPE)
1608                 ndev->dev_type = NTB_DEV_USD;
1609         else
1610                 ndev->dev_type = NTB_DEV_DSD;
1611
1612         ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0;
1613
1614         switch (ppd & SNB_PPD_CONN_TYPE) {
1615         case NTB_CONN_B2B:
1616                 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
1617                 ndev->conn_type = NTB_CONN_B2B;
1618                 break;
1619         case NTB_CONN_RP:
1620                 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
1621                 ndev->conn_type = NTB_CONN_RP;
1622                 break;
1623         case NTB_CONN_TRANSPARENT:
1624                 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
1625                 ndev->conn_type = NTB_CONN_TRANSPARENT;
1626                 /*
1627                  * This mode is default to USD/DSP. HW does not report
1628                  * properly in transparent mode as it has no knowledge of
1629                  * NTB. We will just force correct here.
1630                  */
1631                 ndev->dev_type = NTB_DEV_USD;
1632
1633                 /*
1634                  * This is a way for transparent BAR to figure out if we
1635                  * are doing split BAR or not. There is no way for the hw
1636                  * on the transparent side to know and set the PPD.
1637                  */
1638                 bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM);
1639                 bars = hweight32(bars_mask);
1640                 if (bars == (HSX_SPLITBAR_MAX_MW + 1))
1641                         ndev->split_bar = 1;
1642
1643                 break;
1644         default:
1645                 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd);
1646                 return -ENODEV;
1647         }
1648
1649         ntb_max_mw_detect(ndev);
1650
1651         return 0;
1652 }
1653
1654 static int ntb_atom_detect(struct ntb_device *ndev)
1655 {
1656         int rc;
1657         u32 ppd;
1658
1659         ndev->hw_type = BWD_HW;
1660         ndev->limits.max_mw = BWD_MAX_MW;
1661
1662         rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd);
1663         if (rc)
1664                 return rc;
1665
1666         switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) {
1667         case NTB_CONN_B2B:
1668                 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
1669                 ndev->conn_type = NTB_CONN_B2B;
1670                 break;
1671         case NTB_CONN_RP:
1672         default:
1673                 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
1674                 return -EINVAL;
1675         }
1676
1677         if (ppd & BWD_PPD_DEV_TYPE)
1678                 ndev->dev_type = NTB_DEV_DSD;
1679         else
1680                 ndev->dev_type = NTB_DEV_USD;
1681
1682         return 0;
1683 }
1684
1685 static int ntb_device_detect(struct ntb_device *ndev)
1686 {
1687         int rc;
1688
1689         if (is_ntb_xeon(ndev))
1690                 rc = ntb_xeon_detect(ndev);
1691         else if (is_ntb_atom(ndev))
1692                 rc = ntb_atom_detect(ndev);
1693         else
1694                 rc = -ENODEV;
1695
1696         dev_info(&ndev->pdev->dev, "Device Type = %s\n",
1697                  ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
1698
1699         return 0;
1700 }
1701
1702 static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1703 {
1704         struct ntb_device *ndev;
1705         int rc, i;
1706
1707         ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1708         if (!ndev)
1709                 return -ENOMEM;
1710
1711         ndev->pdev = pdev;
1712
1713         ntb_set_errata_flags(ndev);
1714
1715         ndev->link_status = NTB_LINK_DOWN;
1716         pci_set_drvdata(pdev, ndev);
1717         ntb_setup_debugfs(ndev);
1718
1719         rc = pci_enable_device(pdev);
1720         if (rc)
1721                 goto err;
1722
1723         pci_set_master(ndev->pdev);
1724
1725         rc = ntb_device_detect(ndev);
1726         if (rc)
1727                 goto err;
1728
1729         ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw),
1730                            GFP_KERNEL);
1731         if (!ndev->mw) {
1732                 rc = -ENOMEM;
1733                 goto err1;
1734         }
1735
1736         if (ndev->split_bar)
1737                 rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK,
1738                                                   KBUILD_MODNAME);
1739         else
1740                 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK,
1741                                                   KBUILD_MODNAME);
1742
1743         if (rc)
1744                 goto err2;
1745
1746         ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1747         if (!ndev->reg_base) {
1748                 dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1749                 rc = -EIO;
1750                 goto err3;
1751         }
1752
1753         for (i = 0; i < ndev->limits.max_mw; i++) {
1754                 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1755
1756                 /*
1757                  * with the errata we need to steal last of the memory
1758                  * windows for workarounds and they point to MMIO registers.
1759                  */
1760                 if ((ndev->wa_flags & WA_SNB_ERR) &&
1761                     (i == (ndev->limits.max_mw - 1))) {
1762                         ndev->mw[i].vbase =
1763                                 ioremap_nocache(pci_resource_start(pdev,
1764                                                         MW_TO_BAR(i)),
1765                                                 ndev->mw[i].bar_sz);
1766                 } else {
1767                         ndev->mw[i].vbase =
1768                                 ioremap_wc(pci_resource_start(pdev,
1769                                                         MW_TO_BAR(i)),
1770                                            ndev->mw[i].bar_sz);
1771                 }
1772
1773                 dev_info(&pdev->dev, "MW %d size %llu\n", i,
1774                          (unsigned long long) ndev->mw[i].bar_sz);
1775                 if (!ndev->mw[i].vbase) {
1776                         dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1777                                  MW_TO_BAR(i));
1778                         rc = -EIO;
1779                         goto err4;
1780                 }
1781         }
1782
1783         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1784         if (rc) {
1785                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1786                 if (rc)
1787                         goto err4;
1788
1789                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1790         }
1791
1792         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1793         if (rc) {
1794                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1795                 if (rc)
1796                         goto err4;
1797
1798                 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1799         }
1800
1801         rc = ntb_device_setup(ndev);
1802         if (rc)
1803                 goto err4;
1804
1805         rc = ntb_create_callbacks(ndev);
1806         if (rc)
1807                 goto err5;
1808
1809         rc = ntb_setup_interrupts(ndev);
1810         if (rc)
1811                 goto err6;
1812
1813         /* The scratchpad registers keep the values between rmmod/insmod,
1814          * blast them now
1815          */
1816         for (i = 0; i < ndev->limits.max_spads; i++) {
1817                 ntb_write_local_spad(ndev, i, 0);
1818                 ntb_write_remote_spad(ndev, i, 0);
1819         }
1820
1821         rc = ntb_transport_init(pdev);
1822         if (rc)
1823                 goto err7;
1824
1825         ntb_hw_link_up(ndev);
1826
1827         return 0;
1828
1829 err7:
1830         ntb_free_interrupts(ndev);
1831 err6:
1832         ntb_free_callbacks(ndev);
1833 err5:
1834         ntb_device_free(ndev);
1835 err4:
1836         for (i--; i >= 0; i--)
1837                 iounmap(ndev->mw[i].vbase);
1838         iounmap(ndev->reg_base);
1839 err3:
1840         if (ndev->split_bar)
1841                 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
1842         else
1843                 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1844 err2:
1845         kfree(ndev->mw);
1846 err1:
1847         pci_disable_device(pdev);
1848 err:
1849         ntb_free_debugfs(ndev);
1850         kfree(ndev);
1851
1852         dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1853         return rc;
1854 }
1855
1856 static void ntb_pci_remove(struct pci_dev *pdev)
1857 {
1858         struct ntb_device *ndev = pci_get_drvdata(pdev);
1859         int i;
1860
1861         ntb_hw_link_down(ndev);
1862
1863         ntb_transport_free(ndev->ntb_transport);
1864
1865         ntb_free_interrupts(ndev);
1866         ntb_free_callbacks(ndev);
1867         ntb_device_free(ndev);
1868
1869         /* need to reset max_mw limits so we can unmap properly */
1870         if (ndev->hw_type == SNB_HW)
1871                 ntb_max_mw_detect(ndev);
1872
1873         for (i = 0; i < ndev->limits.max_mw; i++)
1874                 iounmap(ndev->mw[i].vbase);
1875
1876         kfree(ndev->mw);
1877         iounmap(ndev->reg_base);
1878         if (ndev->split_bar)
1879                 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
1880         else
1881                 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1882         pci_disable_device(pdev);
1883         ntb_free_debugfs(ndev);
1884         kfree(ndev);
1885 }
1886
1887 static struct pci_driver ntb_pci_driver = {
1888         .name = KBUILD_MODNAME,
1889         .id_table = ntb_pci_tbl,
1890         .probe = ntb_pci_probe,
1891         .remove = ntb_pci_remove,
1892 };
1893
1894 module_pci_driver(ntb_pci_driver);