drivers/ntb/hw/intel/ntb_hw_intel.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   9  *
  10  *   This program is free software; you can redistribute it and/or modify
  11  *   it under the terms of version 2 of the GNU General Public License as
  12  *   published by the Free Software Foundation.
  13  *
  14  *   BSD LICENSE
  15  *
  16  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  17  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  18  *
  19  *   Redistribution and use in source and binary forms, with or without
  20  *   modification, are permitted provided that the following conditions
  21  *   are met:
  22  *
  23  *     * Redistributions of source code must retain the above copyright
  24  *       notice, this list of conditions and the following disclaimer.
  25  *     * Redistributions in binary form must reproduce the above copy
  26  *       notice, this list of conditions and the following disclaimer in
  27  *       the documentation and/or other materials provided with the
  28  *       distribution.
  29  *     * Neither the name of Intel Corporation nor the names of its
  30  *       contributors may be used to endorse or promote products derived
  31  *       from this software without specific prior written permission.
  32  *
  33  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44  *
  45  * Intel PCIe NTB Linux driver
  46  *
  47  * Contact Information:
  48  * Jon Mason <jon.mason@intel.com>
  49  */
  50
  51 #include <linux/debugfs.h>
  52 #include <linux/delay.h>
  53 #include <linux/init.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/module.h>
  56 #include <linux/pci.h>
  57 #include <linux/random.h>
  58 #include <linux/slab.h>
  59 #include <linux/ntb.h>
  60
  61 #include "ntb_hw_intel.h"
  62
  63 #define NTB_NAME        "ntb_hw_intel"
  64 #define NTB_DESC        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  65 #define NTB_VER         "2.0"
  66
  67 MODULE_DESCRIPTION(NTB_DESC);
  68 MODULE_VERSION(NTB_VER);
  69 MODULE_LICENSE("Dual BSD/GPL");
  70 MODULE_AUTHOR("Intel Corporation");
  71
  72 #define bar0_off(base, bar) ((base) + ((bar) << 2))
  73 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  74
  75 static const struct intel_ntb_reg atom_reg;
  76 static const struct intel_ntb_alt_reg atom_pri_reg;
  77 static const struct intel_ntb_alt_reg atom_sec_reg;
  78 static const struct intel_ntb_alt_reg atom_b2b_reg;
  79 static const struct intel_ntb_xlat_reg atom_pri_xlat;
  80 static const struct intel_ntb_xlat_reg atom_sec_xlat;
  81 static const struct intel_ntb_reg xeon_reg;
  82 static const struct intel_ntb_alt_reg xeon_pri_reg;
  83 static const struct intel_ntb_alt_reg xeon_sec_reg;
  84 static const struct intel_ntb_alt_reg xeon_b2b_reg;
  85 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  86 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  87 static struct intel_b2b_addr xeon_b2b_usd_addr;
  88 static struct intel_b2b_addr xeon_b2b_dsd_addr;
  89 static const struct ntb_dev_ops intel_ntb_ops;
  90
  91 static const struct file_operations intel_ntb_debugfs_info;
  92 static struct dentry *debugfs_dir;
  93
  94 static int b2b_mw_idx = -1;
  95 module_param(b2b_mw_idx, int, 0644);
  96 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
  97                  "value of zero or positive starts from first mw idx, and a "
  98                  "negative value starts from last mw idx.  Both sides MUST "
  99                  "set the same value here!");
 100
 101 static unsigned int b2b_mw_share;
 102 module_param(b2b_mw_share, uint, 0644);
 103 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
 104                  "ntb so that the peer ntb only occupies the first half of "
 105                  "the mw, so the second half can still be used as a mw.  Both "
 106                  "sides MUST set the same value here!");
 107
 108 module_param_named(xeon_b2b_usd_bar2_addr64,
 109                    xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
 110 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 111                  "XEON B2B USD BAR 2 64-bit address");
 112
 113 module_param_named(xeon_b2b_usd_bar4_addr64,
 114                    xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
 115 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 116                  "XEON B2B USD BAR 4 64-bit address");
 117
 118 module_param_named(xeon_b2b_usd_bar4_addr32,
 119                    xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
 120 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 121                  "XEON B2B USD split-BAR 4 32-bit address");
 122
 123 module_param_named(xeon_b2b_usd_bar5_addr32,
 124                    xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
 125 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 126                  "XEON B2B USD split-BAR 5 32-bit address");
 127
 128 module_param_named(xeon_b2b_dsd_bar2_addr64,
 129                    xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
 130 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 131                  "XEON B2B DSD BAR 2 64-bit address");
 132
 133 module_param_named(xeon_b2b_dsd_bar4_addr64,
 134                    xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
 135 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 136                  "XEON B2B DSD BAR 4 64-bit address");
 137
 138 module_param_named(xeon_b2b_dsd_bar4_addr32,
 139                    xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
 140 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 141                  "XEON B2B DSD split-BAR 4 32-bit address");
 142
 143 module_param_named(xeon_b2b_dsd_bar5_addr32,
 144                    xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
 145 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 146                  "XEON B2B DSD split-BAR 5 32-bit address");
 147
 148 #ifndef ioread64
 149 #ifdef readq
 150 #define ioread64 readq
 151 #else
 152 #define ioread64 _ioread64
 153 static inline u64 _ioread64(void __iomem *mmio)
 154 {
 155         u64 low, high;
 156
 157         low = ioread32(mmio);
 158         high = ioread32(mmio + sizeof(u32));
 159         return low | (high << 32);
 160 }
 161 #endif
 162 #endif
 163
 164 #ifndef iowrite64
 165 #ifdef writeq
 166 #define iowrite64 writeq
 167 #else
 168 #define iowrite64 _iowrite64
 169 static inline void _iowrite64(u64 val, void __iomem *mmio)
 170 {
 171         iowrite32(val, mmio);
 172         iowrite32(val >> 32, mmio + sizeof(u32));
 173 }
 174 #endif
 175 #endif
 176
 177 static inline int pdev_is_atom(struct pci_dev *pdev)
 178 {
 179         switch (pdev->device) {
 180         case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
 181                 return 1;
 182         }
 183         return 0;
 184 }
 185
 186 static inline int pdev_is_xeon(struct pci_dev *pdev)
 187 {
 188         switch (pdev->device) {
 189         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 190         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 191         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 192         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 193         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 194         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 195         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 196         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 197         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 198         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 199         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 200         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 201                 return 1;
 202         }
 203         return 0;
 204 }
 205
 206 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
 207 {
 208         ndev->unsafe_flags = 0;
 209         ndev->unsafe_flags_ignore = 0;
 210
 211         /* Only B2B has a workaround to avoid SDOORBELL */
 212         if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
 213                 if (!ntb_topo_is_b2b(ndev->ntb.topo))
 214                         ndev->unsafe_flags |= NTB_UNSAFE_DB;
 215
 216         /* No low level workaround to avoid SB01BASE */
 217         if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
 218                 ndev->unsafe_flags |= NTB_UNSAFE_DB;
 219                 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
 220         }
 221 }
 222
 223 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
 224                                  unsigned long flag)
 225 {
 226         return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
 227 }
 228
 229 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
 230                                      unsigned long flag)
 231 {
 232         flag &= ndev->unsafe_flags;
 233         ndev->unsafe_flags_ignore |= flag;
 234
 235         return !!flag;
 236 }
 237
 238 static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
 239 {
 240         if (idx < 0 || idx > ndev->mw_count)
 241                 return -EINVAL;
 242         return ndev->reg->mw_bar[idx];
 243 }
 244
 245 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
 246                                phys_addr_t *db_addr, resource_size_t *db_size,
 247                                phys_addr_t reg_addr, unsigned long reg)
 248 {
 249         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
 250
 251         if (db_addr) {
 252                 *db_addr = reg_addr + reg;
 253                 dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
 254         }
 255
 256         if (db_size) {
 257                 *db_size = ndev->reg->db_size;
 258                 dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
 259         }
 260
 261         return 0;
 262 }
 263
 264 static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
 265                                void __iomem *mmio)
 266 {
 267         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
 268
 269         return ndev->reg->db_ioread(mmio);
 270 }
 271
 272 static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
 273                                 void __iomem *mmio)
 274 {
 275         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
 276
 277         if (db_bits & ~ndev->db_valid_mask)
 278                 return -EINVAL;
 279
 280         ndev->reg->db_iowrite(db_bits, mmio);
 281
 282         return 0;
 283 }
 284
 285 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 286                                    void __iomem *mmio)
 287 {
 288         unsigned long irqflags;
 289
 290         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
 291
 292         if (db_bits & ~ndev->db_valid_mask)
 293                 return -EINVAL;
 294
 295         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 296         {
 297                 ndev->db_mask |= db_bits;
 298                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 299         }
 300         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 301
 302         return 0;
 303 }
 304
 305 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 306                                      void __iomem *mmio)
 307 {
 308         unsigned long irqflags;
 309
 310         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
 311
 312         if (db_bits & ~ndev->db_valid_mask)
 313                 return -EINVAL;
 314
 315         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 316         {
 317                 ndev->db_mask &= ~db_bits;
 318                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 319         }
 320         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 321
 322         return 0;
 323 }
 324
 325 static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
 326 {
 327         u64 shift, mask;
 328
 329         shift = ndev->db_vec_shift;
 330         mask = BIT_ULL(shift) - 1;
 331
 332         return mask << (shift * db_vector);
 333 }
 334
 335 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
 336                                  phys_addr_t *spad_addr, phys_addr_t reg_addr,
 337                                  unsigned long reg)
 338 {
 339         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
 340
 341         if (idx < 0 || idx >= ndev->spad_count)
 342                 return -EINVAL;
 343
 344         if (spad_addr) {
 345                 *spad_addr = reg_addr + reg + (idx << 2);
 346                 dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
 347         }
 348
 349         return 0;
 350 }
 351
 352 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
 353                                  void __iomem *mmio)
 354 {
 355         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
 356
 357         if (idx < 0 || idx >= ndev->spad_count)
 358                 return 0;
 359
 360         return ioread32(mmio + (idx << 2));
 361 }
 362
 363 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
 364                                   void __iomem *mmio)
 365 {
 366         WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
 367
 368         if (idx < 0 || idx >= ndev->spad_count)
 369                 return -EINVAL;
 370
 371         iowrite32(val, mmio + (idx << 2));
 372
 373         return 0;
 374 }
 375
 376 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
 377 {
 378         u64 vec_mask;
 379
 380         vec_mask = ndev_vec_mask(ndev, vec);
 381
 382         dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
 383
 384         ndev->last_ts = jiffies;
 385
 386         if (vec_mask & ndev->db_link_mask) {
 387                 if (ndev->reg->poll_link(ndev))
 388                         ntb_link_event(&ndev->ntb);
 389         }
 390
 391         if (vec_mask & ndev->db_valid_mask)
 392                 ntb_db_event(&ndev->ntb, vec);
 393
 394         return IRQ_HANDLED;
 395 }
 396
 397 static irqreturn_t ndev_vec_isr(int irq, void *dev)
 398 {
 399         struct intel_ntb_vec *nvec = dev;
 400
 401         return ndev_interrupt(nvec->ndev, nvec->num);
 402 }
 403
 404 static irqreturn_t ndev_irq_isr(int irq, void *dev)
 405 {
 406         struct intel_ntb_dev *ndev = dev;
 407
 408         return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
 409 }
 410
 411 static int ndev_init_isr(struct intel_ntb_dev *ndev,
 412                          int msix_min, int msix_max,
 413                          int msix_shift, int total_shift)
 414 {
 415         struct pci_dev *pdev;
 416         int rc, i, msix_count, node;
 417
 418         pdev = ndev_pdev(ndev);
 419
 420         node = dev_to_node(&pdev->dev);
 421
 422         /* Mask all doorbell interrupts */
 423         ndev->db_mask = ndev->db_valid_mask;
 424         ndev->reg->db_iowrite(ndev->db_mask,
 425                               ndev->self_mmio +
 426                               ndev->self_reg->db_mask);
 427
 428         /* Try to set up msix irq */
 429
 430         ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
 431                                  GFP_KERNEL, node);
 432         if (!ndev->vec)
 433                 goto err_msix_vec_alloc;
 434
 435         ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
 436                                   GFP_KERNEL, node);
 437         if (!ndev->msix)
 438                 goto err_msix_alloc;
 439
 440         for (i = 0; i < msix_max; ++i)
 441                 ndev->msix[i].entry = i;
 442
 443         msix_count = pci_enable_msix_range(pdev, ndev->msix,
 444                                            msix_min, msix_max);
 445         if (msix_count < 0)
 446                 goto err_msix_enable;
 447
 448         for (i = 0; i < msix_count; ++i) {
 449                 ndev->vec[i].ndev = ndev;
 450                 ndev->vec[i].num = i;
 451                 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 452                                  "ndev_vec_isr", &ndev->vec[i]);
 453                 if (rc)
 454                         goto err_msix_request;
 455         }
 456
 457         dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
 458         ndev->db_vec_count = msix_count;
 459         ndev->db_vec_shift = msix_shift;
 460         return 0;
 461
 462 err_msix_request:
 463         while (i-- > 0)
 464                 free_irq(ndev->msix[i].vector, ndev);
 465         pci_disable_msix(pdev);
 466 err_msix_enable:
 467         kfree(ndev->msix);
 468 err_msix_alloc:
 469         kfree(ndev->vec);
 470 err_msix_vec_alloc:
 471         ndev->msix = NULL;
 472         ndev->vec = NULL;
 473
 474         /* Try to set up msi irq */
 475
 476         rc = pci_enable_msi(pdev);
 477         if (rc)
 478                 goto err_msi_enable;
 479
 480         rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 481                          "ndev_irq_isr", ndev);
 482         if (rc)
 483                 goto err_msi_request;
 484
 485         dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
 486         ndev->db_vec_count = 1;
 487         ndev->db_vec_shift = total_shift;
 488         return 0;
 489
 490 err_msi_request:
 491         pci_disable_msi(pdev);
 492 err_msi_enable:
 493
 494         /* Try to set up intx irq */
 495
 496         pci_intx(pdev, 1);
 497
 498         rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 499                          "ndev_irq_isr", ndev);
 500         if (rc)
 501                 goto err_intx_request;
 502
 503         dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
 504         ndev->db_vec_count = 1;
 505         ndev->db_vec_shift = total_shift;
 506         return 0;
 507
 508 err_intx_request:
 509         return rc;
 510 }
 511
 512 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
 513 {
 514         struct pci_dev *pdev;
 515         int i;
 516
 517         pdev = ndev_pdev(ndev);
 518
 519         /* Mask all doorbell interrupts */
 520         ndev->db_mask = ndev->db_valid_mask;
 521         ndev->reg->db_iowrite(ndev->db_mask,
 522                               ndev->self_mmio +
 523                               ndev->self_reg->db_mask);
 524
 525         if (ndev->msix) {
 526                 i = ndev->db_vec_count;
 527                 while (i--)
 528                         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 529                 pci_disable_msix(pdev);
 530                 kfree(ndev->msix);
 531                 kfree(ndev->vec);
 532         } else {
 533                 free_irq(pdev->irq, ndev);
 534                 if (pci_dev_msi_enabled(pdev))
 535                         pci_disable_msi(pdev);
 536         }
 537 }
 538
 539 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 540                                  size_t count, loff_t *offp)
 541 {
 542         struct intel_ntb_dev *ndev;
 543         void __iomem *mmio;
 544         char *buf;
 545         size_t buf_size;
 546         ssize_t ret, off;
 547         union { u64 v64; u32 v32; u16 v16; } u;
 548
 549         ndev = filp->private_data;
 550         mmio = ndev->self_mmio;
 551
 552         buf_size = min(count, 0x800ul);
 553
 554         buf = kmalloc(buf_size, GFP_KERNEL);
 555         if (!buf)
 556                 return -ENOMEM;
 557
 558         off = 0;
 559
 560         off += scnprintf(buf + off, buf_size - off,
 561                          "NTB Device Information:\n");
 562
 563         off += scnprintf(buf + off, buf_size - off,
 564                          "Connection Topology -\t%s\n",
 565                          ntb_topo_string(ndev->ntb.topo));
 566
 567         off += scnprintf(buf + off, buf_size - off,
 568                          "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
 569         off += scnprintf(buf + off, buf_size - off,
 570                          "B2B MW Idx -\t\t%d\n", ndev->b2b_idx);
 571         off += scnprintf(buf + off, buf_size - off,
 572                          "BAR4 Split -\t\t%s\n",
 573                          ndev->bar4_split ? "yes" : "no");
 574
 575         off += scnprintf(buf + off, buf_size - off,
 576                          "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 577         off += scnprintf(buf + off, buf_size - off,
 578                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 579
 580         if (!ndev->reg->link_is_up(ndev)) {
 581                 off += scnprintf(buf + off, buf_size - off,
 582                                  "Link Status -\t\tDown\n");
 583         } else {
 584                 off += scnprintf(buf + off, buf_size - off,
 585                                  "Link Status -\t\tUp\n");
 586                 off += scnprintf(buf + off, buf_size - off,
 587                                  "Link Speed -\t\tPCI-E Gen %u\n",
 588                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 589                 off += scnprintf(buf + off, buf_size - off,
 590                                  "Link Width -\t\tx%u\n",
 591                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 592         }
 593
 594         off += scnprintf(buf + off, buf_size - off,
 595                          "Memory Window Count -\t%u\n", ndev->mw_count);
 596         off += scnprintf(buf + off, buf_size - off,
 597                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 598         off += scnprintf(buf + off, buf_size - off,
 599                          "Doorbell Count -\t%u\n", ndev->db_count);
 600         off += scnprintf(buf + off, buf_size - off,
 601                          "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 602         off += scnprintf(buf + off, buf_size - off,
 603                          "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 604
 605         off += scnprintf(buf + off, buf_size - off,
 606                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 607         off += scnprintf(buf + off, buf_size - off,
 608                          "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 609         off += scnprintf(buf + off, buf_size - off,
 610                          "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 611
 612         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 613         off += scnprintf(buf + off, buf_size - off,
 614                          "Doorbell Mask -\t\t%#llx\n", u.v64);
 615
 616         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 617         off += scnprintf(buf + off, buf_size - off,
 618                          "Doorbell Bell -\t\t%#llx\n", u.v64);
 619
 620         off += scnprintf(buf + off, buf_size - off,
 621                          "\nNTB Incoming XLAT:\n");
 622
 623         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
 624         off += scnprintf(buf + off, buf_size - off,
 625                          "XLAT23 -\t\t%#018llx\n", u.v64);
 626
 627         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 628         off += scnprintf(buf + off, buf_size - off,
 629                          "XLAT45 -\t\t%#018llx\n", u.v64);
 630
 631         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
 632         off += scnprintf(buf + off, buf_size - off,
 633                          "LMT23 -\t\t\t%#018llx\n", u.v64);
 634
 635         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 636         off += scnprintf(buf + off, buf_size - off,
 637                          "LMT45 -\t\t\t%#018llx\n", u.v64);
 638
 639         if (pdev_is_xeon(ndev->ntb.pdev)) {
 640                 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 641                         off += scnprintf(buf + off, buf_size - off,
 642                                          "\nNTB Outgoing B2B XLAT:\n");
 643
 644                         u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
 645                         off += scnprintf(buf + off, buf_size - off,
 646                                          "B2B XLAT23 -\t\t%#018llx\n", u.v64);
 647
 648                         u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
 649                         off += scnprintf(buf + off, buf_size - off,
 650                                          "B2B XLAT45 -\t\t%#018llx\n", u.v64);
 651
 652                         u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
 653                         off += scnprintf(buf + off, buf_size - off,
 654                                          "B2B LMT23 -\t\t%#018llx\n", u.v64);
 655
 656                         u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
 657                         off += scnprintf(buf + off, buf_size - off,
 658                                          "B2B LMT45 -\t\t%#018llx\n", u.v64);
 659
 660                         off += scnprintf(buf + off, buf_size - off,
 661                                          "\nNTB Secondary BAR:\n");
 662
 663                         u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
 664                         off += scnprintf(buf + off, buf_size - off,
 665                                          "SBAR01 -\t\t%#018llx\n", u.v64);
 666
 667                         u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
 668                         off += scnprintf(buf + off, buf_size - off,
 669                                          "SBAR23 -\t\t%#018llx\n", u.v64);
 670
 671                         u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
 672                         off += scnprintf(buf + off, buf_size - off,
 673                                          "SBAR45 -\t\t%#018llx\n", u.v64);
 674                 }
 675
 676                 off += scnprintf(buf + off, buf_size - off,
 677                                  "\nXEON NTB Statistics:\n");
 678
 679                 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
 680                 off += scnprintf(buf + off, buf_size - off,
 681                                  "Upstream Memory Miss -\t%u\n", u.v16);
 682
 683                 off += scnprintf(buf + off, buf_size - off,
 684                                  "\nXEON NTB Hardware Errors:\n");
 685
 686                 if (!pci_read_config_word(ndev->ntb.pdev,
 687                                           XEON_DEVSTS_OFFSET, &u.v16))
 688                         off += scnprintf(buf + off, buf_size - off,
 689                                          "DEVSTS -\t\t%#06x\n", u.v16);
 690
 691                 if (!pci_read_config_word(ndev->ntb.pdev,
 692                                           XEON_LINK_STATUS_OFFSET, &u.v16))
 693                         off += scnprintf(buf + off, buf_size - off,
 694                                          "LNKSTS -\t\t%#06x\n", u.v16);
 695
 696                 if (!pci_read_config_dword(ndev->ntb.pdev,
 697                                            XEON_UNCERRSTS_OFFSET, &u.v32))
 698                         off += scnprintf(buf + off, buf_size - off,
 699                                          "UNCERRSTS -\t\t%#06x\n", u.v32);
 700
 701                 if (!pci_read_config_dword(ndev->ntb.pdev,
 702                                            XEON_CORERRSTS_OFFSET, &u.v32))
 703                         off += scnprintf(buf + off, buf_size - off,
 704                                          "CORERRSTS -\t\t%#06x\n", u.v32);
 705         }
 706
 707         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 708         kfree(buf);
 709         return ret;
 710 }
 711
 712 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
 713 {
 714         if (!debugfs_dir) {
 715                 ndev->debugfs_dir = NULL;
 716                 ndev->debugfs_info = NULL;
 717         } else {
 718                 ndev->debugfs_dir =
 719                         debugfs_create_dir(ndev_name(ndev), debugfs_dir);
 720                 if (!ndev->debugfs_dir)
 721                         ndev->debugfs_info = NULL;
 722                 else
 723                         ndev->debugfs_info =
 724                                 debugfs_create_file("info", S_IRUSR,
 725                                                     ndev->debugfs_dir, ndev,
 726                                                     &intel_ntb_debugfs_info);
 727         }
 728 }
 729
 730 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
 731 {
 732         debugfs_remove_recursive(ndev->debugfs_dir);
 733 }
 734
 735 static int intel_ntb_mw_count(struct ntb_dev *ntb)
 736 {
 737         return ntb_ndev(ntb)->mw_count;
 738 }
 739
 740 static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
 741                                   phys_addr_t *base,
 742                                   resource_size_t *size,
 743                                   resource_size_t *align,
 744                                   resource_size_t *align_size)
 745 {
 746         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 747         int bar;
 748
 749         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 750                 idx += 1;
 751
 752         bar = ndev_mw_to_bar(ndev, idx);
 753         if (bar < 0)
 754                 return bar;
 755
 756         if (base)
 757                 *base = pci_resource_start(ndev->ntb.pdev, bar) +
 758                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
 759
 760         if (size)
 761                 *size = pci_resource_len(ndev->ntb.pdev, bar) -
 762                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
 763
 764         if (align)
 765                 *align = pci_resource_len(ndev->ntb.pdev, bar);
 766
 767         if (align_size)
 768                 *align_size = 1;
 769
 770         return 0;
 771 }
 772
 773 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
 774                                   dma_addr_t addr, resource_size_t size)
 775 {
 776         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 777         unsigned long base_reg, xlat_reg, limit_reg;
 778         resource_size_t bar_size, mw_size;
 779         void __iomem *mmio;
 780         u64 base, limit, reg_val;
 781         int bar;
 782
 783         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 784                 idx += 1;
 785
 786         bar = ndev_mw_to_bar(ndev, idx);
 787         if (bar < 0)
 788                 return bar;
 789
 790         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 791
 792         if (idx == ndev->b2b_idx)
 793                 mw_size = bar_size - ndev->b2b_off;
 794         else
 795                 mw_size = bar_size;
 796
 797         /* hardware requires that addr is aligned to bar size */
 798         if (addr & (bar_size - 1))
 799                 return -EINVAL;
 800
 801         /* make sure the range fits in the usable mw size */
 802         if (size > mw_size)
 803                 return -EINVAL;
 804
 805         mmio = ndev->self_mmio;
 806         base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
 807         xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
 808         limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
 809
 810         if (bar < 4 || !ndev->bar4_split) {
 811                 base = ioread64(mmio + base_reg);
 812
 813                 /* Set the limit if supported, if size is not mw_size */
 814                 if (limit_reg && size != mw_size)
 815                         limit = base + size;
 816                 else
 817                         limit = 0;
 818
 819                 /* set and verify setting the translation address */
 820                 iowrite64(addr, mmio + xlat_reg);
 821                 reg_val = ioread64(mmio + xlat_reg);
 822                 if (reg_val != addr) {
 823                         iowrite64(0, mmio + xlat_reg);
 824                         return -EIO;
 825                 }
 826
 827                 /* set and verify setting the limit */
 828                 iowrite64(limit, mmio + limit_reg);
 829                 reg_val = ioread64(mmio + limit_reg);
 830                 if (reg_val != limit) {
 831                         iowrite64(base, mmio + limit_reg);
 832                         iowrite64(0, mmio + xlat_reg);
 833                         return -EIO;
 834                 }
 835         } else {
 836                 /* split bar addr range must all be 32 bit */
 837                 if (addr & (~0ull << 32))
 838                         return -EINVAL;
 839                 if ((addr + size) & (~0ull << 32))
 840                         return -EINVAL;
 841
 842                 base = ioread32(mmio + base_reg);
 843
 844                 /* Set the limit if supported, if size is not mw_size */
 845                 if (limit_reg && size != mw_size)
 846                         limit = base + size;
 847                 else
 848                         limit = 0;
 849
 850                 /* set and verify setting the translation address */
 851                 iowrite32(addr, mmio + xlat_reg);
 852                 reg_val = ioread32(mmio + xlat_reg);
 853                 if (reg_val != addr) {
 854                         iowrite32(0, mmio + xlat_reg);
 855                         return -EIO;
 856                 }
 857
 858                 /* set and verify setting the limit */
 859                 iowrite32(limit, mmio + limit_reg);
 860                 reg_val = ioread32(mmio + limit_reg);
 861                 if (reg_val != limit) {
 862                         iowrite32(base, mmio + limit_reg);
 863                         iowrite32(0, mmio + xlat_reg);
 864                         return -EIO;
 865                 }
 866         }
 867
 868         return 0;
 869 }
 870
 871 static int intel_ntb_link_is_up(struct ntb_dev *ntb,
 872                                 enum ntb_speed *speed,
 873                                 enum ntb_width *width)
 874 {
 875         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 876
 877         if (ndev->reg->link_is_up(ndev)) {
 878                 if (speed)
 879                         *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 880                 if (width)
 881                         *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 882                 return 1;
 883         } else {
 884                 /* TODO MAYBE: is it possible to observe the link speed and
 885                  * width while link is training? */
 886                 if (speed)
 887                         *speed = NTB_SPEED_NONE;
 888                 if (width)
 889                         *width = NTB_WIDTH_NONE;
 890                 return 0;
 891         }
 892 }
 893
 894 static int intel_ntb_link_enable(struct ntb_dev *ntb,
 895                                  enum ntb_speed max_speed,
 896                                  enum ntb_width max_width)
 897 {
 898         struct intel_ntb_dev *ndev;
 899         u32 ntb_ctl;
 900
 901         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 902
 903         if (ndev->ntb.topo == NTB_TOPO_SEC)
 904                 return -EINVAL;
 905
 906         dev_dbg(ndev_dev(ndev),
 907                 "Enabling link with max_speed %d max_width %d\n",
 908                 max_speed, max_width);
 909         if (max_speed != NTB_SPEED_AUTO)
 910                 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
 911         if (max_width != NTB_WIDTH_AUTO)
 912                 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
 913
 914         ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
 915         ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 916         ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 917         ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 918         if (ndev->bar4_split)
 919                 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
 920         iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
 921
 922         return 0;
 923 }
 924
 925 static int intel_ntb_link_disable(struct ntb_dev *ntb)
 926 {
 927         struct intel_ntb_dev *ndev;
 928         u32 ntb_cntl;
 929
 930         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 931
 932         if (ndev->ntb.topo == NTB_TOPO_SEC)
 933                 return -EINVAL;
 934
 935         dev_dbg(ndev_dev(ndev), "Disabling link\n");
 936
 937         /* Bring NTB link down */
 938         ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
 939         ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
 940         ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
 941         if (ndev->bar4_split)
 942                 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
 943         ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
 944         iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
 945
 946         return 0;
 947 }
 948
 949 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
 950 {
 951         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
 952 }
 953
 954 static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
 955 {
 956         return ntb_ndev(ntb)->db_valid_mask;
 957 }
 958
 959 static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
 960 {
 961         struct intel_ntb_dev *ndev;
 962
 963         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 964
 965         return ndev->db_vec_count;
 966 }
 967
 968 static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 969 {
 970         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 971
 972         if (db_vector < 0 || db_vector > ndev->db_vec_count)
 973                 return 0;
 974
 975         return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
 976 }
 977
 978 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
 979 {
 980         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 981
 982         return ndev_db_read(ndev,
 983                             ndev->self_mmio +
 984                             ndev->self_reg->db_bell);
 985 }
 986
 987 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 988 {
 989         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 990
 991         return ndev_db_write(ndev, db_bits,
 992                              ndev->self_mmio +
 993                              ndev->self_reg->db_bell);
 994 }
 995
 996 static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 997 {
 998         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 999
1000         return ndev_db_set_mask(ndev, db_bits,
1001                                 ndev->self_mmio +
1002                                 ndev->self_reg->db_mask);
1003 }
1004
1005 static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1006 {
1007         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1008
1009         return ndev_db_clear_mask(ndev, db_bits,
1010                                   ndev->self_mmio +
1011                                   ndev->self_reg->db_mask);
1012 }
1013
1014 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1015                                   phys_addr_t *db_addr,
1016                                   resource_size_t *db_size)
1017 {
1018         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1019
1020         return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1021                             ndev->peer_reg->db_bell);
1022 }
1023
1024 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1025 {
1026         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1027
1028         return ndev_db_write(ndev, db_bits,
1029                              ndev->peer_mmio +
1030                              ndev->peer_reg->db_bell);
1031 }
1032
1033 static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1034 {
1035         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1036 }
1037
1038 static int intel_ntb_spad_count(struct ntb_dev *ntb)
1039 {
1040         struct intel_ntb_dev *ndev;
1041
1042         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1043
1044         return ndev->spad_count;
1045 }
1046
1047 static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1048 {
1049         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1050
1051         return ndev_spad_read(ndev, idx,
1052                               ndev->self_mmio +
1053                               ndev->self_reg->spad);
1054 }
1055
1056 static int intel_ntb_spad_write(struct ntb_dev *ntb,
1057                                 int idx, u32 val)
1058 {
1059         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1060
1061         return ndev_spad_write(ndev, idx, val,
1062                                ndev->self_mmio +
1063                                ndev->self_reg->spad);
1064 }
1065
1066 static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
1067                                     phys_addr_t *spad_addr)
1068 {
1069         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1070
1071         return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
1072                               ndev->peer_reg->spad);
1073 }
1074
1075 static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
1076 {
1077         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1078
1079         return ndev_spad_read(ndev, idx,
1080                               ndev->peer_mmio +
1081                               ndev->peer_reg->spad);
1082 }
1083
1084 static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
1085                                      int idx, u32 val)
1086 {
1087         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1088
1089         return ndev_spad_write(ndev, idx, val,
1090                                ndev->peer_mmio +
1091                                ndev->peer_reg->spad);
1092 }
1093
1094 /* ATOM */
1095
1096 static u64 atom_db_ioread(void __iomem *mmio)
1097 {
1098         return ioread64(mmio);
1099 }
1100
1101 static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1102 {
1103         iowrite64(bits, mmio);
1104 }
1105
1106 static int atom_poll_link(struct intel_ntb_dev *ndev)
1107 {
1108         u32 ntb_ctl;
1109
1110         ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1111
1112         if (ntb_ctl == ndev->ntb_ctl)
1113                 return 0;
1114
1115         ndev->ntb_ctl = ntb_ctl;
1116
1117         ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1118
1119         return 1;
1120 }
1121
1122 static int atom_link_is_up(struct intel_ntb_dev *ndev)
1123 {
1124         return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1125 }
1126
1127 static int atom_link_is_err(struct intel_ntb_dev *ndev)
1128 {
1129         if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1130             & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1131                 return 1;
1132
1133         if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1134             & ATOM_IBIST_ERR_OFLOW)
1135                 return 1;
1136
1137         return 0;
1138 }
1139
1140 static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1141 {
1142         switch (ppd & ATOM_PPD_TOPO_MASK) {
1143         case ATOM_PPD_TOPO_B2B_USD:
1144                 dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
1145                 return NTB_TOPO_B2B_USD;
1146
1147         case ATOM_PPD_TOPO_B2B_DSD:
1148                 dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
1149                 return NTB_TOPO_B2B_DSD;
1150
1151         case ATOM_PPD_TOPO_PRI_USD:
1152         case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1153         case ATOM_PPD_TOPO_SEC_USD:
1154         case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1155                 dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
1156                 return NTB_TOPO_NONE;
1157         }
1158
1159         dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
1160         return NTB_TOPO_NONE;
1161 }
1162
1163 static void atom_link_hb(struct work_struct *work)
1164 {
1165         struct intel_ntb_dev *ndev = hb_ndev(work);
1166         unsigned long poll_ts;
1167         void __iomem *mmio;
1168         u32 status32;
1169
1170         poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1171
1172         /* Delay polling the link status if an interrupt was received,
1173          * unless the cached link status says the link is down.
1174          */
1175         if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1176                 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1177                 return;
1178         }
1179
1180         if (atom_poll_link(ndev))
1181                 ntb_link_event(&ndev->ntb);
1182
1183         if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1184                 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1185                 return;
1186         }
1187
1188         /* Link is down with error: recover the link! */
1189
1190         mmio = ndev->self_mmio;
1191
1192         /* Driver resets the NTB ModPhy lanes - magic! */
1193         iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1194         iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1195         iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1196         iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1197
1198         /* Driver waits 100ms to allow the NTB ModPhy to settle */
1199         msleep(100);
1200
1201         /* Clear AER Errors, write to clear */
1202         status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1203         dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
1204         status32 &= PCI_ERR_COR_REP_ROLL;
1205         iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1206
1207         /* Clear unexpected electrical idle event in LTSSM, write to clear */
1208         status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1209         dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1210         status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1211         iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1212
1213         /* Clear DeSkew Buffer error, write to clear */
1214         status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1215         dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1216         status32 |= ATOM_DESKEWSTS_DBERR;
1217         iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1218
1219         status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1220         dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1221         status32 &= ATOM_IBIST_ERR_OFLOW;
1222         iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1223
1224         /* Releases the NTB state machine to allow the link to retrain */
1225         status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1226         dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1227         status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1228         iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1229
1230         /* There is a potential race between the 2 NTB devices recovering at the
1231          * same time.  If the times are the same, the link will not recover and
1232          * the driver will be stuck in this loop forever.  Add a random interval
1233          * to the recovery time to prevent this race.
1234          */
1235         schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1236                               + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1237 }
1238
1239 static int atom_init_isr(struct intel_ntb_dev *ndev)
1240 {
1241         int rc;
1242
1243         rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1244                            ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1245         if (rc)
1246                 return rc;
1247
1248         /* ATOM doesn't have link status interrupt, poll on that platform */
1249         ndev->last_ts = jiffies;
1250         INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1251         schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1252
1253         return 0;
1254 }
1255
1256 static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1257 {
1258         cancel_delayed_work_sync(&ndev->hb_timer);
1259         ndev_deinit_isr(ndev);
1260 }
1261
1262 static int atom_init_ntb(struct intel_ntb_dev *ndev)
1263 {
1264         ndev->mw_count = ATOM_MW_COUNT;
1265         ndev->spad_count = ATOM_SPAD_COUNT;
1266         ndev->db_count = ATOM_DB_COUNT;
1267
1268         switch (ndev->ntb.topo) {
1269         case NTB_TOPO_B2B_USD:
1270         case NTB_TOPO_B2B_DSD:
1271                 ndev->self_reg = &atom_pri_reg;
1272                 ndev->peer_reg = &atom_b2b_reg;
1273                 ndev->xlat_reg = &atom_sec_xlat;
1274
1275                 /* Enable Bus Master and Memory Space on the secondary side */
1276                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1277                           ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1278
1279                 break;
1280
1281         default:
1282                 return -EINVAL;
1283         }
1284
1285         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1286
1287         return 0;
1288 }
1289
1290 static int atom_init_dev(struct intel_ntb_dev *ndev)
1291 {
1292         u32 ppd;
1293         int rc;
1294
1295         rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1296         if (rc)
1297                 return -EIO;
1298
1299         ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1300         if (ndev->ntb.topo == NTB_TOPO_NONE)
1301                 return -EINVAL;
1302
1303         rc = atom_init_ntb(ndev);
1304         if (rc)
1305                 return rc;
1306
1307         rc = atom_init_isr(ndev);
1308         if (rc)
1309                 return rc;
1310
1311         if (ndev->ntb.topo != NTB_TOPO_SEC) {
1312                 /* Initiate PCI-E link training */
1313                 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1314                                             ppd | ATOM_PPD_INIT_LINK);
1315                 if (rc)
1316                         return rc;
1317         }
1318
1319         return 0;
1320 }
1321
1322 static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1323 {
1324         atom_deinit_isr(ndev);
1325 }
1326
1327 /* XEON */
1328
1329 static u64 xeon_db_ioread(void __iomem *mmio)
1330 {
1331         return (u64)ioread16(mmio);
1332 }
1333
1334 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1335 {
1336         iowrite16((u16)bits, mmio);
1337 }
1338
1339 static int xeon_poll_link(struct intel_ntb_dev *ndev)
1340 {
1341         u16 reg_val;
1342         int rc;
1343
1344         ndev->reg->db_iowrite(ndev->db_link_mask,
1345                               ndev->self_mmio +
1346                               ndev->self_reg->db_bell);
1347
1348         rc = pci_read_config_word(ndev->ntb.pdev,
1349                                   XEON_LINK_STATUS_OFFSET, &reg_val);
1350         if (rc)
1351                 return 0;
1352
1353         if (reg_val == ndev->lnk_sta)
1354                 return 0;
1355
1356         ndev->lnk_sta = reg_val;
1357
1358         return 1;
1359 }
1360
1361 static int xeon_link_is_up(struct intel_ntb_dev *ndev)
1362 {
1363         if (ndev->ntb.topo == NTB_TOPO_SEC)
1364                 return 1;
1365
1366         return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1367 }
1368
1369 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1370 {
1371         switch (ppd & XEON_PPD_TOPO_MASK) {
1372         case XEON_PPD_TOPO_B2B_USD:
1373                 return NTB_TOPO_B2B_USD;
1374
1375         case XEON_PPD_TOPO_B2B_DSD:
1376                 return NTB_TOPO_B2B_DSD;
1377
1378         case XEON_PPD_TOPO_PRI_USD:
1379         case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1380                 return NTB_TOPO_PRI;
1381
1382         case XEON_PPD_TOPO_SEC_USD:
1383         case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1384                 return NTB_TOPO_SEC;
1385         }
1386
1387         return NTB_TOPO_NONE;
1388 }
1389
1390 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1391 {
1392         if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1393                 dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
1394                 return 1;
1395         }
1396         return 0;
1397 }
1398
1399 static int xeon_init_isr(struct intel_ntb_dev *ndev)
1400 {
1401         return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1402                              XEON_DB_MSIX_VECTOR_COUNT,
1403                              XEON_DB_MSIX_VECTOR_SHIFT,
1404                              XEON_DB_TOTAL_SHIFT);
1405 }
1406
1407 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1408 {
1409         ndev_deinit_isr(ndev);
1410 }
1411
1412 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1413                              const struct intel_b2b_addr *addr,
1414                              const struct intel_b2b_addr *peer_addr)
1415 {
1416         struct pci_dev *pdev;
1417         void __iomem *mmio;
1418         resource_size_t bar_size;
1419         phys_addr_t bar_addr;
1420         int b2b_bar;
1421         u8 bar_sz;
1422
1423         pdev = ndev_pdev(ndev);
1424         mmio = ndev->self_mmio;
1425
1426         if (ndev->b2b_idx >= ndev->mw_count) {
1427                 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
1428                 b2b_bar = 0;
1429                 ndev->b2b_off = 0;
1430         } else {
1431                 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1432                 if (b2b_bar < 0)
1433                         return -EIO;
1434
1435                 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
1436
1437                 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1438
1439                 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
1440
1441                 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1442                         dev_dbg(ndev_dev(ndev),
1443                                 "b2b using first half of bar\n");
1444                         ndev->b2b_off = bar_size >> 1;
1445                 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1446                         dev_dbg(ndev_dev(ndev),
1447                                 "b2b using whole bar\n");
1448                         ndev->b2b_off = 0;
1449                         --ndev->mw_count;
1450                 } else {
1451                         dev_dbg(ndev_dev(ndev),
1452                                 "b2b bar size is too small\n");
1453                         return -EIO;
1454                 }
1455         }
1456
1457         /* Reset the secondary bar sizes to match the primary bar sizes,
1458          * except disable or halve the size of the b2b secondary bar.
1459          *
1460          * Note: code for each specific bar size register, because the register
1461          * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1462          */
1463         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1464         dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
1465         if (b2b_bar == 2) {
1466                 if (ndev->b2b_off)
1467                         bar_sz -= 1;
1468                 else
1469                         bar_sz = 0;
1470         }
1471         pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1472         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1473         dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
1474
1475         if (!ndev->bar4_split) {
1476                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1477                 dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
1478                 if (b2b_bar == 4) {
1479                         if (ndev->b2b_off)
1480                                 bar_sz -= 1;
1481                         else
1482                                 bar_sz = 0;
1483                 }
1484                 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1485                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1486                 dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
1487         } else {
1488                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1489                 dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
1490                 if (b2b_bar == 4) {
1491                         if (ndev->b2b_off)
1492                                 bar_sz -= 1;
1493                         else
1494                                 bar_sz = 0;
1495                 }
1496                 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1497                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1498                 dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
1499
1500                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1501                 dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
1502                 if (b2b_bar == 5) {
1503                         if (ndev->b2b_off)
1504                                 bar_sz -= 1;
1505                         else
1506                                 bar_sz = 0;
1507                 }
1508                 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1509                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1510                 dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
1511         }
1512
1513         /* SBAR01 hit by first part of the b2b bar */
1514         if (b2b_bar == 0)
1515                 bar_addr = addr->bar0_addr;
1516         else if (b2b_bar == 2)
1517                 bar_addr = addr->bar2_addr64;
1518         else if (b2b_bar == 4 && !ndev->bar4_split)
1519                 bar_addr = addr->bar4_addr64;
1520         else if (b2b_bar == 4)
1521                 bar_addr = addr->bar4_addr32;
1522         else if (b2b_bar == 5)
1523                 bar_addr = addr->bar5_addr32;
1524         else
1525                 return -EIO;
1526
1527         dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
1528         iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1529
1530         /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1531          * The b2b bar is either disabled above, or configured half-size, and
1532          * it starts at the PBAR xlat + offset.
1533          */
1534
1535         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1536         iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1537         bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1538         dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
1539
1540         if (!ndev->bar4_split) {
1541                 bar_addr = addr->bar4_addr64 +
1542                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1543                 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1544                 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1545                 dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
1546         } else {
1547                 bar_addr = addr->bar4_addr32 +
1548                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1549                 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1550                 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1551                 dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
1552
1553                 bar_addr = addr->bar5_addr32 +
1554                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1555                 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1556                 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1557                 dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
1558         }
1559
1560         /* setup incoming bar limits == base addrs (zero length windows) */
1561
1562         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1563         iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1564         bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1565         dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
1566
1567         if (!ndev->bar4_split) {
1568                 bar_addr = addr->bar4_addr64 +
1569                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1570                 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1571                 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1572                 dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
1573         } else {
1574                 bar_addr = addr->bar4_addr32 +
1575                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1576                 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1577                 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1578                 dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
1579
1580                 bar_addr = addr->bar5_addr32 +
1581                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1582                 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1583                 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1584                 dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
1585         }
1586
1587         /* zero incoming translation addrs */
1588         iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1589
1590         if (!ndev->bar4_split) {
1591                 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1592         } else {
1593                 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1594                 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1595         }
1596
1597         /* zero outgoing translation limits (whole bar size windows) */
1598         iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1599         if (!ndev->bar4_split) {
1600                 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1601         } else {
1602                 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1603                 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1604         }
1605
1606         /* set outgoing translation offsets */
1607         bar_addr = peer_addr->bar2_addr64;
1608         iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1609         bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1610         dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
1611
1612         if (!ndev->bar4_split) {
1613                 bar_addr = peer_addr->bar4_addr64;
1614                 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1615                 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1616                 dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
1617         } else {
1618                 bar_addr = peer_addr->bar4_addr32;
1619                 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1620                 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1621                 dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
1622
1623                 bar_addr = peer_addr->bar5_addr32;
1624                 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1625                 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1626                 dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
1627         }
1628
1629         /* set the translation offset for b2b registers */
1630         if (b2b_bar == 0)
1631                 bar_addr = peer_addr->bar0_addr;
1632         else if (b2b_bar == 2)
1633                 bar_addr = peer_addr->bar2_addr64;
1634         else if (b2b_bar == 4 && !ndev->bar4_split)
1635                 bar_addr = peer_addr->bar4_addr64;
1636         else if (b2b_bar == 4)
1637                 bar_addr = peer_addr->bar4_addr32;
1638         else if (b2b_bar == 5)
1639                 bar_addr = peer_addr->bar5_addr32;
1640         else
1641                 return -EIO;
1642
1643         /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1644         dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
1645         iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1646         iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1647
1648         if (b2b_bar) {
1649                 /* map peer ntb mmio config space registers */
1650                 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1651                                             XEON_B2B_MIN_SIZE);
1652                 if (!ndev->peer_mmio)
1653                         return -EIO;
1654         }
1655
1656         return 0;
1657 }
1658
1659 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1660 {
1661         int rc;
1662         u32 ntb_ctl;
1663
1664         if (ndev->bar4_split)
1665                 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1666         else
1667                 ndev->mw_count = XEON_MW_COUNT;
1668
1669         ndev->spad_count = XEON_SPAD_COUNT;
1670         ndev->db_count = XEON_DB_COUNT;
1671         ndev->db_link_mask = XEON_DB_LINK_BIT;
1672
1673         switch (ndev->ntb.topo) {
1674         case NTB_TOPO_PRI:
1675                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1676                         dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
1677                         return -EINVAL;
1678                 }
1679
1680                 /* enable link to allow secondary side device to appear */
1681                 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1682                 ntb_ctl &= ~NTB_CTL_DISABLE;
1683                 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1684
1685                 /* use half the spads for the peer */
1686                 ndev->spad_count >>= 1;
1687                 ndev->self_reg = &xeon_pri_reg;
1688                 ndev->peer_reg = &xeon_sec_reg;
1689                 ndev->xlat_reg = &xeon_sec_xlat;
1690                 break;
1691
1692         case NTB_TOPO_SEC:
1693                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1694                         dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
1695                         return -EINVAL;
1696                 }
1697                 /* use half the spads for the peer */
1698                 ndev->spad_count >>= 1;
1699                 ndev->self_reg = &xeon_sec_reg;
1700                 ndev->peer_reg = &xeon_pri_reg;
1701                 ndev->xlat_reg = &xeon_pri_xlat;
1702                 break;
1703
1704         case NTB_TOPO_B2B_USD:
1705         case NTB_TOPO_B2B_DSD:
1706                 ndev->self_reg = &xeon_pri_reg;
1707                 ndev->peer_reg = &xeon_b2b_reg;
1708                 ndev->xlat_reg = &xeon_sec_xlat;
1709
1710                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1711                         ndev->peer_reg = &xeon_pri_reg;
1712
1713                         if (b2b_mw_idx < 0)
1714                                 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1715                         else
1716                                 ndev->b2b_idx = b2b_mw_idx;
1717
1718                         dev_dbg(ndev_dev(ndev),
1719                                 "setting up b2b mw idx %d means %d\n",
1720                                 b2b_mw_idx, ndev->b2b_idx);
1721
1722                 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1723                         dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
1724                         ndev->db_count -= 1;
1725                 }
1726
1727                 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1728                         rc = xeon_setup_b2b_mw(ndev,
1729                                                &xeon_b2b_dsd_addr,
1730                                                &xeon_b2b_usd_addr);
1731                 } else {
1732                         rc = xeon_setup_b2b_mw(ndev,
1733                                                &xeon_b2b_usd_addr,
1734                                                &xeon_b2b_dsd_addr);
1735                 }
1736                 if (rc)
1737                         return rc;
1738
1739                 /* Enable Bus Master and Memory Space on the secondary side */
1740                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1741                           ndev->self_mmio + XEON_SPCICMD_OFFSET);
1742
1743                 break;
1744
1745         default:
1746                 return -EINVAL;
1747         }
1748
1749         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1750
1751         ndev->reg->db_iowrite(ndev->db_valid_mask,
1752                               ndev->self_mmio +
1753                               ndev->self_reg->db_mask);
1754
1755         return 0;
1756 }
1757
1758 static int xeon_init_dev(struct intel_ntb_dev *ndev)
1759 {
1760         struct pci_dev *pdev;
1761         u8 ppd;
1762         int rc, mem;
1763
1764         pdev = ndev_pdev(ndev);
1765
1766         switch (pdev->device) {
1767         /* There is a Xeon hardware errata related to writes to SDOORBELL or
1768          * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1769          * which may hang the system.  To workaround this use the second memory
1770          * window to access the interrupt and scratch pad registers on the
1771          * remote system.
1772          */
1773         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1774         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1775         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1776         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1777         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1778         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1779         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1780         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1781         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1782         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1783         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1784         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1785                 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1786                 break;
1787         }
1788
1789         switch (pdev->device) {
1790         /* There is a hardware errata related to accessing any register in
1791          * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1792          */
1793         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1794         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1795         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1796         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1797         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1798         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1799                 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1800                 break;
1801         }
1802
1803         switch (pdev->device) {
1804         /* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1805          * mirrored to the remote system.  Shrink the number of bits by one,
1806          * since bit 14 is the last bit.
1807          */
1808         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1809         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1810         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1811         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1812         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1813         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1814         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1815         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1816         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1817         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1818         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1819         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1820                 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1821                 break;
1822         }
1823
1824         ndev->reg = &xeon_reg;
1825
1826         rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1827         if (rc)
1828                 return -EIO;
1829
1830         ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1831         dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
1832                 ntb_topo_string(ndev->ntb.topo));
1833         if (ndev->ntb.topo == NTB_TOPO_NONE)
1834                 return -EINVAL;
1835
1836         if (ndev->ntb.topo != NTB_TOPO_SEC) {
1837                 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1838                 dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
1839                         ppd, ndev->bar4_split);
1840         } else {
1841                 /* This is a way for transparent BAR to figure out if we are
1842                  * doing split BAR or not. There is no way for the hw on the
1843                  * transparent side to know and set the PPD.
1844                  */
1845                 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1846                 ndev->bar4_split = hweight32(mem) ==
1847                         HSX_SPLIT_BAR_MW_COUNT + 1;
1848                 dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
1849                         mem, ndev->bar4_split);
1850         }
1851
1852         rc = xeon_init_ntb(ndev);
1853         if (rc)
1854                 return rc;
1855
1856         return xeon_init_isr(ndev);
1857 }
1858
1859 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1860 {
1861         xeon_deinit_isr(ndev);
1862 }
1863
1864 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1865 {
1866         int rc;
1867
1868         pci_set_drvdata(pdev, ndev);
1869
1870         rc = pci_enable_device(pdev);
1871         if (rc)
1872                 goto err_pci_enable;
1873
1874         rc = pci_request_regions(pdev, NTB_NAME);
1875         if (rc)
1876                 goto err_pci_regions;
1877
1878         pci_set_master(pdev);
1879
1880         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1881         if (rc) {
1882                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1883                 if (rc)
1884                         goto err_dma_mask;
1885                 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1886         }
1887
1888         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1889         if (rc) {
1890                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1891                 if (rc)
1892                         goto err_dma_mask;
1893                 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1894         }
1895
1896         ndev->self_mmio = pci_iomap(pdev, 0, 0);
1897         if (!ndev->self_mmio) {
1898                 rc = -EIO;
1899                 goto err_mmio;
1900         }
1901         ndev->peer_mmio = ndev->self_mmio;
1902
1903         return 0;
1904
1905 err_mmio:
1906 err_dma_mask:
1907         pci_clear_master(pdev);
1908         pci_release_regions(pdev);
1909 err_pci_regions:
1910         pci_disable_device(pdev);
1911 err_pci_enable:
1912         pci_set_drvdata(pdev, NULL);
1913         return rc;
1914 }
1915
1916 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1917 {
1918         struct pci_dev *pdev = ndev_pdev(ndev);
1919
1920         if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1921                 pci_iounmap(pdev, ndev->peer_mmio);
1922         pci_iounmap(pdev, ndev->self_mmio);
1923
1924         pci_clear_master(pdev);
1925         pci_release_regions(pdev);
1926         pci_disable_device(pdev);
1927         pci_set_drvdata(pdev, NULL);
1928 }
1929
1930 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1931                                     struct pci_dev *pdev)
1932 {
1933         ndev->ntb.pdev = pdev;
1934         ndev->ntb.topo = NTB_TOPO_NONE;
1935         ndev->ntb.ops = &intel_ntb_ops;
1936
1937         ndev->b2b_off = 0;
1938         ndev->b2b_idx = INT_MAX;
1939
1940         ndev->bar4_split = 0;
1941
1942         ndev->mw_count = 0;
1943         ndev->spad_count = 0;
1944         ndev->db_count = 0;
1945         ndev->db_vec_count = 0;
1946         ndev->db_vec_shift = 0;
1947
1948         ndev->ntb_ctl = 0;
1949         ndev->lnk_sta = 0;
1950
1951         ndev->db_valid_mask = 0;
1952         ndev->db_link_mask = 0;
1953         ndev->db_mask = 0;
1954
1955         spin_lock_init(&ndev->db_mask_lock);
1956 }
1957
1958 static int intel_ntb_pci_probe(struct pci_dev *pdev,
1959                                const struct pci_device_id *id)
1960 {
1961         struct intel_ntb_dev *ndev;
1962         int rc, node;
1963
1964         node = dev_to_node(&pdev->dev);
1965
1966         if (pdev_is_atom(pdev)) {
1967                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1968                 if (!ndev) {
1969                         rc = -ENOMEM;
1970                         goto err_ndev;
1971                 }
1972
1973                 ndev_init_struct(ndev, pdev);
1974
1975                 rc = intel_ntb_init_pci(ndev, pdev);
1976                 if (rc)
1977                         goto err_init_pci;
1978
1979                 rc = atom_init_dev(ndev);
1980                 if (rc)
1981                         goto err_init_dev;
1982
1983         } else if (pdev_is_xeon(pdev)) {
1984                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1985                 if (!ndev) {
1986                         rc = -ENOMEM;
1987                         goto err_ndev;
1988                 }
1989
1990                 ndev_init_struct(ndev, pdev);
1991
1992                 rc = intel_ntb_init_pci(ndev, pdev);
1993                 if (rc)
1994                         goto err_init_pci;
1995
1996                 rc = xeon_init_dev(ndev);
1997                 if (rc)
1998                         goto err_init_dev;
1999
2000         } else {
2001                 rc = -EINVAL;
2002                 goto err_ndev;
2003         }
2004
2005         ndev_reset_unsafe_flags(ndev);
2006
2007         ndev->reg->poll_link(ndev);
2008
2009         ndev_init_debugfs(ndev);
2010
2011         rc = ntb_register_device(&ndev->ntb);
2012         if (rc)
2013                 goto err_register;
2014
2015         dev_info(&pdev->dev, "NTB device registered.\n");
2016
2017         return 0;
2018
2019 err_register:
2020         ndev_deinit_debugfs(ndev);
2021         if (pdev_is_atom(pdev))
2022                 atom_deinit_dev(ndev);
2023         else if (pdev_is_xeon(pdev))
2024                 xeon_deinit_dev(ndev);
2025 err_init_dev:
2026         intel_ntb_deinit_pci(ndev);
2027 err_init_pci:
2028         kfree(ndev);
2029 err_ndev:
2030         return rc;
2031 }
2032
2033 static void intel_ntb_pci_remove(struct pci_dev *pdev)
2034 {
2035         struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2036
2037         ntb_unregister_device(&ndev->ntb);
2038         ndev_deinit_debugfs(ndev);
2039         if (pdev_is_atom(pdev))
2040                 atom_deinit_dev(ndev);
2041         else if (pdev_is_xeon(pdev))
2042                 xeon_deinit_dev(ndev);
2043         intel_ntb_deinit_pci(ndev);
2044         kfree(ndev);
2045 }
2046
2047 static const struct intel_ntb_reg atom_reg = {
2048         .poll_link              = atom_poll_link,
2049         .link_is_up             = atom_link_is_up,
2050         .db_ioread              = atom_db_ioread,
2051         .db_iowrite             = atom_db_iowrite,
2052         .db_size                = sizeof(u64),
2053         .ntb_ctl                = ATOM_NTBCNTL_OFFSET,
2054         .mw_bar                 = {2, 4},
2055 };
2056
2057 static const struct intel_ntb_alt_reg atom_pri_reg = {
2058         .db_bell                = ATOM_PDOORBELL_OFFSET,
2059         .db_mask                = ATOM_PDBMSK_OFFSET,
2060         .spad                   = ATOM_SPAD_OFFSET,
2061 };
2062
2063 static const struct intel_ntb_alt_reg atom_b2b_reg = {
2064         .db_bell                = ATOM_B2B_DOORBELL_OFFSET,
2065         .spad                   = ATOM_B2B_SPAD_OFFSET,
2066 };
2067
2068 static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2069         /* FIXME : .bar0_base   = ATOM_SBAR0BASE_OFFSET, */
2070         /* FIXME : .bar2_limit  = ATOM_SBAR2LMT_OFFSET, */
2071         .bar2_xlat              = ATOM_SBAR2XLAT_OFFSET,
2072 };
2073
2074 static const struct intel_ntb_reg xeon_reg = {
2075         .poll_link              = xeon_poll_link,
2076         .link_is_up             = xeon_link_is_up,
2077         .db_ioread              = xeon_db_ioread,
2078         .db_iowrite             = xeon_db_iowrite,
2079         .db_size                = sizeof(u32),
2080         .ntb_ctl                = XEON_NTBCNTL_OFFSET,
2081         .mw_bar                 = {2, 4, 5},
2082 };
2083
2084 static const struct intel_ntb_alt_reg xeon_pri_reg = {
2085         .db_bell                = XEON_PDOORBELL_OFFSET,
2086         .db_mask                = XEON_PDBMSK_OFFSET,
2087         .spad                   = XEON_SPAD_OFFSET,
2088 };
2089
2090 static const struct intel_ntb_alt_reg xeon_sec_reg = {
2091         .db_bell                = XEON_SDOORBELL_OFFSET,
2092         .db_mask                = XEON_SDBMSK_OFFSET,
2093         /* second half of the scratchpads */
2094         .spad                   = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2095 };
2096
2097 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2098         .db_bell                = XEON_B2B_DOORBELL_OFFSET,
2099         .spad                   = XEON_B2B_SPAD_OFFSET,
2100 };
2101
2102 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2103         /* Note: no primary .bar0_base visible to the secondary side.
2104          *
2105          * The secondary side cannot get the base address stored in primary
2106          * bars.  The base address is necessary to set the limit register to
2107          * any value other than zero, or unlimited.
2108          *
2109          * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2110          * window by setting the limit equal to base, nor can it limit the size
2111          * of the memory window by setting the limit to base + size.
2112          */
2113         .bar2_limit             = XEON_PBAR23LMT_OFFSET,
2114         .bar2_xlat              = XEON_PBAR23XLAT_OFFSET,
2115 };
2116
2117 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2118         .bar0_base              = XEON_SBAR0BASE_OFFSET,
2119         .bar2_limit             = XEON_SBAR23LMT_OFFSET,
2120         .bar2_xlat              = XEON_SBAR23XLAT_OFFSET,
2121 };
2122
2123 static struct intel_b2b_addr xeon_b2b_usd_addr = {
2124         .bar2_addr64            = XEON_B2B_BAR2_USD_ADDR64,
2125         .bar4_addr64            = XEON_B2B_BAR4_USD_ADDR64,
2126         .bar4_addr32            = XEON_B2B_BAR4_USD_ADDR32,
2127         .bar5_addr32            = XEON_B2B_BAR5_USD_ADDR32,
2128 };
2129
2130 static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2131         .bar2_addr64            = XEON_B2B_BAR2_DSD_ADDR64,
2132         .bar4_addr64            = XEON_B2B_BAR4_DSD_ADDR64,
2133         .bar4_addr32            = XEON_B2B_BAR4_DSD_ADDR32,
2134         .bar5_addr32            = XEON_B2B_BAR5_DSD_ADDR32,
2135 };
2136
2137 /* operations for primary side of local ntb */
2138 static const struct ntb_dev_ops intel_ntb_ops = {
2139         .mw_count               = intel_ntb_mw_count,
2140         .mw_get_range           = intel_ntb_mw_get_range,
2141         .mw_set_trans           = intel_ntb_mw_set_trans,
2142         .link_is_up             = intel_ntb_link_is_up,
2143         .link_enable            = intel_ntb_link_enable,
2144         .link_disable           = intel_ntb_link_disable,
2145         .db_is_unsafe           = intel_ntb_db_is_unsafe,
2146         .db_valid_mask          = intel_ntb_db_valid_mask,
2147         .db_vector_count        = intel_ntb_db_vector_count,
2148         .db_vector_mask         = intel_ntb_db_vector_mask,
2149         .db_read                = intel_ntb_db_read,
2150         .db_clear               = intel_ntb_db_clear,
2151         .db_set_mask            = intel_ntb_db_set_mask,
2152         .db_clear_mask          = intel_ntb_db_clear_mask,
2153         .peer_db_addr           = intel_ntb_peer_db_addr,
2154         .peer_db_set            = intel_ntb_peer_db_set,
2155         .spad_is_unsafe         = intel_ntb_spad_is_unsafe,
2156         .spad_count             = intel_ntb_spad_count,
2157         .spad_read              = intel_ntb_spad_read,
2158         .spad_write             = intel_ntb_spad_write,
2159         .peer_spad_addr         = intel_ntb_peer_spad_addr,
2160         .peer_spad_read         = intel_ntb_peer_spad_read,
2161         .peer_spad_write        = intel_ntb_peer_spad_write,
2162 };
2163
2164 static const struct file_operations intel_ntb_debugfs_info = {
2165         .owner = THIS_MODULE,
2166         .open = simple_open,
2167         .read = ndev_debugfs_read,
2168 };
2169
2170 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2171         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2172         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2173         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2174         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2175         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2176         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2177         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2178         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2179         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2180         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2181         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2182         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2183         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2184         {0}
2185 };
2186 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2187
2188 static struct pci_driver intel_ntb_pci_driver = {
2189         .name = KBUILD_MODNAME,
2190         .id_table = intel_ntb_pci_tbl,
2191         .probe = intel_ntb_pci_probe,
2192         .remove = intel_ntb_pci_remove,
2193 };
2194
2195 static int __init intel_ntb_pci_driver_init(void)
2196 {
2197         pr_info("%s %s\n", NTB_DESC, NTB_VER);
2198
2199         if (debugfs_initialized())
2200                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2201
2202         return pci_register_driver(&intel_ntb_pci_driver);
2203 }
2204 module_init(intel_ntb_pci_driver_init);
2205
2206 static void __exit intel_ntb_pci_driver_exit(void)
2207 {
2208         pci_unregister_driver(&intel_ntb_pci_driver);
2209
2210         debugfs_remove_recursive(debugfs_dir);
2211 }
2212 module_exit(intel_ntb_pci_driver_exit);
2213