NTB: Handle 64bit BAR sizes

64bit BAR sizes are permissible with an NTB device.  To support them
various modifications and clean-ups were required, most significantly
using 2 32bit scratch pad registers for each BAR.

Also, modify the driver to allow more than 2 Memory Windows.

Signed-off-by: Jon Mason <jon.mason@intel.com>

diff --git a/drivers/ntb/ntb_hw.c b/drivers/ntb/ntb_hw.c
index 195cc51..2dacd19 100644 (file)
--- a/drivers/ntb/ntb_hw.c
+++ b/drivers/ntb/ntb_hw.c
@@ -1027,8 +1027,8 @@ static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
                ndev->mw[i].vbase =
                    ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
                               ndev->mw[i].bar_sz);
-               dev_info(&pdev->dev, "MW %d size %d\n", i,
-                        (u32) pci_resource_len(pdev, MW_TO_BAR(i)));
+               dev_info(&pdev->dev, "MW %d size %llu\n", i,
+                        pci_resource_len(pdev, MW_TO_BAR(i)));
                if (!ndev->mw[i].vbase) {
                        dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
                                 MW_TO_BAR(i));

diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 00f5e80..79a3203 100644 (file)
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -58,7 +58,7 @@
 #include <linux/ntb.h>
 #include "ntb_hw.h"
 
-#define NTB_TRANSPORT_VERSION  2
+#define NTB_TRANSPORT_VERSION  3
 
 static unsigned int transport_mtu = 0x401E;
 module_param(transport_mtu, uint, 0644);
@@ -173,10 +173,13 @@ struct ntb_payload_header {
 
 enum {
        VERSION = 0,
-       MW0_SZ,
-       MW1_SZ,
-       NUM_QPS,
        QP_LINKS,
+       NUM_QPS,
+       NUM_MWS,
+       MW0_SZ_HIGH,
+       MW0_SZ_LOW,
+       MW1_SZ_HIGH,
+       MW1_SZ_LOW,
        MAX_SPAD,
 };
 
@@ -526,6 +529,18 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)
        return 0;
 }
 
+static void ntb_free_mw(struct ntb_transport *nt, int num_mw)
+{
+       struct ntb_transport_mw *mw = &nt->mw[num_mw];
+       struct pci_dev *pdev = ntb_query_pdev(nt->ndev);
+
+       if (!mw->virt_addr)
+               return;
+
+       dma_free_coherent(&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr);
+       mw->virt_addr = NULL;
+}
+
 static void ntb_qp_link_cleanup(struct work_struct *work)
 {
        struct ntb_transport_qp *qp = container_of(work,
@@ -604,25 +619,31 @@ static void ntb_transport_link_work(struct work_struct *work)
        u32 val;
        int rc, i;
 
-       /* send the local info */
-       rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION);
-       if (rc) {
-               dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
-                       0, VERSION);
-               goto out;
-       }
+       /* send the local info, in the opposite order of the way we read it */
+       for (i = 0; i < NTB_NUM_MW; i++) {
+               rc = ntb_write_remote_spad(ndev, MW0_SZ_HIGH + (i * 2),
+                                          ntb_get_mw_size(ndev, i) >> 32);
+               if (rc) {
+                       dev_err(&pdev->dev, "Error writing %u to remote spad %d\n",
+                               (u32)(ntb_get_mw_size(ndev, i) >> 32),
+                               MW0_SZ_HIGH + (i * 2));
+                       goto out;
+               }
 
-       rc = ntb_write_remote_spad(ndev, MW0_SZ, ntb_get_mw_size(ndev, 0));
-       if (rc) {
-               dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
-                       (u32) ntb_get_mw_size(ndev, 0), MW0_SZ);
-               goto out;
+               rc = ntb_write_remote_spad(ndev, MW0_SZ_LOW + (i * 2),
+                                          (u32) ntb_get_mw_size(ndev, i));
+               if (rc) {
+                       dev_err(&pdev->dev, "Error writing %u to remote spad %d\n",
+                               (u32) ntb_get_mw_size(ndev, i),
+                               MW0_SZ_LOW + (i * 2));
+                       goto out;
+               }
        }
 
-       rc = ntb_write_remote_spad(ndev, MW1_SZ, ntb_get_mw_size(ndev, 1));
+       rc = ntb_write_remote_spad(ndev, NUM_MWS, NTB_NUM_MW);
        if (rc) {
                dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
-                       (u32) ntb_get_mw_size(ndev, 1), MW1_SZ);
+                       NTB_NUM_MW, NUM_MWS);
                goto out;
        }
 
@@ -633,16 +654,10 @@ static void ntb_transport_link_work(struct work_struct *work)
                goto out;
        }
 
-       rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val);
-       if (rc) {
-               dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
-               goto out;
-       }
-
-       rc = ntb_write_remote_spad(ndev, QP_LINKS, val);
+       rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION);
        if (rc) {
                dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
-                       val, QP_LINKS);
+                       NTB_TRANSPORT_VERSION, VERSION);
                goto out;
        }
 
@@ -667,33 +682,43 @@ static void ntb_transport_link_work(struct work_struct *work)
                goto out;
        dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
 
-       rc = ntb_read_remote_spad(ndev, MW0_SZ, &val);
+       rc = ntb_read_remote_spad(ndev, NUM_MWS, &val);
        if (rc) {
-               dev_err(&pdev->dev, "Error reading remote spad %d\n", MW0_SZ);
+               dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_MWS);
                goto out;
        }
 
-       if (!val)
+       if (val != NTB_NUM_MW)
                goto out;
-       dev_dbg(&pdev->dev, "Remote MW0 size = %d\n", val);
+       dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
 
-       rc = ntb_set_mw(nt, 0, val);
-       if (rc)
-               goto out;
+       for (i = 0; i < NTB_NUM_MW; i++) {
+               u64 val64;
 
-       rc = ntb_read_remote_spad(ndev, MW1_SZ, &val);
-       if (rc) {
-               dev_err(&pdev->dev, "Error reading remote spad %d\n", MW1_SZ);
-               goto out;
-       }
+               rc = ntb_read_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), &val);
+               if (rc) {
+                       dev_err(&pdev->dev, "Error reading remote spad %d\n",
+                               MW0_SZ_HIGH + (i * 2));
+                       goto out1;
+               }
 
-       if (!val)
-               goto out;
-       dev_dbg(&pdev->dev, "Remote MW1 size = %d\n", val);
+               val64 = (u64) val << 32;
 
-       rc = ntb_set_mw(nt, 1, val);
-       if (rc)
-               goto out;
+               rc = ntb_read_remote_spad(ndev, MW0_SZ_LOW + (i * 2), &val);
+               if (rc) {
+                       dev_err(&pdev->dev, "Error reading remote spad %d\n",
+                               MW0_SZ_LOW + (i * 2));
+                       goto out1;
+               }
+
+               val64 |= val;
+
+               dev_dbg(&pdev->dev, "Remote MW%d size = %llu\n", i, val64);
+
+               rc = ntb_set_mw(nt, i, val64);
+               if (rc)
+                       goto out1;
+       }
 
        nt->transport_link = NTB_LINK_UP;
 
@@ -708,6 +733,9 @@ static void ntb_transport_link_work(struct work_struct *work)
 
        return;
 
+out1:
+       for (i = 0; i < NTB_NUM_MW; i++)
+               ntb_free_mw(nt, i);
 out:
        if (ntb_hw_link_status(ndev))
                schedule_delayed_work(&nt->link_work,
@@ -897,10 +925,7 @@ void ntb_transport_free(void *transport)
        pdev = ntb_query_pdev(nt->ndev);
 
        for (i = 0; i < NTB_NUM_MW; i++)
-               if (nt->mw[i].virt_addr)
-                       dma_free_coherent(&pdev->dev, nt->mw[i].size,
-                                         nt->mw[i].virt_addr,
-                                         nt->mw[i].dma_addr);
+               ntb_free_mw(nt, i);
 
        kfree(nt->qps);
        ntb_unregister_transport(nt->ndev);