NVMe: Always use MSI/MSI-x interrupts

[cascardo/linux.git] / drivers / nvme / host / pci.c

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index f8db70a..4fd733f 100644 (file)
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -723,6 +723,13 @@ static void nvme_complete_rq(struct request *req)
        blk_mq_end_request(req, error);
 }
 
+/* We read the CQE phase first to check if the rest of the entry is valid */
+static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head,
+               u16 phase)
+{
+       return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
+}
+
 static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
 {
        u16 head, phase;
@@ -730,13 +737,10 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
        head = nvmeq->cq_head;
        phase = nvmeq->cq_phase;
 
-       for (;;) {
+       while (nvme_cqe_valid(nvmeq, head, phase)) {
                struct nvme_completion cqe = nvmeq->cqes[head];
-               u16 status = le16_to_cpu(cqe.status);
                struct request *req;
 
-               if ((status & 1) != phase)
-                       break;
                if (++head == nvmeq->q_depth) {
                        head = 0;
                        phase = !phase;
@@ -767,7 +771,7 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
                req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
                if (req->cmd_type == REQ_TYPE_DRV_PRIV && req->special)
                        memcpy(req->special, &cqe, sizeof(cqe));
-               blk_mq_complete_request(req, status >> 1);
+               blk_mq_complete_request(req, le16_to_cpu(cqe.status) >> 1);
 
        }
 
@@ -808,18 +812,16 @@ static irqreturn_t nvme_irq(int irq, void *data)
 static irqreturn_t nvme_irq_check(int irq, void *data)
 {
        struct nvme_queue *nvmeq = data;
-       struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
-       if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
-               return IRQ_NONE;
-       return IRQ_WAKE_THREAD;
+       if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
+               return IRQ_WAKE_THREAD;
+       return IRQ_NONE;
 }
 
 static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 {
        struct nvme_queue *nvmeq = hctx->driver_data;
 
-       if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
-           nvmeq->cq_phase) {
+       if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
                spin_lock_irq(&nvmeq->q_lock);
                __nvme_process_cq(nvmeq, &tag);
                spin_unlock_irq(&nvmeq->q_lock);
@@ -1476,8 +1478,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
        if (result > 0) {
                dev_err(dev->ctrl.device,
                        "Could not set queue count (%d)\n", result);
-               nr_io_queues = 0;
-               result = 0;
+               return 0;
        }
 
        if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
@@ -1511,7 +1512,9 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
         * If we enable msix early due to not intx, disable it again before
         * setting up the full range we need.
         */
-       if (!pdev->irq)
+       if (pdev->msi_enabled)
+               pci_disable_msi(pdev);
+       else if (pdev->msix_enabled)
                pci_disable_msix(pdev);
 
        for (i = 0; i < nr_io_queues; i++)
@@ -1694,7 +1697,6 @@ static int nvme_pci_enable(struct nvme_dev *dev)
        if (pci_enable_device_mem(pdev))
                return result;
 
-       dev->entry[0].vector = pdev->irq;
        pci_set_master(pdev);
 
        if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
@@ -1707,13 +1709,18 @@ static int nvme_pci_enable(struct nvme_dev *dev)
        }
 
        /*
-        * Some devices don't advertse INTx interrupts, pre-enable a single
-        * MSIX vec for setup. We'll adjust this later.
+        * Some devices and/or platforms don't advertise or work with INTx
+        * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
+        * adjust this later.
         */
-       if (!pdev->irq) {
-               result = pci_enable_msix(pdev, dev->entry, 1);
-               if (result < 0)
-                       goto disable;
+       if (pci_enable_msix(pdev, dev->entry, 1)) {
+               pci_enable_msi(pdev);
+               dev->entry[0].vector = pdev->irq;
+       }
+
+       if (!dev->entry[0].vector) {
+               result = -ENODEV;
+               goto disable;
        }
 
        cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
@@ -1857,6 +1864,9 @@ static void nvme_reset_work(struct work_struct *work)
        if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
                nvme_dev_disable(dev, false);
 
+       if (test_bit(NVME_CTRL_REMOVING, &dev->flags))
+               goto out;
+
        set_bit(NVME_CTRL_RESETTING, &dev->flags);
 
        result = nvme_pci_enable(dev);
@@ -2076,11 +2086,10 @@ static void nvme_remove(struct pci_dev *pdev)
 {
        struct nvme_dev *dev = pci_get_drvdata(pdev);
 
-       del_timer_sync(&dev->watchdog_timer);
-
        set_bit(NVME_CTRL_REMOVING, &dev->flags);
        pci_set_drvdata(pdev, NULL);
        flush_work(&dev->async_work);
+       flush_work(&dev->reset_work);
        flush_work(&dev->scan_work);
        nvme_remove_namespaces(&dev->ctrl);
        nvme_uninit_ctrl(&dev->ctrl);