2 * PCIe host controller driver for Tegra SoCs
4 * Copyright (c) 2010, CompuLab, Ltd.
5 * Author: Mike Rapoport <mike@compulab.co.il>
7 * Based on NVIDIA PCIe driver
8 * Copyright (c) 2008-2009, NVIDIA Corporation.
10 * Bits taken from arch/arm/mach-dove/pcie.c
12 * Author: Thierry Reding <treding@nvidia.com>
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
19 * This program is distributed in the hope that it will be useful, but WITHOUT
20 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
21 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
24 * You should have received a copy of the GNU General Public License along
25 * with this program; if not, write to the Free Software Foundation, Inc.,
26 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
29 #include <linux/clk.h>
30 #include <linux/debugfs.h>
31 #include <linux/delay.h>
32 #include <linux/export.h>
33 #include <linux/interrupt.h>
34 #include <linux/irq.h>
35 #include <linux/irqdomain.h>
36 #include <linux/kernel.h>
37 #include <linux/init.h>
38 #include <linux/msi.h>
39 #include <linux/of_address.h>
40 #include <linux/of_pci.h>
41 #include <linux/of_platform.h>
42 #include <linux/pci.h>
43 #include <linux/phy/phy.h>
44 #include <linux/platform_device.h>
45 #include <linux/reset.h>
46 #include <linux/sizes.h>
47 #include <linux/slab.h>
48 #include <linux/vmalloc.h>
49 #include <linux/regulator/consumer.h>
51 #include <soc/tegra/cpuidle.h>
52 #include <soc/tegra/pmc.h>
54 #include <asm/mach/irq.h>
55 #include <asm/mach/map.h>
56 #include <asm/mach/pci.h>
58 #define INT_PCI_MSI_NR (8 * 32)
60 /* register definitions */
62 #define AFI_AXI_BAR0_SZ 0x00
63 #define AFI_AXI_BAR1_SZ 0x04
64 #define AFI_AXI_BAR2_SZ 0x08
65 #define AFI_AXI_BAR3_SZ 0x0c
66 #define AFI_AXI_BAR4_SZ 0x10
67 #define AFI_AXI_BAR5_SZ 0x14
69 #define AFI_AXI_BAR0_START 0x18
70 #define AFI_AXI_BAR1_START 0x1c
71 #define AFI_AXI_BAR2_START 0x20
72 #define AFI_AXI_BAR3_START 0x24
73 #define AFI_AXI_BAR4_START 0x28
74 #define AFI_AXI_BAR5_START 0x2c
76 #define AFI_FPCI_BAR0 0x30
77 #define AFI_FPCI_BAR1 0x34
78 #define AFI_FPCI_BAR2 0x38
79 #define AFI_FPCI_BAR3 0x3c
80 #define AFI_FPCI_BAR4 0x40
81 #define AFI_FPCI_BAR5 0x44
83 #define AFI_CACHE_BAR0_SZ 0x48
84 #define AFI_CACHE_BAR0_ST 0x4c
85 #define AFI_CACHE_BAR1_SZ 0x50
86 #define AFI_CACHE_BAR1_ST 0x54
88 #define AFI_MSI_BAR_SZ 0x60
89 #define AFI_MSI_FPCI_BAR_ST 0x64
90 #define AFI_MSI_AXI_BAR_ST 0x68
92 #define AFI_MSI_VEC0 0x6c
93 #define AFI_MSI_VEC1 0x70
94 #define AFI_MSI_VEC2 0x74
95 #define AFI_MSI_VEC3 0x78
96 #define AFI_MSI_VEC4 0x7c
97 #define AFI_MSI_VEC5 0x80
98 #define AFI_MSI_VEC6 0x84
99 #define AFI_MSI_VEC7 0x88
101 #define AFI_MSI_EN_VEC0 0x8c
102 #define AFI_MSI_EN_VEC1 0x90
103 #define AFI_MSI_EN_VEC2 0x94
104 #define AFI_MSI_EN_VEC3 0x98
105 #define AFI_MSI_EN_VEC4 0x9c
106 #define AFI_MSI_EN_VEC5 0xa0
107 #define AFI_MSI_EN_VEC6 0xa4
108 #define AFI_MSI_EN_VEC7 0xa8
110 #define AFI_CONFIGURATION 0xac
111 #define AFI_CONFIGURATION_EN_FPCI (1 << 0)
113 #define AFI_FPCI_ERROR_MASKS 0xb0
115 #define AFI_INTR_MASK 0xb4
116 #define AFI_INTR_MASK_INT_MASK (1 << 0)
117 #define AFI_INTR_MASK_MSI_MASK (1 << 8)
119 #define AFI_INTR_CODE 0xb8
120 #define AFI_INTR_CODE_MASK 0xf
121 #define AFI_INTR_INI_SLAVE_ERROR 1
122 #define AFI_INTR_INI_DECODE_ERROR 2
123 #define AFI_INTR_TARGET_ABORT 3
124 #define AFI_INTR_MASTER_ABORT 4
125 #define AFI_INTR_INVALID_WRITE 5
126 #define AFI_INTR_LEGACY 6
127 #define AFI_INTR_FPCI_DECODE_ERROR 7
128 #define AFI_INTR_AXI_DECODE_ERROR 8
129 #define AFI_INTR_FPCI_TIMEOUT 9
130 #define AFI_INTR_PE_PRSNT_SENSE 10
131 #define AFI_INTR_PE_CLKREQ_SENSE 11
132 #define AFI_INTR_CLKCLAMP_SENSE 12
133 #define AFI_INTR_RDY4PD_SENSE 13
134 #define AFI_INTR_P2P_ERROR 14
136 #define AFI_INTR_SIGNATURE 0xbc
137 #define AFI_UPPER_FPCI_ADDRESS 0xc0
138 #define AFI_SM_INTR_ENABLE 0xc4
139 #define AFI_SM_INTR_INTA_ASSERT (1 << 0)
140 #define AFI_SM_INTR_INTB_ASSERT (1 << 1)
141 #define AFI_SM_INTR_INTC_ASSERT (1 << 2)
142 #define AFI_SM_INTR_INTD_ASSERT (1 << 3)
143 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
144 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
145 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
146 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
148 #define AFI_AFI_INTR_ENABLE 0xc8
149 #define AFI_INTR_EN_INI_SLVERR (1 << 0)
150 #define AFI_INTR_EN_INI_DECERR (1 << 1)
151 #define AFI_INTR_EN_TGT_SLVERR (1 << 2)
152 #define AFI_INTR_EN_TGT_DECERR (1 << 3)
153 #define AFI_INTR_EN_TGT_WRERR (1 << 4)
154 #define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
155 #define AFI_INTR_EN_AXI_DECERR (1 << 6)
156 #define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
157 #define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
159 #define AFI_PCIE_CONFIG 0x0f8
160 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
161 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
162 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
163 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
164 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
165 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
166 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
167 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
168 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
169 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
171 #define AFI_FUSE 0x104
172 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
174 #define AFI_PEX0_CTRL 0x110
175 #define AFI_PEX1_CTRL 0x118
176 #define AFI_PEX2_CTRL 0x128
177 #define AFI_PEX_CTRL_RST (1 << 0)
178 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
179 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
180 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
182 #define AFI_PLLE_CONTROL 0x160
183 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
184 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
186 #define AFI_PEXBIAS_CTRL_0 0x168
188 #define RP_VEND_XP 0x00000f00
189 #define RP_VEND_XP_DL_UP (1 << 30)
191 #define RP_PRIV_MISC 0x00000fe0
192 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
193 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
195 #define RP_LINK_CONTROL_STATUS 0x00000090
196 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
197 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
199 #define PADS_CTL_SEL 0x0000009c
201 #define PADS_CTL 0x000000a0
202 #define PADS_CTL_IDDQ_1L (1 << 0)
203 #define PADS_CTL_TX_DATA_EN_1L (1 << 6)
204 #define PADS_CTL_RX_DATA_EN_1L (1 << 10)
206 #define PADS_PLL_CTL_TEGRA20 0x000000b8
207 #define PADS_PLL_CTL_TEGRA30 0x000000b4
208 #define PADS_PLL_CTL_RST_B4SM (1 << 1)
209 #define PADS_PLL_CTL_LOCKDET (1 << 8)
210 #define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
211 #define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
212 #define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
213 #define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
214 #define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
215 #define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
216 #define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
217 #define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
219 #define PADS_REFCLK_CFG0 0x000000c8
220 #define PADS_REFCLK_CFG1 0x000000cc
221 #define PADS_REFCLK_BIAS 0x000000d0
224 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
225 * entries, one entry per PCIe port. These field definitions and desired
226 * values aren't in the TRM, but do come from NVIDIA.
228 #define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
229 #define PADS_REFCLK_CFG_E_TERM_SHIFT 7
230 #define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
231 #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
234 struct msi_controller chip;
235 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
236 struct irq_domain *domain;
242 /* used to differentiate between Tegra SoC generations */
243 struct tegra_pcie_soc {
244 unsigned int num_ports;
245 unsigned int msi_base_shift;
248 u32 pads_refclk_cfg0;
249 u32 pads_refclk_cfg1;
250 bool has_pex_clkreq_en;
251 bool has_pex_bias_ctrl;
252 bool has_intr_prsnt_sense;
257 static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
259 return container_of(chip, struct tegra_msi, chip);
269 struct list_head buses;
275 struct resource prefetch;
276 struct resource busn;
288 struct reset_control *pex_rst;
289 struct reset_control *afi_rst;
290 struct reset_control *pcie_xrst;
295 struct tegra_msi msi;
297 struct list_head ports;
300 struct regulator_bulk_data *supplies;
301 unsigned int num_supplies;
303 const struct tegra_pcie_soc *soc;
304 struct dentry *debugfs;
307 struct tegra_pcie_port {
308 struct tegra_pcie *pcie;
309 struct device_node *np;
310 struct list_head list;
311 struct resource regs;
319 struct tegra_pcie_bus {
320 struct vm_struct *area;
321 struct list_head list;
325 static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
327 return sys->private_data;
330 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
331 unsigned long offset)
333 writel(value, pcie->afi + offset);
336 static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
338 return readl(pcie->afi + offset);
341 static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
342 unsigned long offset)
344 writel(value, pcie->pads + offset);
347 static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
349 return readl(pcie->pads + offset);
353 * The configuration space mapping on Tegra is somewhat similar to the ECAM
354 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
355 * register accesses are mapped:
357 * [27:24] extended register number
359 * [15:11] device number
360 * [10: 8] function number
361 * [ 7: 0] register number
363 * Mapping the whole extended configuration space would require 256 MiB of
364 * virtual address space, only a small part of which will actually be used.
365 * To work around this, a 1 MiB of virtual addresses are allocated per bus
366 * when the bus is first accessed. When the physical range is mapped, the
367 * the bus number bits are hidden so that the extended register number bits
368 * appear as bits [19:16]. Therefore the virtual mapping looks like this:
370 * [19:16] extended register number
371 * [15:11] device number
372 * [10: 8] function number
373 * [ 7: 0] register number
375 * This is achieved by stitching together 16 chunks of 64 KiB of physical
376 * address space via the MMU.
378 static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
380 return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
381 (PCI_FUNC(devfn) << 8) | (where & 0xfc);
384 static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
387 pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
388 L_PTE_XN | L_PTE_MT_DEV_SHARED | L_PTE_SHARED);
389 phys_addr_t cs = pcie->cs->start;
390 struct tegra_pcie_bus *bus;
394 bus = kzalloc(sizeof(*bus), GFP_KERNEL);
396 return ERR_PTR(-ENOMEM);
398 INIT_LIST_HEAD(&bus->list);
401 /* allocate 1 MiB of virtual addresses */
402 bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
408 /* map each of the 16 chunks of 64 KiB each */
409 for (i = 0; i < 16; i++) {
410 unsigned long virt = (unsigned long)bus->area->addr +
412 phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
414 err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
416 dev_err(pcie->dev, "ioremap_page_range() failed: %d\n",
425 vunmap(bus->area->addr);
431 static int tegra_pcie_add_bus(struct pci_bus *bus)
433 struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
434 struct tegra_pcie_bus *b;
436 b = tegra_pcie_bus_alloc(pcie, bus->number);
440 list_add_tail(&b->list, &pcie->buses);
445 static void tegra_pcie_remove_bus(struct pci_bus *child)
447 struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
448 struct tegra_pcie_bus *bus, *tmp;
450 list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
451 if (bus->nr == child->number) {
452 vunmap(bus->area->addr);
453 list_del(&bus->list);
460 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
464 struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
465 void __iomem *addr = NULL;
467 if (bus->number == 0) {
468 unsigned int slot = PCI_SLOT(devfn);
469 struct tegra_pcie_port *port;
471 list_for_each_entry(port, &pcie->ports, list) {
472 if (port->index + 1 == slot) {
473 addr = port->base + (where & ~3);
478 struct tegra_pcie_bus *b;
480 list_for_each_entry(b, &pcie->buses, list)
481 if (b->nr == bus->number)
482 addr = (void __iomem *)b->area->addr;
486 "failed to map cfg. space for bus %u\n",
491 addr += tegra_pcie_conf_offset(devfn, where);
497 static struct pci_ops tegra_pcie_ops = {
498 .add_bus = tegra_pcie_add_bus,
499 .remove_bus = tegra_pcie_remove_bus,
500 .map_bus = tegra_pcie_map_bus,
501 .read = pci_generic_config_read32,
502 .write = pci_generic_config_write32,
505 static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
507 unsigned long ret = 0;
509 switch (port->index) {
526 static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
528 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
531 /* pulse reset signal */
532 value = afi_readl(port->pcie, ctrl);
533 value &= ~AFI_PEX_CTRL_RST;
534 afi_writel(port->pcie, value, ctrl);
536 usleep_range(1000, 2000);
538 value = afi_readl(port->pcie, ctrl);
539 value |= AFI_PEX_CTRL_RST;
540 afi_writel(port->pcie, value, ctrl);
543 static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
545 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
546 const struct tegra_pcie_soc *soc = port->pcie->soc;
549 /* enable reference clock */
550 value = afi_readl(port->pcie, ctrl);
551 value |= AFI_PEX_CTRL_REFCLK_EN;
553 if (soc->has_pex_clkreq_en)
554 value |= AFI_PEX_CTRL_CLKREQ_EN;
556 value |= AFI_PEX_CTRL_OVERRIDE_EN;
558 afi_writel(port->pcie, value, ctrl);
560 tegra_pcie_port_reset(port);
563 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
565 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
566 const struct tegra_pcie_soc *soc = port->pcie->soc;
569 /* assert port reset */
570 value = afi_readl(port->pcie, ctrl);
571 value &= ~AFI_PEX_CTRL_RST;
572 afi_writel(port->pcie, value, ctrl);
574 /* disable reference clock */
575 value = afi_readl(port->pcie, ctrl);
577 if (soc->has_pex_clkreq_en)
578 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
580 value &= ~AFI_PEX_CTRL_REFCLK_EN;
581 afi_writel(port->pcie, value, ctrl);
584 static void tegra_pcie_port_free(struct tegra_pcie_port *port)
586 struct tegra_pcie *pcie = port->pcie;
588 devm_iounmap(pcie->dev, port->base);
589 devm_release_mem_region(pcie->dev, port->regs.start,
590 resource_size(&port->regs));
591 list_del(&port->list);
592 devm_kfree(pcie->dev, port);
595 /* Tegra PCIE root complex wrongly reports device class */
596 static void tegra_pcie_fixup_class(struct pci_dev *dev)
598 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
600 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
601 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
602 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
603 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
605 /* Tegra PCIE requires relaxed ordering */
606 static void tegra_pcie_relax_enable(struct pci_dev *dev)
608 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
610 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
612 static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
614 struct tegra_pcie *pcie = sys_to_pcie(sys);
617 sys->mem_offset = pcie->offset.mem;
618 sys->io_offset = pcie->offset.io;
620 err = devm_request_resource(pcie->dev, &iomem_resource, &pcie->io);
624 err = pci_remap_iospace(&pcie->pio, pcie->io.start);
626 pci_add_resource_offset(&sys->resources, &pcie->pio,
629 pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
630 pci_add_resource_offset(&sys->resources, &pcie->prefetch,
632 pci_add_resource(&sys->resources, &pcie->busn);
634 err = devm_request_pci_bus_resources(pcie->dev, &sys->resources);
641 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
643 struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
646 tegra_cpuidle_pcie_irqs_in_use();
648 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
655 static irqreturn_t tegra_pcie_isr(int irq, void *arg)
657 const char *err_msg[] = {
665 "Response decoding error",
666 "AXI response decoding error",
667 "Transaction timeout",
668 "Slot present pin change",
669 "Slot clock request change",
670 "TMS clock ramp change",
671 "TMS ready for power down",
674 struct tegra_pcie *pcie = arg;
677 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
678 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
679 afi_writel(pcie, 0, AFI_INTR_CODE);
681 if (code == AFI_INTR_LEGACY)
684 if (code >= ARRAY_SIZE(err_msg))
688 * do not pollute kernel log with master abort reports since they
689 * happen a lot during enumeration
691 if (code == AFI_INTR_MASTER_ABORT)
692 dev_dbg(pcie->dev, "%s, signature: %08x\n", err_msg[code],
695 dev_err(pcie->dev, "%s, signature: %08x\n", err_msg[code],
698 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
699 code == AFI_INTR_FPCI_DECODE_ERROR) {
700 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
701 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
703 if (code == AFI_INTR_MASTER_ABORT)
704 dev_dbg(pcie->dev, " FPCI address: %10llx\n", address);
706 dev_err(pcie->dev, " FPCI address: %10llx\n", address);
713 * FPCI map is as follows:
714 * - 0xfdfc000000: I/O space
715 * - 0xfdfe000000: type 0 configuration space
716 * - 0xfdff000000: type 1 configuration space
717 * - 0xfe00000000: type 0 extended configuration space
718 * - 0xfe10000000: type 1 extended configuration space
720 static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
722 u32 fpci_bar, size, axi_address;
724 /* Bar 0: type 1 extended configuration space */
725 fpci_bar = 0xfe100000;
726 size = resource_size(pcie->cs);
727 axi_address = pcie->cs->start;
728 afi_writel(pcie, axi_address, AFI_AXI_BAR0_START);
729 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
730 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR0);
732 /* Bar 1: downstream IO bar */
733 fpci_bar = 0xfdfc0000;
734 size = resource_size(&pcie->io);
735 axi_address = pcie->io.start;
736 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
737 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
738 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
740 /* Bar 2: prefetchable memory BAR */
741 fpci_bar = (((pcie->prefetch.start >> 12) & 0x0fffffff) << 4) | 0x1;
742 size = resource_size(&pcie->prefetch);
743 axi_address = pcie->prefetch.start;
744 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
745 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
746 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
748 /* Bar 3: non prefetchable memory BAR */
749 fpci_bar = (((pcie->mem.start >> 12) & 0x0fffffff) << 4) | 0x1;
750 size = resource_size(&pcie->mem);
751 axi_address = pcie->mem.start;
752 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
753 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
754 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
756 /* NULL out the remaining BARs as they are not used */
757 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
758 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
759 afi_writel(pcie, 0, AFI_FPCI_BAR4);
761 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
762 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
763 afi_writel(pcie, 0, AFI_FPCI_BAR5);
765 /* map all upstream transactions as uncached */
766 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
767 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
768 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
769 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
771 /* MSI translations are setup only when needed */
772 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
773 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
774 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
775 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
778 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
780 const struct tegra_pcie_soc *soc = pcie->soc;
783 timeout = jiffies + msecs_to_jiffies(timeout);
785 while (time_before(jiffies, timeout)) {
786 value = pads_readl(pcie, soc->pads_pll_ctl);
787 if (value & PADS_PLL_CTL_LOCKDET)
794 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
796 const struct tegra_pcie_soc *soc = pcie->soc;
800 /* initialize internal PHY, enable up to 16 PCIE lanes */
801 pads_writel(pcie, 0x0, PADS_CTL_SEL);
803 /* override IDDQ to 1 on all 4 lanes */
804 value = pads_readl(pcie, PADS_CTL);
805 value |= PADS_CTL_IDDQ_1L;
806 pads_writel(pcie, value, PADS_CTL);
809 * Set up PHY PLL inputs select PLLE output as refclock,
810 * set TX ref sel to div10 (not div5).
812 value = pads_readl(pcie, soc->pads_pll_ctl);
813 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
814 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
815 pads_writel(pcie, value, soc->pads_pll_ctl);
818 value = pads_readl(pcie, soc->pads_pll_ctl);
819 value &= ~PADS_PLL_CTL_RST_B4SM;
820 pads_writel(pcie, value, soc->pads_pll_ctl);
822 usleep_range(20, 100);
824 /* take PLL out of reset */
825 value = pads_readl(pcie, soc->pads_pll_ctl);
826 value |= PADS_PLL_CTL_RST_B4SM;
827 pads_writel(pcie, value, soc->pads_pll_ctl);
829 /* wait for the PLL to lock */
830 err = tegra_pcie_pll_wait(pcie, 500);
832 dev_err(pcie->dev, "PLL failed to lock: %d\n", err);
836 /* turn off IDDQ override */
837 value = pads_readl(pcie, PADS_CTL);
838 value &= ~PADS_CTL_IDDQ_1L;
839 pads_writel(pcie, value, PADS_CTL);
841 /* enable TX/RX data */
842 value = pads_readl(pcie, PADS_CTL);
843 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
844 pads_writel(pcie, value, PADS_CTL);
849 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
851 const struct tegra_pcie_soc *soc = pcie->soc;
854 /* disable TX/RX data */
855 value = pads_readl(pcie, PADS_CTL);
856 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
857 pads_writel(pcie, value, PADS_CTL);
860 value = pads_readl(pcie, PADS_CTL);
861 value |= PADS_CTL_IDDQ_1L;
862 pads_writel(pcie, PADS_CTL, value);
865 value = pads_readl(pcie, soc->pads_pll_ctl);
866 value &= ~PADS_PLL_CTL_RST_B4SM;
867 pads_writel(pcie, value, soc->pads_pll_ctl);
869 usleep_range(20, 100);
874 static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
876 struct device *dev = port->pcie->dev;
880 for (i = 0; i < port->lanes; i++) {
881 err = phy_power_on(port->phys[i]);
883 dev_err(dev, "failed to power on PHY#%u: %d\n", i,
892 static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
894 struct device *dev = port->pcie->dev;
898 for (i = 0; i < port->lanes; i++) {
899 err = phy_power_off(port->phys[i]);
901 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
910 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
912 const struct tegra_pcie_soc *soc = pcie->soc;
913 struct tegra_pcie_port *port;
916 if (pcie->legacy_phy) {
918 err = phy_power_on(pcie->phy);
920 err = tegra_pcie_phy_enable(pcie);
923 dev_err(pcie->dev, "failed to power on PHY: %d\n", err);
928 list_for_each_entry(port, &pcie->ports, list) {
929 err = tegra_pcie_port_phy_power_on(port);
932 "failed to power on PCIe port %u PHY: %d\n",
938 /* Configure the reference clock driver */
939 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
941 if (soc->num_ports > 2)
942 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
947 static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
949 struct tegra_pcie_port *port;
952 if (pcie->legacy_phy) {
954 err = phy_power_off(pcie->phy);
956 err = tegra_pcie_phy_disable(pcie);
959 dev_err(pcie->dev, "failed to power off PHY: %d\n",
965 list_for_each_entry(port, &pcie->ports, list) {
966 err = tegra_pcie_port_phy_power_off(port);
969 "failed to power off PCIe port %u PHY: %d\n",
978 static int tegra_pcie_enable_controller(struct tegra_pcie *pcie)
980 const struct tegra_pcie_soc *soc = pcie->soc;
981 struct tegra_pcie_port *port;
985 /* enable PLL power down */
987 value = afi_readl(pcie, AFI_PLLE_CONTROL);
988 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
989 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
990 afi_writel(pcie, value, AFI_PLLE_CONTROL);
993 /* power down PCIe slot clock bias pad */
994 if (soc->has_pex_bias_ctrl)
995 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
997 /* configure mode and disable all ports */
998 value = afi_readl(pcie, AFI_PCIE_CONFIG);
999 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1000 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1002 list_for_each_entry(port, &pcie->ports, list)
1003 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1005 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1007 if (soc->has_gen2) {
1008 value = afi_readl(pcie, AFI_FUSE);
1009 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1010 afi_writel(pcie, value, AFI_FUSE);
1012 value = afi_readl(pcie, AFI_FUSE);
1013 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1014 afi_writel(pcie, value, AFI_FUSE);
1017 err = tegra_pcie_phy_power_on(pcie);
1019 dev_err(pcie->dev, "failed to power on PHY(s): %d\n", err);
1023 /* take the PCIe interface module out of reset */
1024 reset_control_deassert(pcie->pcie_xrst);
1026 /* finally enable PCIe */
1027 value = afi_readl(pcie, AFI_CONFIGURATION);
1028 value |= AFI_CONFIGURATION_EN_FPCI;
1029 afi_writel(pcie, value, AFI_CONFIGURATION);
1031 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1032 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1033 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1035 if (soc->has_intr_prsnt_sense)
1036 value |= AFI_INTR_EN_PRSNT_SENSE;
1038 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1039 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1041 /* don't enable MSI for now, only when needed */
1042 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1044 /* disable all exceptions */
1045 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1050 static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1054 /* TODO: disable and unprepare clocks? */
1056 err = tegra_pcie_phy_power_off(pcie);
1058 dev_err(pcie->dev, "failed to power off PHY(s): %d\n", err);
1060 reset_control_assert(pcie->pcie_xrst);
1061 reset_control_assert(pcie->afi_rst);
1062 reset_control_assert(pcie->pex_rst);
1064 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1066 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1068 dev_warn(pcie->dev, "failed to disable regulators: %d\n", err);
1071 static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1073 const struct tegra_pcie_soc *soc = pcie->soc;
1076 reset_control_assert(pcie->pcie_xrst);
1077 reset_control_assert(pcie->afi_rst);
1078 reset_control_assert(pcie->pex_rst);
1080 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1082 /* enable regulators */
1083 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1085 dev_err(pcie->dev, "failed to enable regulators: %d\n", err);
1087 err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCIE,
1091 dev_err(pcie->dev, "powerup sequence failed: %d\n", err);
1095 reset_control_deassert(pcie->afi_rst);
1097 err = clk_prepare_enable(pcie->afi_clk);
1099 dev_err(pcie->dev, "failed to enable AFI clock: %d\n", err);
1103 if (soc->has_cml_clk) {
1104 err = clk_prepare_enable(pcie->cml_clk);
1106 dev_err(pcie->dev, "failed to enable CML clock: %d\n",
1112 err = clk_prepare_enable(pcie->pll_e);
1114 dev_err(pcie->dev, "failed to enable PLLE clock: %d\n", err);
1121 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1123 const struct tegra_pcie_soc *soc = pcie->soc;
1125 pcie->pex_clk = devm_clk_get(pcie->dev, "pex");
1126 if (IS_ERR(pcie->pex_clk))
1127 return PTR_ERR(pcie->pex_clk);
1129 pcie->afi_clk = devm_clk_get(pcie->dev, "afi");
1130 if (IS_ERR(pcie->afi_clk))
1131 return PTR_ERR(pcie->afi_clk);
1133 pcie->pll_e = devm_clk_get(pcie->dev, "pll_e");
1134 if (IS_ERR(pcie->pll_e))
1135 return PTR_ERR(pcie->pll_e);
1137 if (soc->has_cml_clk) {
1138 pcie->cml_clk = devm_clk_get(pcie->dev, "cml");
1139 if (IS_ERR(pcie->cml_clk))
1140 return PTR_ERR(pcie->cml_clk);
1146 static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1148 pcie->pex_rst = devm_reset_control_get(pcie->dev, "pex");
1149 if (IS_ERR(pcie->pex_rst))
1150 return PTR_ERR(pcie->pex_rst);
1152 pcie->afi_rst = devm_reset_control_get(pcie->dev, "afi");
1153 if (IS_ERR(pcie->afi_rst))
1154 return PTR_ERR(pcie->afi_rst);
1156 pcie->pcie_xrst = devm_reset_control_get(pcie->dev, "pcie_x");
1157 if (IS_ERR(pcie->pcie_xrst))
1158 return PTR_ERR(pcie->pcie_xrst);
1163 static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1167 pcie->phy = devm_phy_optional_get(pcie->dev, "pcie");
1168 if (IS_ERR(pcie->phy)) {
1169 err = PTR_ERR(pcie->phy);
1170 dev_err(pcie->dev, "failed to get PHY: %d\n", err);
1174 err = phy_init(pcie->phy);
1176 dev_err(pcie->dev, "failed to initialize PHY: %d\n", err);
1180 pcie->legacy_phy = true;
1185 static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1186 struct device_node *np,
1187 const char *consumer,
1193 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1195 return ERR_PTR(-ENOMEM);
1197 phy = devm_of_phy_get(dev, np, name);
1200 if (IS_ERR(phy) && PTR_ERR(phy) == -ENODEV)
1206 static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1208 struct device *dev = port->pcie->dev;
1213 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1217 for (i = 0; i < port->lanes; i++) {
1218 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1220 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1222 return PTR_ERR(phy);
1225 err = phy_init(phy);
1227 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1232 port->phys[i] = phy;
1238 static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1240 const struct tegra_pcie_soc *soc = pcie->soc;
1241 struct device_node *np = pcie->dev->of_node;
1242 struct tegra_pcie_port *port;
1245 if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
1246 return tegra_pcie_phys_get_legacy(pcie);
1248 list_for_each_entry(port, &pcie->ports, list) {
1249 err = tegra_pcie_port_get_phys(port);
1257 static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1259 struct platform_device *pdev = to_platform_device(pcie->dev);
1260 struct resource *pads, *afi, *res;
1263 err = tegra_pcie_clocks_get(pcie);
1265 dev_err(&pdev->dev, "failed to get clocks: %d\n", err);
1269 err = tegra_pcie_resets_get(pcie);
1271 dev_err(&pdev->dev, "failed to get resets: %d\n", err);
1275 err = tegra_pcie_phys_get(pcie);
1277 dev_err(&pdev->dev, "failed to get PHYs: %d\n", err);
1281 err = tegra_pcie_power_on(pcie);
1283 dev_err(&pdev->dev, "failed to power up: %d\n", err);
1287 pads = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pads");
1288 pcie->pads = devm_ioremap_resource(&pdev->dev, pads);
1289 if (IS_ERR(pcie->pads)) {
1290 err = PTR_ERR(pcie->pads);
1294 afi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "afi");
1295 pcie->afi = devm_ioremap_resource(&pdev->dev, afi);
1296 if (IS_ERR(pcie->afi)) {
1297 err = PTR_ERR(pcie->afi);
1301 /* request configuration space, but remap later, on demand */
1302 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1304 err = -EADDRNOTAVAIL;
1308 pcie->cs = devm_request_mem_region(pcie->dev, res->start,
1309 resource_size(res), res->name);
1311 err = -EADDRNOTAVAIL;
1315 /* request interrupt */
1316 err = platform_get_irq_byname(pdev, "intr");
1318 dev_err(&pdev->dev, "failed to get IRQ: %d\n", err);
1324 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1326 dev_err(&pdev->dev, "failed to register IRQ: %d\n", err);
1333 tegra_pcie_power_off(pcie);
1337 static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1342 free_irq(pcie->irq, pcie);
1344 tegra_pcie_power_off(pcie);
1346 err = phy_exit(pcie->phy);
1348 dev_err(pcie->dev, "failed to teardown PHY: %d\n", err);
1353 static int tegra_msi_alloc(struct tegra_msi *chip)
1357 mutex_lock(&chip->lock);
1359 msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
1360 if (msi < INT_PCI_MSI_NR)
1361 set_bit(msi, chip->used);
1365 mutex_unlock(&chip->lock);
1370 static void tegra_msi_free(struct tegra_msi *chip, unsigned long irq)
1372 struct device *dev = chip->chip.dev;
1374 mutex_lock(&chip->lock);
1376 if (!test_bit(irq, chip->used))
1377 dev_err(dev, "trying to free unused MSI#%lu\n", irq);
1379 clear_bit(irq, chip->used);
1381 mutex_unlock(&chip->lock);
1384 static irqreturn_t tegra_pcie_msi_irq(int irq, void *data)
1386 struct tegra_pcie *pcie = data;
1387 struct tegra_msi *msi = &pcie->msi;
1388 unsigned int i, processed = 0;
1390 for (i = 0; i < 8; i++) {
1391 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1394 unsigned int offset = find_first_bit(®, 32);
1395 unsigned int index = i * 32 + offset;
1398 /* clear the interrupt */
1399 afi_writel(pcie, 1 << offset, AFI_MSI_VEC0 + i * 4);
1401 irq = irq_find_mapping(msi->domain, index);
1403 if (test_bit(index, msi->used))
1404 generic_handle_irq(irq);
1406 dev_info(pcie->dev, "unhandled MSI\n");
1409 * that's weird who triggered this?
1412 dev_info(pcie->dev, "unexpected MSI\n");
1415 /* see if there's any more pending in this vector */
1416 reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1422 return processed > 0 ? IRQ_HANDLED : IRQ_NONE;
1425 static int tegra_msi_setup_irq(struct msi_controller *chip,
1426 struct pci_dev *pdev, struct msi_desc *desc)
1428 struct tegra_msi *msi = to_tegra_msi(chip);
1433 hwirq = tegra_msi_alloc(msi);
1437 irq = irq_create_mapping(msi->domain, hwirq);
1439 tegra_msi_free(msi, hwirq);
1443 irq_set_msi_desc(irq, desc);
1445 msg.address_lo = virt_to_phys((void *)msi->pages);
1446 /* 32 bit address only */
1450 pci_write_msi_msg(irq, &msg);
1455 static void tegra_msi_teardown_irq(struct msi_controller *chip,
1458 struct tegra_msi *msi = to_tegra_msi(chip);
1459 struct irq_data *d = irq_get_irq_data(irq);
1460 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1462 irq_dispose_mapping(irq);
1463 tegra_msi_free(msi, hwirq);
1466 static struct irq_chip tegra_msi_irq_chip = {
1467 .name = "Tegra PCIe MSI",
1468 .irq_enable = pci_msi_unmask_irq,
1469 .irq_disable = pci_msi_mask_irq,
1470 .irq_mask = pci_msi_mask_irq,
1471 .irq_unmask = pci_msi_unmask_irq,
1474 static int tegra_msi_map(struct irq_domain *domain, unsigned int irq,
1475 irq_hw_number_t hwirq)
1477 irq_set_chip_and_handler(irq, &tegra_msi_irq_chip, handle_simple_irq);
1478 irq_set_chip_data(irq, domain->host_data);
1480 tegra_cpuidle_pcie_irqs_in_use();
1485 static const struct irq_domain_ops msi_domain_ops = {
1486 .map = tegra_msi_map,
1489 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1491 struct platform_device *pdev = to_platform_device(pcie->dev);
1492 const struct tegra_pcie_soc *soc = pcie->soc;
1493 struct tegra_msi *msi = &pcie->msi;
1498 mutex_init(&msi->lock);
1500 msi->chip.dev = pcie->dev;
1501 msi->chip.setup_irq = tegra_msi_setup_irq;
1502 msi->chip.teardown_irq = tegra_msi_teardown_irq;
1504 msi->domain = irq_domain_add_linear(pcie->dev->of_node, INT_PCI_MSI_NR,
1505 &msi_domain_ops, &msi->chip);
1507 dev_err(&pdev->dev, "failed to create IRQ domain\n");
1511 err = platform_get_irq_byname(pdev, "msi");
1513 dev_err(&pdev->dev, "failed to get IRQ: %d\n", err);
1519 err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD,
1520 tegra_msi_irq_chip.name, pcie);
1522 dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
1526 /* setup AFI/FPCI range */
1527 msi->pages = __get_free_pages(GFP_KERNEL, 0);
1528 base = virt_to_phys((void *)msi->pages);
1530 afi_writel(pcie, base >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1531 afi_writel(pcie, base, AFI_MSI_AXI_BAR_ST);
1532 /* this register is in 4K increments */
1533 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1535 /* enable all MSI vectors */
1536 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC0);
1537 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC1);
1538 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC2);
1539 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC3);
1540 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC4);
1541 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC5);
1542 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC6);
1543 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC7);
1545 /* and unmask the MSI interrupt */
1546 reg = afi_readl(pcie, AFI_INTR_MASK);
1547 reg |= AFI_INTR_MASK_MSI_MASK;
1548 afi_writel(pcie, reg, AFI_INTR_MASK);
1553 irq_domain_remove(msi->domain);
1557 static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1559 struct tegra_msi *msi = &pcie->msi;
1560 unsigned int i, irq;
1563 /* mask the MSI interrupt */
1564 value = afi_readl(pcie, AFI_INTR_MASK);
1565 value &= ~AFI_INTR_MASK_MSI_MASK;
1566 afi_writel(pcie, value, AFI_INTR_MASK);
1568 /* disable all MSI vectors */
1569 afi_writel(pcie, 0, AFI_MSI_EN_VEC0);
1570 afi_writel(pcie, 0, AFI_MSI_EN_VEC1);
1571 afi_writel(pcie, 0, AFI_MSI_EN_VEC2);
1572 afi_writel(pcie, 0, AFI_MSI_EN_VEC3);
1573 afi_writel(pcie, 0, AFI_MSI_EN_VEC4);
1574 afi_writel(pcie, 0, AFI_MSI_EN_VEC5);
1575 afi_writel(pcie, 0, AFI_MSI_EN_VEC6);
1576 afi_writel(pcie, 0, AFI_MSI_EN_VEC7);
1578 free_pages(msi->pages, 0);
1581 free_irq(msi->irq, pcie);
1583 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1584 irq = irq_find_mapping(msi->domain, i);
1586 irq_dispose_mapping(irq);
1589 irq_domain_remove(msi->domain);
1594 static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1597 struct device_node *np = pcie->dev->of_node;
1599 if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
1602 dev_info(pcie->dev, "4x1, 1x1 configuration\n");
1603 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1607 dev_info(pcie->dev, "2x1, 1x1 configuration\n");
1608 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1611 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1614 dev_info(pcie->dev, "4x1, 2x1 configuration\n");
1615 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1619 dev_info(pcie->dev, "2x3 configuration\n");
1620 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1624 dev_info(pcie->dev, "4x1, 1x2 configuration\n");
1625 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1628 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1631 dev_info(pcie->dev, "single-mode configuration\n");
1632 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1636 dev_info(pcie->dev, "dual-mode configuration\n");
1637 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1646 * Check whether a given set of supplies is available in a device tree node.
1647 * This is used to check whether the new or the legacy device tree bindings
1650 static bool of_regulator_bulk_available(struct device_node *np,
1651 struct regulator_bulk_data *supplies,
1652 unsigned int num_supplies)
1657 for (i = 0; i < num_supplies; i++) {
1658 snprintf(property, 32, "%s-supply", supplies[i].supply);
1660 if (of_find_property(np, property, NULL) == NULL)
1668 * Old versions of the device tree binding for this device used a set of power
1669 * supplies that didn't match the hardware inputs. This happened to work for a
1670 * number of cases but is not future proof. However to preserve backwards-
1671 * compatibility with old device trees, this function will try to use the old
1674 static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1676 struct device_node *np = pcie->dev->of_node;
1678 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1679 pcie->num_supplies = 3;
1680 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1681 pcie->num_supplies = 2;
1683 if (pcie->num_supplies == 0) {
1684 dev_err(pcie->dev, "device %s not supported in legacy mode\n",
1689 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1690 sizeof(*pcie->supplies),
1692 if (!pcie->supplies)
1695 pcie->supplies[0].supply = "pex-clk";
1696 pcie->supplies[1].supply = "vdd";
1698 if (pcie->num_supplies > 2)
1699 pcie->supplies[2].supply = "avdd";
1701 return devm_regulator_bulk_get(pcie->dev, pcie->num_supplies,
1706 * Obtains the list of regulators required for a particular generation of the
1709 * This would've been nice to do simply by providing static tables for use
1710 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
1711 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
1712 * and either seems to be optional depending on which ports are being used.
1714 static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
1716 struct device_node *np = pcie->dev->of_node;
1719 if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
1720 pcie->num_supplies = 7;
1722 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1723 sizeof(*pcie->supplies),
1725 if (!pcie->supplies)
1728 pcie->supplies[i++].supply = "avddio-pex";
1729 pcie->supplies[i++].supply = "dvddio-pex";
1730 pcie->supplies[i++].supply = "avdd-pex-pll";
1731 pcie->supplies[i++].supply = "hvdd-pex";
1732 pcie->supplies[i++].supply = "hvdd-pex-pll-e";
1733 pcie->supplies[i++].supply = "vddio-pex-ctl";
1734 pcie->supplies[i++].supply = "avdd-pll-erefe";
1735 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1736 bool need_pexa = false, need_pexb = false;
1738 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
1739 if (lane_mask & 0x0f)
1742 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
1743 if (lane_mask & 0x30)
1746 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
1747 (need_pexb ? 2 : 0);
1749 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1750 sizeof(*pcie->supplies),
1752 if (!pcie->supplies)
1755 pcie->supplies[i++].supply = "avdd-pex-pll";
1756 pcie->supplies[i++].supply = "hvdd-pex";
1757 pcie->supplies[i++].supply = "vddio-pex-ctl";
1758 pcie->supplies[i++].supply = "avdd-plle";
1761 pcie->supplies[i++].supply = "avdd-pexa";
1762 pcie->supplies[i++].supply = "vdd-pexa";
1766 pcie->supplies[i++].supply = "avdd-pexb";
1767 pcie->supplies[i++].supply = "vdd-pexb";
1769 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1770 pcie->num_supplies = 5;
1772 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1773 sizeof(*pcie->supplies),
1775 if (!pcie->supplies)
1778 pcie->supplies[0].supply = "avdd-pex";
1779 pcie->supplies[1].supply = "vdd-pex";
1780 pcie->supplies[2].supply = "avdd-pex-pll";
1781 pcie->supplies[3].supply = "avdd-plle";
1782 pcie->supplies[4].supply = "vddio-pex-clk";
1785 if (of_regulator_bulk_available(pcie->dev->of_node, pcie->supplies,
1786 pcie->num_supplies))
1787 return devm_regulator_bulk_get(pcie->dev, pcie->num_supplies,
1791 * If not all regulators are available for this new scheme, assume
1792 * that the device tree complies with an older version of the device
1795 dev_info(pcie->dev, "using legacy DT binding for power supplies\n");
1797 devm_kfree(pcie->dev, pcie->supplies);
1798 pcie->num_supplies = 0;
1800 return tegra_pcie_get_legacy_regulators(pcie);
1803 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
1805 struct device_node *np = pcie->dev->of_node, *port;
1806 const struct tegra_pcie_soc *soc = pcie->soc;
1807 struct of_pci_range_parser parser;
1808 struct of_pci_range range;
1809 u32 lanes = 0, mask = 0;
1810 unsigned int lane = 0;
1811 struct resource res;
1814 if (of_pci_range_parser_init(&parser, np)) {
1815 dev_err(pcie->dev, "missing \"ranges\" property\n");
1819 for_each_of_pci_range(&parser, &range) {
1820 err = of_pci_range_to_resource(&range, np, &res);
1824 switch (res.flags & IORESOURCE_TYPE_BITS) {
1826 /* Track the bus -> CPU I/O mapping offset. */
1827 pcie->offset.io = res.start - range.pci_addr;
1829 memcpy(&pcie->pio, &res, sizeof(res));
1830 pcie->pio.name = np->full_name;
1833 * The Tegra PCIe host bridge uses this to program the
1834 * mapping of the I/O space to the physical address,
1835 * so we override the .start and .end fields here that
1836 * of_pci_range_to_resource() converted to I/O space.
1837 * We also set the IORESOURCE_MEM type to clarify that
1838 * the resource is in the physical memory space.
1840 pcie->io.start = range.cpu_addr;
1841 pcie->io.end = range.cpu_addr + range.size - 1;
1842 pcie->io.flags = IORESOURCE_MEM;
1843 pcie->io.name = "I/O";
1845 memcpy(&res, &pcie->io, sizeof(res));
1848 case IORESOURCE_MEM:
1850 * Track the bus -> CPU memory mapping offset. This
1851 * assumes that the prefetchable and non-prefetchable
1852 * regions will be the last of type IORESOURCE_MEM in
1853 * the ranges property.
1855 pcie->offset.mem = res.start - range.pci_addr;
1857 if (res.flags & IORESOURCE_PREFETCH) {
1858 memcpy(&pcie->prefetch, &res, sizeof(res));
1859 pcie->prefetch.name = "prefetchable";
1861 memcpy(&pcie->mem, &res, sizeof(res));
1862 pcie->mem.name = "non-prefetchable";
1868 err = of_pci_parse_bus_range(np, &pcie->busn);
1870 dev_err(pcie->dev, "failed to parse ranges property: %d\n",
1872 pcie->busn.name = np->name;
1873 pcie->busn.start = 0;
1874 pcie->busn.end = 0xff;
1875 pcie->busn.flags = IORESOURCE_BUS;
1878 /* parse root ports */
1879 for_each_child_of_node(np, port) {
1880 struct tegra_pcie_port *rp;
1884 err = of_pci_get_devfn(port);
1886 dev_err(pcie->dev, "failed to parse address: %d\n",
1891 index = PCI_SLOT(err);
1893 if (index < 1 || index > soc->num_ports) {
1894 dev_err(pcie->dev, "invalid port number: %d\n", index);
1900 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
1902 dev_err(pcie->dev, "failed to parse # of lanes: %d\n",
1908 dev_err(pcie->dev, "invalid # of lanes: %u\n", value);
1912 lanes |= value << (index << 3);
1914 if (!of_device_is_available(port)) {
1919 mask |= ((1 << value) - 1) << lane;
1922 rp = devm_kzalloc(pcie->dev, sizeof(*rp), GFP_KERNEL);
1926 err = of_address_to_resource(port, 0, &rp->regs);
1928 dev_err(pcie->dev, "failed to parse address: %d\n",
1933 INIT_LIST_HEAD(&rp->list);
1939 rp->base = devm_ioremap_resource(pcie->dev, &rp->regs);
1940 if (IS_ERR(rp->base))
1941 return PTR_ERR(rp->base);
1943 list_add_tail(&rp->list, &pcie->ports);
1946 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
1948 dev_err(pcie->dev, "invalid lane configuration\n");
1952 err = tegra_pcie_get_regulators(pcie, mask);
1960 * FIXME: If there are no PCIe cards attached, then calling this function
1961 * can result in the increase of the bootup time as there are big timeout
1964 #define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
1965 static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
1967 unsigned int retries = 3;
1968 unsigned long value;
1970 /* override presence detection */
1971 value = readl(port->base + RP_PRIV_MISC);
1972 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
1973 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
1974 writel(value, port->base + RP_PRIV_MISC);
1977 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
1980 value = readl(port->base + RP_VEND_XP);
1982 if (value & RP_VEND_XP_DL_UP)
1985 usleep_range(1000, 2000);
1986 } while (--timeout);
1989 dev_err(port->pcie->dev, "link %u down, retrying\n",
1994 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
1997 value = readl(port->base + RP_LINK_CONTROL_STATUS);
1999 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2002 usleep_range(1000, 2000);
2003 } while (--timeout);
2006 tegra_pcie_port_reset(port);
2007 } while (--retries);
2012 static int tegra_pcie_enable(struct tegra_pcie *pcie)
2014 struct tegra_pcie_port *port, *tmp;
2017 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2018 dev_info(pcie->dev, "probing port %u, using %u lanes\n",
2019 port->index, port->lanes);
2021 tegra_pcie_port_enable(port);
2023 if (tegra_pcie_port_check_link(port))
2026 dev_info(pcie->dev, "link %u down, ignoring\n", port->index);
2028 tegra_pcie_port_disable(port);
2029 tegra_pcie_port_free(port);
2032 memset(&hw, 0, sizeof(hw));
2034 #ifdef CONFIG_PCI_MSI
2035 hw.msi_ctrl = &pcie->msi.chip;
2038 hw.nr_controllers = 1;
2039 hw.private_data = (void **)&pcie;
2040 hw.setup = tegra_pcie_setup;
2041 hw.map_irq = tegra_pcie_map_irq;
2042 hw.ops = &tegra_pcie_ops;
2044 pci_common_init_dev(pcie->dev, &hw);
2049 static const struct tegra_pcie_soc tegra20_pcie = {
2051 .msi_base_shift = 0,
2052 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2053 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2054 .pads_refclk_cfg0 = 0xfa5cfa5c,
2055 .has_pex_clkreq_en = false,
2056 .has_pex_bias_ctrl = false,
2057 .has_intr_prsnt_sense = false,
2058 .has_cml_clk = false,
2062 static const struct tegra_pcie_soc tegra30_pcie = {
2064 .msi_base_shift = 8,
2065 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2066 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2067 .pads_refclk_cfg0 = 0xfa5cfa5c,
2068 .pads_refclk_cfg1 = 0xfa5cfa5c,
2069 .has_pex_clkreq_en = true,
2070 .has_pex_bias_ctrl = true,
2071 .has_intr_prsnt_sense = true,
2072 .has_cml_clk = true,
2076 static const struct tegra_pcie_soc tegra124_pcie = {
2078 .msi_base_shift = 8,
2079 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2080 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2081 .pads_refclk_cfg0 = 0x44ac44ac,
2082 .has_pex_clkreq_en = true,
2083 .has_pex_bias_ctrl = true,
2084 .has_intr_prsnt_sense = true,
2085 .has_cml_clk = true,
2089 static const struct of_device_id tegra_pcie_of_match[] = {
2090 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
2091 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
2092 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
2096 static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2098 struct tegra_pcie *pcie = s->private;
2100 if (list_empty(&pcie->ports))
2103 seq_printf(s, "Index Status\n");
2105 return seq_list_start(&pcie->ports, *pos);
2108 static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2110 struct tegra_pcie *pcie = s->private;
2112 return seq_list_next(v, &pcie->ports, pos);
2115 static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2119 static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2121 bool up = false, active = false;
2122 struct tegra_pcie_port *port;
2125 port = list_entry(v, struct tegra_pcie_port, list);
2127 value = readl(port->base + RP_VEND_XP);
2129 if (value & RP_VEND_XP_DL_UP)
2132 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2134 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2137 seq_printf(s, "%2u ", port->index);
2140 seq_printf(s, "up");
2144 seq_printf(s, ", ");
2146 seq_printf(s, "active");
2149 seq_printf(s, "\n");
2153 static const struct seq_operations tegra_pcie_ports_seq_ops = {
2154 .start = tegra_pcie_ports_seq_start,
2155 .next = tegra_pcie_ports_seq_next,
2156 .stop = tegra_pcie_ports_seq_stop,
2157 .show = tegra_pcie_ports_seq_show,
2160 static int tegra_pcie_ports_open(struct inode *inode, struct file *file)
2162 struct tegra_pcie *pcie = inode->i_private;
2166 err = seq_open(file, &tegra_pcie_ports_seq_ops);
2170 s = file->private_data;
2176 static const struct file_operations tegra_pcie_ports_ops = {
2177 .owner = THIS_MODULE,
2178 .open = tegra_pcie_ports_open,
2180 .llseek = seq_lseek,
2181 .release = seq_release,
2184 static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2186 struct dentry *file;
2188 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2192 file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs,
2193 pcie, &tegra_pcie_ports_ops);
2200 debugfs_remove_recursive(pcie->debugfs);
2201 pcie->debugfs = NULL;
2205 static int tegra_pcie_probe(struct platform_device *pdev)
2207 struct tegra_pcie *pcie;
2210 pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
2214 pcie->soc = of_device_get_match_data(&pdev->dev);
2215 INIT_LIST_HEAD(&pcie->buses);
2216 INIT_LIST_HEAD(&pcie->ports);
2217 pcie->dev = &pdev->dev;
2219 err = tegra_pcie_parse_dt(pcie);
2223 err = tegra_pcie_get_resources(pcie);
2225 dev_err(&pdev->dev, "failed to request resources: %d\n", err);
2229 err = tegra_pcie_enable_controller(pcie);
2233 /* setup the AFI address translations */
2234 tegra_pcie_setup_translations(pcie);
2236 if (IS_ENABLED(CONFIG_PCI_MSI)) {
2237 err = tegra_pcie_enable_msi(pcie);
2240 "failed to enable MSI support: %d\n",
2246 err = tegra_pcie_enable(pcie);
2248 dev_err(&pdev->dev, "failed to enable PCIe ports: %d\n", err);
2252 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2253 err = tegra_pcie_debugfs_init(pcie);
2255 dev_err(&pdev->dev, "failed to setup debugfs: %d\n",
2259 platform_set_drvdata(pdev, pcie);
2263 if (IS_ENABLED(CONFIG_PCI_MSI))
2264 tegra_pcie_disable_msi(pcie);
2266 tegra_pcie_put_resources(pcie);
2270 static struct platform_driver tegra_pcie_driver = {
2272 .name = "tegra-pcie",
2273 .of_match_table = tegra_pcie_of_match,
2274 .suppress_bind_attrs = true,
2276 .probe = tegra_pcie_probe,
2278 builtin_platform_driver(tegra_pcie_driver);