rk_pcie.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2019 Michal Meloun <mmel@FreeBSD.org>
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   25  * SUCH DAMAGE.
   26  *
   27  */
   28 
   29 /* Rockchip PCIe controller driver */
   30 
   31 #include <sys/cdefs.h>
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include <sys/param.h>
   35 #include <sys/systm.h>
   36 #include <sys/bus.h>
   37 #include <sys/gpio.h>
   38 #include <sys/proc.h>
   39 #include <sys/kernel.h>
   40 #include <sys/lock.h>
   41 #include <sys/malloc.h>
   42 #include <sys/module.h>
   43 #include <sys/mutex.h>
   44 #include <sys/rman.h>
   45 
   46 #include <machine/bus.h>
   47 #include <machine/intr.h>
   48 #include <machine/resource.h>
   49 
   50 #include <dev/extres/clk/clk.h>
   51 #include <dev/extres/hwreset/hwreset.h>
   52 #include <dev/extres/phy/phy.h>
   53 #include <dev/extres/regulator/regulator.h>
   54 #include <dev/gpio/gpiobusvar.h>
   55 #include <dev/ofw/ofw_bus.h>
   56 #include <dev/ofw/ofw_bus_subr.h>
   57 #include <dev/ofw/ofw_pci.h>
   58 #include <dev/ofw/ofwpci.h>
   59 #include <dev/pci/pcivar.h>
   60 #include <dev/pci/pcireg.h>
   61 #include <dev/pci/pcib_private.h>
   62 
   63 #include <dev/ofw/ofw_bus.h>
   64 
   65 #include "pcib_if.h"
   66 
   67 #define ATU_CFG_BUS(x)          (((x) & 0x0ff) << 20)
   68 #define ATU_CFG_SLOT(x)         (((x) & 0x01f) << 15)
   69 #define ATU_CFG_FUNC(x)         (((x) & 0x007) << 12)
   70 #define ATU_CFG_REG(x)          (((x) & 0xfff) << 0)
   71 
   72 #define ATU_TYPE_MEM            0x2
   73 #define ATU_TYPE_IO             0x6
   74 #define ATU_TYPE_CFG0           0xA
   75 #define ATU_TYPE_CFG1           0xB
   76 #define ATY_TYPE_NOR_MSG        0xC
   77 
   78 #define ATU_OB_REGIONS          33
   79 #define ATU_OB_REGION_SHIFT     20
   80 #define ATU_OB_REGION_SIZE      (1 << ATU_OB_REGION_SHIFT)
   81 #define ATU_OB_REGION_0_SIZE    (( ATU_OB_REGIONS - 1) * ATU_OB_REGION_SIZE)
   82 
   83 #define ATU_IB_REGIONS          3
   84 
   85 #define PCIE_CLIENT_BASIC_STRAP_CONF            0x000000
   86 #define  STRAP_CONF_GEN_2                               (1 << 7)
   87 #define  STRAP_CONF_MODE_RC                             (1 << 6)
   88 #define  STRAP_CONF_LANES(n)                            ((((n) / 2) & 0x3) << 4)
   89 #define  STRAP_CONF_ARI_EN                              (1 << 3)
   90 #define  STRAP_CONF_SR_IOV_EN                           (1 << 2)
   91 #define  STRAP_CONF_LINK_TRAIN_EN                       (1 << 1)
   92 #define  STRAP_CONF_CONF_EN                             (1 << 0)
   93 #define PCIE_CLIENT_HOT_RESET_CTRL              0x000018
   94 #define  HOT_RESET_CTRL_LINK_DOWN_RESET                 (1 << 1)
   95 #define  HOT_RESET_CTRL_HOT_RESET_IN                    (1 << 0)
   96 #define PCIE_CLIENT_BASIC_STATUS0               0x000044
   97 #define PCIE_CLIENT_BASIC_STATUS1               0x000048
   98 #define  STATUS1_LINK_ST_GET(x)                         (((x) >> 20) & 0x3)
   99 #define   STATUS1_LINK_ST_UP                            3
  100 #define PCIE_CLIENT_INT_MASK                    0x00004C
  101 #define PCIE_CLIENT_INT_STATUS                  0x000050
  102 #define  PCIE_CLIENT_INT_LEGACY_DONE                    (1 << 15)
  103 #define  PCIE_CLIENT_INT_MSG                            (1 << 14)
  104 #define  PCIE_CLIENT_INT_HOT_RST                        (1 << 13)
  105 #define  PCIE_CLIENT_INT_DPA                            (1 << 12)
  106 #define  PCIE_CLIENT_INT_FATAL_ERR                      (1 << 11)
  107 #define  PCIE_CLIENT_INT_NFATAL_ERR                     (1 << 10)
  108 #define  PCIE_CLIENT_INT_CORR_ERR                       (1 << 9)
  109 #define  PCIE_CLIENT_INT_INTD                           (1 << 8)
  110 #define  PCIE_CLIENT_INT_INTC                           (1 << 7)
  111 #define  PCIE_CLIENT_INT_INTB                           (1 << 6)
  112 #define  PCIE_CLIENT_INT_INTA                           (1 << 5)
  113 #define  PCIE_CLIENT_INT_LOCAL                          (1 << 4)
  114 #define  PCIE_CLIENT_INT_UDMA                           (1 << 3)
  115 #define  PCIE_CLIENT_INT_PHY                            (1 << 2)
  116 #define  PCIE_CLIENT_INT_HOT_PLUG                       (1 << 1)
  117 #define  PCIE_CLIENT_INT_PWR_STCG                       (1 << 0)
  118 #define  PCIE_CLIENT_INT_LEGACY                 (PCIE_CLIENT_INT_INTA | \
  119                                                 PCIE_CLIENT_INT_INTB | \
  120                                                 PCIE_CLIENT_INT_INTC | \
  121                                                 PCIE_CLIENT_INT_INTD)
  122 
  123 #define PCIE_CORE_CTRL0                         0x900000
  124 #define  CORE_CTRL_LANES_GET(x)                         (((x) >> 20) & 0x3)
  125 #define PCIE_CORE_CTRL1                         0x900004
  126 #define PCIE_CORE_CONFIG_VENDOR                 0x900044
  127 #define PCIE_CORE_INT_STATUS                    0x90020c
  128 #define  PCIE_CORE_INT_PRFPE                            (1 << 0)
  129 #define  PCIE_CORE_INT_CRFPE                            (1 << 1)
  130 #define  PCIE_CORE_INT_RRPE                             (1 << 2)
  131 #define  PCIE_CORE_INT_PRFO                             (1 << 3)
  132 #define  PCIE_CORE_INT_CRFO                             (1 << 4)
  133 #define  PCIE_CORE_INT_RT                               (1 << 5)
  134 #define  PCIE_CORE_INT_RTR                              (1 << 6)
  135 #define  PCIE_CORE_INT_PE                               (1 << 7)
  136 #define  PCIE_CORE_INT_MTR                              (1 << 8)
  137 #define  PCIE_CORE_INT_UCR                              (1 << 9)
  138 #define  PCIE_CORE_INT_FCE                              (1 << 10)
  139 #define  PCIE_CORE_INT_CT                               (1 << 11)
  140 #define  PCIE_CORE_INT_UTC                              (1 << 18)
  141 #define  PCIE_CORE_INT_MMVC                             (1 << 19)
  142 #define PCIE_CORE_INT_MASK                      0x900210
  143 #define PCIE_CORE_PHY_FUNC_CONF                 0x9002C0
  144 #define PCIE_CORE_RC_BAR_CONF                   0x900300
  145 
  146 #define PCIE_RC_CONFIG_STD_BASE                 0x800000
  147 #define PCIE_RC_CONFIG_PRIV_BASE                0xA00000
  148 #define PCIE_RC_CONFIG_DCSR                     0xA000C8
  149 #define  PCIE_RC_CONFIG_DCSR_MPS_MASK                   (0x7 << 5)
  150 #define  PCIE_RC_CONFIG_DCSR_MPS_128                    (0 << 5)
  151 #define  PCIE_RC_CONFIG_DCSR_MPS_256                    (1 << 5)
  152 #define  PCIE_RC_CONFIG_LINK_CAP                0xA00CC
  153 #define   PCIE_RC_CONFIG_LINK_CAP_L0S                   (1 << 10)
  154 
  155 #define PCIE_RC_CONFIG_LCS                      0xA000D0
  156 #define PCIE_RC_CONFIG_THP_CAP                  0xA00274
  157 #define  PCIE_RC_CONFIG_THP_CAP_NEXT_MASK               0xFFF00000
  158 
  159 #define PCIE_CORE_OB_ADDR0(n)                   (0xC00000 + 0x20 * (n) + 0x00)
  160 #define PCIE_CORE_OB_ADDR1(n)                   (0xC00000 + 0x20 * (n) + 0x04)
  161 #define PCIE_CORE_OB_DESC0(n)                   (0xC00000 + 0x20 * (n) + 0x08)
  162 #define PCIE_CORE_OB_DESC1(n)                   (0xC00000 + 0x20 * (n) + 0x0C)
  163 #define PCIE_CORE_OB_DESC2(n)                   (0xC00000 + 0x20 * (n) + 0x10)
  164 #define PCIE_CORE_OB_DESC3(n)                   (0xC00000 + 0x20 * (n) + 0x14)
  165 
  166 #define PCIE_CORE_IB_ADDR0(n)                   (0xC00800 + 0x8 * (n) + 0x00)
  167 #define PCIE_CORE_IB_ADDR1(n)                   (0xC00800 + 0x8 * (n) + 0x04)
  168 
  169 #define PRIV_CFG_RD4(sc, reg)                                           \
  170     (uint32_t)rk_pcie_local_cfg_read(sc, true, reg, 4)
  171 #define PRIV_CFG_RD2(sc, reg)                                           \
  172     (uint16_t)rk_pcie_local_cfg_read(sc, true, reg, 2)
  173 #define PRIV_CFG_RD1(sc, reg)                                           \
  174     (uint8_t)rk_pcie_local_cfg_read(sc, true, reg, 1)
  175 #define PRIV_CFG_WR4(sc, reg, val)                                      \
  176     rk_pcie_local_cfg_write(sc, true, reg, val, 4)
  177 #define PRIV_CFG_WR2(sc, reg, val)                                      \
  178     rk_pcie_local_cfg_write(sc, true, reg, val, 2)
  179 #define PRIV_CFG_WR1(sc, reg, val)                                      \
  180     rk_pcie_local_cfg_write(sc, true, reg, val, 1)
  181 
  182 #define APB_WR4(_sc, _r, _v)    bus_write_4((_sc)->apb_mem_res, (_r), (_v))
  183 #define APB_RD4(_sc, _r)        bus_read_4((_sc)->apb_mem_res, (_r))
  184 
  185 #define MAX_LANES       4
  186 
  187 #define RK_PCIE_ENABLE_MSI
  188 #define RK_PCIE_ENABLE_MSIX
  189 
  190 struct rk_pcie_softc {
  191         struct ofw_pci_softc    ofw_pci;        /* Must be first */
  192 
  193         struct resource         *axi_mem_res;
  194         struct resource         *apb_mem_res;
  195         struct resource         *client_irq_res;
  196         struct resource         *legacy_irq_res;
  197         struct resource         *sys_irq_res;
  198         void                    *client_irq_cookie;
  199         void                    *legacy_irq_cookie;
  200         void                    *sys_irq_cookie;
  201 
  202         device_t                dev;
  203         phandle_t               node;
  204         struct mtx              mtx;
  205 
  206         struct ofw_pci_range    mem_range;
  207         struct ofw_pci_range    pref_mem_range;
  208         struct ofw_pci_range    io_range;
  209 
  210         bool                    coherent;
  211         bus_dma_tag_t           dmat;
  212 
  213         int                     num_lanes;
  214         bool                    link_is_gen2;
  215         bool                    no_l0s;
  216 
  217         u_int                   bus_start;
  218         u_int                   bus_end;
  219         u_int                   root_bus;
  220         u_int                   sub_bus;
  221 
  222         regulator_t             supply_12v;
  223         regulator_t             supply_3v3;
  224         regulator_t             supply_1v8;
  225         regulator_t             supply_0v9;
  226         hwreset_t               hwreset_core;
  227         hwreset_t               hwreset_mgmt;
  228         hwreset_t               hwreset_mgmt_sticky;
  229         hwreset_t               hwreset_pipe;
  230         hwreset_t               hwreset_pm;
  231         hwreset_t               hwreset_aclk;
  232         hwreset_t               hwreset_pclk;
  233         clk_t                   clk_aclk;
  234         clk_t                   clk_aclk_perf;
  235         clk_t                   clk_hclk;
  236         clk_t                   clk_pm;
  237         phy_t                   phys[MAX_LANES];
  238         gpio_pin_t              gpio_ep;
  239 };
  240 
  241 /* Compatible devices. */
  242 static struct ofw_compat_data compat_data[] = {
  243         {"rockchip,rk3399-pcie", 1},
  244         {NULL,                   0},
  245 };
  246 
  247 static uint32_t
  248 rk_pcie_local_cfg_read(struct rk_pcie_softc *sc, bool priv, u_int reg,
  249     int bytes)
  250 {
  251         uint32_t val;
  252         bus_addr_t base;
  253 
  254         if (priv)
  255                 base = PCIE_RC_CONFIG_PRIV_BASE;
  256         else
  257                 base = PCIE_RC_CONFIG_STD_BASE;
  258 
  259         switch (bytes) {
  260         case 4:
  261                 val = bus_read_4(sc->apb_mem_res, base + reg);
  262                 break;
  263         case 2:
  264                 val = bus_read_2(sc->apb_mem_res, base + reg);
  265                 break;
  266         case 1:
  267                 val = bus_read_1(sc->apb_mem_res, base + reg);
  268                 break;
  269         default:
  270                 val = 0xFFFFFFFF;
  271         }
  272         return (val);
  273 }
  274 
  275 static void
  276 rk_pcie_local_cfg_write(struct rk_pcie_softc *sc, bool priv, u_int reg,
  277     uint32_t val, int bytes)
  278 {
  279         uint32_t val2;
  280         bus_addr_t base;
  281 
  282         if (priv)
  283                 base = PCIE_RC_CONFIG_PRIV_BASE;
  284         else
  285                 base = PCIE_RC_CONFIG_STD_BASE;
  286 
  287         switch (bytes) {
  288         case 4:
  289                 bus_write_4(sc->apb_mem_res, base + reg, val);
  290                 break;
  291         case 2:
  292                 val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
  293                 val2 &= ~(0xffff << ((reg & 3) << 3));
  294                 val2 |= ((val & 0xffff) << ((reg & 3) << 3));
  295                 bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
  296                 break;
  297         case 1:
  298                 val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
  299                 val2 &= ~(0xff << ((reg & 3) << 3));
  300                 val2 |= ((val & 0xff) << ((reg & 3) << 3));
  301                 bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
  302                 break;
  303         }
  304 }
  305 
  306 static bool
  307 rk_pcie_check_dev(struct rk_pcie_softc *sc, u_int bus, u_int slot, u_int func,
  308     u_int reg)
  309 {
  310         uint32_t val;
  311 
  312         if (bus < sc->bus_start || bus > sc->bus_end || slot > PCI_SLOTMAX ||
  313             func > PCI_FUNCMAX || reg > PCIE_REGMAX)
  314                 return (false);
  315 
  316         if (bus == sc->root_bus) {
  317                 /* we have only 1 device with 1 function root port */
  318                 if (slot > 0 || func > 0)
  319                         return (false);
  320                 return (true);
  321         }
  322 
  323         /* link is needed for accessing non-root busses */
  324         val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
  325         if (STATUS1_LINK_ST_GET(val) != STATUS1_LINK_ST_UP)
  326                 return (false);
  327 
  328         /* only one device can be on first subordinate bus */
  329         if (bus == sc->sub_bus  && slot != 0 )
  330                 return (false);
  331         return (true);
  332 }
  333 
  334 static void
  335 rk_pcie_map_out_atu(struct rk_pcie_softc *sc, int idx, int type,
  336    int num_bits, uint64_t pa)
  337 {
  338         uint32_t addr0;
  339         uint64_t max_size __diagused;
  340 
  341         /* Check HW constrains */
  342         max_size = idx == 0 ? ATU_OB_REGION_0_SIZE: ATU_OB_REGION_SIZE;
  343         KASSERT(idx <  ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
  344         KASSERT(num_bits  >= 7 &&  num_bits <= 63,
  345             ("Bit width of region is invalid: %d\n", num_bits));
  346         KASSERT(max_size <= (1ULL << (num_bits + 1)),
  347             ("Bit width is invalid for given region[%d]: %d\n", idx, num_bits));
  348 
  349         addr0 = (uint32_t)pa & 0xFFFFFF00;
  350         addr0 |= num_bits;
  351         APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), addr0);
  352         APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), (uint32_t)(pa >> 32));
  353         APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 | type);
  354         APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
  355 
  356         /* Readback for sync */
  357         APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
  358 }
  359 
  360 static void
  361 rk_pcie_map_cfg_atu(struct rk_pcie_softc *sc, int idx, int type)
  362 {
  363 
  364         /* Check HW constrains */
  365         KASSERT(idx <  ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
  366 
  367         /*
  368          * Config window is only 25 bits width, so we cannot encode full bus
  369          * range into it. Remaining bits of bus number should be taken from
  370          * DESC1 field.
  371          */
  372         APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), 25 - 1);
  373         APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), 0);
  374         APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 |  type);
  375         APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
  376 
  377         /* Readback for sync */
  378         APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
  379 
  380 }
  381 
  382 static void
  383 rk_pcie_map_in_atu(struct rk_pcie_softc *sc, int idx, int num_bits, uint64_t pa)
  384 {
  385         uint32_t addr0;
  386 
  387         /* Check HW constrains */
  388         KASSERT(idx <  ATU_IB_REGIONS, ("Invalid region index: %d\n", idx));
  389         KASSERT(num_bits  >= 7 &&  num_bits <= 63,
  390             ("Bit width of region is invalid: %d\n", num_bits));
  391 
  392         addr0 = (uint32_t)pa & 0xFFFFFF00;
  393         addr0 |= num_bits;
  394         APB_WR4(sc, PCIE_CORE_IB_ADDR0(idx), addr0);
  395         APB_WR4(sc, PCIE_CORE_IB_ADDR1(idx), (uint32_t)(pa >> 32));
  396 
  397         /* Readback for sync */
  398         APB_RD4(sc, PCIE_CORE_IB_ADDR1(idx));
  399 }
  400 
  401 static int
  402 rk_pcie_decode_ranges(struct rk_pcie_softc *sc, struct ofw_pci_range *ranges,
  403      int nranges)
  404 {
  405         int i;
  406 
  407         for (i = 0; i < nranges; i++) {
  408                 switch(ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) {
  409                 case OFW_PCI_PHYS_HI_SPACE_IO:
  410                         if (sc->io_range.size != 0) {
  411                                 device_printf(sc->dev,
  412                                     "Duplicated IO range found in DT\n");
  413                                 return (ENXIO);
  414                         }
  415                         sc->io_range = ranges[i];
  416                         break;
  417                 case OFW_PCI_PHYS_HI_SPACE_MEM32:
  418                 case OFW_PCI_PHYS_HI_SPACE_MEM64:
  419                         if (ranges[i].pci_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) {
  420                                 if (sc->pref_mem_range.size != 0) {
  421                                         device_printf(sc->dev,
  422                                             "Duplicated memory range found "
  423                                             "in DT\n");
  424                                         return (ENXIO);
  425                                 }
  426                                 sc->pref_mem_range = ranges[i];
  427                         } else {
  428                                 if (sc->mem_range.size != 0) {
  429                                         device_printf(sc->dev,
  430                                             "Duplicated memory range found "
  431                                             "in DT\n");
  432                                         return (ENXIO);
  433                                 }
  434                                 sc->mem_range = ranges[i];
  435                         }
  436                 }
  437         }
  438         if (sc->mem_range.size == 0) {
  439                 device_printf(sc->dev,
  440                     " At least memory range should be defined in DT.\n");
  441                 return (ENXIO);
  442         }
  443         return (0);
  444 }
  445 
  446 /*-----------------------------------------------------------------------------
  447  *
  448  *  P C I B   I N T E R F A C E
  449  */
  450 static uint32_t
  451 rk_pcie_read_config(device_t dev, u_int bus, u_int slot,
  452     u_int func, u_int reg, int bytes)
  453 {
  454         struct rk_pcie_softc *sc;
  455         uint32_t d32, data;
  456         uint16_t d16;
  457         uint8_t d8;
  458         uint64_t addr;
  459         int type, ret;
  460 
  461         sc = device_get_softc(dev);
  462 
  463         if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
  464                 return (0xFFFFFFFFU);
  465         if (bus == sc->root_bus)
  466                 return (rk_pcie_local_cfg_read(sc, false, reg, bytes));
  467 
  468         addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
  469             ATU_CFG_REG(reg);
  470         type = bus == sc->sub_bus ? ATU_TYPE_CFG0: ATU_TYPE_CFG1;
  471         rk_pcie_map_cfg_atu(sc, 0, type);
  472 
  473         ret = -1;
  474         switch (bytes) {
  475         case 1:
  476                 ret = bus_peek_1(sc->axi_mem_res, addr, &d8);
  477                 data = d8;
  478                 break;
  479         case 2:
  480                 ret = bus_peek_2(sc->axi_mem_res, addr, &d16);
  481                 data = d16;
  482                 break;
  483         case 4:
  484                 ret = bus_peek_4(sc->axi_mem_res, addr, &d32);
  485                 data = d32;
  486                 break;
  487         }
  488         if (ret != 0)
  489                 data = 0xFFFFFFFF;
  490         return (data);
  491 }
  492 
  493 static void
  494 rk_pcie_write_config(device_t dev, u_int bus, u_int slot,
  495     u_int func, u_int reg, uint32_t val, int bytes)
  496 {
  497         struct rk_pcie_softc *sc;
  498         uint64_t addr;
  499         int type;
  500 
  501         sc = device_get_softc(dev);
  502 
  503         if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
  504                 return;
  505 
  506         if (bus == sc->root_bus)
  507                 return (rk_pcie_local_cfg_write(sc, false,  reg, val, bytes));
  508 
  509         addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
  510             ATU_CFG_REG(reg);
  511         type = bus == sc->sub_bus ? ATU_TYPE_CFG0: ATU_TYPE_CFG1;
  512         rk_pcie_map_cfg_atu(sc, 0, type);
  513 
  514         switch (bytes) {
  515         case 1:
  516                 bus_poke_1(sc->axi_mem_res, addr, (uint8_t)val);
  517                 break;
  518         case 2:
  519                 bus_poke_2(sc->axi_mem_res, addr, (uint16_t)val);
  520                 break;
  521         case 4:
  522                 bus_poke_4(sc->axi_mem_res, addr, val);
  523                 break;
  524         default:
  525                 break;
  526         }
  527 }
  528 
  529 #ifdef RK_PCIE_ENABLE_MSI
  530 static int
  531 rk_pcie_alloc_msi(device_t pci, device_t child, int count,
  532     int maxcount, int *irqs)
  533 {
  534         phandle_t msi_parent;
  535         int rv;
  536 
  537         rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
  538             &msi_parent, NULL);
  539         if (rv != 0)
  540                 return (rv);
  541 
  542         rv = intr_alloc_msi(pci, child, msi_parent, count, maxcount,irqs);
  543         return (rv);
  544 }
  545 
  546 static int
  547 rk_pcie_release_msi(device_t pci, device_t child, int count, int *irqs)
  548 {
  549         phandle_t msi_parent;
  550         int rv;
  551 
  552         rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
  553             &msi_parent, NULL);
  554         if (rv != 0)
  555                 return (rv);
  556         rv = intr_release_msi(pci, child, msi_parent, count, irqs);
  557         return (rv);
  558 }
  559 #endif
  560 
  561 static int
  562 rk_pcie_map_msi(device_t pci, device_t child, int irq, uint64_t *addr,
  563     uint32_t *data)
  564 {
  565         phandle_t msi_parent;
  566         int rv;
  567 
  568         rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
  569             &msi_parent, NULL);
  570         if (rv != 0)
  571                 return (rv);
  572         rv = intr_map_msi(pci, child, msi_parent, irq, addr, data);
  573         return (rv);
  574 }
  575 
  576 #ifdef RK_PCIE_ENABLE_MSIX
  577 static int
  578 rk_pcie_alloc_msix(device_t pci, device_t child, int *irq)
  579 {
  580         phandle_t msi_parent;
  581         int rv;
  582 
  583         rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
  584             &msi_parent, NULL);
  585         if (rv != 0)
  586                 return (rv);
  587         rv = intr_alloc_msix(pci, child, msi_parent, irq);
  588         return (rv);
  589 }
  590 
  591 static int
  592 rk_pcie_release_msix(device_t pci, device_t child, int irq)
  593 {
  594         phandle_t msi_parent;
  595         int rv;
  596 
  597         rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
  598             &msi_parent, NULL);
  599         if (rv != 0)
  600                 return (rv);
  601         rv = intr_release_msix(pci, child, msi_parent, irq);
  602         return (rv);
  603 }
  604 #endif
  605 
  606 static int
  607 rk_pcie_get_id(device_t pci, device_t child, enum pci_id_type type,
  608     uintptr_t *id)
  609 {
  610         phandle_t node;
  611         int rv;
  612         uint32_t rid;
  613         uint16_t pci_rid;
  614 
  615         if (type != PCI_ID_MSI)
  616                 return (pcib_get_id(pci, child, type, id));
  617 
  618         node = ofw_bus_get_node(pci);
  619         pci_rid = pci_get_rid(child);
  620 
  621         rv = ofw_bus_msimap(node, pci_rid, NULL, &rid);
  622         if (rv != 0)
  623                 return (rv);
  624 
  625         *id = rid;
  626         return (0);
  627 }
  628 
  629 static int
  630 rk_pcie_route_interrupt(device_t bus, device_t dev, int pin)
  631 {
  632         struct rk_pcie_softc *sc;
  633         u_int irq;
  634 
  635         sc = device_get_softc(bus);
  636         irq = intr_map_clone_irq(rman_get_start(sc->legacy_irq_res));
  637         device_printf(bus, "route pin %d for device %d.%d to %u\n",
  638                     pin, pci_get_slot(dev), pci_get_function(dev), irq);
  639 
  640         return (irq);
  641 }
  642 
  643 /*-----------------------------------------------------------------------------
  644  *
  645  *  B U S  / D E V I C E   I N T E R F A C E
  646  */
  647 static int
  648 rk_pcie_parse_fdt_resources(struct rk_pcie_softc *sc)
  649 {
  650         int i, rv;
  651         char buf[16];
  652 
  653         /* Regulators. All are optional. */
  654         rv = regulator_get_by_ofw_property(sc->dev, 0,
  655             "vpcie12v-supply", &sc->supply_12v);
  656         if (rv != 0 && rv != ENOENT) {
  657                 device_printf(sc->dev,"Cannot get 'vpcie12' regulator\n");
  658                 return (ENXIO);
  659         }
  660         rv = regulator_get_by_ofw_property(sc->dev, 0,
  661             "vpcie3v3-supply", &sc->supply_3v3);
  662         if (rv != 0 && rv != ENOENT) {
  663                 device_printf(sc->dev,"Cannot get 'vpcie3v3' regulator\n");
  664                 return (ENXIO);
  665         }
  666         rv = regulator_get_by_ofw_property(sc->dev, 0,
  667             "vpcie1v8-supply", &sc->supply_1v8);
  668         if (rv != 0 && rv != ENOENT) {
  669                 device_printf(sc->dev,"Cannot get 'vpcie1v8' regulator\n");
  670                 return (ENXIO);
  671         }
  672         rv = regulator_get_by_ofw_property(sc->dev, 0,
  673             "vpcie0v9-supply", &sc->supply_0v9);
  674         if (rv != 0 && rv != ENOENT) {
  675                 device_printf(sc->dev,"Cannot get 'vpcie0v9' regulator\n");
  676                 return (ENXIO);
  677         }
  678 
  679         /* Resets. */
  680         rv = hwreset_get_by_ofw_name(sc->dev, 0, "core", &sc->hwreset_core);
  681         if (rv != 0) {
  682                 device_printf(sc->dev, "Cannot get 'core' reset\n");
  683                 return (ENXIO);
  684         }
  685         rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt", &sc->hwreset_mgmt);
  686         if (rv != 0) {
  687                 device_printf(sc->dev, "Cannot get 'mgmt' reset\n");
  688                 return (ENXIO);
  689         }
  690         rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt-sticky",
  691             &sc->hwreset_mgmt_sticky);
  692         if (rv != 0) {
  693                 device_printf(sc->dev, "Cannot get 'mgmt-sticky' reset\n");
  694                 return (ENXIO);
  695         }
  696         rv = hwreset_get_by_ofw_name(sc->dev, 0, "pipe", &sc->hwreset_pipe);
  697         if (rv != 0) {
  698                 device_printf(sc->dev, "Cannot get 'pipe' reset\n");
  699                 return (ENXIO);
  700         }
  701         rv = hwreset_get_by_ofw_name(sc->dev, 0, "pm", &sc->hwreset_pm);
  702         if (rv != 0) {
  703                 device_printf(sc->dev, "Cannot get 'pm' reset\n");
  704                 return (ENXIO);
  705         }
  706         rv = hwreset_get_by_ofw_name(sc->dev, 0, "aclk", &sc->hwreset_aclk);
  707         if (rv != 0) {
  708                 device_printf(sc->dev, "Cannot get 'aclk' reset\n");
  709                 return (ENXIO);
  710         }
  711         rv = hwreset_get_by_ofw_name(sc->dev, 0, "pclk", &sc->hwreset_pclk);
  712         if (rv != 0) {
  713                 device_printf(sc->dev, "Cannot get 'pclk' reset\n");
  714                 return (ENXIO);
  715         }
  716 
  717         /* Clocks. */
  718         rv = clk_get_by_ofw_name(sc->dev, 0, "aclk", &sc->clk_aclk);
  719         if (rv != 0) {
  720                 device_printf(sc->dev, "Cannot get 'aclk' clock\n");
  721                 return (ENXIO);
  722         }
  723         rv = clk_get_by_ofw_name(sc->dev, 0, "aclk-perf", &sc->clk_aclk_perf);
  724         if (rv != 0) {
  725                 device_printf(sc->dev, "Cannot get 'aclk-perf' clock\n");
  726                 return (ENXIO);
  727         }
  728         rv = clk_get_by_ofw_name(sc->dev, 0, "hclk", &sc->clk_hclk);
  729         if (rv != 0) {
  730                 device_printf(sc->dev, "Cannot get 'hclk' clock\n");
  731                 return (ENXIO);
  732         }
  733         rv = clk_get_by_ofw_name(sc->dev, 0, "pm", &sc->clk_pm);
  734         if (rv != 0) {
  735                 device_printf(sc->dev, "Cannot get 'pm' clock\n");
  736                 return (ENXIO);
  737         }
  738 
  739         /* Phys. */
  740         for (i = 0; i < MAX_LANES; i++ ) {
  741                 sprintf (buf, "pcie-phy-%d", i);
  742                 rv = phy_get_by_ofw_name(sc->dev, 0, buf, sc->phys + i);
  743                 if (rv != 0) {
  744                         device_printf(sc->dev, "Cannot get '%s' phy\n", buf);
  745                         return (ENXIO);
  746                 }
  747         }
  748 
  749         /* GPIO for PERST#. Optional */
  750         rv = gpio_pin_get_by_ofw_property(sc->dev, sc->node, "ep-gpios",
  751             &sc->gpio_ep);
  752         if (rv != 0 && rv != ENOENT) {
  753                 device_printf(sc->dev, "Cannot get 'ep-gpios' gpio\n");
  754                 return (ENXIO);
  755         }
  756 
  757         return (0);
  758 }
  759 
  760 static int
  761 rk_pcie_enable_resources(struct rk_pcie_softc *sc)
  762 {
  763         int i, rv;
  764         uint32_t val;
  765 
  766         /* Assert all resets */
  767         rv = hwreset_assert(sc->hwreset_pclk);
  768         if (rv != 0) {
  769                 device_printf(sc->dev, "Cannot assert 'pclk' reset\n");
  770                 return (rv);
  771         }
  772         rv = hwreset_assert(sc->hwreset_aclk);
  773         if (rv != 0) {
  774                 device_printf(sc->dev, "Cannot assert 'aclk' reset\n");
  775                 return (rv);
  776         }
  777         rv = hwreset_assert(sc->hwreset_pm);
  778         if (rv != 0) {
  779                 device_printf(sc->dev, "Cannot assert 'pm' reset\n");
  780                 return (rv);
  781         }
  782         rv = hwreset_assert(sc->hwreset_pipe);
  783         if (rv != 0) {
  784                 device_printf(sc->dev, "Cannot assert 'pipe' reset\n");
  785                 return (rv);
  786         }
  787         rv = hwreset_assert(sc->hwreset_mgmt_sticky);
  788         if (rv != 0) {
  789                 device_printf(sc->dev, "Cannot assert 'mgmt_sticky' reset\n");
  790                 return (rv);
  791         }
  792         rv = hwreset_assert(sc->hwreset_mgmt);
  793         if (rv != 0) {
  794                 device_printf(sc->dev, "Cannot assert 'hmgmt' reset\n");
  795                 return (rv);
  796         }
  797         rv = hwreset_assert(sc->hwreset_core);
  798         if (rv != 0) {
  799                 device_printf(sc->dev, "Cannot assert 'hcore' reset\n");
  800                 return (rv);
  801         }
  802         DELAY(10000);
  803 
  804         /* Enable clockls */
  805         rv = clk_enable(sc->clk_aclk);
  806         if (rv != 0) {
  807                 device_printf(sc->dev, "Cannot enable 'aclk' clock\n");
  808                 return (rv);
  809         }
  810         rv = clk_enable(sc->clk_aclk_perf);
  811         if (rv != 0) {
  812                 device_printf(sc->dev, "Cannot enable 'aclk_perf' clock\n");
  813                 return (rv);
  814         }
  815         rv = clk_enable(sc->clk_hclk);
  816         if (rv != 0) {
  817                 device_printf(sc->dev, "Cannot enable 'hclk' clock\n");
  818                 return (rv);
  819         }
  820         rv = clk_enable(sc->clk_pm);
  821         if (rv != 0) {
  822                 device_printf(sc->dev, "Cannot enable 'pm' clock\n");
  823                 return (rv);
  824         }
  825 
  826         /* Power up regulators */
  827         if (sc->supply_12v != NULL) {
  828                 rv = regulator_enable(sc->supply_12v);
  829                 if (rv != 0) {
  830                         device_printf(sc->dev,
  831                             "Cannot enable 'vpcie12' regulator\n");
  832                         return (rv);
  833                 }
  834         }
  835         if (sc->supply_3v3 != NULL) {
  836                 rv = regulator_enable(sc->supply_3v3);
  837                 if (rv != 0) {
  838                         device_printf(sc->dev,
  839                             "Cannot enable 'vpcie3v3' regulator\n");
  840                         return (rv);
  841                 }
  842         }
  843         if (sc->supply_1v8 != NULL) {
  844                 rv = regulator_enable(sc->supply_1v8);
  845                 if (rv != 0) {
  846                         device_printf(sc->dev,
  847                             "Cannot enable 'vpcie1v8' regulator\n");
  848                         return (rv);
  849                 }
  850         }
  851         if (sc->supply_0v9 != NULL) {
  852                 rv = regulator_enable(sc->supply_0v9);
  853                 if (rv != 0) {
  854                         device_printf(sc->dev,
  855                             "Cannot enable 'vpcie1v8' regulator\n");
  856                         return (rv);
  857                 }
  858         }
  859         DELAY(1000);
  860 
  861         /* Deassert basic resets*/
  862         rv = hwreset_deassert(sc->hwreset_pm);
  863         if (rv != 0) {
  864                 device_printf(sc->dev, "Cannot deassert 'pm' reset\n");
  865                 return (rv);
  866         }
  867         rv = hwreset_deassert(sc->hwreset_aclk);
  868         if (rv != 0) {
  869                 device_printf(sc->dev, "Cannot deassert 'aclk' reset\n");
  870                 return (rv);
  871         }
  872         rv = hwreset_deassert(sc->hwreset_pclk);
  873         if (rv != 0) {
  874                 device_printf(sc->dev, "Cannot deassert 'pclk' reset\n");
  875                 return (rv);
  876         }
  877 
  878         /* Set basic PCIe core mode (RC, lanes, gen1 or 2) */
  879         val  = STRAP_CONF_GEN_2 << 16 |
  880             (sc->link_is_gen2 ? STRAP_CONF_GEN_2: 0);
  881         val |= STRAP_CONF_MODE_RC << 16 | STRAP_CONF_MODE_RC;
  882         val |= STRAP_CONF_LANES(~0) << 16 | STRAP_CONF_LANES(sc->num_lanes);
  883         val |= STRAP_CONF_ARI_EN << 16 | STRAP_CONF_ARI_EN;
  884         val |= STRAP_CONF_CONF_EN << 16 | STRAP_CONF_CONF_EN;
  885         APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, val);
  886 
  887         for (i = 0; i < MAX_LANES; i++) {
  888                 rv = phy_enable(sc->phys[i]);
  889                 if (rv != 0) {
  890                         device_printf(sc->dev, "Cannot enable phy %d\n", i);
  891                         return (rv);
  892                 }
  893         }
  894 
  895         /* Deassert rest of resets - order is important ! */
  896         rv = hwreset_deassert(sc->hwreset_mgmt_sticky);
  897         if (rv != 0) {
  898                 device_printf(sc->dev, "Cannot deassert 'mgmt_sticky' reset\n");
  899                 return (rv);
  900         }
  901         rv = hwreset_deassert(sc->hwreset_core);
  902         if (rv != 0) {
  903                 device_printf(sc->dev, "Cannot deassert 'core' reset\n");
  904                 return (rv);
  905         }
  906         rv = hwreset_deassert(sc->hwreset_mgmt);
  907         if (rv != 0) {
  908                 device_printf(sc->dev, "Cannot deassert 'mgmt' reset\n");
  909                 return (rv);
  910         }
  911         rv = hwreset_deassert(sc->hwreset_pipe);
  912         if (rv != 0) {
  913                 device_printf(sc->dev, "Cannot deassert 'pipe' reset\n");
  914                 return (rv);
  915         }
  916         return (0);
  917 }
  918 
  919 static int
  920 rk_pcie_setup_hw(struct rk_pcie_softc *sc)
  921 {
  922         uint32_t val;
  923         int i, rv;
  924 
  925         /* Assert PERST# if defined */
  926         if (sc->gpio_ep != NULL) {
  927                 rv = gpio_pin_set_active(sc->gpio_ep, 0);
  928                 if (rv != 0) {
  929                         device_printf(sc->dev,
  930                             "Cannot clear 'gpio-ep' gpio\n");
  931                         return (rv);
  932                 }
  933         }
  934 
  935         rv = rk_pcie_enable_resources(sc);
  936         if (rv != 0)
  937                 return(rv);
  938 
  939         /* Fix wrong default value for transmited FTS for L0s exit */
  940         val = APB_RD4(sc, PCIE_CORE_CTRL1);
  941         val |= 0xFFFF << 8;
  942         APB_WR4(sc, PCIE_CORE_CTRL1, val);
  943 
  944         /* Setup PCIE Link Status & Control register */
  945         val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
  946         val |= PCIEM_LINK_CTL_COMMON_CLOCK;
  947         APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
  948         val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
  949         val |= PCIEM_LINK_CTL_RCB;
  950         APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
  951 
  952         /* Enable training for GEN1 */
  953         APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF,
  954             STRAP_CONF_LINK_TRAIN_EN << 16 | STRAP_CONF_LINK_TRAIN_EN);
  955 
  956         /* Deassert PERST# if defined */
  957         if (sc->gpio_ep != NULL) {
  958                 rv = gpio_pin_set_active(sc->gpio_ep, 1);
  959                 if (rv != 0) {
  960                         device_printf(sc->dev, "Cannot set 'gpio-ep' gpio\n");
  961                         return (rv);
  962                 }
  963         }
  964 
  965         /* Wait for link */
  966         for (i = 500; i > 0; i--) {
  967                 val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
  968                 if (STATUS1_LINK_ST_GET(val) == STATUS1_LINK_ST_UP)
  969                         break;
  970                 DELAY(1000);
  971         }
  972         if (i <= 0) {
  973                 device_printf(sc->dev,
  974                     "Gen1 link training timeouted: 0x%08X.\n", val);
  975                 return (0);
  976         }
  977 
  978         if (sc->link_is_gen2) {
  979                         val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
  980                         val |= PCIEM_LINK_CTL_RETRAIN_LINK;
  981                         APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
  982 
  983                         /* Wait for link */
  984                         for (i = 500; i > 0; i--) {
  985                                 val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
  986                                 if (STATUS1_LINK_ST_GET(val) ==
  987                                     STATUS1_LINK_ST_UP)
  988                                         break;
  989                                 DELAY(1000);
  990                         }
  991                         if (i <= 0)
  992                                 device_printf(sc->dev, "Gen2 link training "
  993                                     "timeouted: 0x%08X.\n", val);
  994         }
  995 
  996         val = APB_RD4(sc, PCIE_CORE_CTRL0);
  997         val = CORE_CTRL_LANES_GET(val);
  998         if (bootverbose)
  999                 device_printf(sc->dev, "Link width: %d\n", 1 << val);
 1000 
 1001         return (0);
 1002 }
 1003 
 1004 static int
 1005 rk_pcie_setup_sw(struct rk_pcie_softc *sc)
 1006 {
 1007         uint32_t val;
 1008         int i, region;
 1009 
 1010         pcib_bridge_init(sc->dev);
 1011 
 1012         /* Setup config registers */
 1013         APB_WR4(sc, PCIE_CORE_CONFIG_VENDOR, 0x1D87); /* Rockchip vendor ID*/
 1014         PRIV_CFG_WR1(sc, PCIR_CLASS, PCIC_BRIDGE);
 1015         PRIV_CFG_WR1(sc, PCIR_SUBCLASS, PCIS_BRIDGE_PCI);
 1016         PRIV_CFG_WR1(sc, PCIR_PRIBUS_1, sc->root_bus);
 1017         PRIV_CFG_WR1(sc, PCIR_SECBUS_1, sc->sub_bus);
 1018         PRIV_CFG_WR1(sc, PCIR_SUBBUS_1, sc->bus_end);
 1019         PRIV_CFG_WR2(sc, PCIR_COMMAND, PCIM_CMD_MEMEN |
 1020            PCIM_CMD_BUSMASTEREN | PCIM_CMD_SERRESPEN);
 1021 
 1022         /* Don't advertise L1 power substate */
 1023         val = APB_RD4(sc, PCIE_RC_CONFIG_THP_CAP);
 1024         val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
 1025         APB_WR4(sc, PCIE_RC_CONFIG_THP_CAP, val);
 1026 
 1027         /* Don't advertise L0s */
 1028         if (sc->no_l0s) {
 1029                 val = APB_RD4(sc, PCIE_RC_CONFIG_LINK_CAP);
 1030                 val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
 1031                 APB_WR4(sc, PCIE_RC_CONFIG_LINK_CAP_L0S, val);
 1032         }
 1033 
 1034         /*Adjust maximum payload size*/
 1035         val = APB_RD4(sc, PCIE_RC_CONFIG_DCSR);
 1036         val &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
 1037         val |= PCIE_RC_CONFIG_DCSR_MPS_128;
 1038         APB_WR4(sc, PCIE_RC_CONFIG_DCSR, val);
 1039 
 1040         /*
 1041          * Prepare IB ATU
 1042          * map whole address range in 1:1 mappings
 1043          */
 1044         rk_pcie_map_in_atu(sc, 2, 64 - 1, 0);
 1045 
 1046         /* Prepare OB ATU */
 1047         /* - region 0 (32 MB) is used for config access */
 1048         region = 0;
 1049         rk_pcie_map_out_atu(sc, region++, ATU_TYPE_CFG0, 25 - 1, 0);
 1050 
 1051         /* - then map memory (by using 1MB regions */
 1052         for (i = 0; i  < sc->mem_range.size / ATU_OB_REGION_SIZE; i++) {
 1053                 rk_pcie_map_out_atu(sc,  region++, ATU_TYPE_MEM,
 1054                     ATU_OB_REGION_SHIFT - 1,
 1055                     sc->mem_range.pci + ATU_OB_REGION_SIZE * i);
 1056         }
 1057 
 1058         /* - IO space is next, one region typically*/
 1059         for (i = 0; i  < sc->io_range.size / ATU_OB_REGION_SIZE; i++) {
 1060                 rk_pcie_map_out_atu(sc, region++, ATU_TYPE_IO,
 1061                     ATU_OB_REGION_SHIFT - 1,
 1062                     sc->io_range.pci + ATU_OB_REGION_SIZE * i);
 1063         }
 1064         APB_WR4(sc, PCIE_CORE_RC_BAR_CONF, 0);
 1065         return (0);
 1066 }
 1067 
 1068 static int
 1069 rk_pcie_sys_irq(void *arg)
 1070 {
 1071         struct rk_pcie_softc *sc;
 1072         uint32_t irq;
 1073 
 1074         sc = (struct rk_pcie_softc *)arg;
 1075         irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
 1076         if (irq & PCIE_CLIENT_INT_LOCAL) {
 1077                 irq = APB_RD4(sc, PCIE_CORE_INT_STATUS);
 1078                 APB_WR4(sc, PCIE_CORE_INT_STATUS, irq);
 1079                 APB_WR4(sc, PCIE_CLIENT_INT_STATUS, PCIE_CLIENT_INT_LOCAL);
 1080 
 1081                 device_printf(sc->dev, "'sys' interrupt received: 0x%04X\n",
 1082                     irq);
 1083         }
 1084 
 1085         return (FILTER_HANDLED);
 1086 }
 1087 
 1088 static int
 1089 rk_pcie_client_irq(void *arg)
 1090 {
 1091         struct rk_pcie_softc *sc;
 1092         uint32_t irq;
 1093 
 1094         sc = (struct rk_pcie_softc *)arg;
 1095         irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
 1096         /* Clear causes handled by other interrups */
 1097         irq &= ~PCIE_CLIENT_INT_LOCAL;
 1098         irq &= ~PCIE_CLIENT_INT_LEGACY;
 1099         APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
 1100 
 1101         device_printf(sc->dev, "'client' interrupt received: 0x%04X\n", irq);
 1102 
 1103         return (FILTER_HANDLED);
 1104 }
 1105 
 1106 static int
 1107 rk_pcie_legacy_irq(void *arg)
 1108 {
 1109         struct rk_pcie_softc *sc;
 1110         uint32_t irq;
 1111 
 1112         sc = (struct rk_pcie_softc *)arg;
 1113         irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
 1114         irq &= PCIE_CLIENT_INT_LEGACY;
 1115         APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
 1116 
 1117         /* all legacy interrupt are shared, do nothing */
 1118         return (FILTER_STRAY);
 1119 }
 1120 
 1121 static bus_dma_tag_t
 1122 rk_pcie_get_dma_tag(device_t dev, device_t child)
 1123 {
 1124         struct rk_pcie_softc *sc;
 1125 
 1126         sc = device_get_softc(dev);
 1127         return (sc->dmat);
 1128 }
 1129 
 1130 static int
 1131 rk_pcie_probe(device_t dev)
 1132 {
 1133 
 1134         if (!ofw_bus_status_okay(dev))
 1135                 return (ENXIO);
 1136 
 1137         if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
 1138                 return (ENXIO);
 1139 
 1140         device_set_desc(dev, "Rockchip PCIe controller");
 1141         return (BUS_PROBE_DEFAULT);
 1142 }
 1143 
 1144 static int
 1145 rk_pcie_attach(device_t dev)
 1146 {       struct rk_pcie_softc *sc;
 1147         uint32_t val;
 1148         int rv, rid, max_speed;
 1149 
 1150         sc = device_get_softc(dev);
 1151         sc->dev = dev;
 1152         sc->node = ofw_bus_get_node(dev);
 1153 
 1154         mtx_init(&sc->mtx, "rk_pcie_mtx", NULL, MTX_DEF);
 1155 
 1156         /* XXX Should not be this configurable ? */
 1157         sc->bus_start = 0;
 1158         sc->bus_end =  0x1F;
 1159         sc->root_bus = sc->bus_start;
 1160         sc->sub_bus = 1;
 1161 
 1162         /* Read FDT properties */
 1163         rv = rk_pcie_parse_fdt_resources(sc);
 1164         if (rv != 0)
 1165                 goto out;
 1166 
 1167         sc->coherent = OF_hasprop(sc->node, "dma-coherent");
 1168         sc->no_l0s = OF_hasprop(sc->node, "aspm-no-l0s");
 1169         rv = OF_getencprop(sc->node, "num-lanes", &sc->num_lanes,
 1170             sizeof(sc->num_lanes));
 1171         if (rv != sizeof(sc->num_lanes))
 1172                 sc->num_lanes = 1;
 1173         if (sc->num_lanes != 1 && sc->num_lanes != 2 && sc->num_lanes != 4) {
 1174                 device_printf(dev,
 1175                     "invalid number of lanes: %d\n",sc->num_lanes);
 1176                 sc->num_lanes = 0;
 1177                 rv = ENXIO;
 1178                 goto out;
 1179         }
 1180 
 1181         rv = OF_getencprop(sc->node, "max-link-speed", &max_speed,
 1182             sizeof(max_speed));
 1183         if (rv != sizeof(max_speed) || max_speed != 1)
 1184                 sc->link_is_gen2 = true;
 1185         else
 1186                 sc->link_is_gen2 = false;
 1187 
 1188         rv = ofw_bus_find_string_index(sc->node, "reg-names", "axi-base", &rid);
 1189         if (rv != 0) {
 1190                 device_printf(dev, "Cannot get 'axi-base' memory\n");
 1191                 rv = ENXIO;
 1192                 goto out;
 1193         }
 1194         sc->axi_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 1195             RF_ACTIVE);
 1196         if (sc->axi_mem_res == NULL) {
 1197                 device_printf(dev, "Cannot allocate 'axi-base' (rid: %d)\n",
 1198                     rid);
 1199                 rv = ENXIO;
 1200                 goto out;
 1201         }
 1202         rv = ofw_bus_find_string_index(sc->node, "reg-names", "apb-base", &rid);
 1203         if (rv != 0) {
 1204                 device_printf(dev, "Cannot get 'apb-base' memory\n");
 1205                 rv = ENXIO;
 1206                 goto out;
 1207         }
 1208         sc->apb_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 1209             RF_ACTIVE);
 1210         if (sc->apb_mem_res == NULL) {
 1211                 device_printf(dev, "Cannot allocate 'apb-base' (rid: %d)\n",
 1212                     rid);
 1213                 rv = ENXIO;
 1214                 goto out;
 1215         }
 1216 
 1217         rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
 1218             "client", &rid);
 1219         if (rv != 0) {
 1220                 device_printf(dev, "Cannot get 'client' IRQ\n");
 1221                 rv = ENXIO;
 1222                 goto out;
 1223         }
 1224         sc->client_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 1225             RF_ACTIVE | RF_SHAREABLE);
 1226         if (sc->client_irq_res == NULL) {
 1227                 device_printf(dev, "Cannot allocate 'client' IRQ resource\n");
 1228                 rv = ENXIO;
 1229                 goto out;
 1230         }
 1231 
 1232         rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
 1233             "legacy", &rid);
 1234         if (rv != 0) {
 1235                 device_printf(dev, "Cannot get 'legacy' IRQ\n");
 1236                 rv = ENXIO;
 1237                 goto out;
 1238         }
 1239         sc->legacy_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 1240             RF_ACTIVE | RF_SHAREABLE);
 1241         if (sc->legacy_irq_res == NULL) {
 1242                 device_printf(dev, "Cannot allocate 'legacy' IRQ resource\n");
 1243                 rv = ENXIO;
 1244                 goto out;
 1245         }
 1246 
 1247         rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
 1248             "sys", &rid);
 1249         if (rv != 0) {
 1250                 device_printf(dev, "Cannot get 'sys' IRQ\n");
 1251                 rv = ENXIO;
 1252                 goto out;
 1253         }
 1254         sc->sys_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 1255             RF_ACTIVE | RF_SHAREABLE);
 1256         if (sc->sys_irq_res == NULL) {
 1257                 device_printf(dev, "Cannot allocate 'sys' IRQ resource\n");
 1258                 rv = ENXIO;
 1259                 goto out;
 1260         }
 1261 
 1262         if (bootverbose)
 1263                 device_printf(dev, "Bus is%s cache-coherent\n",
 1264                     sc->coherent ? "" : " not");
 1265         rv = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
 1266             1, 0,                               /* alignment, bounds */
 1267             BUS_SPACE_MAXADDR,                  /* lowaddr */
 1268             BUS_SPACE_MAXADDR,                  /* highaddr */
 1269             NULL, NULL,                         /* filter, filterarg */
 1270             BUS_SPACE_MAXSIZE,                  /* maxsize */
 1271             BUS_SPACE_UNRESTRICTED,             /* nsegments */
 1272             BUS_SPACE_MAXSIZE,                  /* maxsegsize */
 1273             sc->coherent ? BUS_DMA_COHERENT : 0, /* flags */
 1274             NULL, NULL,                         /* lockfunc, lockarg */
 1275             &sc->dmat);
 1276         if (rv != 0)
 1277                 goto out;
 1278 
 1279         rv = ofw_pcib_init(dev);
 1280         if (rv != 0)
 1281                 goto out;
 1282 
 1283         rv = rk_pcie_decode_ranges(sc, sc->ofw_pci.sc_range,
 1284             sc->ofw_pci.sc_nrange);
 1285         if (rv != 0)
 1286                 goto out_full;
 1287         rv = rk_pcie_setup_hw(sc);
 1288         if (rv != 0)
 1289                 goto out_full;
 1290 
 1291         rv = rk_pcie_setup_sw(sc);
 1292         if (rv != 0)
 1293                 goto out_full;
 1294 
 1295         rv = bus_setup_intr(dev, sc->client_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
 1296            rk_pcie_client_irq, NULL, sc, &sc->client_irq_cookie);
 1297         if (rv != 0) {
 1298                 device_printf(dev, "cannot setup client interrupt handler\n");
 1299                 rv = ENXIO;
 1300                 goto out_full;
 1301         }
 1302 
 1303         rv = bus_setup_intr(dev, sc->legacy_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
 1304            rk_pcie_legacy_irq, NULL, sc, &sc->legacy_irq_cookie);
 1305         if (rv != 0) {
 1306                 device_printf(dev, "cannot setup client interrupt handler\n");
 1307                 rv = ENXIO;
 1308                 goto out_full;
 1309         }
 1310 
 1311         rv = bus_setup_intr(dev, sc->sys_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
 1312            rk_pcie_sys_irq, NULL, sc, &sc->sys_irq_cookie);
 1313         if (rv != 0) {
 1314                 device_printf(dev, "cannot setup client interrupt handler\n");
 1315                 rv = ENXIO;
 1316                 goto out_full;
 1317         }
 1318 
 1319         /* Enable interrupts */
 1320         val =
 1321             PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR |
 1322             PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA |
 1323             PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG |
 1324             PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_INTA |
 1325             PCIE_CLIENT_INT_INTB | PCIE_CLIENT_INT_INTC |
 1326             PCIE_CLIENT_INT_INTD | PCIE_CLIENT_INT_PHY;
 1327 
 1328         APB_WR4(sc, PCIE_CLIENT_INT_MASK, (val << 16) &  ~val);
 1329 
 1330         val =
 1331             PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE |
 1332             PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO |
 1333             PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR |
 1334             PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR |
 1335             PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE |
 1336             PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC |
 1337             PCIE_CORE_INT_MMVC;
 1338         APB_WR4(sc, PCIE_CORE_INT_MASK, ~(val));
 1339 
 1340         val  = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
 1341         val |= PCIEM_LINK_CTL_LBMIE | PCIEM_LINK_CTL_LABIE;
 1342         APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
 1343 
 1344         DELAY(250000);
 1345         device_add_child(dev, "pci", -1);
 1346         return (bus_generic_attach(dev));
 1347 
 1348 out_full:
 1349         bus_teardown_intr(dev, sc->sys_irq_res, sc->sys_irq_cookie);
 1350         bus_teardown_intr(dev, sc->legacy_irq_res, sc->legacy_irq_cookie);
 1351         bus_teardown_intr(dev, sc->client_irq_res, sc->client_irq_cookie);
 1352         ofw_pcib_fini(dev);
 1353 out:
 1354         bus_dma_tag_destroy(sc->dmat);
 1355         bus_free_resource(dev, SYS_RES_IRQ, sc->sys_irq_res);
 1356         bus_free_resource(dev, SYS_RES_IRQ, sc->legacy_irq_res);
 1357         bus_free_resource(dev, SYS_RES_IRQ, sc->client_irq_res);
 1358         bus_free_resource(dev, SYS_RES_MEMORY, sc->apb_mem_res);
 1359         bus_free_resource(dev, SYS_RES_MEMORY, sc->axi_mem_res);
 1360         /* GPIO */
 1361         gpio_pin_release(sc->gpio_ep);
 1362         /* Phys */
 1363         for (int i = 0; i < MAX_LANES; i++) {
 1364                 phy_release(sc->phys[i]);
 1365         }
 1366         /* Clocks */
 1367         clk_release(sc->clk_aclk);
 1368         clk_release(sc->clk_aclk_perf);
 1369         clk_release(sc->clk_hclk);
 1370         clk_release(sc->clk_pm);
 1371         /* Resets */
 1372         hwreset_release(sc->hwreset_core);
 1373         hwreset_release(sc->hwreset_mgmt);
 1374         hwreset_release(sc->hwreset_pipe);
 1375         hwreset_release(sc->hwreset_pm);
 1376         hwreset_release(sc->hwreset_aclk);
 1377         hwreset_release(sc->hwreset_pclk);
 1378         /* Regulators */
 1379         regulator_release(sc->supply_12v);
 1380         regulator_release(sc->supply_3v3);
 1381         regulator_release(sc->supply_1v8);
 1382         regulator_release(sc->supply_0v9);
 1383         return (rv);
 1384 }
 1385 
 1386 static device_method_t rk_pcie_methods[] = {
 1387         /* Device interface */
 1388         DEVMETHOD(device_probe,         rk_pcie_probe),
 1389         DEVMETHOD(device_attach,        rk_pcie_attach),
 1390 
 1391         /* Bus interface */
 1392         DEVMETHOD(bus_get_dma_tag,      rk_pcie_get_dma_tag),
 1393 
 1394         /* pcib interface */
 1395         DEVMETHOD(pcib_read_config,     rk_pcie_read_config),
 1396         DEVMETHOD(pcib_write_config,    rk_pcie_write_config),
 1397         DEVMETHOD(pcib_route_interrupt, rk_pcie_route_interrupt),
 1398 #ifdef RK_PCIE_ENABLE_MSI
 1399         DEVMETHOD(pcib_alloc_msi,       rk_pcie_alloc_msi),
 1400         DEVMETHOD(pcib_release_msi,     rk_pcie_release_msi),
 1401 #endif
 1402 #ifdef RK_PCIE_ENABLE_MSIX
 1403         DEVMETHOD(pcib_alloc_msix,      rk_pcie_alloc_msix),
 1404         DEVMETHOD(pcib_release_msix,    rk_pcie_release_msix),
 1405 #endif
 1406         DEVMETHOD(pcib_map_msi,         rk_pcie_map_msi),
 1407         DEVMETHOD(pcib_get_id,          rk_pcie_get_id),
 1408 
 1409         /* OFW bus interface */
 1410         DEVMETHOD(ofw_bus_get_compat,   ofw_bus_gen_get_compat),
 1411         DEVMETHOD(ofw_bus_get_model,    ofw_bus_gen_get_model),
 1412         DEVMETHOD(ofw_bus_get_name,     ofw_bus_gen_get_name),
 1413         DEVMETHOD(ofw_bus_get_node,     ofw_bus_gen_get_node),
 1414         DEVMETHOD(ofw_bus_get_type,     ofw_bus_gen_get_type),
 1415 
 1416         DEVMETHOD_END
 1417 };
 1418 
 1419 DEFINE_CLASS_1(pcib, rk_pcie_driver, rk_pcie_methods,
 1420     sizeof(struct rk_pcie_softc), ofw_pcib_driver);
 1421 DRIVER_MODULE( rk_pcie, simplebus, rk_pcie_driver, NULL, NULL);
Cache object: ba3192b4ee7b7e917836643e93e6a028
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/arm64/rockchip/rk_pcie.c

FreeBSD/Linux Kernel Cross Reference
sys/arm64/rockchip/rk_pcie.c
