dwc_otg.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* $FreeBSD$ */
    2 /*-
    3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    4  *
    5  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
    6  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
    7  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  */
   30 
   31 /*
   32  * This file contains the driver for the DesignWare series USB 2.0 OTG
   33  * Controller.
   34  */
   35 
   36 /*
   37  * LIMITATION: Drivers must be bound to all OUT endpoints in the
   38  * active configuration for this driver to work properly. Blocking any
   39  * OUT endpoint will block all OUT endpoints including the control
   40  * endpoint. Usually this is not a problem.
   41  */
   42 
   43 /*
   44  * NOTE: Writing to non-existing registers appears to cause an
   45  * internal reset.
   46  */
   47 
   48 #ifdef USB_GLOBAL_INCLUDE_FILE
   49 #include USB_GLOBAL_INCLUDE_FILE
   50 #else
   51 #include <sys/stdint.h>
   52 #include <sys/stddef.h>
   53 #include <sys/param.h>
   54 #include <sys/queue.h>
   55 #include <sys/types.h>
   56 #include <sys/systm.h>
   57 #include <sys/kernel.h>
   58 #include <sys/bus.h>
   59 #include <sys/module.h>
   60 #include <sys/lock.h>
   61 #include <sys/mutex.h>
   62 #include <sys/condvar.h>
   63 #include <sys/sysctl.h>
   64 #include <sys/sx.h>
   65 #include <sys/unistd.h>
   66 #include <sys/callout.h>
   67 #include <sys/malloc.h>
   68 #include <sys/priv.h>
   69 #include <sys/rman.h>
   70 
   71 #include <dev/usb/usb.h>
   72 #include <dev/usb/usbdi.h>
   73 
   74 #define USB_DEBUG_VAR dwc_otg_debug
   75 
   76 #include <dev/usb/usb_core.h>
   77 #include <dev/usb/usb_debug.h>
   78 #include <dev/usb/usb_busdma.h>
   79 #include <dev/usb/usb_process.h>
   80 #include <dev/usb/usb_transfer.h>
   81 #include <dev/usb/usb_device.h>
   82 #include <dev/usb/usb_hub.h>
   83 #include <dev/usb/usb_util.h>
   84 
   85 #include <dev/usb/usb_controller.h>
   86 #include <dev/usb/usb_bus.h>
   87 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
   88 
   89 #include <dev/usb/controller/dwc_otg.h>
   90 #include <dev/usb/controller/dwc_otgreg.h>
   91 
   92 #define DWC_OTG_BUS2SC(bus) \
   93     __containerof(bus, struct dwc_otg_softc, sc_bus)
   94 
   95 #define DWC_OTG_PC2UDEV(pc) \
   96    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
   97 
   98 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
   99    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
  100    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
  101    GINTSTS_SESSREQINT)
  102 
  103 #ifndef DWC_OTG_PHY_DEFAULT
  104 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
  105 #endif
  106 
  107 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
  108 
  109 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  110     "USB DWC OTG");
  111 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
  112     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
  113 
  114 #ifdef USB_DEBUG
  115 static int dwc_otg_debug = 0;
  116 
  117 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
  118     &dwc_otg_debug, 0, "DWC OTG debug level");
  119 #endif
  120 
  121 #define DWC_OTG_INTR_ENDPT 1
  122 
  123 /* prototypes */
  124 
  125 static const struct usb_bus_methods dwc_otg_bus_methods;
  126 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
  127 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
  128 
  129 static dwc_otg_cmd_t dwc_otg_setup_rx;
  130 static dwc_otg_cmd_t dwc_otg_data_rx;
  131 static dwc_otg_cmd_t dwc_otg_data_tx;
  132 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
  133 
  134 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
  135 static dwc_otg_cmd_t dwc_otg_host_data_tx;
  136 static dwc_otg_cmd_t dwc_otg_host_data_rx;
  137 
  138 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
  139 static void dwc_otg_do_poll(struct usb_bus *);
  140 static void dwc_otg_standard_done(struct usb_xfer *);
  141 static void dwc_otg_root_intr(struct dwc_otg_softc *);
  142 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
  143 
  144 /*
  145  * Here is a configuration that the chip supports.
  146  */
  147 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
  148         [0] = {
  149                 .max_in_frame_size = 64,/* fixed */
  150                 .max_out_frame_size = 64,       /* fixed */
  151                 .is_simplex = 1,
  152                 .support_control = 1,
  153         }
  154 };
  155 
  156 static void
  157 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
  158     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
  159 {
  160         struct dwc_otg_softc *sc;
  161 
  162         sc = DWC_OTG_BUS2SC(udev->bus);
  163 
  164         if (ep_addr < sc->sc_dev_ep_max)
  165                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
  166         else
  167                 *ppf = NULL;
  168 }
  169 
  170 static void
  171 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
  172     uint32_t offset, uint32_t fifo, uint32_t count)
  173 {
  174         uint32_t temp;
  175 
  176         /* round down length to nearest 4-bytes */
  177         temp = count & ~3;
  178 
  179         /* check if we can write the data directly */
  180         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
  181                 struct usb_page_search buf_res;
  182 
  183                 /* pre-subtract length */
  184                 count -= temp;
  185 
  186                 /* iterate buffer list */
  187                 do {
  188                         /* get current buffer pointer */
  189                         usbd_get_page(pc, offset, &buf_res);
  190 
  191                         if (buf_res.length > temp)
  192                                 buf_res.length = temp;
  193 
  194                         /* transfer data into FIFO */
  195                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
  196                             fifo, buf_res.buffer, buf_res.length / 4);
  197 
  198                         offset += buf_res.length;
  199                         fifo += buf_res.length;
  200                         temp -= buf_res.length;
  201                 } while (temp != 0);
  202         }
  203 
  204         /* check for remainder */
  205         if (count != 0) {
  206                 /* clear topmost word before copy */
  207                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
  208 
  209                 /* copy out data */
  210                 usbd_copy_out(pc, offset,
  211                     sc->sc_bounce_buffer, count);
  212 
  213                 /* transfer data into FIFO */
  214                 bus_space_write_region_4(sc->sc_io_tag,
  215                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
  216                     (count + 3) / 4);
  217         }
  218 }
  219 
  220 static void
  221 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
  222     uint32_t offset, uint32_t count)
  223 {
  224         uint32_t temp;
  225 
  226         /* round down length to nearest 4-bytes */
  227         temp = count & ~3;
  228 
  229         /* check if we can read the data directly */
  230         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
  231                 struct usb_page_search buf_res;
  232 
  233                 /* pre-subtract length */
  234                 count -= temp;
  235 
  236                 /* iterate buffer list */
  237                 do {
  238                         /* get current buffer pointer */
  239                         usbd_get_page(pc, offset, &buf_res);
  240 
  241                         if (buf_res.length > temp)
  242                                 buf_res.length = temp;
  243 
  244                         /* transfer data from FIFO */
  245                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
  246                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
  247 
  248                         offset += buf_res.length;
  249                         sc->sc_current_rx_fifo += buf_res.length;
  250                         sc->sc_current_rx_bytes -= buf_res.length;
  251                         temp -= buf_res.length;
  252                 } while (temp != 0);
  253         }
  254 
  255         /* check for remainder */
  256         if (count != 0) {
  257                 /* read data into bounce buffer */
  258                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
  259                         sc->sc_current_rx_fifo,
  260                         sc->sc_bounce_buffer, (count + 3) / 4);
  261 
  262                 /* store data into proper buffer */
  263                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
  264 
  265                 /* round length up to nearest 4 bytes */
  266                 count = (count + 3) & ~3;
  267 
  268                 /* update counters */
  269                 sc->sc_current_rx_bytes -= count;
  270                 sc->sc_current_rx_fifo += count;
  271         }
  272 }
  273 
  274 static void
  275 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
  276 {
  277         uint32_t temp;
  278 
  279         /* reset FIFO */
  280         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
  281 
  282         /* wait for reset to complete */
  283         for (temp = 0; temp != 16; temp++) {
  284                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
  285                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
  286                         break;
  287         }
  288 }
  289 
  290 static int
  291 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
  292 {
  293         struct dwc_otg_profile *pf;
  294         uint32_t fifo_size;
  295         uint32_t fifo_regs;
  296         uint32_t tx_start;
  297         uint8_t x;
  298 
  299         fifo_size = sc->sc_fifo_size;
  300 
  301         /*
  302          * NOTE: Reserved fixed size area at end of RAM, which must
  303          * not be allocated to the FIFOs:
  304          */
  305         fifo_regs = 4 * 16;
  306 
  307         if (fifo_size < fifo_regs) {
  308                 DPRINTF("Too little FIFO\n");
  309                 return (EINVAL);
  310         }
  311 
  312         /* subtract FIFO regs from total once */
  313         fifo_size -= fifo_regs;
  314 
  315         /* split equally for IN and OUT */
  316         fifo_size /= 2;
  317 
  318         /* Align to 4 bytes boundary (refer to PGM) */
  319         fifo_size &= ~3;
  320 
  321         /* set global receive FIFO size */
  322         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
  323 
  324         tx_start = fifo_size;
  325 
  326         if (fifo_size < 64) {
  327                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
  328                 return (EINVAL);
  329         }
  330 
  331         if (mode == DWC_MODE_HOST) {
  332                 /* reset active endpoints */
  333                 sc->sc_active_rx_ep = 0;
  334 
  335                 /* split equally for periodic and non-periodic */
  336                 fifo_size /= 2;
  337 
  338                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
  339 
  340                 /* align to 4 bytes boundary */
  341                 fifo_size &= ~3;
  342 
  343                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
  344                     ((fifo_size / 4) << 16) |
  345                     (tx_start / 4));
  346 
  347                 tx_start += fifo_size;
  348 
  349                 for (x = 0; x != sc->sc_host_ch_max; x++) {
  350                         /* enable all host interrupts */
  351                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
  352                             HCINT_DEFAULT_MASK);
  353                 }
  354 
  355                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
  356                     ((fifo_size / 4) << 16) |
  357                     (tx_start / 4));
  358 
  359                 /* reset host channel state */
  360                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
  361 
  362                 /* enable all host channel interrupts */
  363                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
  364                     (1U << sc->sc_host_ch_max) - 1U);
  365 
  366                 /* enable proper host channel interrupts */
  367                 sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
  368                 sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
  369                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  370         }
  371 
  372         if (mode == DWC_MODE_DEVICE) {
  373             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
  374                 (0x10 << 16) | (tx_start / 4));
  375             fifo_size -= 0x40;
  376             tx_start += 0x40;
  377 
  378             /* setup control endpoint profile */
  379             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
  380 
  381             /* reset active endpoints */
  382             sc->sc_active_rx_ep = 1;
  383 
  384             for (x = 1; x != sc->sc_dev_ep_max; x++) {
  385                 pf = sc->sc_hw_ep_profile + x;
  386 
  387                 pf->usb.max_out_frame_size = 1024 * 3;
  388                 pf->usb.is_simplex = 0; /* assume duplex */
  389                 pf->usb.support_bulk = 1;
  390                 pf->usb.support_interrupt = 1;
  391                 pf->usb.support_isochronous = 1;
  392                 pf->usb.support_out = 1;
  393 
  394                 if (x < sc->sc_dev_in_ep_max) {
  395                         uint32_t limit;
  396 
  397                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
  398                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
  399                             DWC_OTG_TX_MAX_FIFO_SIZE);
  400 
  401                         /* see if there is enough FIFO space */
  402                         if (limit <= fifo_size) {
  403                                 pf->max_buffer = limit;
  404                                 pf->usb.support_in = 1;
  405                         } else {
  406                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
  407                                 if (limit <= fifo_size) {
  408                                         pf->usb.support_in = 1;
  409                                 } else {
  410                                         pf->usb.is_simplex = 1;
  411                                         limit = 0;
  412                                 }
  413                         }
  414                         /* set FIFO size */
  415                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
  416                             ((limit / 4) << 16) | (tx_start / 4));
  417                         tx_start += limit;
  418                         fifo_size -= limit;
  419                         pf->usb.max_in_frame_size = limit;
  420                 } else {
  421                         pf->usb.is_simplex = 1;
  422                 }
  423 
  424                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
  425                     pf->usb.max_in_frame_size,
  426                     pf->usb.max_out_frame_size);
  427             }
  428 
  429             /* enable proper device channel interrupts */
  430             sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
  431             sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
  432             DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  433         }
  434 
  435         /* reset RX FIFO */
  436         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
  437 
  438         if (mode != DWC_MODE_OTG) {
  439                 /* reset all TX FIFOs */
  440                 dwc_otg_tx_fifo_reset(sc,
  441                     GRSTCTL_TXFIFO(0x10) |
  442                     GRSTCTL_TXFFLSH);
  443         } else {
  444                 /* reset active endpoints */
  445                 sc->sc_active_rx_ep = 0;
  446 
  447                 /* reset host channel state */
  448                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
  449         }
  450         return (0);
  451 }
  452 
  453 static uint8_t
  454 dwc_otg_uses_split(struct usb_device *udev)
  455 {
  456         /*
  457          * When a LOW or FULL speed device is connected directly to
  458          * the USB port we don't use split transactions:
  459          */ 
  460         return (udev->speed != USB_SPEED_HIGH &&
  461             udev->parent_hs_hub != NULL &&
  462             udev->parent_hs_hub->parent_hub != NULL);
  463 }
  464 
  465 static void
  466 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
  467 {
  468 
  469   /*
  470    * Disabled until further. Assuming that the register is already
  471    * programmed correctly by the boot loader.
  472    */
  473 #if 0
  474         uint32_t temp;
  475 
  476         /* setup HOST frame interval register, based on existing value */
  477         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
  478         if (temp >= 10000)
  479                 temp /= 1000;
  480         else
  481                 temp /= 125;
  482 
  483         /* figure out nearest X-tal value */
  484         if (temp >= 54)
  485                 temp = 60;      /* MHz */
  486         else if (temp >= 39)
  487                 temp = 48;      /* MHz */
  488         else
  489                 temp = 30;      /* MHz */
  490 
  491         if (sc->sc_flags.status_high_speed)
  492                 temp *= 125;
  493         else
  494                 temp *= 1000;
  495 
  496         DPRINTF("HFIR=0x%08x\n", temp);
  497 
  498         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
  499 #endif
  500 }
  501 
  502 static void
  503 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
  504 {
  505         if (sc->sc_flags.clocks_off &&
  506             sc->sc_flags.port_powered) {
  507                 DPRINTFN(5, "\n");
  508 
  509                 /* TODO - platform specific */
  510 
  511                 sc->sc_flags.clocks_off = 0;
  512         }
  513 }
  514 
  515 static void
  516 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
  517 {
  518         if (!sc->sc_flags.clocks_off) {
  519                 DPRINTFN(5, "\n");
  520 
  521                 /* TODO - platform specific */
  522 
  523                 sc->sc_flags.clocks_off = 1;
  524         }
  525 }
  526 
  527 static void
  528 dwc_otg_pull_up(struct dwc_otg_softc *sc)
  529 {
  530         uint32_t temp;
  531 
  532         /* pullup D+, if possible */
  533 
  534         if (!sc->sc_flags.d_pulled_up &&
  535             sc->sc_flags.port_powered) {
  536                 sc->sc_flags.d_pulled_up = 1;
  537 
  538                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
  539                 temp &= ~DCTL_SFTDISCON;
  540                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
  541         }
  542 }
  543 
  544 static void
  545 dwc_otg_pull_down(struct dwc_otg_softc *sc)
  546 {
  547         uint32_t temp;
  548 
  549         /* pulldown D+, if possible */
  550 
  551         if (sc->sc_flags.d_pulled_up) {
  552                 sc->sc_flags.d_pulled_up = 0;
  553 
  554                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
  555                 temp |= DCTL_SFTDISCON;
  556                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
  557         }
  558 }
  559 
  560 static void
  561 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
  562 {
  563         /* In device mode we don't use the SOF interrupt */
  564         if (sc->sc_flags.status_device_mode != 0)
  565                 return;
  566         /* Ensure the SOF interrupt is not disabled */
  567         sc->sc_needsof = 1;
  568         /* Check if the SOF interrupt is already enabled */
  569         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
  570                 return;
  571         sc->sc_irq_mask |= GINTMSK_SOFMSK;
  572         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  573 }
  574 
  575 static void
  576 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
  577 {
  578         if (sc->sc_flags.status_suspend) {
  579                 /* update status bits */
  580                 sc->sc_flags.status_suspend = 0;
  581                 sc->sc_flags.change_suspend = 1;
  582 
  583                 if (sc->sc_flags.status_device_mode) {
  584                         /*
  585                          * Disable resume interrupt and enable suspend
  586                          * interrupt:
  587                          */
  588                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
  589                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
  590                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  591                 }
  592 
  593                 /* complete root HUB interrupt endpoint */
  594                 dwc_otg_root_intr(sc);
  595         }
  596 }
  597 
  598 static void
  599 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
  600 {
  601         if (!sc->sc_flags.status_suspend) {
  602                 /* update status bits */
  603                 sc->sc_flags.status_suspend = 1;
  604                 sc->sc_flags.change_suspend = 1;
  605 
  606                 if (sc->sc_flags.status_device_mode) {
  607                         /*
  608                          * Disable suspend interrupt and enable resume
  609                          * interrupt:
  610                          */
  611                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
  612                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
  613                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  614                 }
  615 
  616                 /* complete root HUB interrupt endpoint */
  617                 dwc_otg_root_intr(sc);
  618         }
  619 }
  620 
  621 static void
  622 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
  623 {
  624         if (!sc->sc_flags.status_suspend)
  625                 return;
  626 
  627         DPRINTFN(5, "Remote wakeup\n");
  628 
  629         if (sc->sc_flags.status_device_mode) {
  630                 uint32_t temp;
  631 
  632                 /* enable remote wakeup signalling */
  633                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
  634                 temp |= DCTL_RMTWKUPSIG;
  635                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
  636 
  637                 /* Wait 8ms for remote wakeup to complete. */
  638                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
  639 
  640                 temp &= ~DCTL_RMTWKUPSIG;
  641                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
  642         } else {
  643                 /* enable USB port */
  644                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
  645 
  646                 /* wait 10ms */
  647                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
  648 
  649                 /* resume port */
  650                 sc->sc_hprt_val |= HPRT_PRTRES;
  651                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
  652 
  653                 /* Wait 100ms for resume signalling to complete. */
  654                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
  655 
  656                 /* clear suspend and resume */
  657                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
  658                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
  659 
  660                 /* Wait 4ms */
  661                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
  662         }
  663 
  664         /* need to fake resume IRQ */
  665         dwc_otg_resume_irq(sc);
  666 }
  667 
  668 static void
  669 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
  670 {
  671         uint32_t temp;
  672 
  673         DPRINTFN(5, "addr=%d\n", addr);
  674 
  675         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
  676         temp &= ~DCFG_DEVADDR_SET(0x7F);
  677         temp |= DCFG_DEVADDR_SET(addr);
  678         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
  679 }
  680 
  681 static void
  682 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
  683 {
  684         DPRINTFN(5, "RX status clear\n");
  685 
  686         /* enable RX FIFO level interrupt */
  687         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
  688         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  689 
  690         if (sc->sc_current_rx_bytes != 0) {
  691                 /* need to dump remaining data */
  692                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
  693                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
  694                     sc->sc_current_rx_bytes / 4);
  695                 /* clear number of active bytes to receive */
  696                 sc->sc_current_rx_bytes = 0;
  697         }
  698         /* clear cached status */
  699         sc->sc_last_rx_status = 0;
  700 }
  701 
  702 static void
  703 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
  704 {
  705         uint32_t hcint;
  706 
  707         /* clear all pending interrupts */
  708         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
  709         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
  710 
  711         /* clear buffered interrupts */
  712         sc->sc_chan_state[x].hcint = 0;
  713 }
  714 
  715 static uint8_t
  716 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
  717 {
  718         uint32_t temp;
  719 
  720         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
  721 
  722         if (td->ep_type == UE_ISOCHRONOUS) {
  723                 /*
  724                  * NOTE: USB INTERRUPT transactions are executed like
  725                  * USB CONTROL transactions! See the setup standard
  726                  * chain function for more information.
  727                  */
  728                 if (!(temp & GINTSTS_PTXFEMP)) {
  729                         DPRINTF("Periodic TX FIFO is not empty\n");
  730                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
  731                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
  732                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  733                         }
  734                         return (1);     /* busy */
  735                 }
  736         } else {
  737                 if (!(temp & GINTSTS_NPTXFEMP)) {
  738                         DPRINTF("Non-periodic TX FIFO is not empty\n");
  739                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
  740                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
  741                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
  742                         }
  743                         return (1);     /* busy */
  744                 }
  745         }
  746         return (0);     /* ready for transmit */
  747 }
  748 
  749 static uint8_t
  750 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
  751     struct dwc_otg_td *td, uint8_t is_out)
  752 {
  753         uint8_t x;
  754         uint8_t y;
  755         uint8_t z;
  756 
  757         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
  758                 return (0);             /* already allocated */
  759 
  760         /* check if device is suspended */
  761         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
  762                 return (1);             /* busy - cannot transfer data */
  763 
  764         /* compute needed TX FIFO size */
  765         if (is_out != 0) {
  766                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
  767                         return (1);     /* busy - cannot transfer data */
  768         }
  769         z = td->max_packet_count;
  770         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
  771                 /* check if channel is allocated */
  772                 if (sc->sc_chan_state[x].allocated != 0)
  773                         continue;
  774                 /* check if channel is still enabled */
  775                 if (sc->sc_chan_state[x].wait_halted != 0)
  776                         continue;
  777                 /* store channel number */
  778                 td->channel[y++] = x;
  779                 /* check if we got all channels */
  780                 if (y == z)
  781                         break;
  782         }
  783         if (y != z) {
  784                 /* reset channel variable */
  785                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
  786                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
  787                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
  788                 /* wait a bit */
  789                 dwc_otg_enable_sof_irq(sc);
  790                 return (1);     /* busy - not enough channels */
  791         }
  792 
  793         for (y = 0; y != z; y++) {
  794                 x = td->channel[y];
  795 
  796                 /* set allocated */
  797                 sc->sc_chan_state[x].allocated = 1;
  798 
  799                 /* set wait halted */
  800                 sc->sc_chan_state[x].wait_halted = 1;
  801 
  802                 /* clear interrupts */
  803                 dwc_otg_clear_hcint(sc, x);
  804 
  805                 DPRINTF("CH=%d HCCHAR=0x%08x "
  806                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
  807 
  808                 /* set active channel */
  809                 sc->sc_active_rx_ep |= (1 << x);
  810         }
  811         return (0);     /* allocated */
  812 }
  813 
  814 static void
  815 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
  816 {
  817         uint32_t hcchar;
  818         uint8_t x;
  819 
  820         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
  821                 return;         /* already freed */
  822 
  823         /* free channel */
  824         x = td->channel[index];
  825         td->channel[index] = DWC_OTG_MAX_CHANNELS;
  826 
  827         DPRINTF("CH=%d\n", x);
  828 
  829         /*
  830          * We need to let programmed host channels run till complete
  831          * else the host channel will stop functioning.
  832          */
  833         sc->sc_chan_state[x].allocated = 0;
  834 
  835         /* ack any pending messages */
  836         if (sc->sc_last_rx_status != 0 &&
  837             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
  838                 dwc_otg_common_rx_ack(sc);
  839         }
  840 
  841         /* clear active channel */
  842         sc->sc_active_rx_ep &= ~(1 << x);
  843 
  844         /* check if already halted */
  845         if (sc->sc_chan_state[x].wait_halted == 0)
  846                 return;
  847 
  848         /* disable host channel */
  849         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
  850         if (hcchar & HCCHAR_CHENA) {
  851                 DPRINTF("Halting channel %d\n", x);
  852                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
  853                     hcchar | HCCHAR_CHDIS);
  854                 /* don't write HCCHAR until the channel is halted */
  855         } else {
  856                 sc->sc_chan_state[x].wait_halted = 0;
  857         }
  858 }
  859 
  860 static void
  861 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
  862 {
  863         uint8_t x;
  864         for (x = 0; x != td->max_packet_count; x++)
  865                 dwc_otg_host_channel_free_sub(sc, td, x);
  866 }
  867 
  868 static void
  869 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
  870 {
  871         uint8_t x;
  872         /* dump any pending messages */
  873         if (sc->sc_last_rx_status == 0)
  874                 return;
  875         for (x = 0; x != td->max_packet_count; x++) {
  876                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
  877                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
  878                         continue;
  879                 dwc_otg_common_rx_ack(sc);
  880                 break;
  881         }
  882 }
  883 
  884 static uint8_t
  885 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
  886 {
  887         struct usb_device_request req __aligned(4);
  888         uint32_t hcint;
  889         uint32_t hcchar;
  890         uint8_t delta;
  891 
  892         dwc_otg_host_dump_rx(sc, td);
  893 
  894         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
  895                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
  896 
  897                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
  898                     td->channel[0], td->state, hcint,
  899                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
  900                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
  901         } else {
  902                 hcint = 0;
  903                 goto check_state;
  904         }
  905 
  906         if (hcint & (HCINT_RETRY |
  907             HCINT_ACK | HCINT_NYET)) {
  908                 /* give success bits priority over failure bits */
  909         } else if (hcint & HCINT_STALL) {
  910                 DPRINTF("CH=%d STALL\n", td->channel[0]);
  911                 td->error_stall = 1;
  912                 td->error_any = 1;
  913                 goto complete;
  914         } else if (hcint & HCINT_ERRORS) {
  915                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
  916                 td->errcnt++;
  917                 if (td->hcsplt != 0 || td->errcnt >= 3) {
  918                         td->error_any = 1;
  919                         goto complete;
  920                 }
  921         }
  922 
  923         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
  924             HCINT_ACK | HCINT_NYET)) {
  925                 if (!(hcint & HCINT_ERRORS))
  926                         td->errcnt = 0;
  927         }
  928 
  929 check_state:
  930         switch (td->state) {
  931         case DWC_CHAN_ST_START:
  932                 goto send_pkt;
  933 
  934         case DWC_CHAN_ST_WAIT_ANE:
  935                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
  936                         td->did_nak = 1;
  937                         td->tt_scheduled = 0;
  938                         goto send_pkt;
  939                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
  940                         td->offset += td->tx_bytes;
  941                         td->remainder -= td->tx_bytes;
  942                         td->toggle = 1;
  943                         td->tt_scheduled = 0;
  944                         goto complete;
  945                 }
  946                 break;
  947 
  948         case DWC_CHAN_ST_WAIT_S_ANE:
  949                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
  950                         td->did_nak = 1;
  951                         td->tt_scheduled = 0;
  952                         goto send_pkt;
  953                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
  954                         goto send_cpkt;
  955                 }
  956                 break;
  957 
  958         case DWC_CHAN_ST_WAIT_C_ANE:
  959                 if (hcint & HCINT_NYET) {
  960                         goto send_cpkt;
  961                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
  962                         td->did_nak = 1;
  963                         td->tt_scheduled = 0;
  964                         goto send_pkt;
  965                 } else if (hcint & HCINT_ACK) {
  966                         td->offset += td->tx_bytes;
  967                         td->remainder -= td->tx_bytes;
  968                         td->toggle = 1;
  969                         goto complete;
  970                 }
  971                 break;
  972 
  973         case DWC_CHAN_ST_WAIT_C_PKT:
  974                 goto send_cpkt;
  975 
  976         default:
  977                 break;
  978         }
  979         goto busy;
  980 
  981 send_pkt:
  982         /* free existing channel, if any */
  983         dwc_otg_host_channel_free(sc, td);
  984 
  985         if (sizeof(req) != td->remainder) {
  986                 td->error_any = 1;
  987                 goto complete;
  988         }
  989 
  990         if (td->hcsplt != 0) {
  991                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
  992                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
  993                         td->state = DWC_CHAN_ST_START;
  994                         goto busy;
  995                 }
  996                 delta = sc->sc_last_frame_num - td->tt_start_slot;
  997                 if (delta > 5) {
  998                         /* missed it */
  999                         td->tt_scheduled = 0;
 1000                         td->state = DWC_CHAN_ST_START;
 1001                         goto busy;
 1002                 }
 1003         }
 1004 
 1005         /* allocate a new channel */
 1006         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 1007                 td->state = DWC_CHAN_ST_START;
 1008                 goto busy;
 1009         }
 1010 
 1011         if (td->hcsplt != 0) {
 1012                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 1013                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 1014         } else {
 1015                 td->state = DWC_CHAN_ST_WAIT_ANE;
 1016         }
 1017 
 1018         /* copy out control request */
 1019         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 1020 
 1021         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
 1022             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 1023             (1 << HCTSIZ_PKTCNT_SHIFT) |
 1024             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 1025 
 1026         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
 1027 
 1028         hcchar = td->hcchar;
 1029         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 1030         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 1031 
 1032         /* must enable channel before writing data to FIFO */
 1033         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
 1034 
 1035         /* transfer data into FIFO */
 1036         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 1037             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
 1038 
 1039         /* wait until next slot before trying complete split */
 1040         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 1041 
 1042         /* store number of bytes transmitted */
 1043         td->tx_bytes = sizeof(req);
 1044         goto busy;
 1045 
 1046 send_cpkt:
 1047         /* free existing channel, if any */
 1048         dwc_otg_host_channel_free(sc, td);
 1049 
 1050         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 1051         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 1052                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 1053                 goto busy;
 1054         }
 1055         delta = sc->sc_last_frame_num - td->tt_start_slot;
 1056         if (delta > DWC_OTG_TT_SLOT_MAX) {
 1057                 /* we missed the service interval */
 1058                 if (td->ep_type != UE_ISOCHRONOUS)
 1059                         td->error_any = 1;
 1060                 goto complete;
 1061         }
 1062         /* allocate a new channel */
 1063         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 1064                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 1065                 goto busy;
 1066         }
 1067 
 1068         /* wait until next slot before trying complete split */
 1069         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 1070 
 1071         td->hcsplt |= HCSPLT_COMPSPLT;
 1072         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 1073 
 1074         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
 1075             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 1076 
 1077         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
 1078 
 1079         hcchar = td->hcchar;
 1080         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 1081         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 1082 
 1083         /* must enable channel before writing data to FIFO */
 1084         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
 1085 
 1086 busy:
 1087         return (1);     /* busy */
 1088 
 1089 complete:
 1090         dwc_otg_host_channel_free(sc, td);
 1091         return (0);     /* complete */
 1092 }
 1093 
 1094 static uint8_t
 1095 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1096 {
 1097         struct usb_device_request req __aligned(4);
 1098         uint32_t temp;
 1099         uint16_t count;
 1100 
 1101         /* check endpoint status */
 1102 
 1103         if (sc->sc_last_rx_status == 0)
 1104                 goto not_complete;
 1105 
 1106         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 1107                 goto not_complete;
 1108 
 1109         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 1110             GRXSTSRD_STP_DATA) {
 1111                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 1112                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
 1113                         /* release FIFO */
 1114                         dwc_otg_common_rx_ack(sc);
 1115                         goto not_complete;
 1116                 }
 1117                 /* release FIFO */
 1118                 dwc_otg_common_rx_ack(sc);
 1119                 return (0);     /* complete */
 1120         }
 1121 
 1122         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 1123             GRXSTSRD_DPID_DATA0) {
 1124                 /* release FIFO */
 1125                 dwc_otg_common_rx_ack(sc);
 1126                 goto not_complete;
 1127         }
 1128 
 1129         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
 1130 
 1131         /* clear did stall */
 1132         td->did_stall = 0;
 1133 
 1134         /* get the packet byte count */
 1135         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 1136 
 1137         if (count != sizeof(req)) {
 1138                 DPRINTFN(0, "Unsupported SETUP packet "
 1139                     "length, %d bytes\n", count);
 1140                 /* release FIFO */
 1141                 dwc_otg_common_rx_ack(sc);
 1142                 goto not_complete;
 1143         }
 1144 
 1145         /* read FIFO */
 1146         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
 1147 
 1148         /* copy out control request */
 1149         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 1150 
 1151         td->offset = sizeof(req);
 1152         td->remainder = 0;
 1153 
 1154         /* sneak peek the set address */
 1155         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
 1156             (req.bRequest == UR_SET_ADDRESS)) {
 1157                 /* must write address before ZLP */
 1158                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
 1159         }
 1160 
 1161         /* don't send any data by default */
 1162         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
 1163         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
 1164 
 1165         /* reset IN endpoint buffer */
 1166         dwc_otg_tx_fifo_reset(sc,
 1167             GRSTCTL_TXFIFO(0) |
 1168             GRSTCTL_TXFFLSH);
 1169 
 1170         /* acknowledge RX status */
 1171         dwc_otg_common_rx_ack(sc);
 1172         td->did_stall = 1;
 1173 
 1174 not_complete:
 1175         /* abort any ongoing transfer, before enabling again */
 1176         if (!td->did_stall) {
 1177                 td->did_stall = 1;
 1178 
 1179                 DPRINTFN(5, "stalling IN and OUT direction\n");
 1180 
 1181                 temp = sc->sc_out_ctl[0];
 1182 
 1183                 /* set stall after enabling endpoint */
 1184                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
 1185                     temp | DOEPCTL_STALL);
 1186 
 1187                 temp = sc->sc_in_ctl[0];
 1188 
 1189                 /* set stall assuming endpoint is enabled */
 1190                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
 1191                     temp | DIEPCTL_STALL);
 1192         }
 1193         return (1);                     /* not complete */
 1194 }
 1195 
 1196 static uint8_t
 1197 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1198 {
 1199         uint8_t delta;
 1200 
 1201         delta = sc->sc_tmr_val - td->tmr_val;
 1202         if (delta >= 128)
 1203                 return (1);     /* busy */
 1204 
 1205         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
 1206 
 1207         /* set toggle, if any */
 1208         if (td->set_toggle) {
 1209                 td->set_toggle = 0;
 1210                 td->toggle = 1;
 1211         }
 1212         return (0);
 1213 }
 1214 
 1215 static uint8_t
 1216 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1217 {
 1218         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
 1219 
 1220         if (td->ep_type == UE_ISOCHRONOUS) {
 1221                 /* non TT isochronous traffic */
 1222                 if (frame_num & (td->tmr_res - 1))
 1223                         goto busy;
 1224                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
 1225                         goto busy;
 1226                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
 1227                 td->toggle = 0;
 1228                 return (0);
 1229         } else if (td->ep_type == UE_INTERRUPT) {
 1230                 if (!td->tt_scheduled)
 1231                         goto busy;
 1232                 td->tt_scheduled = 0;
 1233                 return (0);
 1234         } else if (td->did_nak != 0) {
 1235                 /* check if we should pause sending queries for 125us */
 1236                 if (td->tmr_res == frame_num) {
 1237                         /* wait a bit */
 1238                         dwc_otg_enable_sof_irq(sc);
 1239                         goto busy;
 1240                 }
 1241         } else if (td->set_toggle) {
 1242                 td->set_toggle = 0;
 1243                 td->toggle = 1;
 1244         }
 1245         /* query for data one more time */
 1246         td->tmr_res = frame_num;
 1247         td->did_nak = 0;
 1248         return (0);
 1249 busy:
 1250         return (1);
 1251 }
 1252 
 1253 static uint8_t
 1254 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
 1255     uint8_t channel)
 1256 {
 1257         uint32_t count;
 1258 
 1259         /* check endpoint status */
 1260         if (sc->sc_last_rx_status == 0)
 1261                 goto busy;
 1262 
 1263         if (channel >= DWC_OTG_MAX_CHANNELS)
 1264                 goto busy;
 1265 
 1266         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
 1267                 goto busy;
 1268 
 1269         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
 1270         case GRXSTSRH_IN_DATA:
 1271 
 1272                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
 1273                     (int)td->state, (int)sc->sc_last_rx_status);
 1274 
 1275                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
 1276                         /*
 1277                          * When using SPLIT transactions on interrupt
 1278                          * endpoints, sometimes data occurs twice.
 1279                          */
 1280                         DPRINTF("Data already received\n");
 1281                         break;
 1282                 }
 1283 
 1284                 /* get the packet byte count */
 1285                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 1286 
 1287                 /* check for ISOCHRONOUS endpoint */
 1288                 if (td->ep_type == UE_ISOCHRONOUS) {
 1289                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 1290                             GRXSTSRD_DPID_DATA0) {
 1291                                 /* more data to be received */
 1292                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
 1293                         } else {
 1294                                 /* all data received */
 1295                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
 1296                                 /* verify the packet byte count */
 1297                                 if (count != td->remainder) {
 1298                                         /* we have a short packet */
 1299                                         td->short_pkt = 1;
 1300                                         td->got_short = 1;
 1301                                 }
 1302                         }
 1303                 } else {
 1304                         /* verify the packet byte count */
 1305                         if (count != td->max_packet_size) {
 1306                                 if (count < td->max_packet_size) {
 1307                                         /* we have a short packet */
 1308                                         td->short_pkt = 1;
 1309                                         td->got_short = 1;
 1310                                 } else {
 1311                                         /* invalid USB packet */
 1312                                         td->error_any = 1;
 1313                           
 1314                                         /* release FIFO */
 1315                                         dwc_otg_common_rx_ack(sc);
 1316                                         goto complete;
 1317                                 }
 1318                         }
 1319                         td->toggle ^= 1;
 1320                         td->tt_scheduled = 0;
 1321                 }
 1322 
 1323                 /* verify the packet byte count */
 1324                 if (count > td->remainder) {
 1325                         /* invalid USB packet */
 1326                         td->error_any = 1;
 1327 
 1328                         /* release FIFO */
 1329                         dwc_otg_common_rx_ack(sc);
 1330                         goto complete;
 1331                 }
 1332 
 1333                 /* read data from FIFO */
 1334                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
 1335 
 1336                 td->remainder -= count;
 1337                 td->offset += count;
 1338                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
 1339                 break;
 1340         default:
 1341                 break;
 1342         }
 1343         /* release FIFO */
 1344         dwc_otg_common_rx_ack(sc);
 1345 busy:
 1346         return (0);
 1347 complete:
 1348         return (1);
 1349 }
 1350 
 1351 static uint8_t
 1352 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1353 {
 1354         uint32_t hcint = 0;
 1355         uint32_t hcchar;
 1356         uint8_t delta;
 1357         uint8_t channel;
 1358         uint8_t x;
 1359 
 1360         for (x = 0; x != td->max_packet_count; x++) {
 1361                 channel = td->channel[x];
 1362                 if (channel >= DWC_OTG_MAX_CHANNELS)
 1363                         continue;
 1364                 hcint |= sc->sc_chan_state[channel].hcint;
 1365 
 1366                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 1367                     channel, td->state, hcint,
 1368                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
 1369                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
 1370 
 1371                 /* check interrupt bits */
 1372                 if (hcint & (HCINT_RETRY |
 1373                     HCINT_ACK | HCINT_NYET)) {
 1374                         /* give success bits priority over failure bits */
 1375                 } else if (hcint & HCINT_STALL) {
 1376                         DPRINTF("CH=%d STALL\n", channel);
 1377                         td->error_stall = 1;
 1378                         td->error_any = 1;
 1379                         goto complete;
 1380                 } else if (hcint & HCINT_ERRORS) {
 1381                         DPRINTF("CH=%d ERROR\n", channel);
 1382                         td->errcnt++;
 1383                         if (td->hcsplt != 0 || td->errcnt >= 3) {
 1384                                 if (td->ep_type != UE_ISOCHRONOUS) {
 1385                                         td->error_any = 1;
 1386                                         goto complete;
 1387                                 }
 1388                         }
 1389                 }
 1390 
 1391                 /* check channels for data, if any */
 1392                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
 1393                         goto complete;
 1394 
 1395                 /* refresh interrupt status */
 1396                 hcint |= sc->sc_chan_state[channel].hcint;
 1397 
 1398                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 1399                     HCINT_ACK | HCINT_NYET)) {
 1400                         if (!(hcint & HCINT_ERRORS))
 1401                                 td->errcnt = 0;
 1402                 }
 1403         }
 1404 
 1405         switch (td->state) {
 1406         case DWC_CHAN_ST_START:
 1407                 if (td->hcsplt != 0)
 1408                         goto receive_spkt;
 1409                 else
 1410                         goto receive_pkt;
 1411 
 1412         case DWC_CHAN_ST_WAIT_ANE:
 1413                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 1414                         if (td->ep_type == UE_INTERRUPT) {
 1415                                 /*
 1416                                  * The USB specification does not
 1417                                  * mandate a particular data toggle
 1418                                  * value for USB INTERRUPT
 1419                                  * transfers. Switch the data toggle
 1420                                  * value to receive the packet
 1421                                  * correctly:
 1422                                  */
 1423                                 if (hcint & HCINT_DATATGLERR) {
 1424                                         DPRINTF("Retrying packet due to "
 1425                                             "data toggle error\n");
 1426                                         td->toggle ^= 1;
 1427                                         goto receive_pkt;
 1428                                 }
 1429                         } else if (td->ep_type == UE_ISOCHRONOUS) {
 1430                                 if (td->hcsplt != 0) {
 1431                                         /*
 1432                                          * Sometimes the complete
 1433                                          * split packet may be queued
 1434                                          * too early and the
 1435                                          * transaction translator will
 1436                                          * return a NAK. Ignore
 1437                                          * this message and retry the
 1438                                          * complete split instead.
 1439                                          */
 1440                                         DPRINTF("Retrying complete split\n");
 1441                                         goto receive_pkt;
 1442                                 }
 1443                                 goto complete;
 1444                         }
 1445                         td->did_nak = 1;
 1446                         td->tt_scheduled = 0;
 1447                         if (td->hcsplt != 0)
 1448                                 goto receive_spkt;
 1449                         else
 1450                                 goto receive_pkt;
 1451                 } else if (hcint & HCINT_NYET) {
 1452                         if (td->hcsplt != 0) {
 1453                                 /* try again */
 1454                                 goto receive_pkt;
 1455                         } else {
 1456                                 /* not a valid token for IN endpoints */
 1457                                 td->error_any = 1;
 1458                                 goto complete;
 1459                         }
 1460                 } else if (hcint & HCINT_ACK) {
 1461                         /* wait for data - ACK arrived first */
 1462                         if (!(hcint & HCINT_SOFTWARE_ONLY))
 1463                                 goto busy;
 1464 
 1465                         if (td->ep_type == UE_ISOCHRONOUS) {
 1466                                 /* check if we are complete */
 1467                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
 1468                                         goto complete;
 1469                                 } else if (td->hcsplt != 0) {
 1470                                         goto receive_pkt;
 1471                                 } else {
 1472                                         /* get more packets */
 1473                                         goto busy;
 1474                                 }
 1475                         } else {
 1476                                 /* check if we are complete */
 1477                                 if ((td->remainder == 0) || (td->got_short != 0)) {
 1478                                         if (td->short_pkt)
 1479                                                 goto complete;
 1480 
 1481                                         /*
 1482                                          * Else need to receive a zero length
 1483                                          * packet.
 1484                                          */
 1485                                 }
 1486                                 td->tt_scheduled = 0;
 1487                                 td->did_nak = 0;
 1488                                 if (td->hcsplt != 0)
 1489                                         goto receive_spkt;
 1490                                 else
 1491                                         goto receive_pkt;
 1492                         }
 1493                 }
 1494                 break;
 1495 
 1496         case DWC_CHAN_ST_WAIT_S_ANE:
 1497                 /*
 1498                  * NOTE: The DWC OTG hardware provides a fake ACK in
 1499                  * case of interrupt and isochronous transfers:
 1500                  */ 
 1501                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 1502                         td->did_nak = 1;
 1503                         td->tt_scheduled = 0;
 1504                         goto receive_spkt;
 1505                 } else if (hcint & HCINT_NYET) {
 1506                         td->tt_scheduled = 0;
 1507                         goto receive_spkt;
 1508                 } else if (hcint & HCINT_ACK) {
 1509                         td->did_nak = 0;
 1510                         goto receive_pkt;
 1511                 }
 1512                 break;
 1513 
 1514         case DWC_CHAN_ST_WAIT_C_PKT:
 1515                 goto receive_pkt;
 1516 
 1517         default:
 1518                 break;
 1519         }
 1520         goto busy;
 1521 
 1522 receive_pkt:
 1523         /* free existing channel, if any */
 1524         dwc_otg_host_channel_free(sc, td);
 1525 
 1526         if (td->hcsplt != 0) {
 1527                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 1528                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 1529                         if (td->ep_type != UE_ISOCHRONOUS) {
 1530                                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 1531                                 goto busy;
 1532                         }
 1533                 }
 1534                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 1535                 if (delta > DWC_OTG_TT_SLOT_MAX) {
 1536                         if (td->ep_type != UE_ISOCHRONOUS) {
 1537                                 /* we missed the service interval */
 1538                                 td->error_any = 1;
 1539                         }
 1540                         goto complete;
 1541                 }
 1542                 /* complete split */
 1543                 td->hcsplt |= HCSPLT_COMPSPLT;
 1544         } else if (dwc_otg_host_rate_check(sc, td)) {
 1545                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 1546                 goto busy;
 1547         }
 1548 
 1549         /* allocate a new channel */
 1550         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 1551                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 1552                 goto busy;
 1553         }
 1554 
 1555         /* set toggle, if any */
 1556         if (td->set_toggle) {
 1557                 td->set_toggle = 0;
 1558                 td->toggle = 1;
 1559         }
 1560 
 1561         td->state = DWC_CHAN_ST_WAIT_ANE;
 1562 
 1563         for (x = 0; x != td->max_packet_count; x++) {
 1564                 channel = td->channel[x];
 1565 
 1566                 /* receive one packet */
 1567                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 1568                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
 1569                     (1 << HCTSIZ_PKTCNT_SHIFT) |
 1570                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
 1571                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
 1572 
 1573                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
 1574 
 1575                 hcchar = td->hcchar;
 1576                 hcchar |= HCCHAR_EPDIR_IN;
 1577 
 1578                 if (td->ep_type == UE_ISOCHRONOUS) {
 1579                         if (td->hcsplt != 0) {
 1580                                 /* continously buffer */
 1581                                 if (sc->sc_last_frame_num & 1)
 1582                                         hcchar &= ~HCCHAR_ODDFRM;
 1583                                 else
 1584                                         hcchar |= HCCHAR_ODDFRM;
 1585                         } else {
 1586                                 /* multi buffer, if any */
 1587                                 if (sc->sc_last_frame_num & 1)
 1588                                         hcchar |= HCCHAR_ODDFRM;
 1589                                 else
 1590                                         hcchar &= ~HCCHAR_ODDFRM;
 1591                         }
 1592                 } else {
 1593                         hcchar &= ~HCCHAR_ODDFRM;
 1594                 }
 1595 
 1596                 /* must enable channel before data can be received */
 1597                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
 1598         }
 1599         /* wait until next slot before trying complete split */
 1600         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 1601 
 1602         goto busy;
 1603 
 1604 receive_spkt:
 1605         /* free existing channel(s), if any */
 1606         dwc_otg_host_channel_free(sc, td);
 1607 
 1608         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 1609         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 1610                 td->state = DWC_CHAN_ST_START;
 1611                 goto busy;
 1612         }
 1613         delta = sc->sc_last_frame_num - td->tt_start_slot;
 1614         if (delta > 5) {
 1615                 /* missed it */
 1616                 td->tt_scheduled = 0;
 1617                 td->state = DWC_CHAN_ST_START;
 1618                 goto busy;
 1619         }
 1620 
 1621         /* allocate a new channel */
 1622         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 1623                 td->state = DWC_CHAN_ST_START;
 1624                 goto busy;
 1625         }
 1626 
 1627         channel = td->channel[0];
 1628 
 1629         td->hcsplt &= ~HCSPLT_COMPSPLT;
 1630         td->state = DWC_CHAN_ST_WAIT_S_ANE;
 1631 
 1632         /* receive one packet */
 1633         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 1634             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
 1635 
 1636         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
 1637 
 1638         /* send after next SOF event */
 1639         if ((sc->sc_last_frame_num & 1) == 0 &&
 1640             td->ep_type == UE_ISOCHRONOUS)
 1641                 td->hcchar |= HCCHAR_ODDFRM;
 1642         else
 1643                 td->hcchar &= ~HCCHAR_ODDFRM;
 1644 
 1645         hcchar = td->hcchar;
 1646         hcchar |= HCCHAR_EPDIR_IN;
 1647 
 1648         /* wait until next slot before trying complete split */
 1649         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 1650 
 1651         /* must enable channel before data can be received */
 1652         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
 1653 busy:
 1654         return (1);     /* busy */
 1655 
 1656 complete:
 1657         dwc_otg_host_channel_free(sc, td);
 1658         return (0);     /* complete */
 1659 }
 1660 
 1661 static uint8_t
 1662 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1663 {
 1664         uint32_t temp;
 1665         uint16_t count;
 1666         uint8_t got_short;
 1667 
 1668         got_short = 0;
 1669 
 1670         /* check endpoint status */
 1671         if (sc->sc_last_rx_status == 0)
 1672                 goto not_complete;
 1673 
 1674         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
 1675                 goto not_complete;
 1676 
 1677         /* check for SETUP packet */
 1678         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
 1679             GRXSTSRD_STP_DATA ||
 1680             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
 1681             GRXSTSRD_STP_COMPLETE) {
 1682                 if (td->remainder == 0) {
 1683                         /*
 1684                          * We are actually complete and have
 1685                          * received the next SETUP
 1686                          */
 1687                         DPRINTFN(5, "faking complete\n");
 1688                         return (0);     /* complete */
 1689                 }
 1690                 /*
 1691                  * USB Host Aborted the transfer.
 1692                  */
 1693                 td->error_any = 1;
 1694                 return (0);             /* complete */
 1695         }
 1696 
 1697         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 1698             GRXSTSRD_OUT_DATA) {
 1699                 /* release FIFO */
 1700                 dwc_otg_common_rx_ack(sc);
 1701                 goto not_complete;
 1702         }
 1703 
 1704         /* get the packet byte count */
 1705         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 1706 
 1707         /* verify the packet byte count */
 1708         if (count != td->max_packet_size) {
 1709                 if (count < td->max_packet_size) {
 1710                         /* we have a short packet */
 1711                         td->short_pkt = 1;
 1712                         got_short = 1;
 1713                 } else {
 1714                         /* invalid USB packet */
 1715                         td->error_any = 1;
 1716 
 1717                         /* release FIFO */
 1718                         dwc_otg_common_rx_ack(sc);
 1719                         return (0);     /* we are complete */
 1720                 }
 1721         }
 1722         /* verify the packet byte count */
 1723         if (count > td->remainder) {
 1724                 /* invalid USB packet */
 1725                 td->error_any = 1;
 1726 
 1727                 /* release FIFO */
 1728                 dwc_otg_common_rx_ack(sc);
 1729                 return (0);             /* we are complete */
 1730         }
 1731 
 1732         /* read data from FIFO */
 1733         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
 1734 
 1735         td->remainder -= count;
 1736         td->offset += count;
 1737 
 1738         /* release FIFO */
 1739         dwc_otg_common_rx_ack(sc);
 1740 
 1741         temp = sc->sc_out_ctl[td->ep_no];
 1742 
 1743         /* check for isochronous mode */
 1744         if ((temp & DIEPCTL_EPTYPE_MASK) ==
 1745             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
 1746                 /* toggle odd or even frame bit */
 1747                 if (temp & DIEPCTL_SETD1PID) {
 1748                         temp &= ~DIEPCTL_SETD1PID;
 1749                         temp |= DIEPCTL_SETD0PID;
 1750                 } else {
 1751                         temp &= ~DIEPCTL_SETD0PID;
 1752                         temp |= DIEPCTL_SETD1PID;
 1753                 }
 1754                 sc->sc_out_ctl[td->ep_no] = temp;
 1755         }
 1756 
 1757         /* check if we are complete */
 1758         if ((td->remainder == 0) || got_short) {
 1759                 if (td->short_pkt) {
 1760                         /* we are complete */
 1761                         return (0);
 1762                 }
 1763                 /* else need to receive a zero length packet */
 1764         }
 1765 
 1766 not_complete:
 1767 
 1768         /* enable SETUP and transfer complete interrupt */
 1769         if (td->ep_no == 0) {
 1770                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
 1771                     DXEPTSIZ_SET_MULTI(3) |
 1772                     DXEPTSIZ_SET_NPKT(1) | 
 1773                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
 1774         } else {
 1775                 /* allow reception of multiple packets */
 1776                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
 1777                     DXEPTSIZ_SET_MULTI(1) |
 1778                     DXEPTSIZ_SET_NPKT(4) | 
 1779                     DXEPTSIZ_SET_NBYTES(4 *
 1780                         ((td->max_packet_size + 3) & ~3)));
 1781         }
 1782         temp = sc->sc_out_ctl[td->ep_no];
 1783         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
 1784             DOEPCTL_EPENA | DOEPCTL_CNAK);
 1785 
 1786         return (1);                     /* not complete */
 1787 }
 1788 
 1789 static uint8_t
 1790 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 1791 {
 1792         uint32_t count;
 1793         uint32_t hcint;
 1794         uint32_t hcchar;
 1795         uint8_t delta;
 1796         uint8_t channel;
 1797         uint8_t x;
 1798 
 1799         dwc_otg_host_dump_rx(sc, td);
 1800 
 1801         /* check that last channel is complete */
 1802         channel = td->channel[td->npkt];
 1803 
 1804         if (channel < DWC_OTG_MAX_CHANNELS) {
 1805                 hcint = sc->sc_chan_state[channel].hcint;
 1806 
 1807                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 1808                     channel, td->state, hcint,
 1809                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
 1810                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
 1811 
 1812                 if (hcint & (HCINT_RETRY |
 1813                     HCINT_ACK | HCINT_NYET)) {
 1814                         /* give success bits priority over failure bits */
 1815                 } else if (hcint & HCINT_STALL) {
 1816                         DPRINTF("CH=%d STALL\n", channel);
 1817                         td->error_stall = 1;
 1818                         td->error_any = 1;
 1819                         goto complete;
 1820                 } else if (hcint & HCINT_ERRORS) {
 1821                         DPRINTF("CH=%d ERROR\n", channel);
 1822                         td->errcnt++;
 1823                         if (td->hcsplt != 0 || td->errcnt >= 3) {
 1824                                 td->error_any = 1;
 1825                                 goto complete;
 1826                         }
 1827                 }
 1828 
 1829                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 1830                     HCINT_ACK | HCINT_NYET)) {
 1831                         if (!(hcint & HCINT_ERRORS))
 1832                                 td->errcnt = 0;
 1833                 }
 1834         } else {
 1835                 hcint = 0;
 1836         }
 1837 
 1838         switch (td->state) {
 1839         case DWC_CHAN_ST_START:
 1840                 goto send_pkt;
 1841 
 1842         case DWC_CHAN_ST_WAIT_ANE:
 1843                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 1844                         td->did_nak = 1;
 1845                         td->tt_scheduled = 0;
 1846                         goto send_pkt;
 1847                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 1848                         td->offset += td->tx_bytes;
 1849                         td->remainder -= td->tx_bytes;
 1850                         td->toggle ^= 1;
 1851                         /* check if next response will be a NAK */
 1852                         if (hcint & HCINT_NYET)
 1853                                 td->did_nak = 1;
 1854                         else
 1855                                 td->did_nak = 0;
 1856                         td->tt_scheduled = 0;
 1857 
 1858                         /* check remainder */
 1859                         if (td->remainder == 0) {
 1860                                 if (td->short_pkt)
 1861                                         goto complete;
 1862 
 1863                                 /*
 1864                                  * Else we need to transmit a short
 1865                                  * packet:
 1866                                  */
 1867                         }
 1868                         goto send_pkt;
 1869                 }
 1870                 break;
 1871 
 1872         case DWC_CHAN_ST_WAIT_S_ANE:
 1873                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 1874                         td->did_nak = 1;
 1875                         td->tt_scheduled = 0;
 1876                         goto send_pkt;
 1877                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 1878                         td->did_nak = 0;
 1879                         goto send_cpkt;
 1880                 }
 1881                 break;
 1882 
 1883         case DWC_CHAN_ST_WAIT_C_ANE:
 1884                 if (hcint & HCINT_NYET) {
 1885                         goto send_cpkt;
 1886                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 1887                         td->did_nak = 1;
 1888                         td->tt_scheduled = 0;
 1889                         goto send_pkt;
 1890                 } else if (hcint & HCINT_ACK) {
 1891                         td->offset += td->tx_bytes;
 1892                         td->remainder -= td->tx_bytes;
 1893                         td->toggle ^= 1;
 1894                         td->did_nak = 0;
 1895                         td->tt_scheduled = 0;
 1896 
 1897                         /* check remainder */
 1898                         if (td->remainder == 0) {
 1899                                 if (td->short_pkt)
 1900                                         goto complete;
 1901 
 1902                                 /* else we need to transmit a short packet */
 1903                         }
 1904                         goto send_pkt;
 1905                 }
 1906                 break;
 1907 
 1908         case DWC_CHAN_ST_WAIT_C_PKT:
 1909                 goto send_cpkt;
 1910 
 1911         case DWC_CHAN_ST_TX_WAIT_ISOC:
 1912                 /* Check if ISOCHRONOUS OUT traffic is complete */
 1913                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
 1914                         break;
 1915 
 1916                 td->offset += td->tx_bytes;
 1917                 td->remainder -= td->tx_bytes;
 1918                 goto complete;
 1919         default:
 1920                 break;
 1921         }
 1922         goto busy;
 1923 
 1924 send_pkt:
 1925         /* free existing channel(s), if any */
 1926         dwc_otg_host_channel_free(sc, td);
 1927 
 1928         if (td->hcsplt != 0) {
 1929                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 1930                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 1931                         td->state = DWC_CHAN_ST_START;
 1932                         goto busy;
 1933                 }
 1934                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 1935                 if (delta > 5) {
 1936                         /* missed it */
 1937                         td->tt_scheduled = 0;
 1938                         td->state = DWC_CHAN_ST_START;
 1939                         goto busy;
 1940                 }
 1941         } else if (dwc_otg_host_rate_check(sc, td)) {
 1942                 td->state = DWC_CHAN_ST_START;
 1943                 goto busy;
 1944         }
 1945 
 1946         /* allocate a new channel */
 1947         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 1948                 td->state = DWC_CHAN_ST_START;
 1949                 goto busy;
 1950         }
 1951 
 1952         /* set toggle, if any */
 1953         if (td->set_toggle) {
 1954                 td->set_toggle = 0;
 1955                 td->toggle = 1;
 1956         }
 1957 
 1958         if (td->ep_type == UE_ISOCHRONOUS) {
 1959                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
 1960                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
 1961                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 1962                 if (td->hcsplt != 0) {
 1963                         /* get maximum transfer length */
 1964                         count = td->remainder;
 1965                         if (count > HCSPLT_XACTLEN_BURST) {
 1966                                 DPRINTF("TT overflow\n");
 1967                                 td->error_any = 1;
 1968                                 goto complete;
 1969                         }
 1970                         /* Update transaction position */
 1971                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
 1972                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
 1973                 }
 1974         } else if (td->hcsplt != 0) {
 1975                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 1976                 /* Wait for ACK/NAK/ERR from TT */
 1977                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 1978         } else {
 1979                 /* Wait for ACK/NAK/STALL from device */
 1980                 td->state = DWC_CHAN_ST_WAIT_ANE;
 1981         }
 1982 
 1983         td->tx_bytes = 0;
 1984 
 1985         for (x = 0; x != td->max_packet_count; x++) {
 1986                 uint32_t rem_bytes;
 1987 
 1988                 channel = td->channel[x];
 1989 
 1990                 /* send one packet at a time */
 1991                 count = td->max_packet_size;
 1992                 rem_bytes = td->remainder - td->tx_bytes;
 1993                 if (rem_bytes < count) {
 1994                         /* we have a short packet */
 1995                         td->short_pkt = 1;
 1996                         count = rem_bytes;
 1997                 }
 1998                 if (count == rem_bytes) {
 1999                         /* last packet */
 2000                         switch (x) {
 2001                         case 0:
 2002                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2003                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
 2004                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
 2005                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
 2006                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
 2007                                 break;
 2008                         case 1:
 2009                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2010                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
 2011                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
 2012                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
 2013                                 break;
 2014                         default:
 2015                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2016                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
 2017                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
 2018                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
 2019                                 break;
 2020                         }
 2021                 } else if (td->ep_type == UE_ISOCHRONOUS &&
 2022                            td->max_packet_count > 1) {
 2023                         /* ISOCHRONOUS multi packet */
 2024                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2025                             (count << HCTSIZ_XFERSIZE_SHIFT) |
 2026                             (1 << HCTSIZ_PKTCNT_SHIFT) |
 2027                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
 2028                 } else {
 2029                         /* TODO: HCTSIZ_DOPNG */
 2030                         /* standard BULK/INTERRUPT/CONTROL packet */
 2031                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2032                             (count << HCTSIZ_XFERSIZE_SHIFT) |
 2033                             (1 << HCTSIZ_PKTCNT_SHIFT) |
 2034                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
 2035                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
 2036                 }
 2037 
 2038                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
 2039 
 2040                 hcchar = td->hcchar;
 2041                 hcchar &= ~HCCHAR_EPDIR_IN;
 2042 
 2043                 /* send after next SOF event */
 2044                 if ((sc->sc_last_frame_num & 1) == 0 &&
 2045                     td->ep_type == UE_ISOCHRONOUS)
 2046                         hcchar |= HCCHAR_ODDFRM;
 2047                 else
 2048                         hcchar &= ~HCCHAR_ODDFRM;
 2049 
 2050                 /* must enable before writing data to FIFO */
 2051                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
 2052 
 2053                 if (count != 0) {
 2054                         /* write data into FIFO */
 2055                         dwc_otg_write_fifo(sc, td->pc, td->offset +
 2056                             td->tx_bytes, DOTG_DFIFO(channel), count);
 2057                 }
 2058 
 2059                 /* store number of bytes transmitted */
 2060                 td->tx_bytes += count;
 2061 
 2062                 /* store last packet index */
 2063                 td->npkt = x;
 2064                 
 2065                 /* check for last packet */
 2066                 if (count == rem_bytes)
 2067                         break;
 2068         }
 2069         goto busy;
 2070 
 2071 send_cpkt:
 2072         /* free existing channel, if any */
 2073         dwc_otg_host_channel_free(sc, td);
 2074 
 2075         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 2076         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 2077                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 2078                 goto busy;
 2079         }
 2080         delta = sc->sc_last_frame_num - td->tt_start_slot;
 2081         if (delta > DWC_OTG_TT_SLOT_MAX) {
 2082                 /* we missed the service interval */
 2083                 if (td->ep_type != UE_ISOCHRONOUS)
 2084                         td->error_any = 1;
 2085                 goto complete;
 2086         }
 2087 
 2088         /* allocate a new channel */
 2089         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 2090                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 2091                 goto busy;
 2092         }
 2093 
 2094         channel = td->channel[0];
 2095 
 2096         td->hcsplt |= HCSPLT_COMPSPLT;
 2097         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 2098 
 2099         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
 2100             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
 2101 
 2102         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
 2103 
 2104         hcchar = td->hcchar;
 2105         hcchar &= ~HCCHAR_EPDIR_IN;
 2106 
 2107         /* receive complete split ASAP */
 2108         if ((sc->sc_last_frame_num & 1) != 0 &&
 2109             td->ep_type == UE_ISOCHRONOUS)
 2110                 hcchar |= HCCHAR_ODDFRM;
 2111         else
 2112                 hcchar &= ~HCCHAR_ODDFRM;
 2113 
 2114         /* must enable channel before data can be received */
 2115         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
 2116 
 2117         /* wait until next slot before trying complete split */
 2118         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 2119 busy:
 2120         return (1);     /* busy */
 2121 
 2122 complete:
 2123         dwc_otg_host_channel_free(sc, td);
 2124         return (0);     /* complete */
 2125 }
 2126 
 2127 static uint8_t
 2128 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 2129 {
 2130         uint32_t max_buffer;
 2131         uint32_t count;
 2132         uint32_t fifo_left;
 2133         uint32_t mpkt;
 2134         uint32_t temp;
 2135         uint8_t to;
 2136 
 2137         to = 3;                         /* don't loop forever! */
 2138 
 2139         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
 2140 
 2141 repeat:
 2142         /* check for endpoint 0 data */
 2143 
 2144         temp = sc->sc_last_rx_status;
 2145 
 2146         if ((td->ep_no == 0) && (temp != 0) &&
 2147             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
 2148                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
 2149                     GRXSTSRD_STP_DATA &&
 2150                     (temp & GRXSTSRD_PKTSTS_MASK) !=
 2151                     GRXSTSRD_STP_COMPLETE) {
 2152                         /* dump data - wrong direction */
 2153                         dwc_otg_common_rx_ack(sc);
 2154                 } else {
 2155                         /*
 2156                          * The current transfer was cancelled
 2157                          * by the USB Host:
 2158                          */
 2159                         td->error_any = 1;
 2160                         return (0);             /* complete */
 2161                 }
 2162         }
 2163 
 2164         /* fill in more TX data, if possible */
 2165         if (td->tx_bytes != 0) {
 2166                 uint16_t cpkt;
 2167 
 2168                 /* check if packets have been transferred */
 2169                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
 2170 
 2171                 /* get current packet number */
 2172                 cpkt = DXEPTSIZ_GET_NPKT(temp);
 2173 
 2174                 if (cpkt >= td->npkt) {
 2175                         fifo_left = 0;
 2176                 } else {
 2177                         if (max_buffer != 0) {
 2178                                 fifo_left = (td->npkt - cpkt) *
 2179                                     td->max_packet_size;
 2180 
 2181                                 if (fifo_left > max_buffer)
 2182                                         fifo_left = max_buffer;
 2183                         } else {
 2184                                 fifo_left = td->max_packet_size;
 2185                         }
 2186                 }
 2187 
 2188                 count = td->tx_bytes;
 2189                 if (count > fifo_left)
 2190                         count = fifo_left;
 2191 
 2192                 if (count != 0) {
 2193                         /* write data into FIFO */
 2194                         dwc_otg_write_fifo(sc, td->pc, td->offset,
 2195                             DOTG_DFIFO(td->ep_no), count);
 2196 
 2197                         td->tx_bytes -= count;
 2198                         td->remainder -= count;
 2199                         td->offset += count;
 2200                         td->npkt = cpkt;
 2201                 }
 2202                 if (td->tx_bytes != 0)
 2203                         goto not_complete;
 2204 
 2205                 /* check remainder */
 2206                 if (td->remainder == 0) {
 2207                         if (td->short_pkt)
 2208                                 return (0);     /* complete */
 2209 
 2210                         /* else we need to transmit a short packet */
 2211                 }
 2212         }
 2213 
 2214         if (!to--)
 2215                 goto not_complete;
 2216 
 2217         /* check if not all packets have been transferred */
 2218         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
 2219 
 2220         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
 2221                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
 2222                     "DIEPCTL=0x%08x\n", td->ep_no,
 2223                     DXEPTSIZ_GET_NPKT(temp),
 2224                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
 2225 
 2226                 goto not_complete;
 2227         }
 2228 
 2229         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
 2230 
 2231         /* try to optimise by sending more data */
 2232         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
 2233                 /* send multiple packets at the same time */
 2234                 mpkt = max_buffer / td->max_packet_size;
 2235 
 2236                 if (mpkt > 0x3FE)
 2237                         mpkt = 0x3FE;
 2238 
 2239                 count = td->remainder;
 2240                 if (count > 0x7FFFFF)
 2241                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
 2242 
 2243                 td->npkt = count / td->max_packet_size;
 2244 
 2245                 /*
 2246                  * NOTE: We could use 0x3FE instead of "mpkt" in the
 2247                  * check below to get more throughput, but then we
 2248                  * have a dependency towards non-generic chip features
 2249                  * to disable the TX-FIFO-EMPTY interrupts on a per
 2250                  * endpoint basis. Increase the maximum buffer size of
 2251                  * the IN endpoint to increase the performance.
 2252                  */
 2253                 if (td->npkt > mpkt) {
 2254                         td->npkt = mpkt;
 2255                         count = td->max_packet_size * mpkt;
 2256                 } else if ((count == 0) || (count % td->max_packet_size)) {
 2257                         /* we are transmitting a short packet */
 2258                         td->npkt++;
 2259                         td->short_pkt = 1;
 2260                 }
 2261         } else {
 2262                 /* send one packet at a time */
 2263                 mpkt = 1;
 2264                 count = td->max_packet_size;
 2265                 if (td->remainder < count) {
 2266                         /* we have a short packet */
 2267                         td->short_pkt = 1;
 2268                         count = td->remainder;
 2269                 }
 2270                 td->npkt = 1;
 2271         }
 2272         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
 2273             DXEPTSIZ_SET_MULTI(1) |
 2274             DXEPTSIZ_SET_NPKT(td->npkt) | 
 2275             DXEPTSIZ_SET_NBYTES(count));
 2276 
 2277         /* make room for buffering */
 2278         td->npkt += mpkt;
 2279 
 2280         temp = sc->sc_in_ctl[td->ep_no];
 2281 
 2282         /* check for isochronous mode */
 2283         if ((temp & DIEPCTL_EPTYPE_MASK) ==
 2284             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
 2285                 /* toggle odd or even frame bit */
 2286                 if (temp & DIEPCTL_SETD1PID) {
 2287                         temp &= ~DIEPCTL_SETD1PID;
 2288                         temp |= DIEPCTL_SETD0PID;
 2289                 } else {
 2290                         temp &= ~DIEPCTL_SETD0PID;
 2291                         temp |= DIEPCTL_SETD1PID;
 2292                 }
 2293                 sc->sc_in_ctl[td->ep_no] = temp;
 2294         }
 2295 
 2296         /* must enable before writing data to FIFO */
 2297         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
 2298             DIEPCTL_EPENA | DIEPCTL_CNAK);
 2299 
 2300         td->tx_bytes = count;
 2301 
 2302         /* check remainder */
 2303         if (td->tx_bytes == 0 &&
 2304             td->remainder == 0) {
 2305                 if (td->short_pkt)
 2306                         return (0);     /* complete */
 2307 
 2308                 /* else we need to transmit a short packet */
 2309         }
 2310         goto repeat;
 2311 
 2312 not_complete:
 2313         return (1);                     /* not complete */
 2314 }
 2315 
 2316 static uint8_t
 2317 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 2318 {
 2319         uint32_t temp;
 2320 
 2321         /*
 2322          * If all packets are transferred we are complete:
 2323          */
 2324         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
 2325 
 2326         /* check that all packets have been transferred */
 2327         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
 2328                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
 2329                 goto not_complete;
 2330         }
 2331         return (0);
 2332 
 2333 not_complete:
 2334 
 2335         /* we only want to know if there is a SETUP packet or free IN packet */
 2336 
 2337         temp = sc->sc_last_rx_status;
 2338 
 2339         if ((td->ep_no == 0) && (temp != 0) &&
 2340             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
 2341                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
 2342                     GRXSTSRD_STP_DATA ||
 2343                     (temp & GRXSTSRD_PKTSTS_MASK) ==
 2344                     GRXSTSRD_STP_COMPLETE) {
 2345                         DPRINTFN(5, "faking complete\n");
 2346                         /*
 2347                          * Race condition: We are complete!
 2348                          */
 2349                         return (0);
 2350                 } else {
 2351                         /* dump data - wrong direction */
 2352                         dwc_otg_common_rx_ack(sc);
 2353                 }
 2354         }
 2355         return (1);                     /* not complete */
 2356 }
 2357 
 2358 static void
 2359 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
 2360 {
 2361         struct dwc_otg_td *td;
 2362         uint8_t toggle;
 2363         uint8_t tmr_val;
 2364         uint8_t tmr_res;
 2365 
 2366         DPRINTFN(9, "\n");
 2367 
 2368         td = xfer->td_transfer_cache;
 2369         if (td == NULL)
 2370                 return;
 2371 
 2372         while (1) {
 2373                 if ((td->func) (sc, td)) {
 2374                         /* operation in progress */
 2375                         break;
 2376                 }
 2377                 if (((void *)td) == xfer->td_transfer_last) {
 2378                         goto done;
 2379                 }
 2380                 if (td->error_any) {
 2381                         goto done;
 2382                 } else if (td->remainder > 0) {
 2383                         /*
 2384                          * We had a short transfer. If there is no alternate
 2385                          * next, stop processing !
 2386                          */
 2387                         if (!td->alt_next)
 2388                                 goto done;
 2389                 }
 2390 
 2391                 /*
 2392                  * Fetch the next transfer descriptor and transfer
 2393                  * some flags to the next transfer descriptor
 2394                  */
 2395                 tmr_res = td->tmr_res;
 2396                 tmr_val = td->tmr_val;
 2397                 toggle = td->toggle;
 2398                 td = td->obj_next;
 2399                 xfer->td_transfer_cache = td;
 2400                 td->toggle = toggle;    /* transfer toggle */
 2401                 td->tmr_res = tmr_res;
 2402                 td->tmr_val = tmr_val;
 2403         }
 2404         return;
 2405 
 2406 done:
 2407         xfer->td_transfer_cache = NULL;
 2408         sc->sc_xfer_complete = 1;
 2409 }
 2410 
 2411 static uint8_t
 2412 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
 2413 {
 2414         struct dwc_otg_td *td;
 2415 
 2416         DPRINTFN(9, "\n");
 2417 
 2418         td = xfer->td_transfer_cache;
 2419         if (td == NULL) {
 2420                 /* compute all actual lengths */
 2421                 dwc_otg_standard_done(xfer);
 2422                 return (1);
 2423         }
 2424         return (0);
 2425 }
 2426 
 2427 static void
 2428 dwc_otg_timer(void *_sc)
 2429 {
 2430         struct dwc_otg_softc *sc = _sc;
 2431 
 2432         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 2433 
 2434         DPRINTF("\n");
 2435 
 2436         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 2437 
 2438         /* increment timer value */
 2439         sc->sc_tmr_val++;
 2440 
 2441         /* enable SOF interrupt, which will poll jobs */
 2442         dwc_otg_enable_sof_irq(sc);
 2443 
 2444         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 2445 
 2446         if (sc->sc_timer_active) {
 2447                 /* restart timer */
 2448                 usb_callout_reset(&sc->sc_timer,
 2449                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
 2450                     &dwc_otg_timer, sc);
 2451         }
 2452 }
 2453 
 2454 static void
 2455 dwc_otg_timer_start(struct dwc_otg_softc *sc)
 2456 {
 2457         if (sc->sc_timer_active != 0)
 2458                 return;
 2459 
 2460         sc->sc_timer_active = 1;
 2461 
 2462         /* restart timer */
 2463         usb_callout_reset(&sc->sc_timer,
 2464             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
 2465             &dwc_otg_timer, sc);
 2466 }
 2467 
 2468 static void
 2469 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
 2470 {
 2471         if (sc->sc_timer_active == 0)
 2472                 return;
 2473 
 2474         sc->sc_timer_active = 0;
 2475 
 2476         /* stop timer */
 2477         usb_callout_stop(&sc->sc_timer);
 2478 }
 2479 
 2480 static uint16_t
 2481 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
 2482 {
 2483         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
 2484                 pinfo->slot_index++;
 2485         return (pinfo->slot_index);
 2486 }
 2487 
 2488 static uint8_t
 2489 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
 2490 {
 2491         TAILQ_HEAD(, usb_xfer) head;
 2492         struct usb_xfer *xfer;
 2493         struct usb_xfer *xfer_next;
 2494         struct dwc_otg_td *td;
 2495         uint16_t temp;
 2496         uint16_t slot;
 2497 
 2498         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
 2499 
 2500         if (sc->sc_last_frame_num == temp)
 2501                 return (0);
 2502 
 2503         sc->sc_last_frame_num = temp;
 2504 
 2505         TAILQ_INIT(&head);
 2506 
 2507         if ((temp & 7) == 0) {
 2508                 /* reset the schedule */
 2509                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
 2510 
 2511                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2512                         td = xfer->td_transfer_cache;
 2513                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
 2514                                 continue;
 2515 
 2516                         /* check for IN direction */
 2517                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
 2518                                 continue;
 2519 
 2520                         sc->sc_needsof = 1;
 2521 
 2522                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
 2523                                 continue;
 2524 
 2525                         /* compute slot */
 2526                         slot = dwc_otg_compute_isoc_rx_tt_slot(
 2527                             sc->sc_tt_info + td->tt_index);
 2528                         if (slot > 3) {
 2529                                 /* 
 2530                                  * Not enough time to get complete
 2531                                  * split executed.
 2532                                  */
 2533                                 continue;
 2534                         }
 2535                         /* Delayed start */
 2536                         td->tt_start_slot = temp + slot;
 2537                         td->tt_scheduled = 1;
 2538                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2539                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2540                 }
 2541 
 2542                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2543                         td = xfer->td_transfer_cache;
 2544                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
 2545                                 continue;
 2546 
 2547                         /* check for OUT direction */
 2548                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
 2549                                 continue;
 2550 
 2551                         sc->sc_needsof = 1;
 2552 
 2553                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
 2554                                 continue;
 2555 
 2556                         /* Start ASAP */
 2557                         td->tt_start_slot = temp;
 2558                         td->tt_scheduled = 1;
 2559                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2560                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2561                 }
 2562 
 2563                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2564                         td = xfer->td_transfer_cache;
 2565                         if (td == NULL || td->ep_type != UE_INTERRUPT)
 2566                                 continue;
 2567 
 2568                         if (td->tt_scheduled != 0) {
 2569                                 sc->sc_needsof = 1;
 2570                                 continue;
 2571                         }
 2572 
 2573                         if (dwc_otg_host_rate_check_interrupt(sc, td))
 2574                                 continue;
 2575 
 2576                         if (td->hcsplt == 0) {
 2577                                 sc->sc_needsof = 1;
 2578                                 td->tt_scheduled = 1;
 2579                                 continue;
 2580                         }
 2581 
 2582                         /* start ASAP */
 2583                         td->tt_start_slot = temp;
 2584                         sc->sc_needsof = 1;
 2585                         td->tt_scheduled = 1;
 2586                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2587                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2588                 }
 2589 
 2590                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2591                         td = xfer->td_transfer_cache;
 2592                         if (td == NULL ||
 2593                             td->ep_type != UE_CONTROL) {
 2594                                 continue;
 2595                         }
 2596 
 2597                         sc->sc_needsof = 1;
 2598 
 2599                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
 2600                                 continue;
 2601 
 2602                         /* start ASAP */
 2603                         td->tt_start_slot = temp;
 2604                         td->tt_scheduled = 1;
 2605                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2606                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2607                 }
 2608         }
 2609         if ((temp & 7) < 6) {
 2610                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2611                         td = xfer->td_transfer_cache;
 2612                         if (td == NULL ||
 2613                             td->ep_type != UE_BULK) {
 2614                                 continue;
 2615                         }
 2616 
 2617                         sc->sc_needsof = 1;
 2618 
 2619                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
 2620                                 continue;
 2621 
 2622                         /* start ASAP */
 2623                         td->tt_start_slot = temp;
 2624                         td->tt_scheduled = 1;
 2625                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2626                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2627                 }
 2628         }
 2629 
 2630         /* Put TT transfers in execution order at the end */
 2631         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
 2632 
 2633         /* move all TT transfers in front, keeping the current order */
 2634         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2635                 td = xfer->td_transfer_cache;
 2636                 if (td == NULL || td->hcsplt == 0)
 2637                         continue;
 2638                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2639                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2640         }
 2641         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
 2642         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
 2643 
 2644         /* put non-TT non-ISOCHRONOUS transfers last */
 2645         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
 2646                 td = xfer->td_transfer_cache;
 2647                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
 2648                         continue;
 2649                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
 2650                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
 2651         }
 2652         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
 2653 
 2654         if ((temp & 7) == 0) {
 2655                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
 2656                     (int)temp, (int)sc->sc_needsof);
 2657 
 2658                 /* update SOF IRQ mask */
 2659                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
 2660                         if (sc->sc_needsof == 0) {
 2661                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK; 
 2662                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2663                         }
 2664                 } else {
 2665                         if (sc->sc_needsof != 0) {
 2666                                 sc->sc_irq_mask |= GINTMSK_SOFMSK; 
 2667                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2668                         }
 2669                 }
 2670 
 2671                 /* clear need SOF flag */
 2672                 sc->sc_needsof = 0;
 2673         }
 2674         return (1);
 2675 }
 2676 
 2677 static void
 2678 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
 2679 {
 2680         struct usb_xfer *xfer;
 2681         uint32_t count;
 2682         uint32_t temp;
 2683         uint32_t haint;
 2684         uint8_t got_rx_status;
 2685         uint8_t x;
 2686 
 2687         if (sc->sc_flags.status_device_mode == 0) {
 2688                 /*
 2689                  * Update host transfer schedule, so that new
 2690                  * transfers can be issued:
 2691                  */
 2692                 dwc_otg_update_host_transfer_schedule_locked(sc);
 2693         }
 2694         count = 0;
 2695 repeat:
 2696         if (++count == 16) {
 2697                 /* give other interrupts a chance */
 2698                 DPRINTF("Yield\n");
 2699                 return;
 2700         }
 2701 
 2702         /* get all host channel interrupts */
 2703         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
 2704         while (1) {
 2705                 x = ffs(haint) - 1;
 2706                 if (x >= sc->sc_host_ch_max)
 2707                         break;
 2708                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 2709                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
 2710                 temp &= ~HCINT_SOFTWARE_ONLY;
 2711                 sc->sc_chan_state[x].hcint |= temp;
 2712                 haint &= ~(1U << x);
 2713         }
 2714 
 2715         if (sc->sc_last_rx_status == 0) {
 2716                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 2717                 if (temp & GINTSTS_RXFLVL) {
 2718                         /* pop current status */
 2719                         sc->sc_last_rx_status =
 2720                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
 2721                 }
 2722 
 2723                 if (sc->sc_last_rx_status != 0) {
 2724                         uint8_t ep_no;
 2725 
 2726                         temp = sc->sc_last_rx_status &
 2727                             GRXSTSRD_PKTSTS_MASK;
 2728 
 2729                         /* non-data messages we simply skip */
 2730                         if (temp != GRXSTSRD_STP_DATA &&
 2731                             temp != GRXSTSRD_STP_COMPLETE &&
 2732                             temp != GRXSTSRD_OUT_DATA) {
 2733                                 /* check for halted channel */
 2734                                 if (temp == GRXSTSRH_HALTED) {
 2735                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
 2736                                         sc->sc_chan_state[ep_no].wait_halted = 0;
 2737                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
 2738                                 }
 2739                                 /* store bytes and FIFO offset */
 2740                                 sc->sc_current_rx_bytes = 0;
 2741                                 sc->sc_current_rx_fifo = 0;
 2742 
 2743                                 /* acknowledge status */
 2744                                 dwc_otg_common_rx_ack(sc);
 2745                                 goto repeat;
 2746                         }
 2747 
 2748                         temp = GRXSTSRD_BCNT_GET(
 2749                             sc->sc_last_rx_status);
 2750                         ep_no = GRXSTSRD_CHNUM_GET(
 2751                             sc->sc_last_rx_status);
 2752 
 2753                         /* store bytes and FIFO offset */
 2754                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
 2755                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
 2756 
 2757                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
 2758 
 2759                         /* check if we should dump the data */
 2760                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
 2761                                 dwc_otg_common_rx_ack(sc);
 2762                                 goto repeat;
 2763                         }
 2764 
 2765                         got_rx_status = 1;
 2766 
 2767                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
 2768                             sc->sc_last_rx_status, ep_no,
 2769                             (sc->sc_last_rx_status >> 15) & 3,
 2770                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
 2771                             (sc->sc_last_rx_status >> 17) & 15);
 2772                 } else {
 2773                         got_rx_status = 0;
 2774                 }
 2775         } else {
 2776                 uint8_t ep_no;
 2777 
 2778                 ep_no = GRXSTSRD_CHNUM_GET(
 2779                     sc->sc_last_rx_status);
 2780 
 2781                 /* check if we should dump the data */
 2782                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
 2783                         dwc_otg_common_rx_ack(sc);
 2784                         goto repeat;
 2785                 }
 2786 
 2787                 got_rx_status = 1;
 2788         }
 2789 
 2790         /* execute FIFOs */
 2791         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
 2792                 dwc_otg_xfer_do_fifo(sc, xfer);
 2793 
 2794         if (got_rx_status) {
 2795                 /* check if data was consumed */
 2796                 if (sc->sc_last_rx_status == 0)
 2797                         goto repeat;
 2798 
 2799                 /* disable RX FIFO level interrupt */
 2800                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
 2801                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2802         }
 2803 }
 2804 
 2805 static void
 2806 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
 2807 {
 2808         struct usb_xfer *xfer;
 2809 repeat:
 2810         /* scan for completion events */
 2811         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
 2812                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
 2813                         goto repeat;
 2814         }
 2815 }
 2816 
 2817 static void
 2818 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
 2819 {
 2820         DPRINTFN(5, "vbus = %u\n", is_on);
 2821 
 2822         /*
 2823          * If the USB host mode is forced, then assume VBUS is always
 2824          * present else rely on the input to this function:
 2825          */
 2826         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
 2827                 if (!sc->sc_flags.status_vbus) {
 2828                         sc->sc_flags.status_vbus = 1;
 2829 
 2830                         /* complete root HUB interrupt endpoint */
 2831 
 2832                         dwc_otg_root_intr(sc);
 2833                 }
 2834         } else {
 2835                 if (sc->sc_flags.status_vbus) {
 2836                         sc->sc_flags.status_vbus = 0;
 2837                         sc->sc_flags.status_bus_reset = 0;
 2838                         sc->sc_flags.status_suspend = 0;
 2839                         sc->sc_flags.change_suspend = 0;
 2840                         sc->sc_flags.change_connect = 1;
 2841 
 2842                         /* complete root HUB interrupt endpoint */
 2843 
 2844                         dwc_otg_root_intr(sc);
 2845                 }
 2846         }
 2847 }
 2848 
 2849 int
 2850 dwc_otg_filter_interrupt(void *arg)
 2851 {
 2852         struct dwc_otg_softc *sc = arg;
 2853         int retval = FILTER_HANDLED;
 2854         uint32_t status;
 2855 
 2856         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 2857 
 2858         /* read and clear interrupt status */
 2859         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 2860 
 2861         /* clear interrupts we are handling here */
 2862         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
 2863 
 2864         /* check for USB state change interrupts */
 2865         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
 2866                 retval = FILTER_SCHEDULE_THREAD;
 2867 
 2868         /* clear FIFO empty interrupts */
 2869         if (status & sc->sc_irq_mask &
 2870             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
 2871                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
 2872                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2873         }
 2874         /* clear all IN endpoint interrupts */
 2875         if (status & GINTSTS_IEPINT) {
 2876                 uint32_t temp;
 2877                 uint8_t x;
 2878 
 2879                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
 2880                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
 2881                         /*
 2882                          * NOTE: Need to clear all interrupt bits,
 2883                          * because some appears to be unmaskable and
 2884                          * can cause an interrupt loop:
 2885                          */
 2886                         if (temp != 0)
 2887                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
 2888                 }
 2889         }
 2890 
 2891         /* poll FIFOs, if any */
 2892         dwc_otg_interrupt_poll_locked(sc);
 2893 
 2894         if (sc->sc_xfer_complete != 0)
 2895                 retval = FILTER_SCHEDULE_THREAD;
 2896 
 2897         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 2898 
 2899         return (retval);
 2900 }
 2901 
 2902 void
 2903 dwc_otg_interrupt(void *arg)
 2904 {
 2905         struct dwc_otg_softc *sc = arg;
 2906         uint32_t status;
 2907 
 2908         USB_BUS_LOCK(&sc->sc_bus);
 2909         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 2910 
 2911         /* read and clear interrupt status */
 2912         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 2913 
 2914         /* clear interrupts we are handling here */
 2915         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
 2916 
 2917         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
 2918             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
 2919             DWC_OTG_READ_4(sc, DOTG_HFNUM));
 2920 
 2921         if (status & GINTSTS_USBRST) {
 2922                 /* set correct state */
 2923                 sc->sc_flags.status_device_mode = 1;
 2924                 sc->sc_flags.status_bus_reset = 0;
 2925                 sc->sc_flags.status_suspend = 0;
 2926                 sc->sc_flags.change_suspend = 0;
 2927                 sc->sc_flags.change_connect = 1;
 2928 
 2929                 /* Disable SOF interrupt */
 2930                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
 2931                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2932 
 2933                 /* complete root HUB interrupt endpoint */
 2934                 dwc_otg_root_intr(sc);
 2935         }
 2936 
 2937         /* check for any bus state change interrupts */
 2938         if (status & GINTSTS_ENUMDONE) {
 2939                 uint32_t temp;
 2940 
 2941                 DPRINTFN(5, "end of reset\n");
 2942 
 2943                 /* set correct state */
 2944                 sc->sc_flags.status_device_mode = 1;
 2945                 sc->sc_flags.status_bus_reset = 1;
 2946                 sc->sc_flags.status_suspend = 0;
 2947                 sc->sc_flags.change_suspend = 0;
 2948                 sc->sc_flags.change_connect = 1;
 2949                 sc->sc_flags.status_low_speed = 0;
 2950                 sc->sc_flags.port_enabled = 1;
 2951 
 2952                 /* reset FIFOs */
 2953                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
 2954 
 2955                 /* reset function address */
 2956                 dwc_otg_set_address(sc, 0);
 2957 
 2958                 /* figure out enumeration speed */
 2959                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
 2960                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
 2961                         sc->sc_flags.status_high_speed = 1;
 2962                 else
 2963                         sc->sc_flags.status_high_speed = 0;
 2964 
 2965                 /*
 2966                  * Disable resume and SOF interrupt, and enable
 2967                  * suspend and RX frame interrupt:
 2968                  */
 2969                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
 2970                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 2971                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 2972 
 2973                 /* complete root HUB interrupt endpoint */
 2974                 dwc_otg_root_intr(sc);
 2975         }
 2976 
 2977         if (status & GINTSTS_PRTINT) {
 2978                 uint32_t hprt;
 2979 
 2980                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
 2981 
 2982                 /* clear change bits */
 2983                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
 2984                     HPRT_PRTPWR | HPRT_PRTENCHNG |
 2985                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
 2986                     sc->sc_hprt_val);
 2987 
 2988                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
 2989 
 2990                 sc->sc_flags.status_device_mode = 0;
 2991 
 2992                 if (hprt & HPRT_PRTCONNSTS)
 2993                         sc->sc_flags.status_bus_reset = 1;
 2994                 else
 2995                         sc->sc_flags.status_bus_reset = 0;
 2996 
 2997                 if ((hprt & HPRT_PRTENCHNG) &&
 2998                     (hprt & HPRT_PRTENA) == 0)
 2999                         sc->sc_flags.change_enabled = 1;
 3000 
 3001                 if (hprt & HPRT_PRTENA)
 3002                         sc->sc_flags.port_enabled = 1;
 3003                 else
 3004                         sc->sc_flags.port_enabled = 0;
 3005 
 3006                 if (hprt & HPRT_PRTOVRCURRCHNG)
 3007                         sc->sc_flags.change_over_current = 1;
 3008 
 3009                 if (hprt & HPRT_PRTOVRCURRACT)
 3010                         sc->sc_flags.port_over_current = 1;
 3011                 else
 3012                         sc->sc_flags.port_over_current = 0;
 3013 
 3014                 if (hprt & HPRT_PRTPWR)
 3015                         sc->sc_flags.port_powered = 1;
 3016                 else
 3017                         sc->sc_flags.port_powered = 0;
 3018 
 3019                 if (((hprt & HPRT_PRTSPD_MASK)
 3020                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
 3021                         sc->sc_flags.status_low_speed = 1;
 3022                 else
 3023                         sc->sc_flags.status_low_speed = 0;
 3024 
 3025                 if (((hprt & HPRT_PRTSPD_MASK)
 3026                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
 3027                         sc->sc_flags.status_high_speed = 1;
 3028                 else
 3029                         sc->sc_flags.status_high_speed = 0;
 3030 
 3031                 if (hprt & HPRT_PRTCONNDET)
 3032                         sc->sc_flags.change_connect = 1;
 3033 
 3034                 if (hprt & HPRT_PRTSUSP)
 3035                         dwc_otg_suspend_irq(sc);
 3036                 else
 3037                         dwc_otg_resume_irq(sc);
 3038 
 3039                 /* complete root HUB interrupt endpoint */
 3040                 dwc_otg_root_intr(sc);
 3041 
 3042                 /* update host frame interval */
 3043                 dwc_otg_update_host_frame_interval(sc);
 3044         }
 3045 
 3046         /*
 3047          * If resume and suspend is set at the same time we interpret
 3048          * that like RESUME. Resume is set when there is at least 3
 3049          * milliseconds of inactivity on the USB BUS.
 3050          */
 3051         if (status & GINTSTS_WKUPINT) {
 3052                 DPRINTFN(5, "resume interrupt\n");
 3053 
 3054                 dwc_otg_resume_irq(sc);
 3055 
 3056         } else if (status & GINTSTS_USBSUSP) {
 3057                 DPRINTFN(5, "suspend interrupt\n");
 3058 
 3059                 dwc_otg_suspend_irq(sc);
 3060         }
 3061         /* check VBUS */
 3062         if (status & (GINTSTS_USBSUSP |
 3063             GINTSTS_USBRST |
 3064             GINTMSK_OTGINTMSK |
 3065             GINTSTS_SESSREQINT)) {
 3066                 uint32_t temp;
 3067 
 3068                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
 3069 
 3070                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
 3071 
 3072                 dwc_otg_vbus_interrupt(sc,
 3073                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
 3074         }
 3075 
 3076         if (sc->sc_xfer_complete != 0) {
 3077                 sc->sc_xfer_complete = 0;
 3078 
 3079                 /* complete FIFOs, if any */
 3080                 dwc_otg_interrupt_complete_locked(sc);
 3081         }
 3082         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 3083         USB_BUS_UNLOCK(&sc->sc_bus);
 3084 }
 3085 
 3086 static void
 3087 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
 3088 {
 3089         struct dwc_otg_td *td;
 3090 
 3091         /* get current Transfer Descriptor */
 3092         td = temp->td_next;
 3093         temp->td = td;
 3094 
 3095         /* prepare for next TD */
 3096         temp->td_next = td->obj_next;
 3097 
 3098         /* fill out the Transfer Descriptor */
 3099         td->func = temp->func;
 3100         td->pc = temp->pc;
 3101         td->offset = temp->offset;
 3102         td->remainder = temp->len;
 3103         td->tx_bytes = 0;
 3104         td->error_any = 0;
 3105         td->error_stall = 0;
 3106         td->npkt = 0;
 3107         td->did_stall = temp->did_stall;
 3108         td->short_pkt = temp->short_pkt;
 3109         td->alt_next = temp->setup_alt_next;
 3110         td->set_toggle = 0;
 3111         td->got_short = 0;
 3112         td->did_nak = 0;
 3113         td->channel[0] = DWC_OTG_MAX_CHANNELS;
 3114         td->channel[1] = DWC_OTG_MAX_CHANNELS;
 3115         td->channel[2] = DWC_OTG_MAX_CHANNELS;
 3116         td->state = 0;
 3117         td->errcnt = 0;
 3118         td->tt_scheduled = 0;
 3119         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
 3120 }
 3121 
 3122 static void
 3123 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
 3124 {
 3125         struct dwc_otg_std_temp temp;
 3126         struct dwc_otg_td *td;
 3127         uint32_t x;
 3128         uint8_t need_sync;
 3129         uint8_t is_host;
 3130 
 3131         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
 3132             xfer->address, UE_GET_ADDR(xfer->endpointno),
 3133             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
 3134 
 3135         temp.max_frame_size = xfer->max_frame_size;
 3136 
 3137         td = xfer->td_start[0];
 3138         xfer->td_transfer_first = td;
 3139         xfer->td_transfer_cache = td;
 3140 
 3141         /* setup temp */
 3142 
 3143         temp.pc = NULL;
 3144         temp.td = NULL;
 3145         temp.td_next = xfer->td_start[0];
 3146         temp.offset = 0;
 3147         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
 3148             xfer->flags_int.isochronous_xfr;
 3149         temp.did_stall = !xfer->flags_int.control_stall;
 3150 
 3151         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
 3152 
 3153         /* check if we should prepend a setup message */
 3154 
 3155         if (xfer->flags_int.control_xfr) {
 3156                 if (xfer->flags_int.control_hdr) {
 3157                         if (is_host)
 3158                                 temp.func = &dwc_otg_host_setup_tx;
 3159                         else
 3160                                 temp.func = &dwc_otg_setup_rx;
 3161 
 3162                         temp.len = xfer->frlengths[0];
 3163                         temp.pc = xfer->frbuffers + 0;
 3164                         temp.short_pkt = temp.len ? 1 : 0;
 3165 
 3166                         /* check for last frame */
 3167                         if (xfer->nframes == 1) {
 3168                                 /* no STATUS stage yet, SETUP is last */
 3169                                 if (xfer->flags_int.control_act)
 3170                                         temp.setup_alt_next = 0;
 3171                         }
 3172 
 3173                         dwc_otg_setup_standard_chain_sub(&temp);
 3174                 }
 3175                 x = 1;
 3176         } else {
 3177                 x = 0;
 3178         }
 3179 
 3180         if (x != xfer->nframes) {
 3181                 if (xfer->endpointno & UE_DIR_IN) {
 3182                         if (is_host) {
 3183                                 temp.func = &dwc_otg_host_data_rx;
 3184                                 need_sync = 0;
 3185                         } else {
 3186                                 temp.func = &dwc_otg_data_tx;
 3187                                 need_sync = 1;
 3188                         }
 3189                 } else {
 3190                         if (is_host) {
 3191                                 temp.func = &dwc_otg_host_data_tx;
 3192                                 need_sync = 0;
 3193                         } else {
 3194                                 temp.func = &dwc_otg_data_rx;
 3195                                 need_sync = 0;
 3196                         }
 3197                 }
 3198 
 3199                 /* setup "pc" pointer */
 3200                 temp.pc = xfer->frbuffers + x;
 3201         } else {
 3202                 need_sync = 0;
 3203         }
 3204         while (x != xfer->nframes) {
 3205                 /* DATA0 / DATA1 message */
 3206 
 3207                 temp.len = xfer->frlengths[x];
 3208 
 3209                 x++;
 3210 
 3211                 if (x == xfer->nframes) {
 3212                         if (xfer->flags_int.control_xfr) {
 3213                                 if (xfer->flags_int.control_act) {
 3214                                         temp.setup_alt_next = 0;
 3215                                 }
 3216                         } else {
 3217                                 temp.setup_alt_next = 0;
 3218                         }
 3219                 }
 3220                 if (temp.len == 0) {
 3221                         /* make sure that we send an USB packet */
 3222 
 3223                         temp.short_pkt = 0;
 3224 
 3225                 } else {
 3226                         /* regular data transfer */
 3227 
 3228                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
 3229                 }
 3230 
 3231                 dwc_otg_setup_standard_chain_sub(&temp);
 3232 
 3233                 if (xfer->flags_int.isochronous_xfr) {
 3234                         temp.offset += temp.len;
 3235                 } else {
 3236                         /* get next Page Cache pointer */
 3237                         temp.pc = xfer->frbuffers + x;
 3238                 }
 3239         }
 3240 
 3241         if (xfer->flags_int.control_xfr) {
 3242                 /* always setup a valid "pc" pointer for status and sync */
 3243                 temp.pc = xfer->frbuffers + 0;
 3244                 temp.len = 0;
 3245                 temp.short_pkt = 0;
 3246                 temp.setup_alt_next = 0;
 3247 
 3248                 /* check if we need to sync */
 3249                 if (need_sync) {
 3250                         /* we need a SYNC point after TX */
 3251                         temp.func = &dwc_otg_data_tx_sync;
 3252                         dwc_otg_setup_standard_chain_sub(&temp);
 3253                 }
 3254 
 3255                 /* check if we should append a status stage */
 3256                 if (!xfer->flags_int.control_act) {
 3257                         /*
 3258                          * Send a DATA1 message and invert the current
 3259                          * endpoint direction.
 3260                          */
 3261                         if (xfer->endpointno & UE_DIR_IN) {
 3262                                 if (is_host) {
 3263                                         temp.func = &dwc_otg_host_data_tx;
 3264                                         need_sync = 0;
 3265                                 } else {
 3266                                         temp.func = &dwc_otg_data_rx;
 3267                                         need_sync = 0;
 3268                                 }
 3269                         } else {
 3270                                 if (is_host) {
 3271                                         temp.func = &dwc_otg_host_data_rx;
 3272                                         need_sync = 0;
 3273                                 } else {
 3274                                         temp.func = &dwc_otg_data_tx;
 3275                                         need_sync = 1;
 3276                                 }
 3277                         }
 3278 
 3279                         dwc_otg_setup_standard_chain_sub(&temp);
 3280 
 3281                         /* data toggle should be DATA1 */
 3282                         td = temp.td;
 3283                         td->set_toggle = 1;
 3284 
 3285                         if (need_sync) {
 3286                                 /* we need a SYNC point after TX */
 3287                                 temp.func = &dwc_otg_data_tx_sync;
 3288                                 dwc_otg_setup_standard_chain_sub(&temp);
 3289                         }
 3290                 }
 3291         } else {
 3292                 /* check if we need to sync */
 3293                 if (need_sync) {
 3294                         temp.pc = xfer->frbuffers + 0;
 3295                         temp.len = 0;
 3296                         temp.short_pkt = 0;
 3297                         temp.setup_alt_next = 0;
 3298 
 3299                         /* we need a SYNC point after TX */
 3300                         temp.func = &dwc_otg_data_tx_sync;
 3301                         dwc_otg_setup_standard_chain_sub(&temp);
 3302                 }
 3303         }
 3304 
 3305         /* must have at least one frame! */
 3306         td = temp.td;
 3307         xfer->td_transfer_last = td;
 3308 
 3309         if (is_host) {
 3310                 struct dwc_otg_softc *sc;
 3311                 uint32_t hcchar;
 3312                 uint32_t hcsplt;
 3313 
 3314                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
 3315 
 3316                 /* get first again */
 3317                 td = xfer->td_transfer_first;
 3318                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
 3319 
 3320                 hcchar =
 3321                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
 3322                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
 3323                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
 3324                         HCCHAR_CHENA;
 3325 
 3326                 /*
 3327                  * We are not always able to meet the timing
 3328                  * requirements of the USB interrupt endpoint's
 3329                  * complete split token, when doing transfers going
 3330                  * via a transaction translator. Use the CONTROL
 3331                  * transfer type instead of the INTERRUPT transfer
 3332                  * type in general, as a means to workaround
 3333                  * that. This trick should work for both FULL and LOW
 3334                  * speed USB traffic going through a TT. For non-TT
 3335                  * traffic it works as well. The reason for using
 3336                  * CONTROL type instead of BULK is that some TTs might
 3337                  * reject LOW speed BULK traffic.
 3338                  */
 3339                 if (td->ep_type == UE_INTERRUPT)
 3340                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
 3341                 else
 3342                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
 3343 
 3344                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
 3345                         hcchar |= HCCHAR_EPDIR_IN;
 3346 
 3347                 switch (xfer->xroot->udev->speed) {
 3348                 case USB_SPEED_LOW:
 3349                         hcchar |= HCCHAR_LSPDDEV;
 3350                         /* FALLTHROUGH */
 3351                 case USB_SPEED_FULL:
 3352                         /* check if root HUB port is running High Speed */
 3353                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
 3354                                 hcsplt = HCSPLT_SPLTENA |
 3355                                     (xfer->xroot->udev->hs_port_no <<
 3356                                     HCSPLT_PRTADDR_SHIFT) |
 3357                                     (xfer->xroot->udev->hs_hub_addr <<
 3358                                     HCSPLT_HUBADDR_SHIFT);
 3359                         } else {
 3360                                 hcsplt = 0;
 3361                         }
 3362                         if (td->ep_type == UE_INTERRUPT) {
 3363                                 uint32_t ival;
 3364                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
 3365                                 if (ival == 0)
 3366                                         ival = 1;
 3367                                 else if (ival > 127)
 3368                                         ival = 127;
 3369                                 td->tmr_val = sc->sc_tmr_val + ival;
 3370                                 td->tmr_res = ival;
 3371                         } else if (td->ep_type == UE_ISOCHRONOUS) {
 3372                                 td->tmr_res = 1;
 3373                                 td->tmr_val = sc->sc_last_frame_num;
 3374                                 if (td->hcchar & HCCHAR_EPDIR_IN)
 3375                                         td->tmr_val++;
 3376                         } else {
 3377                                 td->tmr_val = 0;
 3378                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
 3379                         }
 3380                         break;
 3381                 case USB_SPEED_HIGH:
 3382                         hcsplt = 0;
 3383                         if (td->ep_type == UE_INTERRUPT) {
 3384                                 uint32_t ival;
 3385                                 hcchar |= ((xfer->max_packet_count & 3)
 3386                                     << HCCHAR_MC_SHIFT);
 3387                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
 3388                                 if (ival == 0)
 3389                                         ival = 1;
 3390                                 else if (ival > 127)
 3391                                         ival = 127;
 3392                                 td->tmr_val = sc->sc_tmr_val + ival;
 3393                                 td->tmr_res = ival;
 3394                         } else if (td->ep_type == UE_ISOCHRONOUS) {
 3395                                 hcchar |= ((xfer->max_packet_count & 3)
 3396                                     << HCCHAR_MC_SHIFT);
 3397                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
 3398                                 td->tmr_val = sc->sc_last_frame_num;
 3399                                 if (td->hcchar & HCCHAR_EPDIR_IN)
 3400                                         td->tmr_val += td->tmr_res;
 3401 
 3402                         } else {
 3403                                 td->tmr_val = 0;
 3404                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
 3405                         }
 3406                         break;
 3407                 default:
 3408                         hcsplt = 0;
 3409                         td->tmr_val = 0;
 3410                         td->tmr_res = 0;
 3411                         break;
 3412                 }
 3413 
 3414                 /* store configuration in all TD's */
 3415                 while (1) {
 3416                         td->hcchar = hcchar;
 3417                         td->hcsplt = hcsplt;
 3418 
 3419                         if (((void *)td) == xfer->td_transfer_last)
 3420                                 break;
 3421 
 3422                         td = td->obj_next;
 3423                 }
 3424         }
 3425 }
 3426 
 3427 static void
 3428 dwc_otg_timeout(void *arg)
 3429 {
 3430         struct usb_xfer *xfer = arg;
 3431 
 3432         DPRINTF("xfer=%p\n", xfer);
 3433 
 3434         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
 3435 
 3436         /* transfer is transferred */
 3437         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
 3438 }
 3439 
 3440 static void
 3441 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
 3442 {
 3443         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
 3444 
 3445         DPRINTFN(9, "\n");
 3446 
 3447         /*
 3448          * Poll one time in device mode, which will turn on the
 3449          * endpoint interrupts. Else wait for SOF interrupt in host
 3450          * mode.
 3451          */
 3452         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 3453 
 3454         if (sc->sc_flags.status_device_mode != 0) {
 3455                 dwc_otg_xfer_do_fifo(sc, xfer);
 3456                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
 3457                         goto done;
 3458         } else {
 3459                 struct dwc_otg_td *td = xfer->td_transfer_cache;
 3460                 if (td->ep_type == UE_ISOCHRONOUS &&
 3461                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
 3462                         /*
 3463                          * Need to start ISOCHRONOUS OUT transfer ASAP
 3464                          * because execution is delayed by one 125us
 3465                          * microframe:
 3466                          */
 3467                         dwc_otg_xfer_do_fifo(sc, xfer);
 3468                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
 3469                                 goto done;
 3470                 }
 3471         }
 3472 
 3473         /* put transfer on interrupt queue */
 3474         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 3475 
 3476         /* start timeout, if any */
 3477         if (xfer->timeout != 0) {
 3478                 usbd_transfer_timeout_ms(xfer,
 3479                     &dwc_otg_timeout, xfer->timeout);
 3480         }
 3481 
 3482         if (sc->sc_flags.status_device_mode != 0)
 3483                 goto done;
 3484 
 3485         /* enable SOF interrupt, if any */
 3486         dwc_otg_enable_sof_irq(sc);
 3487 done:
 3488         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 3489 }
 3490 
 3491 static void
 3492 dwc_otg_root_intr(struct dwc_otg_softc *sc)
 3493 {
 3494         DPRINTFN(9, "\n");
 3495 
 3496         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 3497 
 3498         /* set port bit */
 3499         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
 3500 
 3501         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
 3502             sizeof(sc->sc_hub_idata));
 3503 }
 3504 
 3505 static usb_error_t
 3506 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
 3507 {
 3508         struct dwc_otg_td *td;
 3509         uint32_t len;
 3510         usb_error_t error;
 3511 
 3512         DPRINTFN(9, "\n");
 3513 
 3514         td = xfer->td_transfer_cache;
 3515 
 3516         do {
 3517                 len = td->remainder;
 3518 
 3519                 /* store last data toggle */
 3520                 xfer->endpoint->toggle_next = td->toggle;
 3521 
 3522                 if (xfer->aframes != xfer->nframes) {
 3523                         /*
 3524                          * Verify the length and subtract
 3525                          * the remainder from "frlengths[]":
 3526                          */
 3527                         if (len > xfer->frlengths[xfer->aframes]) {
 3528                                 td->error_any = 1;
 3529                         } else {
 3530                                 xfer->frlengths[xfer->aframes] -= len;
 3531                         }
 3532                 }
 3533                 /* Check for transfer error */
 3534                 if (td->error_any) {
 3535                         /* the transfer is finished */
 3536                         error = (td->error_stall ?
 3537                             USB_ERR_STALLED : USB_ERR_IOERROR);
 3538                         td = NULL;
 3539                         break;
 3540                 }
 3541                 /* Check for short transfer */
 3542                 if (len > 0) {
 3543                         if (xfer->flags_int.short_frames_ok ||
 3544                             xfer->flags_int.isochronous_xfr) {
 3545                                 /* follow alt next */
 3546                                 if (td->alt_next) {
 3547                                         td = td->obj_next;
 3548                                 } else {
 3549                                         td = NULL;
 3550                                 }
 3551                         } else {
 3552                                 /* the transfer is finished */
 3553                                 td = NULL;
 3554                         }
 3555                         error = 0;
 3556                         break;
 3557                 }
 3558                 td = td->obj_next;
 3559 
 3560                 /* this USB frame is complete */
 3561                 error = 0;
 3562                 break;
 3563 
 3564         } while (0);
 3565 
 3566         /* update transfer cache */
 3567 
 3568         xfer->td_transfer_cache = td;
 3569 
 3570         return (error);
 3571 }
 3572 
 3573 static void
 3574 dwc_otg_standard_done(struct usb_xfer *xfer)
 3575 {
 3576         usb_error_t err = 0;
 3577 
 3578         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
 3579             xfer, xfer->endpoint);
 3580 
 3581         /* reset scanner */
 3582 
 3583         xfer->td_transfer_cache = xfer->td_transfer_first;
 3584 
 3585         if (xfer->flags_int.control_xfr) {
 3586                 if (xfer->flags_int.control_hdr) {
 3587                         err = dwc_otg_standard_done_sub(xfer);
 3588                 }
 3589                 xfer->aframes = 1;
 3590 
 3591                 if (xfer->td_transfer_cache == NULL) {
 3592                         goto done;
 3593                 }
 3594         }
 3595         while (xfer->aframes != xfer->nframes) {
 3596                 err = dwc_otg_standard_done_sub(xfer);
 3597                 xfer->aframes++;
 3598 
 3599                 if (xfer->td_transfer_cache == NULL) {
 3600                         goto done;
 3601                 }
 3602         }
 3603 
 3604         if (xfer->flags_int.control_xfr &&
 3605             !xfer->flags_int.control_act) {
 3606                 err = dwc_otg_standard_done_sub(xfer);
 3607         }
 3608 done:
 3609         dwc_otg_device_done(xfer, err);
 3610 }
 3611 
 3612 /*------------------------------------------------------------------------*
 3613  *      dwc_otg_device_done
 3614  *
 3615  * NOTE: this function can be called more than one time on the
 3616  * same USB transfer!
 3617  *------------------------------------------------------------------------*/
 3618 static void
 3619 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
 3620 {
 3621         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
 3622 
 3623         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
 3624             xfer, xfer->endpoint, error);
 3625 
 3626         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 3627 
 3628         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 3629                 /* Interrupts are cleared by the interrupt handler */
 3630         } else {
 3631                 struct dwc_otg_td *td;
 3632 
 3633                 td = xfer->td_transfer_cache;
 3634                 if (td != NULL)
 3635                         dwc_otg_host_channel_free(sc, td);
 3636         }
 3637         /* dequeue transfer and start next transfer */
 3638         usbd_transfer_done(xfer, error);
 3639 
 3640         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 3641 }
 3642 
 3643 static void
 3644 dwc_otg_xfer_stall(struct usb_xfer *xfer)
 3645 {
 3646         dwc_otg_device_done(xfer, USB_ERR_STALLED);
 3647 }
 3648 
 3649 static void
 3650 dwc_otg_set_stall(struct usb_device *udev,
 3651     struct usb_endpoint *ep, uint8_t *did_stall)
 3652 {
 3653         struct dwc_otg_softc *sc;
 3654         uint32_t temp;
 3655         uint32_t reg;
 3656         uint8_t ep_no;
 3657 
 3658         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
 3659 
 3660         /* check mode */
 3661         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
 3662                 /* not supported */
 3663                 return;
 3664         }
 3665 
 3666         sc = DWC_OTG_BUS2SC(udev->bus);
 3667 
 3668         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 3669 
 3670         /* get endpoint address */
 3671         ep_no = ep->edesc->bEndpointAddress;
 3672 
 3673         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
 3674 
 3675         if (ep_no & UE_DIR_IN) {
 3676                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
 3677                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
 3678         } else {
 3679                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
 3680                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
 3681         }
 3682 
 3683         /* disable and stall endpoint */
 3684         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
 3685         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
 3686 
 3687         /* clear active OUT ep */
 3688         if (!(ep_no & UE_DIR_IN)) {
 3689                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
 3690 
 3691                 if (sc->sc_last_rx_status != 0 &&
 3692                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
 3693                     sc->sc_last_rx_status)) {
 3694                         /* dump data */
 3695                         dwc_otg_common_rx_ack(sc);
 3696                         /* poll interrupt */
 3697                         dwc_otg_interrupt_poll_locked(sc);
 3698                         dwc_otg_interrupt_complete_locked(sc);
 3699                 }
 3700         }
 3701         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 3702 }
 3703 
 3704 static void
 3705 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
 3706     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
 3707 {
 3708         uint32_t reg;
 3709         uint32_t temp;
 3710 
 3711         if (ep_type == UE_CONTROL) {
 3712                 /* clearing stall is not needed */
 3713                 return;
 3714         }
 3715 
 3716         if (ep_dir) {
 3717                 reg = DOTG_DIEPCTL(ep_no);
 3718         } else {
 3719                 reg = DOTG_DOEPCTL(ep_no);
 3720                 sc->sc_active_rx_ep |= (1U << ep_no);
 3721         }
 3722 
 3723         /* round up and mask away the multiplier count */
 3724         mps = (mps + 3) & 0x7FC;
 3725 
 3726         if (ep_type == UE_BULK) {
 3727                 temp = DIEPCTL_EPTYPE_SET(
 3728                     DIEPCTL_EPTYPE_BULK) |
 3729                     DIEPCTL_USBACTEP;
 3730         } else if (ep_type == UE_INTERRUPT) {
 3731                 temp = DIEPCTL_EPTYPE_SET(
 3732                     DIEPCTL_EPTYPE_INTERRUPT) |
 3733                     DIEPCTL_USBACTEP;
 3734         } else {
 3735                 temp = DIEPCTL_EPTYPE_SET(
 3736                     DIEPCTL_EPTYPE_ISOC) |
 3737                     DIEPCTL_USBACTEP;
 3738         }
 3739 
 3740         temp |= DIEPCTL_MPS_SET(mps);
 3741         temp |= DIEPCTL_TXFNUM_SET(ep_no);
 3742 
 3743         if (ep_dir)
 3744                 sc->sc_in_ctl[ep_no] = temp;
 3745         else
 3746                 sc->sc_out_ctl[ep_no] = temp;
 3747 
 3748         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
 3749         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
 3750         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
 3751 
 3752         /* we only reset the transmit FIFO */
 3753         if (ep_dir) {
 3754                 dwc_otg_tx_fifo_reset(sc,
 3755                     GRSTCTL_TXFIFO(ep_no) |
 3756                     GRSTCTL_TXFFLSH);
 3757 
 3758                 DWC_OTG_WRITE_4(sc,
 3759                     DOTG_DIEPTSIZ(ep_no), 0);
 3760         }
 3761 
 3762         /* poll interrupt */
 3763         dwc_otg_interrupt_poll_locked(sc);
 3764         dwc_otg_interrupt_complete_locked(sc);
 3765 }
 3766 
 3767 static void
 3768 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
 3769 {
 3770         struct dwc_otg_softc *sc;
 3771         struct usb_endpoint_descriptor *ed;
 3772 
 3773         DPRINTFN(5, "endpoint=%p\n", ep);
 3774 
 3775         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
 3776 
 3777         /* check mode */
 3778         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
 3779                 /* not supported */
 3780                 return;
 3781         }
 3782         /* get softc */
 3783         sc = DWC_OTG_BUS2SC(udev->bus);
 3784 
 3785         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 3786 
 3787         /* get endpoint descriptor */
 3788         ed = ep->edesc;
 3789 
 3790         /* reset endpoint */
 3791         dwc_otg_clear_stall_sub_locked(sc,
 3792             UGETW(ed->wMaxPacketSize),
 3793             (ed->bEndpointAddress & UE_ADDR),
 3794             (ed->bmAttributes & UE_XFERTYPE),
 3795             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
 3796 
 3797         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 3798 }
 3799 
 3800 static void
 3801 dwc_otg_device_state_change(struct usb_device *udev)
 3802 {
 3803         struct dwc_otg_softc *sc;
 3804         uint8_t x;
 3805 
 3806         /* check mode */
 3807         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
 3808                 /* not supported */
 3809                 return;
 3810         }
 3811 
 3812         /* get softc */
 3813         sc = DWC_OTG_BUS2SC(udev->bus);
 3814 
 3815         /* deactivate all other endpoint but the control endpoint */
 3816         if (udev->state == USB_STATE_CONFIGURED ||
 3817             udev->state == USB_STATE_ADDRESSED) {
 3818                 USB_BUS_LOCK(&sc->sc_bus);
 3819 
 3820                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
 3821                         if (x < sc->sc_dev_in_ep_max) {
 3822                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
 3823                                     DIEPCTL_EPDIS);
 3824                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
 3825                         }
 3826 
 3827                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
 3828                             DOEPCTL_EPDIS);
 3829                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
 3830                 }
 3831                 USB_BUS_UNLOCK(&sc->sc_bus);
 3832         }
 3833 }
 3834 
 3835 int
 3836 dwc_otg_init(struct dwc_otg_softc *sc)
 3837 {
 3838         uint32_t temp;
 3839         int err;
 3840 
 3841         DPRINTF("start\n");
 3842 
 3843         sc->sc_io_tag = rman_get_bustag(sc->sc_io_res);
 3844         sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res);
 3845         sc->sc_io_size = rman_get_size(sc->sc_io_res);
 3846 
 3847         /* set up the bus structure */
 3848         sc->sc_bus.devices = sc->sc_devices;
 3849         sc->sc_bus.devices_max = DWC_OTG_MAX_DEVICES;
 3850         sc->sc_bus.dma_bits = 32;
 3851         sc->sc_bus.usbrev = USB_REV_2_0;
 3852         sc->sc_bus.methods = &dwc_otg_bus_methods;
 3853 
 3854         /* get all DMA memory */
 3855         if (usb_bus_mem_alloc_all(&sc->sc_bus,
 3856             USB_GET_DMA_TAG(sc->sc_bus.parent), NULL)) {
 3857                 return (ENOMEM);
 3858         }
 3859 
 3860         sc->sc_bus.bdev = device_add_child(sc->sc_bus.parent, "usbus", -1);
 3861         if (sc->sc_bus.bdev == NULL)
 3862                 return (ENXIO);
 3863 
 3864         device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus);
 3865 
 3866         err = bus_setup_intr(sc->sc_bus.parent, sc->sc_irq_res,
 3867             INTR_TYPE_TTY | INTR_MPSAFE, &dwc_otg_filter_interrupt,
 3868             &dwc_otg_interrupt, sc, &sc->sc_intr_hdl);
 3869         if (err) {
 3870                 sc->sc_intr_hdl = NULL;
 3871                 return (ENXIO);
 3872         }
 3873 
 3874         usb_callout_init_mtx(&sc->sc_timer,
 3875             &sc->sc_bus.bus_mtx, 0);
 3876 
 3877         USB_BUS_LOCK(&sc->sc_bus);
 3878 
 3879         /* turn on clocks */
 3880         dwc_otg_clocks_on(sc);
 3881 
 3882         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
 3883         DPRINTF("Version = 0x%08x\n", temp);
 3884 
 3885         /* disconnect */
 3886         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
 3887             DCTL_SFTDISCON);
 3888 
 3889         /* wait for host to detect disconnect */
 3890         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
 3891 
 3892         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 3893             GRSTCTL_CSFTRST);
 3894 
 3895         /* wait a little bit for block to reset */
 3896         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
 3897 
 3898         switch (sc->sc_mode) {
 3899         case DWC_MODE_DEVICE:
 3900                 temp = GUSBCFG_FORCEDEVMODE;
 3901                 break;
 3902         case DWC_MODE_HOST:
 3903                 temp = GUSBCFG_FORCEHOSTMODE;
 3904                 break;
 3905         default:
 3906                 temp = 0;
 3907                 break;
 3908         }
 3909 
 3910         if (sc->sc_phy_type == 0)
 3911                 sc->sc_phy_type = dwc_otg_phy_type + 1;
 3912         if (sc->sc_phy_bits == 0)
 3913                 sc->sc_phy_bits = 16;
 3914 
 3915         /* select HSIC, ULPI, UTMI+ or internal PHY mode */
 3916         switch (sc->sc_phy_type) {
 3917         case DWC_OTG_PHY_HSIC:
 3918                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
 3919                     GUSBCFG_PHYIF |
 3920                     GUSBCFG_TRD_TIM_SET(5) | temp);
 3921                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
 3922                     0x000000EC);
 3923 
 3924                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
 3925                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
 3926                     temp & ~GLPMCFG_HSIC_CONN);
 3927                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
 3928                     temp | GLPMCFG_HSIC_CONN);
 3929                 break;
 3930         case DWC_OTG_PHY_ULPI:
 3931                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
 3932                     GUSBCFG_ULPI_UTMI_SEL |
 3933                     GUSBCFG_TRD_TIM_SET(5) | temp);
 3934                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
 3935 
 3936                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
 3937                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
 3938                     temp & ~GLPMCFG_HSIC_CONN);
 3939                 break;
 3940         case DWC_OTG_PHY_UTMI:
 3941                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
 3942                     (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
 3943                     GUSBCFG_TRD_TIM_SET(5) | temp);
 3944                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
 3945 
 3946                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
 3947                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
 3948                     temp & ~GLPMCFG_HSIC_CONN);
 3949                 break;
 3950         case DWC_OTG_PHY_INTERNAL:
 3951                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
 3952                     GUSBCFG_PHYSEL |
 3953                     GUSBCFG_TRD_TIM_SET(5) | temp);
 3954                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
 3955 
 3956                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
 3957                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
 3958                     temp & ~GLPMCFG_HSIC_CONN);
 3959 
 3960                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
 3961                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
 3962                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
 3963                     DOTG_GGPIO_PWRDWN);
 3964                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
 3965                 break;
 3966         default:
 3967                 break;
 3968         }
 3969 
 3970         /* clear global nak */
 3971         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
 3972             DCTL_CGOUTNAK |
 3973             DCTL_CGNPINNAK);
 3974 
 3975         /* disable USB port */
 3976         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
 3977 
 3978         /* wait 10ms */
 3979         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 3980 
 3981         /* enable USB port */
 3982         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 3983 
 3984         /* wait 10ms */
 3985         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 3986 
 3987         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
 3988 
 3989         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
 3990 
 3991         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
 3992 
 3993         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
 3994 
 3995         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
 3996                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
 3997 
 3998         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
 3999 
 4000         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
 4001                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
 4002 
 4003         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
 4004 
 4005         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
 4006 
 4007         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
 4008             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
 4009             sc->sc_host_ch_max);
 4010 
 4011         /* setup FIFO */
 4012         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
 4013                 USB_BUS_UNLOCK(&sc->sc_bus);
 4014                 return (EINVAL);
 4015         }
 4016 
 4017         /* enable interrupts */
 4018         sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
 4019         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 4020 
 4021         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
 4022                 /* enable all endpoint interrupts */
 4023                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
 4024                 if (temp & GHWCFG2_MPI) {
 4025                         uint8_t x;
 4026 
 4027                         DPRINTF("Disable Multi Process Interrupts\n");
 4028 
 4029                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
 4030                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
 4031                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
 4032                         }
 4033                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
 4034                 }
 4035                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
 4036                     DIEPMSK_XFERCOMPLMSK);
 4037                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
 4038                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
 4039         }
 4040 
 4041         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
 4042                 /* setup clocks */
 4043                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
 4044                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
 4045                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
 4046                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
 4047         }
 4048 
 4049         /* only enable global IRQ */
 4050         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
 4051             GAHBCFG_GLBLINTRMSK);
 4052 
 4053         /* turn off clocks */
 4054         dwc_otg_clocks_off(sc);
 4055 
 4056         /* read initial VBUS state */
 4057 
 4058         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
 4059 
 4060         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
 4061 
 4062         dwc_otg_vbus_interrupt(sc,
 4063             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
 4064 
 4065         USB_BUS_UNLOCK(&sc->sc_bus);
 4066 
 4067         /* catch any lost interrupts */
 4068 
 4069         dwc_otg_do_poll(&sc->sc_bus);
 4070 
 4071         return (0);                     /* success */
 4072 }
 4073 
 4074 void
 4075 dwc_otg_uninit(struct dwc_otg_softc *sc)
 4076 {
 4077         USB_BUS_LOCK(&sc->sc_bus);
 4078 
 4079         /* stop host timer */
 4080         dwc_otg_timer_stop(sc);
 4081 
 4082         /* set disconnect */
 4083         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
 4084             DCTL_SFTDISCON);
 4085 
 4086         /* turn off global IRQ */
 4087         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
 4088 
 4089         sc->sc_flags.port_enabled = 0;
 4090         sc->sc_flags.port_powered = 0;
 4091         sc->sc_flags.status_vbus = 0;
 4092         sc->sc_flags.status_bus_reset = 0;
 4093         sc->sc_flags.status_suspend = 0;
 4094         sc->sc_flags.change_suspend = 0;
 4095         sc->sc_flags.change_connect = 1;
 4096 
 4097         dwc_otg_pull_down(sc);
 4098         dwc_otg_clocks_off(sc);
 4099 
 4100         USB_BUS_UNLOCK(&sc->sc_bus);
 4101 
 4102         usb_callout_drain(&sc->sc_timer);
 4103 }
 4104 
 4105 static void
 4106 dwc_otg_suspend(struct dwc_otg_softc *sc)
 4107 {
 4108         return;
 4109 }
 4110 
 4111 static void
 4112 dwc_otg_resume(struct dwc_otg_softc *sc)
 4113 {
 4114         return;
 4115 }
 4116 
 4117 static void
 4118 dwc_otg_do_poll(struct usb_bus *bus)
 4119 {
 4120         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
 4121 
 4122         USB_BUS_LOCK(&sc->sc_bus);
 4123         USB_BUS_SPIN_LOCK(&sc->sc_bus);
 4124         dwc_otg_interrupt_poll_locked(sc);
 4125         dwc_otg_interrupt_complete_locked(sc);
 4126         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
 4127         USB_BUS_UNLOCK(&sc->sc_bus);
 4128 }
 4129 
 4130 /*------------------------------------------------------------------------*
 4131  * DWC OTG bulk support
 4132  * DWC OTG control support
 4133  * DWC OTG interrupt support
 4134  *------------------------------------------------------------------------*/
 4135 static void
 4136 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
 4137 {
 4138 }
 4139 
 4140 static void
 4141 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
 4142 {
 4143         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
 4144 }
 4145 
 4146 static void
 4147 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
 4148 {
 4149 }
 4150 
 4151 static void
 4152 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
 4153 {
 4154         /* setup TDs */
 4155         dwc_otg_setup_standard_chain(xfer);
 4156         dwc_otg_start_standard_chain(xfer);
 4157 }
 4158 
 4159 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
 4160 {
 4161         .open = dwc_otg_device_non_isoc_open,
 4162         .close = dwc_otg_device_non_isoc_close,
 4163         .enter = dwc_otg_device_non_isoc_enter,
 4164         .start = dwc_otg_device_non_isoc_start,
 4165 };
 4166 
 4167 /*------------------------------------------------------------------------*
 4168  * DWC OTG full speed isochronous support
 4169  *------------------------------------------------------------------------*/
 4170 static void
 4171 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
 4172 {
 4173 }
 4174 
 4175 static void
 4176 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
 4177 {
 4178         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
 4179 }
 4180 
 4181 static void
 4182 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
 4183 {
 4184 }
 4185 
 4186 static void
 4187 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
 4188 {
 4189         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
 4190         uint32_t temp;
 4191         uint32_t framenum;
 4192 
 4193         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
 4194             xfer, xfer->endpoint->isoc_next, xfer->nframes);
 4195 
 4196         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
 4197                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
 4198 
 4199                 /* get the current frame index */
 4200                 framenum = (temp & HFNUM_FRNUM_MASK);
 4201         } else {
 4202                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
 4203 
 4204                 /* get the current frame index */
 4205                 framenum = DSTS_SOFFN_GET(temp);
 4206         }
 4207 
 4208         /*
 4209          * Check if port is doing 8000 or 1000 frames per second:
 4210          */
 4211         if (sc->sc_flags.status_high_speed)
 4212                 framenum /= 8;
 4213 
 4214         if (usbd_xfer_get_isochronous_start_frame(
 4215             xfer, framenum, 0, 1, DWC_OTG_FRAME_MASK, NULL))
 4216                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
 4217 
 4218         /* setup TDs */
 4219         dwc_otg_setup_standard_chain(xfer);
 4220 
 4221         /* start TD chain */
 4222         dwc_otg_start_standard_chain(xfer);
 4223 }
 4224 
 4225 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
 4226 {
 4227         .open = dwc_otg_device_isoc_open,
 4228         .close = dwc_otg_device_isoc_close,
 4229         .enter = dwc_otg_device_isoc_enter,
 4230         .start = dwc_otg_device_isoc_start,
 4231 };
 4232 
 4233 /*------------------------------------------------------------------------*
 4234  * DWC OTG root control support
 4235  *------------------------------------------------------------------------*
 4236  * Simulate a hardware HUB by handling all the necessary requests.
 4237  *------------------------------------------------------------------------*/
 4238 
 4239 static const struct usb_device_descriptor dwc_otg_devd = {
 4240         .bLength = sizeof(struct usb_device_descriptor),
 4241         .bDescriptorType = UDESC_DEVICE,
 4242         .bcdUSB = {0x00, 0x02},
 4243         .bDeviceClass = UDCLASS_HUB,
 4244         .bDeviceSubClass = UDSUBCLASS_HUB,
 4245         .bDeviceProtocol = UDPROTO_HSHUBSTT,
 4246         .bMaxPacketSize = 64,
 4247         .bcdDevice = {0x00, 0x01},
 4248         .iManufacturer = 1,
 4249         .iProduct = 2,
 4250         .bNumConfigurations = 1,
 4251 };
 4252 
 4253 static const struct dwc_otg_config_desc dwc_otg_confd = {
 4254         .confd = {
 4255                 .bLength = sizeof(struct usb_config_descriptor),
 4256                 .bDescriptorType = UDESC_CONFIG,
 4257                 .wTotalLength[0] = sizeof(dwc_otg_confd),
 4258                 .bNumInterface = 1,
 4259                 .bConfigurationValue = 1,
 4260                 .iConfiguration = 0,
 4261                 .bmAttributes = UC_SELF_POWERED,
 4262                 .bMaxPower = 0,
 4263         },
 4264         .ifcd = {
 4265                 .bLength = sizeof(struct usb_interface_descriptor),
 4266                 .bDescriptorType = UDESC_INTERFACE,
 4267                 .bNumEndpoints = 1,
 4268                 .bInterfaceClass = UICLASS_HUB,
 4269                 .bInterfaceSubClass = UISUBCLASS_HUB,
 4270                 .bInterfaceProtocol = 0,
 4271         },
 4272         .endpd = {
 4273                 .bLength = sizeof(struct usb_endpoint_descriptor),
 4274                 .bDescriptorType = UDESC_ENDPOINT,
 4275                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
 4276                 .bmAttributes = UE_INTERRUPT,
 4277                 .wMaxPacketSize[0] = 8,
 4278                 .bInterval = 255,
 4279         },
 4280 };
 4281 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
 4282 
 4283 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
 4284         .bDescLength = sizeof(dwc_otg_hubd),
 4285         .bDescriptorType = UDESC_HUB,
 4286         .bNbrPorts = 1,
 4287         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
 4288         .bPwrOn2PwrGood = 50,
 4289         .bHubContrCurrent = 0,
 4290         .DeviceRemovable = {0},         /* port is removable */
 4291 };
 4292 
 4293 #define STRING_VENDOR \
 4294   "D\0W\0C\0O\0T\0G"
 4295 
 4296 #define STRING_PRODUCT \
 4297   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 4298 
 4299 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
 4300 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
 4301 
 4302 static usb_error_t
 4303 dwc_otg_roothub_exec(struct usb_device *udev,
 4304     struct usb_device_request *req, const void **pptr, uint16_t *plength)
 4305 {
 4306         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
 4307         const void *ptr;
 4308         uint16_t len;
 4309         uint16_t value;
 4310         uint16_t index;
 4311         usb_error_t err;
 4312 
 4313         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 4314 
 4315         /* buffer reset */
 4316         ptr = (const void *)&sc->sc_hub_temp;
 4317         len = 0;
 4318         err = 0;
 4319 
 4320         value = UGETW(req->wValue);
 4321         index = UGETW(req->wIndex);
 4322 
 4323         /* demultiplex the control request */
 4324 
 4325         switch (req->bmRequestType) {
 4326         case UT_READ_DEVICE:
 4327                 switch (req->bRequest) {
 4328                 case UR_GET_DESCRIPTOR:
 4329                         goto tr_handle_get_descriptor;
 4330                 case UR_GET_CONFIG:
 4331                         goto tr_handle_get_config;
 4332                 case UR_GET_STATUS:
 4333                         goto tr_handle_get_status;
 4334                 default:
 4335                         goto tr_stalled;
 4336                 }
 4337                 break;
 4338 
 4339         case UT_WRITE_DEVICE:
 4340                 switch (req->bRequest) {
 4341                 case UR_SET_ADDRESS:
 4342                         goto tr_handle_set_address;
 4343                 case UR_SET_CONFIG:
 4344                         goto tr_handle_set_config;
 4345                 case UR_CLEAR_FEATURE:
 4346                         goto tr_valid;  /* nop */
 4347                 case UR_SET_DESCRIPTOR:
 4348                         goto tr_valid;  /* nop */
 4349                 case UR_SET_FEATURE:
 4350                 default:
 4351                         goto tr_stalled;
 4352                 }
 4353                 break;
 4354 
 4355         case UT_WRITE_ENDPOINT:
 4356                 switch (req->bRequest) {
 4357                 case UR_CLEAR_FEATURE:
 4358                         switch (UGETW(req->wValue)) {
 4359                         case UF_ENDPOINT_HALT:
 4360                                 goto tr_handle_clear_halt;
 4361                         case UF_DEVICE_REMOTE_WAKEUP:
 4362                                 goto tr_handle_clear_wakeup;
 4363                         default:
 4364                                 goto tr_stalled;
 4365                         }
 4366                         break;
 4367                 case UR_SET_FEATURE:
 4368                         switch (UGETW(req->wValue)) {
 4369                         case UF_ENDPOINT_HALT:
 4370                                 goto tr_handle_set_halt;
 4371                         case UF_DEVICE_REMOTE_WAKEUP:
 4372                                 goto tr_handle_set_wakeup;
 4373                         default:
 4374                                 goto tr_stalled;
 4375                         }
 4376                         break;
 4377                 case UR_SYNCH_FRAME:
 4378                         goto tr_valid;  /* nop */
 4379                 default:
 4380                         goto tr_stalled;
 4381                 }
 4382                 break;
 4383 
 4384         case UT_READ_ENDPOINT:
 4385                 switch (req->bRequest) {
 4386                 case UR_GET_STATUS:
 4387                         goto tr_handle_get_ep_status;
 4388                 default:
 4389                         goto tr_stalled;
 4390                 }
 4391                 break;
 4392 
 4393         case UT_WRITE_INTERFACE:
 4394                 switch (req->bRequest) {
 4395                 case UR_SET_INTERFACE:
 4396                         goto tr_handle_set_interface;
 4397                 case UR_CLEAR_FEATURE:
 4398                         goto tr_valid;  /* nop */
 4399                 case UR_SET_FEATURE:
 4400                 default:
 4401                         goto tr_stalled;
 4402                 }
 4403                 break;
 4404 
 4405         case UT_READ_INTERFACE:
 4406                 switch (req->bRequest) {
 4407                 case UR_GET_INTERFACE:
 4408                         goto tr_handle_get_interface;
 4409                 case UR_GET_STATUS:
 4410                         goto tr_handle_get_iface_status;
 4411                 default:
 4412                         goto tr_stalled;
 4413                 }
 4414                 break;
 4415 
 4416         case UT_WRITE_CLASS_INTERFACE:
 4417         case UT_WRITE_VENDOR_INTERFACE:
 4418                 /* XXX forward */
 4419                 break;
 4420 
 4421         case UT_READ_CLASS_INTERFACE:
 4422         case UT_READ_VENDOR_INTERFACE:
 4423                 /* XXX forward */
 4424                 break;
 4425 
 4426         case UT_WRITE_CLASS_DEVICE:
 4427                 switch (req->bRequest) {
 4428                 case UR_CLEAR_FEATURE:
 4429                         goto tr_valid;
 4430                 case UR_SET_DESCRIPTOR:
 4431                 case UR_SET_FEATURE:
 4432                         break;
 4433                 default:
 4434                         goto tr_stalled;
 4435                 }
 4436                 break;
 4437 
 4438         case UT_WRITE_CLASS_OTHER:
 4439                 switch (req->bRequest) {
 4440                 case UR_CLEAR_FEATURE:
 4441                         goto tr_handle_clear_port_feature;
 4442                 case UR_SET_FEATURE:
 4443                         goto tr_handle_set_port_feature;
 4444                 case UR_CLEAR_TT_BUFFER:
 4445                 case UR_RESET_TT:
 4446                 case UR_STOP_TT:
 4447                         goto tr_valid;
 4448 
 4449                 default:
 4450                         goto tr_stalled;
 4451                 }
 4452                 break;
 4453 
 4454         case UT_READ_CLASS_OTHER:
 4455                 switch (req->bRequest) {
 4456                 case UR_GET_TT_STATE:
 4457                         goto tr_handle_get_tt_state;
 4458                 case UR_GET_STATUS:
 4459                         goto tr_handle_get_port_status;
 4460                 default:
 4461                         goto tr_stalled;
 4462                 }
 4463                 break;
 4464 
 4465         case UT_READ_CLASS_DEVICE:
 4466                 switch (req->bRequest) {
 4467                 case UR_GET_DESCRIPTOR:
 4468                         goto tr_handle_get_class_descriptor;
 4469                 case UR_GET_STATUS:
 4470                         goto tr_handle_get_class_status;
 4471 
 4472                 default:
 4473                         goto tr_stalled;
 4474                 }
 4475                 break;
 4476         default:
 4477                 goto tr_stalled;
 4478         }
 4479         goto tr_valid;
 4480 
 4481 tr_handle_get_descriptor:
 4482         switch (value >> 8) {
 4483         case UDESC_DEVICE:
 4484                 if (value & 0xff) {
 4485                         goto tr_stalled;
 4486                 }
 4487                 len = sizeof(dwc_otg_devd);
 4488                 ptr = (const void *)&dwc_otg_devd;
 4489                 goto tr_valid;
 4490         case UDESC_CONFIG:
 4491                 if (value & 0xff) {
 4492                         goto tr_stalled;
 4493                 }
 4494                 len = sizeof(dwc_otg_confd);
 4495                 ptr = (const void *)&dwc_otg_confd;
 4496                 goto tr_valid;
 4497         case UDESC_STRING:
 4498                 switch (value & 0xff) {
 4499                 case 0:         /* Language table */
 4500                         len = sizeof(usb_string_lang_en);
 4501                         ptr = (const void *)&usb_string_lang_en;
 4502                         goto tr_valid;
 4503 
 4504                 case 1:         /* Vendor */
 4505                         len = sizeof(dwc_otg_vendor);
 4506                         ptr = (const void *)&dwc_otg_vendor;
 4507                         goto tr_valid;
 4508 
 4509                 case 2:         /* Product */
 4510                         len = sizeof(dwc_otg_product);
 4511                         ptr = (const void *)&dwc_otg_product;
 4512                         goto tr_valid;
 4513                 default:
 4514                         break;
 4515                 }
 4516                 break;
 4517         default:
 4518                 goto tr_stalled;
 4519         }
 4520         goto tr_stalled;
 4521 
 4522 tr_handle_get_config:
 4523         len = 1;
 4524         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
 4525         goto tr_valid;
 4526 
 4527 tr_handle_get_status:
 4528         len = 2;
 4529         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
 4530         goto tr_valid;
 4531 
 4532 tr_handle_set_address:
 4533         if (value & 0xFF00) {
 4534                 goto tr_stalled;
 4535         }
 4536         sc->sc_rt_addr = value;
 4537         goto tr_valid;
 4538 
 4539 tr_handle_set_config:
 4540         if (value >= 2) {
 4541                 goto tr_stalled;
 4542         }
 4543         sc->sc_conf = value;
 4544         goto tr_valid;
 4545 
 4546 tr_handle_get_interface:
 4547         len = 1;
 4548         sc->sc_hub_temp.wValue[0] = 0;
 4549         goto tr_valid;
 4550 
 4551 tr_handle_get_tt_state:
 4552 tr_handle_get_class_status:
 4553 tr_handle_get_iface_status:
 4554 tr_handle_get_ep_status:
 4555         len = 2;
 4556         USETW(sc->sc_hub_temp.wValue, 0);
 4557         goto tr_valid;
 4558 
 4559 tr_handle_set_halt:
 4560 tr_handle_set_interface:
 4561 tr_handle_set_wakeup:
 4562 tr_handle_clear_wakeup:
 4563 tr_handle_clear_halt:
 4564         goto tr_valid;
 4565 
 4566 tr_handle_clear_port_feature:
 4567         if (index != 1)
 4568                 goto tr_stalled;
 4569 
 4570         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
 4571 
 4572         switch (value) {
 4573         case UHF_PORT_SUSPEND:
 4574                 dwc_otg_wakeup_peer(sc);
 4575                 break;
 4576 
 4577         case UHF_PORT_ENABLE:
 4578                 if (sc->sc_flags.status_device_mode == 0) {
 4579                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
 4580                             sc->sc_hprt_val | HPRT_PRTENA);
 4581                 }
 4582                 sc->sc_flags.port_enabled = 0;
 4583                 break;
 4584 
 4585         case UHF_C_PORT_RESET:
 4586                 sc->sc_flags.change_reset = 0;
 4587                 break;
 4588 
 4589         case UHF_C_PORT_ENABLE:
 4590                 sc->sc_flags.change_enabled = 0;
 4591                 break;
 4592 
 4593         case UHF_C_PORT_OVER_CURRENT:
 4594                 sc->sc_flags.change_over_current = 0;
 4595                 break;
 4596 
 4597         case UHF_PORT_TEST:
 4598         case UHF_PORT_INDICATOR:
 4599                 /* nops */
 4600                 break;
 4601 
 4602         case UHF_PORT_POWER:
 4603                 sc->sc_flags.port_powered = 0;
 4604                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
 4605                         sc->sc_hprt_val = 0;
 4606                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
 4607                 }
 4608                 dwc_otg_pull_down(sc);
 4609                 dwc_otg_clocks_off(sc);
 4610                 break;
 4611 
 4612         case UHF_C_PORT_CONNECTION:
 4613                 /* clear connect change flag */
 4614                 sc->sc_flags.change_connect = 0;
 4615                 break;
 4616 
 4617         case UHF_C_PORT_SUSPEND:
 4618                 sc->sc_flags.change_suspend = 0;
 4619                 break;
 4620 
 4621         default:
 4622                 err = USB_ERR_IOERROR;
 4623                 goto done;
 4624         }
 4625         goto tr_valid;
 4626 
 4627 tr_handle_set_port_feature:
 4628         if (index != 1) {
 4629                 goto tr_stalled;
 4630         }
 4631         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
 4632 
 4633         switch (value) {
 4634         case UHF_PORT_ENABLE:
 4635                 break;
 4636 
 4637         case UHF_PORT_SUSPEND:
 4638                 if (sc->sc_flags.status_device_mode == 0) {
 4639                         /* set suspend BIT */
 4640                         sc->sc_hprt_val |= HPRT_PRTSUSP;
 4641                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 4642 
 4643                         /* generate HUB suspend event */
 4644                         dwc_otg_suspend_irq(sc);
 4645                 }
 4646                 break;
 4647 
 4648         case UHF_PORT_RESET:
 4649                 if (sc->sc_flags.status_device_mode == 0) {
 4650                         DPRINTF("PORT RESET\n");
 4651 
 4652                         /* enable PORT reset */
 4653                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
 4654 
 4655                         /* Wait 62.5ms for reset to complete */
 4656                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
 4657 
 4658                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 4659 
 4660                         /* Wait 62.5ms for reset to complete */
 4661                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
 4662 
 4663                         /* reset FIFOs */
 4664                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
 4665 
 4666                         sc->sc_flags.change_reset = 1;
 4667                 } else {
 4668                         err = USB_ERR_IOERROR;
 4669                 }
 4670                 break;
 4671 
 4672         case UHF_PORT_TEST:
 4673         case UHF_PORT_INDICATOR:
 4674                 /* nops */
 4675                 break;
 4676         case UHF_PORT_POWER:
 4677                 sc->sc_flags.port_powered = 1;
 4678                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
 4679                         sc->sc_hprt_val |= HPRT_PRTPWR;
 4680                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 4681                 }
 4682                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
 4683                         /* pull up D+, if any */
 4684                         dwc_otg_pull_up(sc);
 4685                 }
 4686                 break;
 4687         default:
 4688                 err = USB_ERR_IOERROR;
 4689                 goto done;
 4690         }
 4691         goto tr_valid;
 4692 
 4693 tr_handle_get_port_status:
 4694 
 4695         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
 4696 
 4697         if (index != 1)
 4698                 goto tr_stalled;
 4699 
 4700         if (sc->sc_flags.status_vbus)
 4701                 dwc_otg_clocks_on(sc);
 4702         else
 4703                 dwc_otg_clocks_off(sc);
 4704 
 4705         /* Select Device Side Mode */
 4706 
 4707         if (sc->sc_flags.status_device_mode) {
 4708                 value = UPS_PORT_MODE_DEVICE;
 4709                 dwc_otg_timer_stop(sc);
 4710         } else {
 4711                 value = 0;
 4712                 dwc_otg_timer_start(sc);
 4713         }
 4714 
 4715         if (sc->sc_flags.status_high_speed)
 4716                 value |= UPS_HIGH_SPEED;
 4717         else if (sc->sc_flags.status_low_speed)
 4718                 value |= UPS_LOW_SPEED;
 4719 
 4720         if (sc->sc_flags.port_powered)
 4721                 value |= UPS_PORT_POWER;
 4722 
 4723         if (sc->sc_flags.port_enabled)
 4724                 value |= UPS_PORT_ENABLED;
 4725 
 4726         if (sc->sc_flags.port_over_current)
 4727                 value |= UPS_OVERCURRENT_INDICATOR;
 4728 
 4729         if (sc->sc_flags.status_vbus &&
 4730             sc->sc_flags.status_bus_reset)
 4731                 value |= UPS_CURRENT_CONNECT_STATUS;
 4732 
 4733         if (sc->sc_flags.status_suspend)
 4734                 value |= UPS_SUSPEND;
 4735 
 4736         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
 4737 
 4738         value = 0;
 4739 
 4740         if (sc->sc_flags.change_enabled)
 4741                 value |= UPS_C_PORT_ENABLED;
 4742         if (sc->sc_flags.change_connect)
 4743                 value |= UPS_C_CONNECT_STATUS;
 4744         if (sc->sc_flags.change_suspend)
 4745                 value |= UPS_C_SUSPEND;
 4746         if (sc->sc_flags.change_reset)
 4747                 value |= UPS_C_PORT_RESET;
 4748         if (sc->sc_flags.change_over_current)
 4749                 value |= UPS_C_OVERCURRENT_INDICATOR;
 4750 
 4751         USETW(sc->sc_hub_temp.ps.wPortChange, value);
 4752         len = sizeof(sc->sc_hub_temp.ps);
 4753         goto tr_valid;
 4754 
 4755 tr_handle_get_class_descriptor:
 4756         if (value & 0xFF) {
 4757                 goto tr_stalled;
 4758         }
 4759         ptr = (const void *)&dwc_otg_hubd;
 4760         len = sizeof(dwc_otg_hubd);
 4761         goto tr_valid;
 4762 
 4763 tr_stalled:
 4764         err = USB_ERR_STALLED;
 4765 tr_valid:
 4766 done:
 4767         *plength = len;
 4768         *pptr = ptr;
 4769         return (err);
 4770 }
 4771 
 4772 static void
 4773 dwc_otg_xfer_setup(struct usb_setup_params *parm)
 4774 {
 4775         struct usb_xfer *xfer;
 4776         void *last_obj;
 4777         uint32_t ntd;
 4778         uint32_t n;
 4779         uint8_t ep_no;
 4780         uint8_t ep_type;
 4781 
 4782         xfer = parm->curr_xfer;
 4783 
 4784         /*
 4785          * NOTE: This driver does not use any of the parameters that
 4786          * are computed from the following values. Just set some
 4787          * reasonable dummies:
 4788          */
 4789         parm->hc_max_packet_size = 0x500;
 4790         parm->hc_max_packet_count = 3;
 4791         parm->hc_max_frame_size = 3 * 0x500;
 4792 
 4793         usbd_transfer_setup_sub(parm);
 4794 
 4795         /*
 4796          * compute maximum number of TDs
 4797          */
 4798         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
 4799 
 4800         if (ep_type == UE_CONTROL) {
 4801                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
 4802                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
 4803         } else {
 4804                 ntd = xfer->nframes + 1 /* SYNC */ ;
 4805         }
 4806 
 4807         /*
 4808          * check if "usbd_transfer_setup_sub" set an error
 4809          */
 4810         if (parm->err)
 4811                 return;
 4812 
 4813         /*
 4814          * allocate transfer descriptors
 4815          */
 4816         last_obj = NULL;
 4817 
 4818         ep_no = xfer->endpointno & UE_ADDR;
 4819 
 4820         /*
 4821          * Check for a valid endpoint profile in USB device mode:
 4822          */
 4823         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 4824                 const struct usb_hw_ep_profile *pf;
 4825 
 4826                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
 4827 
 4828                 if (pf == NULL) {
 4829                         /* should not happen */
 4830                         parm->err = USB_ERR_INVAL;
 4831                         return;
 4832                 }
 4833         }
 4834 
 4835         /* align data */
 4836         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 4837 
 4838         for (n = 0; n != ntd; n++) {
 4839                 struct dwc_otg_td *td;
 4840 
 4841                 if (parm->buf) {
 4842                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
 4843 
 4844                         /* compute shared bandwidth resource index for TT */
 4845                         if (dwc_otg_uses_split(parm->udev)) {
 4846                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
 4847                                         td->tt_index = parm->udev->device_index;
 4848                                 else
 4849                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
 4850                         } else {
 4851                                 td->tt_index = parm->udev->device_index;
 4852                         }
 4853 
 4854                         /* init TD */
 4855                         td->max_packet_size = xfer->max_packet_size;
 4856                         td->max_packet_count = xfer->max_packet_count;
 4857                         /* range check */
 4858                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
 4859                                 td->max_packet_count = 1;
 4860                         td->ep_no = ep_no;
 4861                         td->ep_type = ep_type;
 4862                         td->obj_next = last_obj;
 4863 
 4864                         last_obj = td;
 4865                 }
 4866                 parm->size[0] += sizeof(*td);
 4867         }
 4868 
 4869         xfer->td_start[0] = last_obj;
 4870 }
 4871 
 4872 static void
 4873 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
 4874 {
 4875         return;
 4876 }
 4877 
 4878 static void
 4879 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
 4880     struct usb_endpoint *ep)
 4881 {
 4882         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
 4883 
 4884         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
 4885             ep, udev->address,
 4886             edesc->bEndpointAddress, udev->flags.usb_mode,
 4887             sc->sc_rt_addr, udev->device_index);
 4888 
 4889         if (udev->device_index != sc->sc_rt_addr) {
 4890                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
 4891                         if (udev->speed != USB_SPEED_FULL &&
 4892                             udev->speed != USB_SPEED_HIGH) {
 4893                                 /* not supported */
 4894                                 return;
 4895                         }
 4896                 } else {
 4897                         if (udev->speed == USB_SPEED_HIGH &&
 4898                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
 4899                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
 4900                                 /* not supported */
 4901                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
 4902                                     "endpoint not supported\n");
 4903                                 return;
 4904                         }
 4905                 }
 4906                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
 4907                         ep->methods = &dwc_otg_device_isoc_methods;
 4908                 else
 4909                         ep->methods = &dwc_otg_device_non_isoc_methods;
 4910         }
 4911 }
 4912 
 4913 static void
 4914 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
 4915 {
 4916         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
 4917 
 4918         switch (state) {
 4919         case USB_HW_POWER_SUSPEND:
 4920                 dwc_otg_suspend(sc);
 4921                 break;
 4922         case USB_HW_POWER_SHUTDOWN:
 4923                 dwc_otg_uninit(sc);
 4924                 break;
 4925         case USB_HW_POWER_RESUME:
 4926                 dwc_otg_resume(sc);
 4927                 break;
 4928         default:
 4929                 break;
 4930         }
 4931 }
 4932 
 4933 static void
 4934 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
 4935 {
 4936         /* DMA delay - wait until any use of memory is finished */
 4937         *pus = (2125);                  /* microseconds */
 4938 }
 4939 
 4940 static void
 4941 dwc_otg_device_resume(struct usb_device *udev)
 4942 {
 4943         DPRINTF("\n");
 4944 
 4945         /* poll all transfers again to restart resumed ones */
 4946         dwc_otg_do_poll(udev->bus);
 4947 }
 4948 
 4949 static void
 4950 dwc_otg_device_suspend(struct usb_device *udev)
 4951 {
 4952         DPRINTF("\n");
 4953 }
 4954 
 4955 static const struct usb_bus_methods dwc_otg_bus_methods =
 4956 {
 4957         .endpoint_init = &dwc_otg_ep_init,
 4958         .xfer_setup = &dwc_otg_xfer_setup,
 4959         .xfer_unsetup = &dwc_otg_xfer_unsetup,
 4960         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
 4961         .xfer_stall = &dwc_otg_xfer_stall,
 4962         .set_stall = &dwc_otg_set_stall,
 4963         .clear_stall = &dwc_otg_clear_stall,
 4964         .roothub_exec = &dwc_otg_roothub_exec,
 4965         .xfer_poll = &dwc_otg_do_poll,
 4966         .device_state_change = &dwc_otg_device_state_change,
 4967         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
 4968         .get_dma_delay = &dwc_otg_get_dma_delay,
 4969         .device_resume = &dwc_otg_device_resume,
 4970         .device_suspend = &dwc_otg_device_suspend,
 4971 };
Cache object: 307abac40f7514c0dda3178baa11745c
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/usb/controller/dwc_otg.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/usb/controller/dwc_otg.c
