mvsata.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $NetBSD: mvsata.c,v 1.62 2021/12/05 04:37:11 msaitoh Exp $      */
    2 /*
    3  * Copyright (c) 2008 KIYOHARA Takashi
    4  * All rights reserved.
    5  *
    6  * Redistribution and use in source and binary forms, with or without
    7  * modification, are permitted provided that the following conditions
    8  * are met:
    9  * 1. Redistributions of source code must retain the above copyright
   10  *    notice, this list of conditions and the following disclaimer.
   11  * 2. Redistributions in binary form must reproduce the above copyright
   12  *    notice, this list of conditions and the following disclaimer in the
   13  *    documentation and/or other materials provided with the distribution.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   18  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
   19  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   20  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   21  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
   23  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
   24  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   25  * POSSIBILITY OF SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __KERNEL_RCSID(0, "$NetBSD: mvsata.c,v 1.62 2021/12/05 04:37:11 msaitoh Exp $");
   30 
   31 #include "opt_mvsata.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/buf.h>
   35 #include <sys/bus.h>
   36 #include <sys/cpu.h>
   37 #include <sys/device.h>
   38 #include <sys/disklabel.h>
   39 #include <sys/errno.h>
   40 #include <sys/kernel.h>
   41 #include <sys/malloc.h>
   42 #include <sys/proc.h>
   43 
   44 #include <machine/vmparam.h>
   45 
   46 #include <dev/ata/atareg.h>
   47 #include <dev/ata/atavar.h>
   48 #include <dev/ic/wdcvar.h>
   49 #include <dev/ata/satafisvar.h>
   50 #include <dev/ata/satafisreg.h>
   51 #include <dev/ata/satapmpreg.h>
   52 #include <dev/ata/satareg.h>
   53 #include <dev/ata/satavar.h>
   54 
   55 #include <dev/scsipi/scsi_all.h>        /* for SCSI status */
   56 
   57 #include "atapibus.h"
   58 
   59 #include <dev/pci/pcidevs.h> /* XXX should not be here */
   60 
   61 /*
   62  * Nice things to do:
   63  *
   64  * - MSI/MSI-X support - though on some models MSI actually doesn't work
   65  *   even when hardware claims to support it, according to FreeBSD/OpenBSD
   66  * - move pci-specific code to the pci attach code
   67  * - mvsata(4) use 64-bit DMA on hardware which claims to support it
   68  *   - e.g. AHA1430SA does not really work, crash in mvsata_intr() on boot
   69  */
   70 
   71 #include <dev/ic/mvsatareg.h>
   72 #include <dev/ic/mvsatavar.h>
   73 
   74 #define MVSATA_DEV(sc)          ((sc)->sc_wdcdev.sc_atac.atac_dev)
   75 #define MVSATA_DEV2(mvport)     ((mvport)->port_ata_channel.ch_atac->atac_dev)
   76 
   77 #define MVSATA_HC_READ_4(hc, reg) \
   78         bus_space_read_4((hc)->hc_iot, (hc)->hc_ioh, (reg))
   79 #define MVSATA_HC_WRITE_4(hc, reg, val) \
   80         bus_space_write_4((hc)->hc_iot, (hc)->hc_ioh, (reg), (val))
   81 #define MVSATA_EDMA_READ_4(mvport, reg) \
   82         bus_space_read_4((mvport)->port_iot, (mvport)->port_ioh, (reg))
   83 #define MVSATA_EDMA_WRITE_4(mvport, reg, val) \
   84         bus_space_write_4((mvport)->port_iot, (mvport)->port_ioh, (reg), (val))
   85 #define MVSATA_WDC_READ_2(mvport, reg) \
   86         bus_space_read_2((mvport)->port_iot, (mvport)->port_ioh, \
   87         SHADOW_REG_BLOCK_OFFSET + (reg))
   88 #define MVSATA_WDC_READ_1(mvport, reg) \
   89         bus_space_read_1((mvport)->port_iot, (mvport)->port_ioh, \
   90         SHADOW_REG_BLOCK_OFFSET + (reg))
   91 #define MVSATA_WDC_WRITE_2(mvport, reg, val) \
   92         bus_space_write_2((mvport)->port_iot, (mvport)->port_ioh, \
   93         SHADOW_REG_BLOCK_OFFSET + (reg), (val))
   94 #define MVSATA_WDC_WRITE_1(mvport, reg, val) \
   95         bus_space_write_1((mvport)->port_iot, (mvport)->port_ioh, \
   96         SHADOW_REG_BLOCK_OFFSET + (reg), (val))
   97 
   98 #ifdef MVSATA_DEBUG
   99 
  100 #define DEBUG_INTR   0x01
  101 #define DEBUG_XFERS  0x02
  102 #define DEBUG_FUNCS  0x08
  103 #define DEBUG_PROBE  0x10
  104 
  105 #define DPRINTF(n,x)    if (mvsata_debug & (n)) printf x
  106 int     mvsata_debug = 0;
  107 #else
  108 #define DPRINTF(n,x)
  109 #endif
  110 
  111 #define ATA_DELAY               10000   /* 10s for a drive I/O */
  112 #define ATAPI_DELAY             10      /* 10 ms, this is used only before
  113                                            sending a cmd */
  114 #define ATAPI_MODE_DELAY        1000    /* 1s, timeout for SET_FEATURE cmds */
  115 
  116 #define MVSATA_MAX_SEGS         (MAXPHYS / PAGE_SIZE + 1)
  117 #define MVSATA_EPRD_MAX_SIZE    (sizeof(struct eprd) * MVSATA_MAX_SEGS)
  118 
  119 
  120 static void mvsata_probe_drive(struct ata_channel *);
  121 
  122 #ifndef MVSATA_WITHOUTDMA
  123 static void mvsata_reset_channel(struct ata_channel *, int);
  124 static void mvsata_bio(struct ata_drive_datas *, struct ata_xfer *);
  125 static void mvsata_reset_drive(struct ata_drive_datas *, int, uint32_t *);
  126 static void mvsata_exec_command(struct ata_drive_datas *, struct ata_xfer *);
  127 static int mvsata_addref(struct ata_drive_datas *);
  128 static void mvsata_delref(struct ata_drive_datas *);
  129 static void mvsata_killpending(struct ata_drive_datas *);
  130 
  131 #if NATAPIBUS > 0
  132 static void mvsata_atapibus_attach(struct atabus_softc *);
  133 static void mvsata_atapi_scsipi_request(struct scsipi_channel *,
  134                                         scsipi_adapter_req_t, void *);
  135 static void mvsata_atapi_minphys(struct buf *);
  136 static void mvsata_atapi_probe_device(struct atapibus_softc *, int);
  137 static void mvsata_atapi_kill_pending(struct scsipi_periph *);
  138 #endif
  139 #endif
  140 
  141 static void mvsata_setup_channel(struct ata_channel *);
  142 
  143 #ifndef MVSATA_WITHOUTDMA
  144 static int mvsata_bio_start(struct ata_channel *, struct ata_xfer *);
  145 static int mvsata_bio_intr(struct ata_channel *, struct ata_xfer *, int);
  146 static int mvsata_bio_poll(struct ata_channel *, struct ata_xfer *);
  147 static void mvsata_bio_kill_xfer(struct ata_channel *, struct ata_xfer *, int);
  148 static void mvsata_bio_done(struct ata_channel *, struct ata_xfer *);
  149 static int mvsata_bio_ready(struct mvsata_port *, struct ata_bio *, int,
  150                             int);
  151 static int mvsata_wdc_cmd_start(struct ata_channel *, struct ata_xfer *);
  152 static int mvsata_wdc_cmd_intr(struct ata_channel *, struct ata_xfer *, int);
  153 static int mvsata_wdc_cmd_poll(struct ata_channel *, struct ata_xfer *);
  154 static void mvsata_wdc_cmd_kill_xfer(struct ata_channel *, struct ata_xfer *,
  155                                      int);
  156 static void mvsata_wdc_cmd_done(struct ata_channel *, struct ata_xfer *);
  157 static void mvsata_wdc_cmd_done_end(struct ata_channel *, struct ata_xfer *);
  158 #if NATAPIBUS > 0
  159 static int mvsata_atapi_start(struct ata_channel *, struct ata_xfer *);
  160 static int mvsata_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
  161 static int mvsata_atapi_poll(struct ata_channel *, struct ata_xfer *);
  162 static void mvsata_atapi_kill_xfer(struct ata_channel *, struct ata_xfer *,
  163                                    int);
  164 static void mvsata_atapi_reset(struct ata_channel *, struct ata_xfer *);
  165 static void mvsata_atapi_phase_complete(struct ata_xfer *, int);
  166 static void mvsata_atapi_done(struct ata_channel *, struct ata_xfer *);
  167 static void mvsata_atapi_polldsc(void *);
  168 #endif
  169 
  170 static int mvsata_edma_enqueue(struct mvsata_port *, struct ata_xfer *);
  171 static int mvsata_edma_handle(struct mvsata_port *, struct ata_xfer *);
  172 static int mvsata_edma_wait(struct mvsata_port *, struct ata_xfer *, int);
  173 static void mvsata_edma_rqq_remove(struct mvsata_port *, struct ata_xfer *);
  174 #if NATAPIBUS > 0
  175 static int mvsata_bdma_init(struct mvsata_port *, struct ata_xfer *);
  176 static void mvsata_bdma_start(struct mvsata_port *);
  177 #endif
  178 #endif
  179 
  180 static int mvsata_nondma_handle(struct mvsata_port *);
  181 
  182 static int mvsata_port_init(struct mvsata_hc *, int);
  183 static int mvsata_wdc_reg_init(struct mvsata_port *, struct wdc_regs *);
  184 #ifndef MVSATA_WITHOUTDMA
  185 static void mvsata_channel_recover(struct ata_channel *, int, uint32_t);
  186 static void *mvsata_edma_resource_prepare(struct mvsata_port *, bus_dma_tag_t,
  187                                           bus_dmamap_t *, size_t, int);
  188 static void mvsata_edma_resource_purge(struct mvsata_port *, bus_dma_tag_t,
  189                                        bus_dmamap_t, void *);
  190 static int mvsata_dma_bufload(struct mvsata_port *, int, void *, size_t, int);
  191 static inline void mvsata_dma_bufunload(struct mvsata_port *, int, int);
  192 #endif
  193 
  194 static void mvsata_hreset_port(struct mvsata_port *);
  195 static void mvsata_reset_port(struct mvsata_port *);
  196 static void mvsata_reset_hc(struct mvsata_hc *);
  197 static uint32_t mvsata_softreset(struct mvsata_port *, int);
  198 #ifndef MVSATA_WITHOUTDMA
  199 static void mvsata_edma_reset_qptr(struct mvsata_port *);
  200 static inline void mvsata_edma_enable(struct mvsata_port *);
  201 static void mvsata_edma_disable(struct mvsata_port *, int, int);
  202 static void mvsata_edma_config(struct mvsata_port *, enum mvsata_edmamode);
  203 
  204 static void mvsata_edma_setup_crqb(struct mvsata_port *, int,
  205                                    struct ata_xfer *);
  206 #endif
  207 static uint32_t mvsata_read_preamps_gen1(struct mvsata_port *);
  208 static void mvsata_fix_phy_gen1(struct mvsata_port *);
  209 static void mvsata_devconn_gen1(struct mvsata_port *);
  210 
  211 static uint32_t mvsata_read_preamps_gen2(struct mvsata_port *);
  212 static void mvsata_fix_phy_gen2(struct mvsata_port *);
  213 #ifndef MVSATA_WITHOUTDMA
  214 static void mvsata_edma_setup_crqb_gen2e(struct mvsata_port *, int,
  215                                          struct ata_xfer *);
  216 
  217 #ifdef MVSATA_DEBUG
  218 static void mvsata_print_crqb(struct mvsata_port *, int);
  219 static void mvsata_print_crpb(struct mvsata_port *, int);
  220 static void mvsata_print_eprd(struct mvsata_port *, int);
  221 #endif
  222 
  223 static const struct ata_bustype mvsata_ata_bustype = {
  224         .bustype_type = SCSIPI_BUSTYPE_ATA,
  225         .ata_bio = mvsata_bio,
  226         .ata_reset_drive = mvsata_reset_drive,
  227         .ata_reset_channel = mvsata_reset_channel,
  228         .ata_exec_command = mvsata_exec_command,
  229         .ata_get_params = ata_get_params,
  230         .ata_addref = mvsata_addref,
  231         .ata_delref = mvsata_delref,
  232         .ata_killpending = mvsata_killpending,
  233         .ata_recovery = mvsata_channel_recover,
  234 };
  235 
  236 #if NATAPIBUS > 0
  237 static const struct scsipi_bustype mvsata_atapi_bustype = {
  238         .bustype_type = SCSIPI_BUSTYPE_ATAPI,
  239         .bustype_cmd = atapi_scsipi_cmd,
  240         .bustype_interpret_sense = atapi_interpret_sense,
  241         .bustype_printaddr = atapi_print_addr,
  242         .bustype_kill_pending = mvsata_atapi_kill_pending,
  243         .bustype_async_event_xfer_mode = NULL,
  244 };
  245 #endif /* NATAPIBUS */
  246 #endif
  247 
  248 static void
  249 mvsata_pmp_select(struct mvsata_port *mvport, int pmpport)
  250 {
  251         uint32_t ifctl;
  252 
  253         KASSERT(pmpport < PMP_MAX_DRIVES);
  254 #if defined(DIAGNOSTIC) || defined(MVSATA_DEBUG)
  255         if ((MVSATA_EDMA_READ_4(mvport, EDMA_CMD) & EDMA_CMD_EENEDMA) != 0) {
  256                 panic("EDMA enabled");
  257         }
  258 #endif
  259 
  260         ifctl = MVSATA_EDMA_READ_4(mvport, SATA_SATAICTL);
  261         ifctl &= ~0xf;
  262         ifctl |= pmpport;
  263         MVSATA_EDMA_WRITE_4(mvport, SATA_SATAICTL, ifctl);
  264 }
  265 
  266 int
  267 mvsata_attach(struct mvsata_softc *sc, const struct mvsata_product *product,
  268               int (*mvsata_sreset)(struct mvsata_softc *),
  269               int (*mvsata_misc_reset)(struct mvsata_softc *),
  270               int read_pre_amps)
  271 {
  272         struct mvsata_hc *mvhc;
  273         struct mvsata_port *mvport;
  274         uint32_t (*read_preamps)(struct mvsata_port *) = NULL;
  275         void (*_fix_phy)(struct mvsata_port *) = NULL;
  276 #ifndef MVSATA_WITHOUTDMA
  277         void (*edma_setup_crqb)
  278             (struct mvsata_port *, int, struct ata_xfer *) = NULL;
  279 #endif
  280         int hc, port, channel;
  281 
  282         aprint_normal_dev(MVSATA_DEV(sc), "Gen%s, %dhc, %dport/hc\n",
  283             (product->generation == gen1) ? "I" :
  284             ((product->generation == gen2) ? "II" : "IIe"),
  285             product->hc, product->port);
  286 
  287 
  288         switch (product->generation) {
  289         case gen1:
  290                 mvsata_sreset = NULL;
  291                 read_pre_amps = 1;      /* MUST */
  292                 read_preamps = mvsata_read_preamps_gen1;
  293                 _fix_phy = mvsata_fix_phy_gen1;
  294 #ifndef MVSATA_WITHOUTDMA
  295                 edma_setup_crqb = mvsata_edma_setup_crqb;
  296 #endif
  297                 break;
  298 
  299         case gen2:
  300                 read_preamps = mvsata_read_preamps_gen2;
  301                 _fix_phy = mvsata_fix_phy_gen2;
  302 #ifndef MVSATA_WITHOUTDMA
  303                 edma_setup_crqb = mvsata_edma_setup_crqb;
  304 #endif
  305                 break;
  306 
  307         case gen2e:
  308                 read_preamps = mvsata_read_preamps_gen2;
  309                 _fix_phy = mvsata_fix_phy_gen2;
  310 #ifndef MVSATA_WITHOUTDMA
  311                 edma_setup_crqb = mvsata_edma_setup_crqb_gen2e;
  312                 sc->sc_wdcdev.sc_atac.atac_cap |= ATAC_CAP_NCQ;
  313 #endif
  314                 break;
  315         }
  316 
  317         sc->sc_gen = product->generation;
  318         sc->sc_hc = product->hc;
  319         sc->sc_port = product->port;
  320         sc->sc_flags = product->flags;
  321 
  322 #ifdef MVSATA_WITHOUTDMA
  323         sc->sc_wdcdev.sc_atac.atac_cap |= ATAC_CAP_DATA16;
  324 #else
  325         sc->sc_edma_setup_crqb = edma_setup_crqb;
  326         sc->sc_wdcdev.sc_atac.atac_cap |=
  327             (ATAC_CAP_DATA16 | ATAC_CAP_DMA | ATAC_CAP_UDMA);
  328 #endif
  329         sc->sc_wdcdev.sc_atac.atac_pio_cap = 4;
  330 #ifdef MVSATA_WITHOUTDMA
  331         sc->sc_wdcdev.sc_atac.atac_dma_cap = 0;
  332         sc->sc_wdcdev.sc_atac.atac_udma_cap = 0;
  333 #else
  334         sc->sc_wdcdev.sc_atac.atac_dma_cap = 2;
  335         sc->sc_wdcdev.sc_atac.atac_udma_cap = 6;
  336 #endif
  337         sc->sc_wdcdev.sc_atac.atac_channels = sc->sc_ata_channels;
  338         sc->sc_wdcdev.sc_atac.atac_nchannels = sc->sc_hc * sc->sc_port;
  339 #ifndef MVSATA_WITHOUTDMA
  340         sc->sc_wdcdev.sc_atac.atac_bustype_ata = &mvsata_ata_bustype;
  341 #if NATAPIBUS > 0
  342         sc->sc_wdcdev.sc_atac.atac_atapibus_attach = mvsata_atapibus_attach;
  343 #endif
  344 #endif
  345         sc->sc_wdcdev.wdc_maxdrives = 1;        /* SATA is always 1 drive */
  346         sc->sc_wdcdev.sc_atac.atac_probe = mvsata_probe_drive;
  347         sc->sc_wdcdev.sc_atac.atac_set_modes = mvsata_setup_channel;
  348 
  349         sc->sc_wdc_regs =
  350             malloc(sizeof(struct wdc_regs) * product->hc * product->port,
  351             M_DEVBUF, M_WAITOK);
  352         sc->sc_wdcdev.regs = sc->sc_wdc_regs;
  353 
  354         for (hc = 0; hc < sc->sc_hc; hc++) {
  355                 mvhc = &sc->sc_hcs[hc];
  356                 mvhc->hc = hc;
  357                 mvhc->hc_sc = sc;
  358                 mvhc->hc_iot = sc->sc_iot;
  359                 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
  360                     hc * SATAHC_REGISTER_SIZE, SATAHC_REGISTER_SIZE,
  361                     &mvhc->hc_ioh)) {
  362                         aprint_error_dev(MVSATA_DEV(sc),
  363                             "can't subregion SATAHC %d registers\n", hc);
  364                         continue;
  365                 }
  366 
  367                 for (port = 0; port < sc->sc_port; port++)
  368                         if (mvsata_port_init(mvhc, port) == 0) {
  369                                 int pre_amps;
  370 
  371                                 mvport = mvhc->hc_ports[port];
  372                                 pre_amps = read_pre_amps ?
  373                                     read_preamps(mvport) : 0x00000720;
  374                                 mvport->_fix_phy_param.pre_amps = pre_amps;
  375                                 mvport->_fix_phy_param._fix_phy = _fix_phy;
  376 
  377                                 if (!mvsata_sreset)
  378                                         mvsata_reset_port(mvport);
  379                         }
  380 
  381                 if (!mvsata_sreset)
  382                         mvsata_reset_hc(mvhc);
  383         }
  384         if (mvsata_sreset)
  385                 mvsata_sreset(sc);
  386 
  387         if (mvsata_misc_reset)
  388                 mvsata_misc_reset(sc);
  389 
  390         for (hc = 0; hc < sc->sc_hc; hc++)
  391                 for (port = 0; port < sc->sc_port; port++) {
  392                         mvport = sc->sc_hcs[hc].hc_ports[port];
  393                         if (mvport == NULL)
  394                                 continue;
  395                         if (mvsata_sreset)
  396                                 mvport->_fix_phy_param._fix_phy(mvport);
  397                 }
  398         for (channel = 0; channel < sc->sc_hc * sc->sc_port; channel++)
  399                 wdcattach(sc->sc_ata_channels[channel]);
  400 
  401         return 0;
  402 }
  403 
  404 int
  405 mvsata_intr(struct mvsata_hc *mvhc)
  406 {
  407         struct mvsata_softc *sc = mvhc->hc_sc;
  408         struct mvsata_port *mvport;
  409         uint32_t cause;
  410         int port, handled = 0;
  411 
  412         cause = MVSATA_HC_READ_4(mvhc, SATAHC_IC);
  413 
  414         DPRINTF(DEBUG_INTR, ("%s:%d: mvsata_intr: cause=0x%08x\n",
  415             device_xname(MVSATA_DEV(sc)), mvhc->hc, cause));
  416 
  417         if (cause & SATAHC_IC_SAINTCOAL)
  418                 MVSATA_HC_WRITE_4(mvhc, SATAHC_IC, ~SATAHC_IC_SAINTCOAL);
  419         cause &= ~SATAHC_IC_SAINTCOAL;
  420 
  421         for (port = 0; port < sc->sc_port; port++) {
  422                 mvport = mvhc->hc_ports[port];
  423 
  424                 if (cause & SATAHC_IC_DONE(port)) {
  425 #ifndef MVSATA_WITHOUTDMA
  426                         handled = mvsata_edma_handle(mvport, NULL);
  427 #endif
  428                         MVSATA_HC_WRITE_4(mvhc, SATAHC_IC,
  429                             ~SATAHC_IC_DONE(port));
  430                 }
  431 
  432                 if (cause & SATAHC_IC_SADEVINTERRUPT(port)) {
  433                         (void) mvsata_nondma_handle(mvport);
  434                         MVSATA_HC_WRITE_4(mvhc, SATAHC_IC,
  435                             ~SATAHC_IC_SADEVINTERRUPT(port));
  436                         handled = 1;
  437                 }
  438         }
  439 
  440         return handled;
  441 }
  442 
  443 static int
  444 mvsata_nondma_handle(struct mvsata_port *mvport)
  445 {
  446         struct ata_channel *chp = &mvport->port_ata_channel;
  447         struct ata_xfer *xfer;
  448         int ret;
  449 
  450         /*
  451          * The chip doesn't support several pending non-DMA commands,
  452          * and the ata middle layer never issues several non-NCQ commands,
  453          * so there must be exactly one active command at this moment.
  454          */
  455         xfer = ata_queue_get_active_xfer(chp);
  456         if (xfer == NULL) {
  457                 /* Can happen after error recovery, ignore */
  458                 DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
  459                     ("%s:%d: %s: intr without xfer\n",
  460                     device_xname(MVSATA_DEV2(mvport)), chp->ch_channel,
  461                     __func__));
  462                 return 0;
  463         }
  464 
  465         ret = xfer->ops->c_intr(chp, xfer, 1);
  466         return (ret);
  467 }
  468 
  469 int
  470 mvsata_error(struct mvsata_port *mvport)
  471 {
  472         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
  473         uint32_t cause;
  474 
  475         cause = MVSATA_EDMA_READ_4(mvport, EDMA_IEC);
  476         /*
  477          * We must ack SATA_SE and SATA_FISIC before acking corresponding bits
  478          * in EDMA_IEC.
  479          */
  480         if (cause & EDMA_IE_SERRINT) {
  481                 MVSATA_EDMA_WRITE_4(mvport, SATA_SE,
  482                     MVSATA_EDMA_READ_4(mvport, SATA_SEIM));
  483         }
  484         if (cause & EDMA_IE_ETRANSINT) {
  485                 MVSATA_EDMA_WRITE_4(mvport, SATA_FISIC,
  486                     ~MVSATA_EDMA_READ_4(mvport, SATA_FISIM));
  487         }
  488         MVSATA_EDMA_WRITE_4(mvport, EDMA_IEC, ~cause);
  489 
  490         DPRINTF(DEBUG_INTR, ("%s:%d:%d:"
  491             " mvsata_error: cause=0x%08x, mask=0x%08x, status=0x%08x\n",
  492             device_xname(MVSATA_DEV2(mvport)), mvport->port_hc->hc,
  493             mvport->port, cause, MVSATA_EDMA_READ_4(mvport, EDMA_IEM),
  494             MVSATA_EDMA_READ_4(mvport, EDMA_S)));
  495 
  496         cause &= MVSATA_EDMA_READ_4(mvport, EDMA_IEM);
  497         if (!cause)
  498                 return 0;
  499 
  500         if (cause & EDMA_IE_EDEVDIS) {
  501                 aprint_normal("%s:%d:%d: device disconnect\n",
  502                     device_xname(MVSATA_DEV2(mvport)),
  503                     mvport->port_hc->hc, mvport->port);
  504         }
  505         if (cause & EDMA_IE_EDEVCON) {
  506                 if (sc->sc_gen == gen1)
  507                         mvsata_devconn_gen1(mvport);
  508 
  509                 DPRINTF(DEBUG_INTR, ("    device connected\n"));
  510         }
  511 
  512 #ifndef MVSATA_WITHOUTDMA
  513         if ((sc->sc_gen == gen1 && cause & EDMA_IE_ETRANSINT) ||
  514             (sc->sc_gen != gen1 && cause & EDMA_IE_ESELFDIS)) {
  515                 switch (mvport->port_edmamode_curr) {
  516                 case dma:
  517                 case queued:
  518                 case ncq:
  519                         mvsata_edma_reset_qptr(mvport);
  520                         mvsata_edma_enable(mvport);
  521                         if (cause & EDMA_IE_EDEVERR)
  522                                 break;
  523 
  524                         /* FALLTHROUGH */
  525 
  526                 case nodma:
  527                 default:
  528                         DPRINTF(DEBUG_INTR,
  529                             ("%s:%d:%d: EDMA self disable happen 0x%x\n",
  530                             device_xname(MVSATA_DEV2(mvport)),
  531                             mvport->port_hc->hc, mvport->port, cause));
  532                         break;
  533                 }
  534         }
  535 #endif
  536         if (cause & EDMA_IE_ETRANSINT) {
  537                 /* hot plug the Port Multiplier */
  538                 aprint_normal("%s:%d:%d: detect Port Multiplier?\n",
  539                     device_xname(MVSATA_DEV2(mvport)),
  540                     mvport->port_hc->hc, mvport->port);
  541         }
  542         if (cause & EDMA_IE_EDEVERR) {
  543                 struct ata_channel *chp = &mvport->port_ata_channel;
  544 
  545                 aprint_error("%s:%d:%d: device error, recovering\n",
  546                     device_xname(MVSATA_DEV2(mvport)),
  547                     mvport->port_hc->hc, mvport->port);
  548 
  549                 ata_channel_lock(chp);
  550                 ata_thread_run(chp, 0, ATACH_TH_RECOVERY,
  551                     ATACH_ERR_ST(0, WDCS_ERR));
  552                 ata_channel_unlock(chp);
  553         }
  554 
  555         return 1;
  556 }
  557 
  558 #ifndef MVSATA_WITHOUTDMA
  559 static void
  560 mvsata_channel_recover(struct ata_channel *chp, int flags, uint32_t tfd)
  561 {
  562         struct mvsata_port * const mvport = (struct mvsata_port *)chp;
  563         int drive;
  564 
  565         ata_channel_lock_owned(chp);
  566 
  567         if (chp->ch_ndrives > PMP_PORT_CTL) {
  568                 /* Get PM port number for the device in error. This device
  569                  * doesn't seem to have dedicated register for this, so just
  570                  * assume last selected port was the one. */
  571                 /* XXX FIS-based switching */
  572                 drive = MVSATA_EDMA_READ_4(mvport, SATA_SATAICTL) & 0xf;
  573         } else
  574                 drive = 0;
  575 
  576         /*
  577          * Controller doesn't need any special action. Simply execute
  578          * READ LOG EXT for NCQ to unblock device processing, then continue
  579          * as if nothing happened.
  580          */
  581 
  582         ata_recovery_resume(chp, drive, tfd, AT_POLL);
  583 
  584         /* Drive unblocked, back to normal operation */
  585         return;
  586 }
  587 #endif /* !MVSATA_WITHOUTDMA */
  588 
  589 /*
  590  * ATA callback entry points
  591  */
  592 
  593 static void
  594 mvsata_probe_drive(struct ata_channel *chp)
  595 {
  596         struct mvsata_port * const mvport = (struct mvsata_port *)chp;
  597         uint32_t sstat, sig;
  598 
  599         ata_channel_lock(chp);
  600 
  601         sstat = sata_reset_interface(chp, mvport->port_iot,
  602             mvport->port_sata_scontrol, mvport->port_sata_sstatus, AT_WAIT);
  603         switch (sstat) {
  604         case SStatus_DET_DEV:
  605                 mvsata_pmp_select(mvport, PMP_PORT_CTL);
  606                 sig = mvsata_softreset(mvport, AT_WAIT);
  607                 sata_interpret_sig(chp, 0, sig);
  608                 break;
  609         default:
  610                 break;
  611         }
  612 
  613         ata_channel_unlock(chp);
  614 }
  615 
  616 #ifndef MVSATA_WITHOUTDMA
  617 static void
  618 mvsata_reset_drive(struct ata_drive_datas *drvp, int flags, uint32_t *sigp)
  619 {
  620         struct ata_channel *chp = drvp->chnl_softc;
  621         struct mvsata_port *mvport = (struct mvsata_port *)chp;
  622         uint32_t edma_c;
  623         uint32_t sig;
  624 
  625         ata_channel_lock_owned(chp);
  626 
  627         edma_c = MVSATA_EDMA_READ_4(mvport, EDMA_CMD);
  628 
  629         DPRINTF(DEBUG_FUNCS,
  630             ("%s:%d: mvsata_reset_drive: drive=%d (EDMA %sactive)\n",
  631             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel, drvp->drive,
  632             (edma_c & EDMA_CMD_EENEDMA) ? "" : "not "));
  633 
  634         if (edma_c & EDMA_CMD_EENEDMA)
  635                 mvsata_edma_disable(mvport, 10000, flags);
  636 
  637         mvsata_pmp_select(mvport, drvp->drive);
  638 
  639         sig = mvsata_softreset(mvport, flags);
  640 
  641         if (sigp)
  642                 *sigp = sig;
  643 
  644         if (edma_c & EDMA_CMD_EENEDMA) {
  645                 mvsata_edma_reset_qptr(mvport);
  646                 mvsata_edma_enable(mvport);
  647         }
  648 }
  649 
  650 static void
  651 mvsata_reset_channel(struct ata_channel *chp, int flags)
  652 {
  653         struct mvsata_port *mvport = (struct mvsata_port *)chp;
  654         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
  655         uint32_t sstat, ctrl;
  656 
  657         DPRINTF(DEBUG_FUNCS, ("%s: mvsata_reset_channel: channel=%d\n",
  658             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel));
  659 
  660         ata_channel_lock_owned(chp);
  661 
  662         mvsata_hreset_port(mvport);
  663         sstat = sata_reset_interface(chp, mvport->port_iot,
  664             mvport->port_sata_scontrol, mvport->port_sata_sstatus, flags);
  665 
  666         if (flags & AT_WAIT && sstat == SStatus_DET_DEV_NE &&
  667             sc->sc_gen != gen1) {
  668                 /* Downgrade to GenI */
  669                 const uint32_t val = SControl_IPM_NONE | SControl_SPD_ANY |
  670                     SControl_DET_DISABLE;
  671 
  672                 bus_space_write_4(mvport->port_iot,
  673                     mvport->port_sata_scontrol, 0, val);
  674 
  675                 ctrl = MVSATA_EDMA_READ_4(mvport, SATA_SATAICFG);
  676                 ctrl &= ~(1 << 17);     /* Disable GenII */
  677                 MVSATA_EDMA_WRITE_4(mvport, SATA_SATAICFG, ctrl);
  678 
  679                 mvsata_hreset_port(mvport);
  680                 sata_reset_interface(chp, mvport->port_iot,
  681                     mvport->port_sata_scontrol, mvport->port_sata_sstatus,
  682                     flags);
  683         }
  684 
  685         ata_kill_active(chp, KILL_RESET, flags);
  686 
  687         mvsata_edma_config(mvport, mvport->port_edmamode_curr);
  688         mvsata_edma_reset_qptr(mvport);
  689         mvsata_edma_enable(mvport);
  690 }
  691 
  692 static int
  693 mvsata_addref(struct ata_drive_datas *drvp)
  694 {
  695 
  696         return 0;
  697 }
  698 
  699 static void
  700 mvsata_delref(struct ata_drive_datas *drvp)
  701 {
  702 
  703         return;
  704 }
  705 
  706 static void
  707 mvsata_killpending(struct ata_drive_datas *drvp)
  708 {
  709 
  710         return;
  711 }
  712 
  713 #if NATAPIBUS > 0
  714 static void
  715 mvsata_atapibus_attach(struct atabus_softc *ata_sc)
  716 {
  717         struct ata_channel *chp = ata_sc->sc_chan;
  718         struct atac_softc *atac = chp->ch_atac;
  719         struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
  720         struct scsipi_channel *chan = &chp->ch_atapi_channel;
  721 
  722         /*
  723          * Fill in the scsipi_adapter.
  724          */
  725         adapt->adapt_dev = atac->atac_dev;
  726         adapt->adapt_nchannels = atac->atac_nchannels;
  727         adapt->adapt_request = mvsata_atapi_scsipi_request;
  728         adapt->adapt_minphys = mvsata_atapi_minphys;
  729         atac->atac_atapi_adapter.atapi_probe_device = mvsata_atapi_probe_device;
  730 
  731         /*
  732          * Fill in the scsipi_channel.
  733          */
  734         memset(chan, 0, sizeof(*chan));
  735         chan->chan_adapter = adapt;
  736         chan->chan_bustype = &mvsata_atapi_bustype;
  737         chan->chan_channel = chp->ch_channel;
  738         chan->chan_flags = SCSIPI_CHAN_OPENINGS;
  739         chan->chan_openings = 1;
  740         chan->chan_max_periph = 1;
  741         chan->chan_ntargets = 1;
  742         chan->chan_nluns = 1;
  743 
  744         chp->atapibus = config_found(ata_sc->sc_dev, chan, atapiprint,
  745             CFARGS(.iattr = "atapi"));
  746 }
  747 
  748 static void
  749 mvsata_atapi_minphys(struct buf *bp)
  750 {
  751 
  752         if (bp->b_bcount > MAXPHYS)
  753                 bp->b_bcount = MAXPHYS;
  754         minphys(bp);
  755 }
  756 
  757 static void
  758 mvsata_atapi_probe_device(struct atapibus_softc *sc, int target)
  759 {
  760         struct scsipi_channel *chan = sc->sc_channel;
  761         struct scsipi_periph *periph;
  762         struct ataparams ids;
  763         struct ataparams *id = &ids;
  764         struct mvsata_softc *mvc =
  765             device_private(chan->chan_adapter->adapt_dev);
  766         struct atac_softc *atac = &mvc->sc_wdcdev.sc_atac;
  767         struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
  768         struct ata_drive_datas *drvp = &chp->ch_drive[target];
  769         struct scsipibus_attach_args sa;
  770         char serial_number[21], model[41], firmware_revision[9];
  771         int s;
  772 
  773         /* skip if already attached */
  774         if (scsipi_lookup_periph(chan, target, 0) != NULL)
  775                 return;
  776 
  777         /* if no ATAPI device detected at attach time, skip */
  778         if (drvp->drive_type != ATA_DRIVET_ATAPI) {
  779                 DPRINTF(DEBUG_PROBE, ("%s:%d: mvsata_atapi_probe_device:"
  780                     " drive %d not present\n",
  781                     device_xname(atac->atac_dev), chp->ch_channel, target));
  782                 return;
  783         }
  784 
  785         /* Some ATAPI devices need a bit more time after software reset. */
  786         delay(5000);
  787         if (ata_get_params(drvp, AT_WAIT, id) == 0) {
  788 #ifdef ATAPI_DEBUG_PROBE
  789                 printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
  790                     device_xname(sc->sc_dev), target,
  791                     id->atap_config & ATAPI_CFG_CMD_MASK,
  792                     id->atap_config & ATAPI_CFG_DRQ_MASK);
  793 #endif
  794                 periph = scsipi_alloc_periph(M_WAITOK);
  795                 periph->periph_dev = NULL;
  796                 periph->periph_channel = chan;
  797                 periph->periph_switch = &atapi_probe_periphsw;
  798                 periph->periph_target = target;
  799                 periph->periph_lun = 0;
  800                 periph->periph_quirks = PQUIRK_ONLYBIG;
  801 
  802 #ifdef SCSIPI_DEBUG
  803                 if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
  804                     SCSIPI_DEBUG_TARGET == target)
  805                         periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
  806 #endif
  807                 periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
  808                 if (id->atap_config & ATAPI_CFG_REMOV)
  809                         periph->periph_flags |= PERIPH_REMOVABLE;
  810                 if (periph->periph_type == T_SEQUENTIAL) {
  811                         s = splbio();
  812                         drvp->drive_flags |= ATA_DRIVE_ATAPIDSCW;
  813                         splx(s);
  814                 }
  815 
  816                 sa.sa_periph = periph;
  817                 sa.sa_inqbuf.type = ATAPI_CFG_TYPE(id->atap_config);
  818                 sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
  819                     T_REMOV : T_FIXED;
  820                 strnvisx(model, sizeof(model), id->atap_model, 40,
  821                     VIS_TRIM|VIS_SAFE|VIS_OCTAL);
  822                 strnvisx(serial_number, sizeof(serial_number), id->atap_serial,
  823                     20, VIS_TRIM|VIS_SAFE|VIS_OCTAL);
  824                 strnvisx(firmware_revision, sizeof(firmware_revision),
  825                     id->atap_revision, 8, VIS_TRIM|VIS_SAFE|VIS_OCTAL);
  826                 sa.sa_inqbuf.vendor = model;
  827                 sa.sa_inqbuf.product = serial_number;
  828                 sa.sa_inqbuf.revision = firmware_revision;
  829 
  830                 /*
  831                  * Determine the operating mode capabilities of the device.
  832                  */
  833                 if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
  834                         periph->periph_cap |= PERIPH_CAP_CMD16;
  835                 /* XXX This is gross. */
  836                 periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
  837 
  838                 drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
  839 
  840                 if (drvp->drv_softc)
  841                         ata_probe_caps(drvp);
  842                 else {
  843                         s = splbio();
  844                         drvp->drive_type = ATA_DRIVET_NONE;
  845                         splx(s);
  846                 }
  847         } else {
  848                 DPRINTF(DEBUG_PROBE, ("%s:%d: mvsata_atapi_probe_device:"
  849                     " ATAPI_IDENTIFY_DEVICE failed for drive %d: error\n",
  850                     device_xname(atac->atac_dev), chp->ch_channel, target));
  851                 s = splbio();
  852                 drvp->drive_type = ATA_DRIVET_NONE;
  853                 splx(s);
  854         }
  855 }
  856 
  857 /*
  858  * Kill off all pending xfers for a periph.
  859  *
  860  * Must be called at splbio().
  861  */
  862 static void
  863 mvsata_atapi_kill_pending(struct scsipi_periph *periph)
  864 {
  865         struct atac_softc *atac =
  866             device_private(periph->periph_channel->chan_adapter->adapt_dev);
  867         struct ata_channel *chp =
  868             atac->atac_channels[periph->periph_channel->chan_channel];
  869 
  870         ata_kill_pending(&chp->ch_drive[periph->periph_target]);
  871 }
  872 #endif  /* NATAPIBUS > 0 */
  873 #endif  /* MVSATA_WITHOUTDMA */
  874 
  875 
  876 /*
  877  * mvsata_setup_channel()
  878  *   Setup EDMA registers and prepare/purge DMA resources.
  879  *   We assuming already stopped the EDMA.
  880  */
  881 static void
  882 mvsata_setup_channel(struct ata_channel *chp)
  883 {
  884 #ifndef MVSATA_WITHOUTDMA
  885         struct mvsata_port *mvport = (struct mvsata_port *)chp;
  886         struct ata_drive_datas *drvp;
  887         int drive, s;
  888         uint32_t edma_mode = nodma;
  889         int i;
  890         const int crqb_size = sizeof(union mvsata_crqb) * MVSATA_EDMAQ_LEN;
  891         const int crpb_size = sizeof(struct crpb) * MVSATA_EDMAQ_LEN;
  892         const int eprd_buf_size = MVSATA_EPRD_MAX_SIZE * MVSATA_EDMAQ_LEN;
  893 
  894         DPRINTF(DEBUG_FUNCS, ("%s:%d: mvsata_setup_channel: ",
  895             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel));
  896 
  897         for (drive = 0; drive < chp->ch_ndrives; drive++) {
  898                 drvp = &chp->ch_drive[drive];
  899 
  900                 /* If no drive, skip */
  901                 if (drvp->drive_type == ATA_DRIVET_NONE)
  902                         continue;
  903 
  904                 if (drvp->drive_flags & ATA_DRIVE_UDMA) {
  905                         /* use Ultra/DMA */
  906                         s = splbio();
  907                         drvp->drive_flags &= ~ATA_DRIVE_DMA;
  908                         splx(s);
  909                 }
  910 
  911                 if (drvp->drive_flags & (ATA_DRIVE_UDMA | ATA_DRIVE_DMA)) {
  912                         if (drvp->drive_flags & ATA_DRIVE_NCQ)
  913                                 edma_mode = ncq;
  914                         else if (drvp->drive_type == ATA_DRIVET_ATA)
  915                                 edma_mode = dma;
  916                 }
  917         }
  918 
  919         DPRINTF(DEBUG_FUNCS,
  920             ("EDMA %sactive mode\n", (edma_mode == nodma) ? "not " : ""));
  921 
  922         if (edma_mode == nodma) {
  923 no_edma:
  924                 if (mvport->port_crqb != NULL)
  925                         mvsata_edma_resource_purge(mvport, mvport->port_dmat,
  926                             mvport->port_crqb_dmamap, mvport->port_crqb);
  927                 if (mvport->port_crpb != NULL)
  928                         mvsata_edma_resource_purge(mvport, mvport->port_dmat,
  929                             mvport->port_crpb_dmamap, mvport->port_crpb);
  930                 if (mvport->port_eprd != NULL)
  931                         mvsata_edma_resource_purge(mvport, mvport->port_dmat,
  932                             mvport->port_eprd_dmamap, mvport->port_eprd);
  933 
  934                 return;
  935         }
  936 
  937         if (mvport->port_crqb == NULL)
  938                 mvport->port_crqb = mvsata_edma_resource_prepare(mvport,
  939                     mvport->port_dmat, &mvport->port_crqb_dmamap, crqb_size, 1);
  940         if (mvport->port_crpb == NULL)
  941                 mvport->port_crpb = mvsata_edma_resource_prepare(mvport,
  942                     mvport->port_dmat, &mvport->port_crpb_dmamap, crpb_size, 0);
  943         if (mvport->port_eprd == NULL) {
  944                 mvport->port_eprd = mvsata_edma_resource_prepare(mvport,
  945                     mvport->port_dmat, &mvport->port_eprd_dmamap, eprd_buf_size,
  946                     1);
  947                 for (i = 0; i < MVSATA_EDMAQ_LEN; i++) {
  948                         mvport->port_reqtbl[i].eprd_offset =
  949                             i * MVSATA_EPRD_MAX_SIZE;
  950                         mvport->port_reqtbl[i].eprd = mvport->port_eprd +
  951                             i * MVSATA_EPRD_MAX_SIZE / sizeof(struct eprd);
  952                 }
  953         }
  954 
  955         if (mvport->port_crqb == NULL || mvport->port_crpb == NULL ||
  956             mvport->port_eprd == NULL) {
  957                 aprint_error_dev(MVSATA_DEV2(mvport),
  958                     "channel %d: can't use EDMA\n", chp->ch_channel);
  959                 s = splbio();
  960                 for (drive = 0; drive < chp->ch_ndrives; drive++) {
  961                         drvp = &chp->ch_drive[drive];
  962 
  963                         /* If no drive, skip */
  964                         if (drvp->drive_type == ATA_DRIVET_NONE)
  965                                 continue;
  966 
  967                         drvp->drive_flags &= ~(ATA_DRIVE_UDMA | ATA_DRIVE_DMA);
  968                 }
  969                 splx(s);
  970                 goto no_edma;
  971         }
  972 
  973         mvsata_edma_config(mvport, edma_mode);
  974         mvsata_edma_reset_qptr(mvport);
  975         mvsata_edma_enable(mvport);
  976 #endif
  977 }
  978 
  979 #ifndef MVSATA_WITHOUTDMA
  980 static const struct ata_xfer_ops mvsata_bio_xfer_ops = {
  981         .c_start = mvsata_bio_start,
  982         .c_intr = mvsata_bio_intr,
  983         .c_poll = mvsata_bio_poll,
  984         .c_abort = mvsata_bio_done,
  985         .c_kill_xfer = mvsata_bio_kill_xfer,
  986 };
  987 
  988 static void
  989 mvsata_bio(struct ata_drive_datas *drvp, struct ata_xfer *xfer)
  990 {
  991         struct ata_channel *chp = drvp->chnl_softc;
  992         struct atac_softc *atac = chp->ch_atac;
  993         struct ata_bio *ata_bio = &xfer->c_bio;
  994 
  995         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
  996             ("%s:%d: mvsata_bio: drive=%d, blkno=%" PRId64
  997             ", bcount=%ld\n", device_xname(atac->atac_dev), chp->ch_channel,
  998             drvp->drive, ata_bio->blkno, ata_bio->bcount));
  999 
 1000         if (atac->atac_cap & ATAC_CAP_NOIRQ)
 1001                 ata_bio->flags |= ATA_POLL;
 1002         if (ata_bio->flags & ATA_POLL)
 1003                 xfer->c_flags |= C_POLL;
 1004         if ((drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) &&
 1005             (ata_bio->flags & ATA_SINGLE) == 0)
 1006                 xfer->c_flags |= C_DMA;
 1007         xfer->c_drive = drvp->drive;
 1008         xfer->c_databuf = ata_bio->databuf;
 1009         xfer->c_bcount = ata_bio->bcount;
 1010         xfer->ops = &mvsata_bio_xfer_ops;
 1011         ata_exec_xfer(chp, xfer);
 1012 }
 1013 
 1014 static int
 1015 mvsata_bio_start(struct ata_channel *chp, struct ata_xfer *xfer)
 1016 {
 1017         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1018         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 1019         struct atac_softc *atac = chp->ch_atac;
 1020         struct wdc_softc *wdc = CHAN_TO_WDC(chp);
 1021         struct ata_bio *ata_bio = &xfer->c_bio;
 1022         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 1023         int wait_flags = (xfer->c_flags & C_POLL) ? AT_POLL : 0;
 1024         u_int16_t cyl;
 1025         u_int8_t head, sect, cmd = 0;
 1026         int nblks, error, tfd;
 1027 
 1028         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, ("%s:%d: mvsata_bio_start: drive=%d\n",
 1029             device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive));
 1030 
 1031         ata_channel_lock_owned(chp);
 1032 
 1033         if (xfer->c_flags & C_DMA)
 1034                 if (drvp->n_xfers <= NXFER)
 1035                         drvp->n_xfers++;
 1036 
 1037         /*
 1038          *
 1039          * When starting a multi-sector transfer, or doing single-sector
 1040          * transfers...
 1041          */
 1042         if (xfer->c_skip == 0 || (ata_bio->flags & ATA_SINGLE) != 0) {
 1043                 if (ata_bio->flags & ATA_SINGLE)
 1044                         nblks = 1;
 1045                 else
 1046                         nblks = xfer->c_bcount / drvp->lp->d_secsize;
 1047                 /* Check for bad sectors and adjust transfer, if necessary. */
 1048                 if ((drvp->lp->d_flags & D_BADSECT) != 0) {
 1049                         long blkdiff;
 1050                         int i;
 1051 
 1052                         for (i = 0; (blkdiff = drvp->badsect[i]) != -1;
 1053                             i++) {
 1054                                 blkdiff -= ata_bio->blkno;
 1055                                 if (blkdiff < 0)
 1056                                         continue;
 1057                                 if (blkdiff == 0)
 1058                                         /* Replace current block of transfer. */
 1059                                         ata_bio->blkno =
 1060                                             drvp->lp->d_secperunit -
 1061                                             drvp->lp->d_nsectors - i - 1;
 1062                                 if (blkdiff < nblks) {
 1063                                         /* Bad block inside transfer. */
 1064                                         ata_bio->flags |= ATA_SINGLE;
 1065                                         nblks = 1;
 1066                                 }
 1067                                 break;
 1068                         }
 1069                         /* Transfer is okay now. */
 1070                 }
 1071                 if (xfer->c_flags & C_DMA) {
 1072                         enum mvsata_edmamode dmamode;
 1073 
 1074                         ata_bio->nblks = nblks;
 1075                         ata_bio->nbytes = xfer->c_bcount;
 1076 
 1077                         /* switch to appropriate dma mode if necessary */
 1078                         dmamode = (xfer->c_flags & C_NCQ) ? ncq : dma;
 1079                         if (mvport->port_edmamode_curr != dmamode)
 1080                                 mvsata_edma_config(mvport, dmamode);
 1081 
 1082                         if (xfer->c_flags & C_POLL)
 1083                                 sc->sc_enable_intr(mvport, 0 /*off*/);
 1084                         error = mvsata_edma_enqueue(mvport, xfer);
 1085                         if (error) {
 1086                                 if (error == EINVAL) {
 1087                                         /*
 1088                                          * We can't do DMA on this transfer
 1089                                          * for some reason.  Fall back to
 1090                                          * PIO.
 1091                                          */
 1092                                         xfer->c_flags &= ~C_DMA;
 1093                                         error = 0;
 1094                                         goto do_pio;
 1095                                 }
 1096                                 if (error == EBUSY) {
 1097                                         aprint_error_dev(atac->atac_dev,
 1098                                             "channel %d: EDMA Queue full\n",
 1099                                             chp->ch_channel);
 1100                                         /*
 1101                                          * XXX: Perhaps, after it waits for
 1102                                          * a while, it is necessary to call
 1103                                          * bio_start again.
 1104                                          */
 1105                                 }
 1106                                 ata_bio->error = ERR_DMA;
 1107                                 ata_bio->r_error = 0;
 1108                                 return ATASTART_ABORT;
 1109                         }
 1110                         chp->ch_flags |= ATACH_DMA_WAIT;
 1111                         /* wait for irq */
 1112                         goto intr;
 1113                 } /* else not DMA */
 1114 do_pio:
 1115                 if (ata_bio->flags & ATA_LBA48) {
 1116                         sect = 0;
 1117                         cyl =  0;
 1118                         head = 0;
 1119                 } else if (ata_bio->flags & ATA_LBA) {
 1120                         sect = (ata_bio->blkno >> 0) & 0xff;
 1121                         cyl = (ata_bio->blkno >> 8) & 0xffff;
 1122                         head = (ata_bio->blkno >> 24) & 0x0f;
 1123                         head |= WDSD_LBA;
 1124                 } else {
 1125                         int blkno = ata_bio->blkno;
 1126                         sect = blkno % drvp->lp->d_nsectors;
 1127                         sect++; /* Sectors begin with 1, not 0. */
 1128                         blkno /= drvp->lp->d_nsectors;
 1129                         head = blkno % drvp->lp->d_ntracks;
 1130                         blkno /= drvp->lp->d_ntracks;
 1131                         cyl = blkno;
 1132                         head |= WDSD_CHS;
 1133                 }
 1134                 ata_bio->nblks = uimin(nblks, drvp->multi);
 1135                 ata_bio->nbytes = ata_bio->nblks * drvp->lp->d_secsize;
 1136                 KASSERT(nblks == 1 || (ata_bio->flags & ATA_SINGLE) == 0);
 1137                 if (ata_bio->nblks > 1)
 1138                         cmd = (ata_bio->flags & ATA_READ) ?
 1139                             WDCC_READMULTI : WDCC_WRITEMULTI;
 1140                 else
 1141                         cmd = (ata_bio->flags & ATA_READ) ?
 1142                             WDCC_READ : WDCC_WRITE;
 1143 
 1144                 /* EDMA disable, if enabled this channel. */
 1145                 KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
 1146                 if (mvport->port_edmamode_curr != nodma)
 1147                         mvsata_edma_disable(mvport, 10 /* ms */, wait_flags);
 1148 
 1149                 mvsata_pmp_select(mvport, xfer->c_drive);
 1150 
 1151                 /* Do control operations specially. */
 1152                 if (__predict_false(drvp->state < READY)) {
 1153                         /*
 1154                          * Actually, we want to be careful not to mess with
 1155                          * the control state if the device is currently busy,
 1156                          * but we can assume that we never get to this point
 1157                          * if that's the case.
 1158                          */
 1159                         /*
 1160                          * If it's not a polled command, we need the kernel
 1161                          * thread
 1162                          */
 1163                         if ((xfer->c_flags & C_POLL) == 0
 1164                             && !ata_is_thread_run(chp))
 1165                                 return ATASTART_TH;
 1166 
 1167                         if (mvsata_bio_ready(mvport, ata_bio, xfer->c_drive,
 1168                             (xfer->c_flags & C_POLL) ? AT_POLL : 0) != 0) {
 1169                                 return ATASTART_ABORT;
 1170                         }
 1171                 }
 1172 
 1173                 /* Initiate command! */
 1174                 MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 1175                 switch(wdc_wait_for_ready(chp, ATA_DELAY, wait_flags, &tfd)) {
 1176                 case WDCWAIT_OK:
 1177                         break;
 1178                 case WDCWAIT_TOUT:
 1179                         goto timeout;
 1180                 case WDCWAIT_THR:
 1181                         return ATASTART_TH;
 1182                 }
 1183                 if (ata_bio->flags & ATA_LBA48)
 1184                         wdccommandext(chp, 0, atacmd_to48(cmd),
 1185                             ata_bio->blkno, nblks, 0, WDSD_LBA);
 1186                 else
 1187                         wdccommand(chp, 0, cmd, cyl,
 1188                             head, sect, nblks,
 1189                             (drvp->lp->d_type == DKTYPE_ST506) ?
 1190                             drvp->lp->d_precompcyl / 4 : 0);
 1191         } else if (ata_bio->nblks > 1) {
 1192                 /* The number of blocks in the last stretch may be smaller. */
 1193                 nblks = xfer->c_bcount / drvp->lp->d_secsize;
 1194                 if (ata_bio->nblks > nblks) {
 1195                         ata_bio->nblks = nblks;
 1196                         ata_bio->nbytes = xfer->c_bcount;
 1197                 }
 1198         }
 1199         /* If this was a write and not using DMA, push the data. */
 1200         if ((ata_bio->flags & ATA_READ) == 0) {
 1201                 /*
 1202                  * we have to busy-wait here, we can't rely on running in
 1203                  * thread context.
 1204                  */
 1205                 if (wdc_wait_for_drq(chp, ATA_DELAY, AT_POLL, &tfd) != 0) {
 1206                         aprint_error_dev(atac->atac_dev,
 1207                             "channel %d: drive %d timeout waiting for DRQ,"
 1208                             " st=0x%02x, err=0x%02x\n",
 1209                             chp->ch_channel, xfer->c_drive, ATACH_ST(tfd),
 1210                             ATACH_ERR(tfd));
 1211                         ata_bio->error = TIMEOUT;
 1212                         return ATASTART_ABORT;
 1213                 }
 1214                 if (ATACH_ST(tfd) & WDCS_ERR) {
 1215                         ata_bio->error = ERROR;
 1216                         ata_bio->r_error = ATACH_ERR(tfd);
 1217                         mvsata_bio_done(chp, xfer);
 1218                         return ATASTART_ABORT;
 1219                 }
 1220 
 1221                 wdc->dataout_pio(chp, drvp->drive_flags,
 1222                     (char *)xfer->c_databuf + xfer->c_skip, ata_bio->nbytes);
 1223         }
 1224 
 1225 intr:
 1226         KASSERTMSG(((xfer->c_flags & C_DMA) != 0)
 1227                 == (mvport->port_edmamode_curr != nodma),
 1228                 "DMA mode mismatch: flags %x vs edmamode %d != %d",
 1229                 xfer->c_flags, mvport->port_edmamode_curr, nodma); 
 1230 
 1231         /* Wait for IRQ (either real or polled) */
 1232         if ((ata_bio->flags & ATA_POLL) != 0) {
 1233                 /* start timeout machinery */
 1234                 callout_reset(&chp->c_timo_callout,
 1235                     mstohz(ATA_DELAY), wdctimeout, chp);
 1236                 return ATASTART_POLL;
 1237         } else
 1238                 return ATASTART_STARTED;
 1239 
 1240 timeout:
 1241         aprint_error_dev(atac->atac_dev,
 1242             "channel %d: drive %d not ready, st=0x%02x, err=0x%02x\n",
 1243             chp->ch_channel, xfer->c_drive, ATACH_ST(tfd), ATACH_ERR(tfd));
 1244         ata_bio->error = TIMEOUT;
 1245         return ATASTART_ABORT;
 1246 }
 1247 
 1248 static int
 1249 mvsata_bio_poll(struct ata_channel *chp, struct ata_xfer *xfer)
 1250 {
 1251         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1252         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 1253 
 1254         /* Wait for at last 400ns for status bit to be valid */
 1255         delay(1);
 1256         if (chp->ch_flags & ATACH_DMA_WAIT) {
 1257                 mvsata_edma_wait(mvport, xfer, ATA_DELAY);
 1258                 sc->sc_enable_intr(mvport, 1 /*on*/);
 1259                 chp->ch_flags &= ~ATACH_DMA_WAIT;
 1260         }
 1261 
 1262         mvsata_bio_intr(chp, xfer, 0);
 1263 
 1264         return (xfer->c_bio.flags & ATA_ITSDONE) ? ATAPOLL_DONE : ATAPOLL_AGAIN;
 1265 }
 1266 
 1267 static int
 1268 mvsata_bio_intr(struct ata_channel *chp, struct ata_xfer *xfer, int intr_arg)
 1269 {
 1270         struct atac_softc *atac = chp->ch_atac;
 1271         struct wdc_softc *wdc = CHAN_TO_WDC(chp);
 1272         struct ata_bio *ata_bio = &xfer->c_bio;
 1273         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 1274         int irq = ISSET(xfer->c_flags, (C_POLL|C_TIMEOU)) ? 0 : 1;
 1275         int tfd = 0;
 1276 
 1277         if (ISSET(xfer->c_flags, C_DMA|C_RECOVERED) && irq) {
 1278                 /* Invoked via mvsata_edma_handle() or recovery */
 1279                 tfd = intr_arg;
 1280 
 1281                 if (tfd > 0 && ata_bio->error == NOERROR) {
 1282                         if (ATACH_ST(tfd) & WDCS_ERR)
 1283                                 ata_bio->error = ERROR;
 1284                         if (ATACH_ST(tfd) & WDCS_BSY)
 1285                                 ata_bio->error = TIMEOUT;
 1286                         ata_bio->r_error = ATACH_ERR(tfd);
 1287                 }
 1288         }
 1289 
 1290         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, ("%s:%d: %s: drive=%d\n",
 1291             device_xname(atac->atac_dev), chp->ch_channel, __func__,
 1292             xfer->c_drive));
 1293 
 1294         /* Cleanup EDMA if invoked from wdctimeout()/ata_timeout() */
 1295         if (ISSET(xfer->c_flags, C_TIMEOU) && ISSET(xfer->c_flags, C_DMA)
 1296             && !ISSET(xfer->c_flags, C_POLL)) {
 1297                 mvsata_edma_rqq_remove((struct mvsata_port *)chp, xfer);
 1298         }
 1299 
 1300         ata_channel_lock(chp);
 1301 
 1302         chp->ch_flags &= ~(ATACH_DMA_WAIT);
 1303 
 1304         /*
 1305          * If we missed an interrupt transfer, reset and restart.
 1306          * Don't try to continue transfer, we may have missed cycles.
 1307          */
 1308         if (xfer->c_flags & C_TIMEOU) {
 1309                 ata_bio->error = TIMEOUT;
 1310                 ata_channel_unlock(chp);
 1311                 mvsata_bio_done(chp, xfer);
 1312                 return 1;
 1313         }
 1314 
 1315         /* Is it not a transfer, but a control operation? */
 1316         if (!(xfer->c_flags & C_DMA) && drvp->state < READY) {
 1317                 aprint_error_dev(atac->atac_dev,
 1318                     "channel %d: drive %d bad state %d in %s\n",
 1319                     chp->ch_channel, xfer->c_drive, drvp->state, __func__);
 1320                 panic("%s: bad state", __func__);
 1321         }
 1322 
 1323         /* Ack interrupt done by wdc_wait_for_unbusy */
 1324         if (!(xfer->c_flags & C_DMA) &&
 1325             (wdc_wait_for_unbusy(chp, (irq == 0) ? ATA_DELAY : 0, AT_POLL, &tfd)
 1326                                                         == WDCWAIT_TOUT)) {
 1327                 if (irq && (xfer->c_flags & C_TIMEOU) == 0) {
 1328                         ata_channel_unlock(chp);
 1329                         return 0;       /* IRQ was not for us */
 1330                 }
 1331                 aprint_error_dev(atac->atac_dev,
 1332                     "channel %d: drive %d timeout, c_bcount=%d, c_skip%d\n",
 1333                     chp->ch_channel, xfer->c_drive, xfer->c_bcount,
 1334                     xfer->c_skip);
 1335                 ata_bio->error = TIMEOUT;
 1336                 ata_channel_unlock(chp);
 1337                 mvsata_bio_done(chp, xfer);
 1338                 return 1;
 1339         }
 1340 
 1341         if (xfer->c_flags & C_DMA) {
 1342                 if (ata_bio->error == NOERROR)
 1343                         goto end;
 1344                 if (ata_bio->error == ERR_DMA) {
 1345                         ata_dmaerr(drvp,
 1346                             (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 1347                         ata_channel_unlock(chp);
 1348                         goto err;
 1349                 }
 1350         }
 1351 
 1352         /* if we had an error, end */
 1353         if (ata_bio->error != NOERROR) {
 1354                 ata_channel_unlock(chp);
 1355 err:
 1356                 mvsata_bio_done(chp, xfer);
 1357                 return 1;
 1358         }
 1359 
 1360         /* If this was a read and not using DMA, fetch the data. */
 1361         if ((ata_bio->flags & ATA_READ) != 0) {
 1362                 if ((ATACH_ST(tfd) & WDCS_DRQ) != WDCS_DRQ) {
 1363                         aprint_error_dev(atac->atac_dev,
 1364                             "channel %d: drive %d read intr before drq\n",
 1365                             chp->ch_channel, xfer->c_drive);
 1366                         ata_bio->error = TIMEOUT;
 1367                         ata_channel_unlock(chp);
 1368                         mvsata_bio_done(chp, xfer);
 1369                         return 1;
 1370                 }
 1371                 wdc->datain_pio(chp, drvp->drive_flags,
 1372                     (char *)xfer->c_databuf + xfer->c_skip, ata_bio->nbytes);
 1373         }
 1374 
 1375 end:
 1376         ata_bio->blkno += ata_bio->nblks;
 1377         ata_bio->blkdone += ata_bio->nblks;
 1378         xfer->c_skip += ata_bio->nbytes;
 1379         xfer->c_bcount -= ata_bio->nbytes;
 1380 
 1381         /* See if this transfer is complete. */
 1382         if (xfer->c_bcount > 0) {
 1383                 if ((ata_bio->flags & ATA_POLL) == 0) {
 1384                         /* Start the next operation */
 1385                         ata_xfer_start(xfer);
 1386                 } else {
 1387                         /*
 1388                          * Let ata_xfer_start() do the loop;
 1389                          * see mvsata_bio_poll().
 1390                          */
 1391                 }
 1392                 ata_channel_unlock(chp);
 1393         } else { /* Done with this transfer */
 1394                 ata_bio->error = NOERROR;
 1395                 ata_channel_unlock(chp);
 1396                 mvsata_bio_done(chp, xfer);
 1397         }
 1398         return 1;
 1399 }
 1400 
 1401 static void
 1402 mvsata_bio_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
 1403 {
 1404         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1405         struct atac_softc *atac = chp->ch_atac;
 1406         struct ata_bio *ata_bio = &xfer->c_bio;
 1407         int drive = xfer->c_drive;
 1408         bool deactivate = true;
 1409 
 1410         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1411             ("%s:%d: mvsata_bio_kill_xfer: drive=%d\n",
 1412             device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive));
 1413 
 1414         /* EDMA restart, if enabled */
 1415         if (!(xfer->c_flags & C_DMA) && mvport->port_edmamode_curr != nodma) {
 1416                 mvsata_edma_reset_qptr(mvport);
 1417                 mvsata_edma_enable(mvport);
 1418         }
 1419 
 1420         ata_bio->flags |= ATA_ITSDONE;
 1421         switch (reason) {
 1422         case KILL_GONE_INACTIVE:
 1423                 deactivate = false;
 1424                 /* FALLTHROUGH */
 1425         case KILL_GONE:
 1426                 ata_bio->error = ERR_NODEV;
 1427                 break;
 1428         case KILL_RESET:
 1429                 ata_bio->error = ERR_RESET;
 1430                 break;
 1431         case KILL_REQUEUE:
 1432                 ata_bio->error = REQUEUE;
 1433                 break;
 1434         default:
 1435                 aprint_error_dev(atac->atac_dev,
 1436                     "mvsata_bio_kill_xfer: unknown reason %d\n", reason);
 1437                 panic("mvsata_bio_kill_xfer");
 1438         }
 1439         ata_bio->r_error = WDCE_ABRT;
 1440 
 1441         if (deactivate)
 1442                 ata_deactivate_xfer(chp, xfer);
 1443 
 1444         (*chp->ch_drive[drive].drv_done)(chp->ch_drive[drive].drv_softc, xfer);
 1445 }
 1446 
 1447 static void
 1448 mvsata_bio_done(struct ata_channel *chp, struct ata_xfer *xfer)
 1449 {
 1450         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1451         struct ata_bio *ata_bio = &xfer->c_bio;
 1452         int drive = xfer->c_drive;
 1453         bool iserror = (ata_bio->error != NOERROR);
 1454 
 1455         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1456             ("%s:%d: mvsata_bio_done: drive=%d, flags=0x%x\n",
 1457             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel, xfer->c_drive,
 1458             (u_int)xfer->c_flags));
 1459 
 1460         /* EDMA restart, if enabled */
 1461         if (!(xfer->c_flags & C_DMA) && mvport->port_edmamode_curr != nodma) {
 1462                 mvsata_edma_reset_qptr(mvport);
 1463                 mvsata_edma_enable(mvport);
 1464         }
 1465 
 1466         if (ata_waitdrain_xfer_check(chp, xfer))
 1467                 return;
 1468 
 1469         /* feed back residual bcount to our caller */
 1470         ata_bio->bcount = xfer->c_bcount;
 1471 
 1472         /* mark controller inactive and free xfer */
 1473         ata_deactivate_xfer(chp, xfer);
 1474 
 1475         ata_bio->flags |= ATA_ITSDONE;
 1476         (*chp->ch_drive[drive].drv_done)(chp->ch_drive[drive].drv_softc, xfer);
 1477         if (!iserror)
 1478                 atastart(chp);
 1479 }
 1480 
 1481 static int
 1482 mvsata_bio_ready(struct mvsata_port *mvport, struct ata_bio *ata_bio, int drive,
 1483                  int flags)
 1484 {
 1485         struct ata_channel *chp = &mvport->port_ata_channel;
 1486         struct atac_softc *atac = chp->ch_atac;
 1487         struct ata_drive_datas *drvp = &chp->ch_drive[drive];
 1488         const char *errstring;
 1489         int tfd;
 1490 
 1491         flags |= AT_POLL;       /* XXX */
 1492 
 1493         ata_channel_lock_owned(chp);
 1494 
 1495         /*
 1496          * disable interrupts, all commands here should be quick
 1497          * enough to be able to poll, and we don't go here that often
 1498          */
 1499         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT | WDCTL_IDS);
 1500         MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 1501         DELAY(10);
 1502         errstring = "wait";
 1503         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1504                 goto ctrltimeout;
 1505         wdccommandshort(chp, 0, WDCC_RECAL);
 1506         /* Wait for at least 400ns for status bit to be valid */
 1507         DELAY(1);
 1508         errstring = "recal";
 1509         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1510                 goto ctrltimeout;
 1511         if (ATACH_ST(tfd) & (WDCS_ERR | WDCS_DWF))
 1512                 goto ctrlerror;
 1513         /* Don't try to set modes if controller can't be adjusted */
 1514         if (atac->atac_set_modes == NULL)
 1515                 goto geometry;
 1516         /* Also don't try if the drive didn't report its mode */
 1517         if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0)
 1518                 goto geometry;
 1519         wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 1520             0x08 | drvp->PIO_mode, WDSF_SET_MODE);
 1521         errstring = "piomode-bio";
 1522         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1523                 goto ctrltimeout;
 1524         if (ATACH_ST(tfd) & (WDCS_ERR | WDCS_DWF))
 1525                 goto ctrlerror;
 1526         if (drvp->drive_flags & ATA_DRIVE_UDMA)
 1527                 wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 1528                     0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
 1529         else if (drvp->drive_flags & ATA_DRIVE_DMA)
 1530                 wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 1531                     0x20 | drvp->DMA_mode, WDSF_SET_MODE);
 1532         else
 1533                 goto geometry;
 1534         errstring = "dmamode-bio";
 1535         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1536                 goto ctrltimeout;
 1537         if (ATACH_ST(tfd) & (WDCS_ERR | WDCS_DWF))
 1538                 goto ctrlerror;
 1539 geometry:
 1540         if (ata_bio->flags & ATA_LBA)
 1541                 goto multimode;
 1542         wdccommand(chp, 0, WDCC_IDP, drvp->lp->d_ncylinders,
 1543             drvp->lp->d_ntracks - 1, 0, drvp->lp->d_nsectors,
 1544             (drvp->lp->d_type == DKTYPE_ST506) ?
 1545             drvp->lp->d_precompcyl / 4 : 0);
 1546         errstring = "geometry";
 1547         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1548                 goto ctrltimeout;
 1549         if (ATACH_ST(tfd) & (WDCS_ERR | WDCS_DWF))
 1550                 goto ctrlerror;
 1551 multimode:
 1552         if (drvp->multi == 1)
 1553                 goto ready;
 1554         wdccommand(chp, 0, WDCC_SETMULTI, 0, 0, 0, drvp->multi, 0);
 1555         errstring = "setmulti";
 1556         if (wdcwait(chp, WDCS_DRDY, WDCS_DRDY, ATA_DELAY, flags, &tfd))
 1557                 goto ctrltimeout;
 1558         if (ATACH_ST(tfd) & (WDCS_ERR | WDCS_DWF))
 1559                 goto ctrlerror;
 1560 ready:
 1561         drvp->state = READY;
 1562         /*
 1563          * The drive is usable now
 1564          */
 1565         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 1566         delay(10);      /* some drives need a little delay here */
 1567         return 0;
 1568 
 1569 ctrltimeout:
 1570         aprint_error_dev(atac->atac_dev, "channel %d: drive %d %s timed out\n",
 1571             chp->ch_channel, drive, errstring);
 1572         ata_bio->error = TIMEOUT;
 1573         goto ctrldone;
 1574 ctrlerror:
 1575         aprint_error_dev(atac->atac_dev, "channel %d: drive %d %s ",
 1576             chp->ch_channel, drive, errstring);
 1577         if (ATACH_ST(tfd) & WDCS_DWF) {
 1578                 aprint_error("drive fault\n");
 1579                 ata_bio->error = ERR_DF;
 1580         } else {
 1581                 ata_bio->r_error = ATACH_ERR(tfd);
 1582                 ata_bio->error = ERROR;
 1583                 aprint_error("error (%x)\n", ata_bio->r_error);
 1584         }
 1585 ctrldone:
 1586         drvp->state = 0;
 1587         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 1588         return -1;
 1589 }
 1590 
 1591 static const struct ata_xfer_ops mvsata_wdc_cmd_xfer_ops = {
 1592         .c_start = mvsata_wdc_cmd_start,
 1593         .c_intr = mvsata_wdc_cmd_intr,
 1594         .c_poll = mvsata_wdc_cmd_poll,
 1595         .c_abort = mvsata_wdc_cmd_done,
 1596         .c_kill_xfer = mvsata_wdc_cmd_kill_xfer,
 1597 };
 1598 
 1599 static void
 1600 mvsata_exec_command(struct ata_drive_datas *drvp, struct ata_xfer *xfer)
 1601 {
 1602         struct ata_channel *chp = drvp->chnl_softc;
 1603         struct ata_command *ata_c = &xfer->c_ata_c;
 1604 
 1605         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1606             ("%s:%d: mvsata_exec_command: drive=%d, bcount=%d,"
 1607             " r_lba=0x%012"PRIx64", r_count=0x%04x, r_features=0x%04x,"
 1608             " r_device=0x%02x, r_command=0x%02x\n",
 1609             device_xname(MVSATA_DEV2((struct mvsata_port *)chp)),
 1610             chp->ch_channel,
 1611             drvp->drive, ata_c->bcount, ata_c->r_lba, ata_c->r_count,
 1612             ata_c->r_features, ata_c->r_device, ata_c->r_command));
 1613 
 1614         if (ata_c->flags & AT_POLL)
 1615                 xfer->c_flags |= C_POLL;
 1616         if (ata_c->flags & AT_WAIT)
 1617                 xfer->c_flags |= C_WAIT;
 1618         xfer->c_drive = drvp->drive;
 1619         xfer->c_databuf = ata_c->data;
 1620         xfer->c_bcount = ata_c->bcount;
 1621         xfer->ops = &mvsata_wdc_cmd_xfer_ops;
 1622 
 1623         ata_exec_xfer(chp, xfer);
 1624 }
 1625 
 1626 static int
 1627 mvsata_wdc_cmd_start(struct ata_channel *chp, struct ata_xfer *xfer)
 1628 {
 1629         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1630         int drive = xfer->c_drive;
 1631         int wait_flags = (xfer->c_flags & C_POLL) ? AT_POLL : 0;
 1632         struct ata_command *ata_c = &xfer->c_ata_c;
 1633         int tfd;
 1634 
 1635         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1636             ("%s:%d: mvsata_cmd_start: drive=%d\n",
 1637             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel, drive));
 1638 
 1639         ata_channel_lock_owned(chp);
 1640 
 1641         /* First, EDMA disable, if enabled this channel. */
 1642         KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
 1643         if (mvport->port_edmamode_curr != nodma)
 1644                 mvsata_edma_disable(mvport, 10 /* ms */, wait_flags);
 1645 
 1646         mvsata_pmp_select(mvport, drive);
 1647 
 1648         MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 1649         switch(wdcwait(chp, ata_c->r_st_bmask | WDCS_DRQ,
 1650             ata_c->r_st_bmask, ata_c->timeout, wait_flags, &tfd)) {
 1651         case WDCWAIT_OK:
 1652                 break;
 1653         case WDCWAIT_TOUT:
 1654                 ata_c->flags |= AT_TIMEOU;
 1655                 return ATASTART_ABORT;
 1656         case WDCWAIT_THR:
 1657                 return ATASTART_TH;
 1658         }
 1659         if (ata_c->flags & AT_POLL)
 1660                 /* polled command, disable interrupts */
 1661                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT | WDCTL_IDS);
 1662         if ((ata_c->flags & AT_LBA48) != 0) {
 1663                 wdccommandext(chp, 0, ata_c->r_command,
 1664                     ata_c->r_lba, ata_c->r_count, ata_c->r_features,
 1665                     ata_c->r_device & ~0x10);
 1666         } else {
 1667                 wdccommand(chp, 0, ata_c->r_command,
 1668                     (ata_c->r_lba >> 8) & 0xffff,
 1669                     (((ata_c->flags & AT_LBA) != 0) ? WDSD_LBA : 0) |
 1670                     ((ata_c->r_lba >> 24) & 0x0f),
 1671                     ata_c->r_lba & 0xff,
 1672                     ata_c->r_count & 0xff,
 1673                     ata_c->r_features & 0xff);
 1674         }
 1675 
 1676         if ((ata_c->flags & AT_POLL) == 0) {
 1677                 callout_reset(&chp->c_timo_callout, ata_c->timeout / 1000 * hz,
 1678                     wdctimeout, chp);
 1679                 return ATASTART_STARTED;
 1680         }
 1681 
 1682         return ATASTART_POLL;
 1683 }
 1684 
 1685 static int
 1686 mvsata_wdc_cmd_poll(struct ata_channel *chp, struct ata_xfer *xfer)
 1687 {
 1688         /*
 1689          * Polled command. Wait for drive ready or drq. Done in intr().
 1690          * Wait for at last 400ns for status bit to be valid.
 1691          */
 1692         delay(10);      /* 400ns delay */
 1693         mvsata_wdc_cmd_intr(chp, xfer, 0);
 1694         return ATAPOLL_DONE;
 1695 }
 1696 
 1697 static int
 1698 mvsata_wdc_cmd_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
 1699 {
 1700         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1701         struct wdc_softc *wdc = CHAN_TO_WDC(chp);
 1702         struct ata_command *ata_c = &xfer->c_ata_c;
 1703         int bcount = ata_c->bcount;
 1704         char *data = ata_c->data;
 1705         int wflags;
 1706         int drive_flags;
 1707         int tfd;
 1708 
 1709         ata_channel_lock(chp);
 1710 
 1711         if (ata_c->r_command == WDCC_IDENTIFY ||
 1712             ata_c->r_command == ATAPI_IDENTIFY_DEVICE)
 1713                 /*
 1714                  * The IDENTIFY data has been designed as an array of
 1715                  * u_int16_t, so we can byteswap it on the fly.
 1716                  * Historically it's what we have always done so keeping it
 1717                  * here ensure binary backward compatibility.
 1718                  */
 1719                 drive_flags = ATA_DRIVE_NOSTREAM |
 1720                     chp->ch_drive[xfer->c_drive].drive_flags;
 1721         else
 1722                 /*
 1723                  * Other data structure are opaque and should be transferred
 1724                  * as is.
 1725                  */
 1726                 drive_flags = chp->ch_drive[xfer->c_drive].drive_flags;
 1727 
 1728         if ((ata_c->flags & (AT_WAIT | AT_POLL)) == (AT_WAIT | AT_POLL))
 1729                 /* both wait and poll, we can kpause here */
 1730                 wflags = AT_WAIT | AT_POLL;
 1731         else
 1732                 wflags = AT_POLL;
 1733 
 1734 again:
 1735         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, ("%s:%d: %s: drive=%d\n",
 1736             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel,
 1737             __func__, xfer->c_drive));
 1738 
 1739         /*
 1740          * after a ATAPI_SOFT_RESET, the device will have released the bus.
 1741          * Reselect again, it doesn't hurt for others commands, and the time
 1742          * penalty for the extra register write is acceptable,
 1743          * wdc_exec_command() isn't called often (mostly for autoconfig)
 1744          */
 1745         if ((xfer->c_flags & C_ATAPI) != 0) {
 1746                 MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 1747         }
 1748         if ((ata_c->flags & AT_XFDONE) != 0) {
 1749                 /*
 1750                  * We have completed a data xfer. The drive should now be
 1751                  * in its initial state
 1752                  */
 1753                 if (wdcwait(chp, ata_c->r_st_bmask | WDCS_DRQ,
 1754                     ata_c->r_st_bmask, (irq == 0) ? ata_c->timeout : 0,
 1755                     wflags, &tfd) ==  WDCWAIT_TOUT) {
 1756                         if (irq && (xfer->c_flags & C_TIMEOU) == 0) {
 1757                                 ata_channel_unlock(chp);
 1758                                 return 0;       /* IRQ was not for us */
 1759                         }
 1760                         ata_c->flags |= AT_TIMEOU;
 1761                 }
 1762                 goto out;
 1763         }
 1764         if (wdcwait(chp, ata_c->r_st_pmask, ata_c->r_st_pmask,
 1765             (irq == 0) ? ata_c->timeout : 0, wflags, &tfd) == WDCWAIT_TOUT) {
 1766                 if (irq && (xfer->c_flags & C_TIMEOU) == 0) {
 1767                         ata_channel_unlock(chp);
 1768                         return 0;       /* IRQ was not for us */
 1769                 }
 1770                 ata_c->flags |= AT_TIMEOU;
 1771                 goto out;
 1772         }
 1773         delay(20);      /* XXXXX: Delay more times. */
 1774         if (ata_c->flags & AT_READ) {
 1775                 if ((ATACH_ST(tfd) & WDCS_DRQ) == 0) {
 1776                         ata_c->flags |= AT_TIMEOU;
 1777                         goto out;
 1778                 }
 1779                 wdc->datain_pio(chp, drive_flags, data, bcount);
 1780                 /* at this point the drive should be in its initial state */
 1781                 ata_c->flags |= AT_XFDONE;
 1782                 /*
 1783                  * XXX checking the status register again here cause some
 1784                  * hardware to timeout.
 1785                  */
 1786         } else if (ata_c->flags & AT_WRITE) {
 1787                 if ((ATACH_ST(tfd) & WDCS_DRQ) == 0) {
 1788                         ata_c->flags |= AT_TIMEOU;
 1789                         goto out;
 1790                 }
 1791                 wdc->dataout_pio(chp, drive_flags, data, bcount);
 1792                 ata_c->flags |= AT_XFDONE;
 1793                 if ((ata_c->flags & AT_POLL) == 0) {
 1794                         callout_reset(&chp->c_timo_callout,
 1795                             mstohz(ata_c->timeout), wdctimeout, chp);
 1796                         ata_channel_unlock(chp);
 1797                         return 1;
 1798                 } else
 1799                         goto again;
 1800         }
 1801 out:
 1802         if (ATACH_ST(tfd) & WDCS_DWF)
 1803                 ata_c->flags |= AT_DF;
 1804         if (ATACH_ST(tfd) & WDCS_ERR) {
 1805                 ata_c->flags |= AT_ERROR;
 1806                 ata_c->r_error = ATACH_ERR(tfd);
 1807         }
 1808         ata_channel_unlock(chp);
 1809         mvsata_wdc_cmd_done(chp, xfer);
 1810 
 1811         if ((ATACH_ST(tfd) & WDCS_ERR) == 0)
 1812                 atastart(chp);
 1813 
 1814         return 1;
 1815 }
 1816 
 1817 static void
 1818 mvsata_wdc_cmd_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer,
 1819                          int reason)
 1820 {
 1821         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1822         struct ata_command *ata_c = &xfer->c_ata_c;
 1823         bool deactivate = true;
 1824 
 1825         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1826             ("%s:%d: mvsata_cmd_kill_xfer: drive=%d\n",
 1827             device_xname(MVSATA_DEV2(mvport)), chp->ch_channel, xfer->c_drive));
 1828 
 1829         switch (reason) {
 1830         case KILL_GONE_INACTIVE:
 1831                 deactivate = false;
 1832                 /* FALLTHROUGH */
 1833         case KILL_GONE:
 1834                 ata_c->flags |= AT_GONE;
 1835                 break;
 1836         case KILL_RESET:
 1837                 ata_c->flags |= AT_RESET;
 1838                 break;
 1839         case KILL_REQUEUE:
 1840                 panic("%s: not supposed to be requeued\n", __func__);
 1841                 break;
 1842         default:
 1843                 aprint_error_dev(MVSATA_DEV2(mvport),
 1844                     "mvsata_cmd_kill_xfer: unknown reason %d\n", reason);
 1845                 panic("mvsata_cmd_kill_xfer");
 1846         }
 1847 
 1848         mvsata_wdc_cmd_done_end(chp, xfer);
 1849 
 1850         if (deactivate)
 1851                 ata_deactivate_xfer(chp, xfer);
 1852 }
 1853 
 1854 static void
 1855 mvsata_wdc_cmd_done(struct ata_channel *chp, struct ata_xfer *xfer)
 1856 {
 1857         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1858         struct atac_softc *atac = chp->ch_atac;
 1859         struct ata_command *ata_c = &xfer->c_ata_c;
 1860 
 1861         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 1862             ("%s:%d: mvsata_cmd_done: drive=%d, flags=0x%x\n",
 1863             device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
 1864             ata_c->flags));
 1865 
 1866         if (ata_waitdrain_xfer_check(chp, xfer))
 1867                 return;
 1868 
 1869         if ((ata_c->flags & AT_READREG) != 0 &&
 1870             device_is_active(atac->atac_dev) &&
 1871             (ata_c->flags & (AT_ERROR | AT_DF)) == 0) {
 1872                 ata_c->r_status = MVSATA_WDC_READ_1(mvport, SRB_CS);
 1873                 ata_c->r_error = MVSATA_WDC_READ_1(mvport, SRB_FE);
 1874                 ata_c->r_count = MVSATA_WDC_READ_1(mvport, SRB_SC);
 1875                 ata_c->r_lba =
 1876                     (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAL) << 0;
 1877                 ata_c->r_lba |=
 1878                     (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAM) << 8;
 1879                 ata_c->r_lba |=
 1880                     (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAH) << 16;
 1881                 ata_c->r_device = MVSATA_WDC_READ_1(mvport, SRB_H);
 1882                 if ((ata_c->flags & AT_LBA48) != 0) {
 1883                         if ((ata_c->flags & AT_POLL) != 0) {
 1884                                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS,
 1885                                     WDCTL_HOB|WDCTL_4BIT|WDCTL_IDS);
 1886                         } else {
 1887                                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS,
 1888                                     WDCTL_HOB|WDCTL_4BIT);
 1889                         }
 1890                         ata_c->r_count |=
 1891                             MVSATA_WDC_READ_1(mvport, SRB_SC) << 8;
 1892                         ata_c->r_lba |=
 1893                             (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAL) << 24;
 1894                         ata_c->r_lba |=
 1895                             (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAM) << 32;
 1896                         ata_c->r_lba |=
 1897                             (uint64_t)MVSATA_WDC_READ_1(mvport, SRB_LBAH) << 40;
 1898                         if ((ata_c->flags & AT_POLL) != 0) {
 1899                                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS,
 1900                                     WDCTL_4BIT|WDCTL_IDS);
 1901                         } else {
 1902                                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS,
 1903                                     WDCTL_4BIT);
 1904                         }
 1905                 } else {
 1906                         ata_c->r_lba |=
 1907                             (uint64_t)(ata_c->r_device & 0x0f) << 24;
 1908                 }
 1909         }
 1910 
 1911         if (ata_c->flags & AT_POLL) {
 1912                 /* enable interrupts */
 1913                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 1914                 delay(10);      /* some drives need a little delay here */
 1915         }
 1916 
 1917         mvsata_wdc_cmd_done_end(chp, xfer);
 1918 
 1919         ata_deactivate_xfer(chp, xfer);
 1920 }
 1921 
 1922 static void
 1923 mvsata_wdc_cmd_done_end(struct ata_channel *chp, struct ata_xfer *xfer)
 1924 {
 1925         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 1926         struct ata_command *ata_c = &xfer->c_ata_c;
 1927 
 1928         /* EDMA restart, if enabled */
 1929         if (mvport->port_edmamode_curr != nodma) {
 1930                 mvsata_edma_reset_qptr(mvport);
 1931                 mvsata_edma_enable(mvport);
 1932         }
 1933 
 1934         ata_c->flags |= AT_DONE;
 1935 }
 1936 
 1937 #if NATAPIBUS > 0
 1938 static const struct ata_xfer_ops mvsata_atapi_xfer_ops = {
 1939         .c_start = mvsata_atapi_start,
 1940         .c_intr = mvsata_atapi_intr,
 1941         .c_poll = mvsata_atapi_poll,
 1942         .c_abort = mvsata_atapi_reset,
 1943         .c_kill_xfer = mvsata_atapi_kill_xfer,
 1944 };
 1945 
 1946 static void
 1947 mvsata_atapi_scsipi_request(struct scsipi_channel *chan,
 1948                             scsipi_adapter_req_t req, void *arg)
 1949 {
 1950         struct scsipi_adapter *adapt = chan->chan_adapter;
 1951         struct scsipi_periph *periph;
 1952         struct scsipi_xfer *sc_xfer;
 1953         struct mvsata_softc *sc = device_private(adapt->adapt_dev);
 1954         struct atac_softc *atac = &sc->sc_wdcdev.sc_atac;
 1955         struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
 1956         struct ata_xfer *xfer;
 1957         int drive, s;
 1958 
 1959         switch (req) {
 1960         case ADAPTER_REQ_RUN_XFER:
 1961                 sc_xfer = arg;
 1962                 periph = sc_xfer->xs_periph;
 1963                 drive = periph->periph_target;
 1964 
 1965                 if (!device_is_active(atac->atac_dev)) {
 1966                         sc_xfer->error = XS_DRIVER_STUFFUP;
 1967                         scsipi_done(sc_xfer);
 1968                         return;
 1969                 }
 1970                 xfer = ata_get_xfer(chp, false);
 1971                 if (xfer == NULL) {
 1972                         sc_xfer->error = XS_RESOURCE_SHORTAGE;
 1973                         scsipi_done(sc_xfer);
 1974                         return;
 1975                 }
 1976 
 1977                 if (sc_xfer->xs_control & XS_CTL_POLL)
 1978                         xfer->c_flags |= C_POLL;
 1979                 xfer->c_drive = drive;
 1980                 xfer->c_flags |= C_ATAPI;
 1981                 xfer->c_databuf = sc_xfer->data;
 1982                 xfer->c_bcount = sc_xfer->datalen;
 1983                 xfer->ops = &mvsata_atapi_xfer_ops;
 1984                 xfer->c_scsipi = sc_xfer;
 1985                 xfer->c_atapi.c_dscpoll = 0;
 1986                 s = splbio();
 1987                 ata_exec_xfer(chp, xfer);
 1988 #ifdef DIAGNOSTIC
 1989                 if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
 1990                     (sc_xfer->xs_status & XS_STS_DONE) == 0)
 1991                         panic("mvsata_atapi_scsipi_request:"
 1992                             " polled command not done");
 1993 #endif
 1994                 splx(s);
 1995                 return;
 1996 
 1997         default:
 1998                 /* Not supported, nothing to do. */
 1999                 ;
 2000         }
 2001 }
 2002 
 2003 static int
 2004 mvsata_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
 2005 {
 2006         struct mvsata_softc *sc = (struct mvsata_softc *)chp->ch_atac;
 2007         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 2008         struct atac_softc *atac = &sc->sc_wdcdev.sc_atac;
 2009         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2010         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 2011         const int wait_flags = (xfer->c_flags & C_POLL) ? AT_POLL : 0;
 2012         const char *errstring;
 2013         int tfd;
 2014 
 2015         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2016             ("%s:%d:%d: mvsata_atapi_start: scsi flags 0x%x\n",
 2017             device_xname(chp->ch_atac->atac_dev), chp->ch_channel,
 2018             xfer->c_drive, sc_xfer->xs_control));
 2019 
 2020         ata_channel_lock_owned(chp);
 2021 
 2022         KASSERT((chp->ch_flags  & ATACH_NCQ) == 0);
 2023         if (mvport->port_edmamode_curr != nodma)
 2024                 mvsata_edma_disable(mvport, 10 /* ms */, wait_flags);
 2025 
 2026         mvsata_pmp_select(mvport, xfer->c_drive);
 2027 
 2028         if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
 2029                 drvp->n_xfers++;
 2030 
 2031         /* Do control operations specially. */
 2032         if (__predict_false(drvp->state < READY)) {
 2033                 /* If it's not a polled command, we need the kernel thread */
 2034                 if ((sc_xfer->xs_control & XS_CTL_POLL) == 0
 2035                     && !ata_is_thread_run(chp))
 2036                         return ATASTART_TH;
 2037 
 2038                 /*
 2039                  * disable interrupts, all commands here should be quick
 2040                  * enough to be able to poll, and we don't go here that often
 2041                  */
 2042                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT | WDCTL_IDS);
 2043 
 2044                 MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 2045                 /* Don't try to set mode if controller can't be adjusted */
 2046                 if (atac->atac_set_modes == NULL)
 2047                         goto ready;
 2048                 /* Also don't try if the drive didn't report its mode */
 2049                 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0)
 2050                         goto ready;
 2051                 errstring = "unbusy";
 2052                 if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd))
 2053                         goto timeout;
 2054                 wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 2055                     0x08 | drvp->PIO_mode, WDSF_SET_MODE);
 2056                 errstring = "piomode-atapi";
 2057                 if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
 2058                     &tfd))
 2059                         goto timeout;
 2060                 if (ATACH_ST(tfd) & WDCS_ERR) {
 2061                         if (ATACH_ERR(tfd) == WDCE_ABRT) {
 2062                                 /*
 2063                                  * Some ATAPI drives reject PIO settings.
 2064                                  * Fall back to PIO mode 3 since that's the
 2065                                  * minimum for ATAPI.
 2066                                  */
 2067                                 aprint_error_dev(atac->atac_dev,
 2068                                     "channel %d drive %d: PIO mode %d rejected,"
 2069                                     " falling back to PIO mode 3\n",
 2070                                     chp->ch_channel, xfer->c_drive,
 2071                                     drvp->PIO_mode);
 2072                                 if (drvp->PIO_mode > 3)
 2073                                         drvp->PIO_mode = 3;
 2074                         } else
 2075                                 goto error;
 2076                 }
 2077                 if (drvp->drive_flags & ATA_DRIVE_UDMA)
 2078                         wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 2079                             0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
 2080                 else
 2081                 if (drvp->drive_flags & ATA_DRIVE_DMA)
 2082                         wdccommand(chp, 0, SET_FEATURES, 0, 0, 0,
 2083                             0x20 | drvp->DMA_mode, WDSF_SET_MODE);
 2084                 else
 2085                         goto ready;
 2086                 errstring = "dmamode-atapi";
 2087                 if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
 2088                     &tfd))
 2089                         goto timeout;
 2090                 if (ATACH_ST(tfd) & WDCS_ERR) {
 2091                         if (ATACH_ERR(tfd) == WDCE_ABRT) {
 2092                                 if (drvp->drive_flags & ATA_DRIVE_UDMA)
 2093                                         goto error;
 2094                                 else {
 2095                                         /*
 2096                                          * The drive rejected our DMA setting.
 2097                                          * Fall back to mode 1.
 2098                                          */
 2099                                         aprint_error_dev(atac->atac_dev,
 2100                                             "channel %d drive %d:"
 2101                                             " DMA mode %d rejected,"
 2102                                             " falling back to DMA mode 0\n",
 2103                                             chp->ch_channel, xfer->c_drive,
 2104                                             drvp->DMA_mode);
 2105                                         if (drvp->DMA_mode > 0)
 2106                                                 drvp->DMA_mode = 0;
 2107                                 }
 2108                         } else
 2109                                 goto error;
 2110                 }
 2111 ready:
 2112                 drvp->state = READY;
 2113                 MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 2114                 delay(10); /* some drives need a little delay here */
 2115         }
 2116         MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 2117         if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd) != 0) {
 2118                 aprint_error_dev(atac->atac_dev, "not ready, st = %02x\n",
 2119                     ATACH_ST(tfd));
 2120                 sc_xfer->error = XS_TIMEOUT;
 2121                 return ATASTART_ABORT;
 2122         }
 2123 
 2124         /* start timeout machinery */
 2125         if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
 2126                 callout_reset(&chp->c_timo_callout, mstohz(sc_xfer->timeout),
 2127                     wdctimeout, chp);
 2128 
 2129         /*
 2130          * Even with WDCS_ERR, the device should accept a command packet
 2131          * Limit length to what can be stuffed into the cylinder register
 2132          * (16 bits).  Some CD-ROMs seem to interpret '' as 65536,
 2133          * but not all devices do that and it's not obvious from the
 2134          * ATAPI spec that that behaviour should be expected.  If more
 2135          * data is necessary, multiple data transfer phases will be done.
 2136          */
 2137 
 2138         wdccommand(chp, 0, ATAPI_PKT_CMD,
 2139             xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff, 0, 0, 0,
 2140             (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA : 0);
 2141 
 2142         /*
 2143          * If there is no interrupt for CMD input, busy-wait for it (done in
 2144          * the interrupt routine. Poll routine will exit early in this case.
 2145          */
 2146         if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
 2147             ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL))
 2148                 return ATASTART_POLL;
 2149         else
 2150                 return ATASTART_STARTED;
 2151 
 2152 timeout:
 2153         aprint_error_dev(atac->atac_dev, "channel %d drive %d: %s timed out\n",
 2154             chp->ch_channel, xfer->c_drive, errstring);
 2155         sc_xfer->error = XS_TIMEOUT;
 2156         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 2157         delay(10);              /* some drives need a little delay here */
 2158         return ATASTART_ABORT;
 2159 
 2160 error:
 2161         aprint_error_dev(atac->atac_dev,
 2162             "channel %d drive %d: %s error (0x%x)\n",
 2163             chp->ch_channel, xfer->c_drive, errstring, ATACH_ERR(tfd));
 2164         sc_xfer->error = XS_SHORTSENSE;
 2165         sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
 2166         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 2167         delay(10);              /* some drives need a little delay here */
 2168         return ATASTART_ABORT;
 2169 }
 2170 
 2171 static int
 2172 mvsata_atapi_poll(struct ata_channel *chp, struct ata_xfer *xfer)
 2173 {
 2174         /*
 2175          * If there is no interrupt for CMD input, busy-wait for it (done in
 2176          * the interrupt routine. If it is a polled command, call the interrupt
 2177          * routine until command is done.
 2178          */
 2179         const bool poll = ((xfer->c_scsipi->xs_control & XS_CTL_POLL) != 0);
 2180 
 2181         /* Wait for at last 400ns for status bit to be valid */
 2182         DELAY(1);
 2183         mvsata_atapi_intr(chp, xfer, 0);
 2184 
 2185         if (!poll)
 2186                 return ATAPOLL_DONE;
 2187 
 2188         if (chp->ch_flags & ATACH_DMA_WAIT) {
 2189                 wdc_dmawait(chp, xfer, xfer->c_scsipi->timeout);
 2190                 chp->ch_flags &= ~ATACH_DMA_WAIT;
 2191         }
 2192 
 2193         while ((xfer->c_scsipi->xs_status & XS_STS_DONE) == 0) {
 2194                 /* Wait for at last 400ns for status bit to be valid */
 2195                 DELAY(1);
 2196                 mvsata_atapi_intr(chp, xfer, 0);
 2197         }
 2198 
 2199         return ATAPOLL_DONE;
 2200 }
 2201 
 2202 static int
 2203 mvsata_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
 2204 {
 2205         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 2206         struct atac_softc *atac = chp->ch_atac;
 2207         struct wdc_softc *wdc = CHAN_TO_WDC(chp);
 2208         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2209         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 2210         int len, phase, ire, error, retries=0, i;
 2211         int tfd;
 2212         void *cmd;
 2213 
 2214         ata_channel_lock(chp);
 2215 
 2216         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2217             ("%s:%d:%d: mvsata_atapi_intr\n",
 2218             device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive));
 2219 
 2220         /* Is it not a transfer, but a control operation? */
 2221         if (drvp->state < READY) {
 2222                 aprint_error_dev(atac->atac_dev,
 2223                     "channel %d drive %d: bad state %d\n",
 2224                     chp->ch_channel, xfer->c_drive, drvp->state);
 2225                 panic("mvsata_atapi_intr: bad state");
 2226         }
 2227         /*
 2228          * If we missed an interrupt in a PIO transfer, reset and restart.
 2229          * Don't try to continue transfer, we may have missed cycles.
 2230          */
 2231         if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
 2232                 ata_channel_unlock(chp);
 2233                 sc_xfer->error = XS_TIMEOUT;
 2234                 mvsata_atapi_reset(chp, xfer);
 2235                 return 1;
 2236         }
 2237 
 2238         /* Ack interrupt done in wdc_wait_for_unbusy */
 2239         MVSATA_WDC_WRITE_1(mvport, SRB_H, WDSD_IBM);
 2240         if (wdc_wait_for_unbusy(chp,
 2241             (irq == 0) ? sc_xfer->timeout : 0, AT_POLL, &tfd) == WDCWAIT_TOUT) {
 2242                 if (irq && (xfer->c_flags & C_TIMEOU) == 0) {
 2243                         ata_channel_unlock(chp);
 2244                         return 0; /* IRQ was not for us */
 2245                 }
 2246                 aprint_error_dev(atac->atac_dev,
 2247                     "channel %d: device timeout, c_bcount=%d, c_skip=%d\n",
 2248                     chp->ch_channel, xfer->c_bcount, xfer->c_skip);
 2249                 if (xfer->c_flags & C_DMA)
 2250                         ata_dmaerr(drvp,
 2251                             (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2252                 sc_xfer->error = XS_TIMEOUT;
 2253                 ata_channel_unlock(chp);
 2254                 mvsata_atapi_reset(chp, xfer);
 2255                 return 1;
 2256         }
 2257 
 2258         /*
 2259          * If we missed an IRQ and were using DMA, flag it as a DMA error
 2260          * and reset device.
 2261          */
 2262         if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
 2263                 ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2264                 sc_xfer->error = XS_RESET;
 2265                 ata_channel_unlock(chp);
 2266                 mvsata_atapi_reset(chp, xfer);
 2267                 return (1);
 2268         }
 2269         /*
 2270          * if the request sense command was aborted, report the short sense
 2271          * previously recorded, else continue normal processing
 2272          */
 2273 
 2274 again:
 2275         len = MVSATA_WDC_READ_1(mvport, SRB_LBAM) +
 2276             256 * MVSATA_WDC_READ_1(mvport, SRB_LBAH);
 2277         ire = MVSATA_WDC_READ_1(mvport, SRB_SC);
 2278         phase = (ire & (WDCI_CMD | WDCI_IN)) | (ATACH_ST(tfd) & WDCS_DRQ);
 2279         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, (
 2280             "mvsata_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x ire 0x%x :",
 2281             xfer->c_bcount, len, ATACH_ST(tfd), ATACH_ERR(tfd), ire));
 2282 
 2283         switch (phase) {
 2284         case PHASE_CMDOUT:
 2285                 cmd = sc_xfer->cmd;
 2286                 DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, ("PHASE_CMDOUT\n"));
 2287                 /* Init the DMA channel if necessary */
 2288                 if (xfer->c_flags & C_DMA) {
 2289                         error = mvsata_bdma_init(mvport, xfer);
 2290                         if (error) {
 2291                                 if (error == EINVAL) {
 2292                                         /*
 2293                                          * We can't do DMA on this transfer
 2294                                          * for some reason.  Fall back to PIO.
 2295                                          */
 2296                                         xfer->c_flags &= ~C_DMA;
 2297                                         error = 0;
 2298                                 } else {
 2299                                         sc_xfer->error = XS_DRIVER_STUFFUP;
 2300                                         break;
 2301                                 }
 2302                         }
 2303                 }
 2304 
 2305                 /* send packet command */
 2306                 /* Commands are 12 or 16 bytes long. It's 32-bit aligned */
 2307                 wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
 2308 
 2309                 /* Start the DMA channel if necessary */
 2310                 if (xfer->c_flags & C_DMA) {
 2311                         mvsata_bdma_start(mvport);
 2312                         chp->ch_flags |= ATACH_DMA_WAIT;
 2313                 }
 2314                 ata_channel_unlock(chp);
 2315                 return 1;
 2316 
 2317         case PHASE_DATAOUT:
 2318                 /* write data */
 2319                 DPRINTF(DEBUG_XFERS, ("PHASE_DATAOUT\n"));
 2320                 if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
 2321                     (xfer->c_flags & C_DMA) != 0) {
 2322                         aprint_error_dev(atac->atac_dev,
 2323                             "channel %d drive %d: bad data phase DATAOUT\n",
 2324                             chp->ch_channel, xfer->c_drive);
 2325                         if (xfer->c_flags & C_DMA)
 2326                                 ata_dmaerr(drvp,
 2327                                     (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2328                         sc_xfer->error = XS_TIMEOUT;
 2329                         ata_channel_unlock(chp);
 2330                         mvsata_atapi_reset(chp, xfer);
 2331                         return 1;
 2332                 }
 2333                 xfer->c_atapi.c_lenoff = len - xfer->c_bcount;
 2334                 if (xfer->c_bcount < len) {
 2335                         aprint_error_dev(atac->atac_dev, "channel %d drive %d:"
 2336                             " warning: write only %d of %d requested bytes\n",
 2337                             chp->ch_channel, xfer->c_drive, xfer->c_bcount,
 2338                             len);
 2339                         len = xfer->c_bcount;
 2340                 }
 2341 
 2342                 wdc->dataout_pio(chp, drvp->drive_flags,
 2343                     (char *)xfer->c_databuf + xfer->c_skip, len);
 2344 
 2345                 for (i = xfer->c_atapi.c_lenoff; i > 0; i -= 2)
 2346                         MVSATA_WDC_WRITE_2(mvport, SRB_PIOD, 0);
 2347 
 2348                 xfer->c_skip += len;
 2349                 xfer->c_bcount -= len;
 2350                 ata_channel_unlock(chp);
 2351                 return 1;
 2352 
 2353         case PHASE_DATAIN:
 2354                 /* Read data */
 2355                 DPRINTF(DEBUG_XFERS, ("PHASE_DATAIN\n"));
 2356                 if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
 2357                     (xfer->c_flags & C_DMA) != 0) {
 2358                         aprint_error_dev(atac->atac_dev,
 2359                             "channel %d drive %d: bad data phase DATAIN\n",
 2360                             chp->ch_channel, xfer->c_drive);
 2361                         if (xfer->c_flags & C_DMA)
 2362                                 ata_dmaerr(drvp,
 2363                                     (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2364                         ata_channel_unlock(chp);
 2365                         sc_xfer->error = XS_TIMEOUT;
 2366                         mvsata_atapi_reset(chp, xfer);
 2367                         return 1;
 2368                 }
 2369                 xfer->c_atapi.c_lenoff = len - xfer->c_bcount;
 2370                 if (xfer->c_bcount < len) {
 2371                         aprint_error_dev(atac->atac_dev, "channel %d drive %d:"
 2372                             " warning: reading only %d of %d bytes\n",
 2373                             chp->ch_channel, xfer->c_drive, xfer->c_bcount,
 2374                             len);
 2375                         len = xfer->c_bcount;
 2376                 }
 2377 
 2378                 wdc->datain_pio(chp, drvp->drive_flags,
 2379                     (char *)xfer->c_databuf + xfer->c_skip, len);
 2380 
 2381                 if (xfer->c_atapi.c_lenoff > 0)
 2382                         wdcbit_bucket(chp, len - xfer->c_bcount);
 2383 
 2384                 xfer->c_skip += len;
 2385                 xfer->c_bcount -= len;
 2386                 ata_channel_unlock(chp);
 2387                 return 1;
 2388 
 2389         case PHASE_ABORTED:
 2390         case PHASE_COMPLETED:
 2391                 DPRINTF(DEBUG_XFERS, ("PHASE_COMPLETED\n"));
 2392                 if (xfer->c_flags & C_DMA)
 2393                         xfer->c_bcount -= sc_xfer->datalen;
 2394                 sc_xfer->resid = xfer->c_bcount;
 2395                 /* this will unlock channel lock too */
 2396                 mvsata_atapi_phase_complete(xfer, tfd);
 2397                 return 1;
 2398 
 2399         default:
 2400                 if (++retries<500) {
 2401                         DELAY(100);
 2402                         tfd = ATACH_ERR_ST(
 2403                             MVSATA_WDC_READ_1(mvport, SRB_FE),
 2404                             MVSATA_WDC_READ_1(mvport, SRB_CS)
 2405                         );
 2406                         goto again;
 2407                 }
 2408                 aprint_error_dev(atac->atac_dev,
 2409                     "channel %d drive %d: unknown phase 0x%x\n",
 2410                     chp->ch_channel, xfer->c_drive, phase);
 2411                 if (ATACH_ST(tfd) & WDCS_ERR) {
 2412                         sc_xfer->error = XS_SHORTSENSE;
 2413                         sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
 2414                 } else {
 2415                         if (xfer->c_flags & C_DMA)
 2416                                 ata_dmaerr(drvp,
 2417                                     (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2418                         sc_xfer->error = XS_RESET;
 2419                         ata_channel_unlock(chp);
 2420                         mvsata_atapi_reset(chp, xfer);
 2421                         return (1);
 2422                 }
 2423         }
 2424         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2425             ("mvsata_atapi_intr: %s (end), error 0x%x "
 2426             "sense 0x%x\n", __func__,
 2427             sc_xfer->error, sc_xfer->sense.atapi_sense));
 2428         ata_channel_unlock(chp);
 2429         mvsata_atapi_done(chp, xfer);
 2430         return 1;
 2431 }
 2432 
 2433 static void
 2434 mvsata_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer,
 2435                        int reason)
 2436 {
 2437         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 2438         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2439         bool deactivate = true;
 2440 
 2441         /* remove this command from xfer queue */
 2442         switch (reason) {
 2443         case KILL_GONE_INACTIVE:
 2444                 deactivate = false;
 2445                 /* FALLTHROUGH */
 2446         case KILL_GONE:
 2447                 sc_xfer->error = XS_DRIVER_STUFFUP;
 2448                 break;
 2449         case KILL_RESET:
 2450                 sc_xfer->error = XS_RESET;
 2451                 break;
 2452         case KILL_REQUEUE:
 2453                 sc_xfer->error = XS_REQUEUE;
 2454                 break;
 2455         default:
 2456                 aprint_error_dev(MVSATA_DEV2(mvport),
 2457                     "mvsata_atapi_kill_xfer: unknown reason %d\n", reason);
 2458                 panic("mvsata_atapi_kill_xfer");
 2459         }
 2460 
 2461         if (deactivate)
 2462                 ata_deactivate_xfer(chp, xfer);
 2463 
 2464         ata_free_xfer(chp, xfer);
 2465         scsipi_done(sc_xfer);
 2466 }
 2467 
 2468 static void
 2469 mvsata_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
 2470 {
 2471         struct mvsata_port *mvport = (struct mvsata_port *)chp;
 2472         struct atac_softc *atac = chp->ch_atac;
 2473         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 2474         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2475         int tfd;
 2476 
 2477         ata_channel_lock(chp);
 2478 
 2479         mvsata_pmp_select(mvport, xfer->c_drive);
 2480 
 2481         wdccommandshort(chp, 0, ATAPI_SOFT_RESET);
 2482         drvp->state = 0;
 2483         if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL, &tfd) != 0) {
 2484                 printf("%s:%d:%d: reset failed\n", device_xname(atac->atac_dev),
 2485                     chp->ch_channel, xfer->c_drive);
 2486                 sc_xfer->error = XS_SELTIMEOUT;
 2487         }
 2488 
 2489         ata_channel_unlock(chp);
 2490 
 2491         mvsata_atapi_done(chp, xfer);
 2492         return;
 2493 }
 2494 
 2495 static void
 2496 mvsata_atapi_phase_complete(struct ata_xfer *xfer, int tfd)
 2497 {
 2498         struct ata_channel *chp = xfer->c_chp;
 2499         struct atac_softc *atac = chp->ch_atac;
 2500         struct wdc_softc *wdc = CHAN_TO_WDC(chp);
 2501         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2502         struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
 2503 
 2504         ata_channel_lock_owned(chp);
 2505 
 2506         /* wait for DSC if needed */
 2507         if (drvp->drive_flags & ATA_DRIVE_ATAPIDSCW) {
 2508                 DPRINTF(DEBUG_XFERS,
 2509                     ("%s:%d:%d: mvsata_atapi_phase_complete: polldsc %d\n",
 2510                     device_xname(atac->atac_dev), chp->ch_channel,
 2511                     xfer->c_drive, xfer->c_atapi.c_dscpoll));
 2512                 if (cold)
 2513                         panic("mvsata_atapi_phase_complete: cold");
 2514 
 2515                 if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10, AT_POLL, &tfd) ==
 2516                     WDCWAIT_TOUT) {
 2517                         /* 10ms not enough, try again in 1 tick */
 2518                         if (xfer->c_atapi.c_dscpoll++ >
 2519                             mstohz(sc_xfer->timeout)) {
 2520                                 aprint_error_dev(atac->atac_dev,
 2521                                     "channel %d: wait_for_dsc failed\n",
 2522                                     chp->ch_channel);
 2523                                 ata_channel_unlock(chp);
 2524                                 sc_xfer->error = XS_TIMEOUT;
 2525                                 mvsata_atapi_reset(chp, xfer);
 2526                         } else {
 2527                                 callout_reset(&chp->c_timo_callout, 1,
 2528                                     mvsata_atapi_polldsc, chp);
 2529                                 ata_channel_unlock(chp);
 2530                         }
 2531                         return;
 2532                 }
 2533         }
 2534 
 2535         /*
 2536          * Some drive occasionally set WDCS_ERR with
 2537          * "ATA illegal length indication" in the error
 2538          * register. If we read some data the sense is valid
 2539          * anyway, so don't report the error.
 2540          */
 2541         if (ATACH_ST(tfd) & WDCS_ERR &&
 2542             ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
 2543             sc_xfer->resid == sc_xfer->datalen)) {
 2544                 /* save the short sense */
 2545                 sc_xfer->error = XS_SHORTSENSE;
 2546                 sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
 2547                 if ((sc_xfer->xs_periph->periph_quirks & PQUIRK_NOSENSE) == 0) {
 2548                         /* ask scsipi to send a REQUEST_SENSE */
 2549                         sc_xfer->error = XS_BUSY;
 2550                         sc_xfer->status = SCSI_CHECK;
 2551                 } else
 2552                     if (wdc->dma_status & (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
 2553                         ata_dmaerr(drvp,
 2554                             (xfer->c_flags & C_POLL) ? AT_POLL : 0);
 2555                         sc_xfer->error = XS_RESET;
 2556                         ata_channel_unlock(chp);
 2557                         mvsata_atapi_reset(chp, xfer);
 2558                         return;
 2559                 }
 2560         }
 2561         if (xfer->c_bcount != 0) {
 2562                 DPRINTF(DEBUG_XFERS, ("%s:%d:%d: mvsata_atapi_intr:"
 2563                     " bcount value is %d after io\n",
 2564                     device_xname(atac->atac_dev), chp->ch_channel,
 2565                     xfer->c_drive, xfer->c_bcount));
 2566         }
 2567 #ifdef DIAGNOSTIC
 2568         if (xfer->c_bcount < 0) {
 2569                 aprint_error_dev(atac->atac_dev,
 2570                     "channel %d drive %d: mvsata_atapi_intr:"
 2571                     " warning: bcount value is %d after io\n",
 2572                     chp->ch_channel, xfer->c_drive, xfer->c_bcount);
 2573         }
 2574 #endif
 2575 
 2576         DPRINTF(DEBUG_XFERS,
 2577             ("%s:%d:%d: mvsata_atapi_phase_complete:"
 2578             " mvsata_atapi_done(), error 0x%x sense 0x%x\n",
 2579             device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
 2580             sc_xfer->error, sc_xfer->sense.atapi_sense));
 2581         ata_channel_unlock(chp);
 2582         mvsata_atapi_done(chp, xfer);
 2583 }
 2584 
 2585 static void
 2586 mvsata_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
 2587 {
 2588         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2589         bool iserror = (sc_xfer->error != XS_NOERROR);
 2590 
 2591         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2592             ("%s:%d:%d: mvsata_atapi_done: flags 0x%x\n",
 2593             device_xname(chp->ch_atac->atac_dev), chp->ch_channel,
 2594             xfer->c_drive, (u_int)xfer->c_flags));
 2595 
 2596         if (ata_waitdrain_xfer_check(chp, xfer))
 2597                 return;
 2598 
 2599         /* mark controller inactive and free the command */
 2600         ata_deactivate_xfer(chp, xfer);
 2601 
 2602         ata_free_xfer(chp, xfer);
 2603 
 2604         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2605             ("%s:%d: mvsata_atapi_done: scsipi_done\n",
 2606             device_xname(chp->ch_atac->atac_dev), chp->ch_channel));
 2607         scsipi_done(sc_xfer);
 2608         DPRINTF(DEBUG_FUNCS,
 2609             ("%s:%d: atastart from wdc_atapi_done, flags 0x%x\n",
 2610             device_xname(chp->ch_atac->atac_dev), chp->ch_channel,
 2611             chp->ch_flags));
 2612         if (!iserror)
 2613                 atastart(chp);
 2614 }
 2615 
 2616 static void
 2617 mvsata_atapi_polldsc(void *arg)
 2618 {
 2619         struct ata_channel *chp = arg;
 2620         struct ata_xfer *xfer = ata_queue_get_active_xfer(chp);
 2621 
 2622         KASSERT(xfer != NULL);
 2623 
 2624         ata_channel_lock(chp);
 2625 
 2626         /* this will unlock channel lock too */
 2627         mvsata_atapi_phase_complete(xfer, 0);
 2628 }
 2629 #endif  /* NATAPIBUS > 0 */
 2630 
 2631 
 2632 /*
 2633  * XXXX: Shall we need lock for race condition in mvsata_edma_enqueue{,_gen2}(),
 2634  * if supported queuing command by atabus?  The race condition will not happen
 2635  * if this is called only to the thread of atabus.
 2636  */
 2637 static int
 2638 mvsata_edma_enqueue(struct mvsata_port *mvport, struct ata_xfer *xfer)
 2639 {
 2640         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 2641         struct ata_bio *ata_bio = &xfer->c_bio;
 2642         void *databuf = (uint8_t *)xfer->c_databuf + xfer->c_skip;
 2643         struct eprd *eprd;
 2644         bus_addr_t crqb_base_addr;
 2645         bus_dmamap_t data_dmamap;
 2646         uint32_t reg;
 2647         int erqqip, erqqop, next, rv, i;
 2648 
 2649         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS, ("%s:%d:%d: mvsata_edma_enqueue:"
 2650             " blkno=0x%" PRIx64 ", nbytes=%d, flags=0x%x\n",
 2651             device_xname(MVSATA_DEV2(mvport)), mvport->port_hc->hc,
 2652             mvport->port, ata_bio->blkno, ata_bio->nbytes, ata_bio->flags));
 2653 
 2654         reg = MVSATA_EDMA_READ_4(mvport, EDMA_REQQOP);
 2655         erqqop = (reg & EDMA_REQQP_ERQQP_MASK) >> EDMA_REQQP_ERQQP_SHIFT;
 2656         reg = MVSATA_EDMA_READ_4(mvport, EDMA_REQQIP);
 2657         erqqip = (reg & EDMA_REQQP_ERQQP_MASK) >> EDMA_REQQP_ERQQP_SHIFT;
 2658         next = erqqip;
 2659         MVSATA_EDMAQ_INC(next);
 2660         if (next == erqqop) {
 2661                 /* queue full */
 2662                 return EBUSY;
 2663         }
 2664         DPRINTF(DEBUG_XFERS,
 2665             ("    erqqip=%d, quetag=%d\n", erqqip, xfer->c_slot));
 2666 
 2667         rv = mvsata_dma_bufload(mvport, xfer->c_slot, databuf, ata_bio->nbytes,
 2668             ata_bio->flags);
 2669         if (rv != 0)
 2670                 return rv;
 2671 
 2672         /* setup EDMA Physical Region Descriptors (ePRD) Table Data */
 2673         data_dmamap = mvport->port_reqtbl[xfer->c_slot].data_dmamap;
 2674         eprd = mvport->port_reqtbl[xfer->c_slot].eprd;
 2675         for (i = 0; i < data_dmamap->dm_nsegs; i++) {
 2676                 bus_addr_t ds_addr = data_dmamap->dm_segs[i].ds_addr;
 2677                 bus_size_t ds_len = data_dmamap->dm_segs[i].ds_len;
 2678 
 2679                 eprd->prdbal = htole32(ds_addr & EPRD_PRDBAL_MASK);
 2680                 eprd->bytecount = htole32(EPRD_BYTECOUNT(ds_len));
 2681                 eprd->eot = htole16(0);
 2682                 eprd->prdbah = htole32((ds_addr >> 16) >> 16);
 2683                 eprd++;
 2684         }
 2685         (eprd - 1)->eot |= htole16(EPRD_EOT);
 2686 #ifdef MVSATA_DEBUG
 2687         if (mvsata_debug >= 3)
 2688                 mvsata_print_eprd(mvport, xfer->c_slot);
 2689 #endif
 2690         bus_dmamap_sync(mvport->port_dmat, mvport->port_eprd_dmamap,
 2691             mvport->port_reqtbl[xfer->c_slot].eprd_offset, MVSATA_EPRD_MAX_SIZE,
 2692             BUS_DMASYNC_PREWRITE);
 2693 
 2694         /* setup EDMA Command Request Block (CRQB) Data */
 2695         sc->sc_edma_setup_crqb(mvport, erqqip, xfer);
 2696 #ifdef MVSATA_DEBUG
 2697         if (mvsata_debug >= 3)
 2698                 mvsata_print_crqb(mvport, erqqip);
 2699 #endif
 2700         bus_dmamap_sync(mvport->port_dmat, mvport->port_crqb_dmamap,
 2701             erqqip * sizeof(union mvsata_crqb),
 2702             sizeof(union mvsata_crqb), BUS_DMASYNC_PREWRITE);
 2703 
 2704         MVSATA_EDMAQ_INC(erqqip);
 2705 
 2706         crqb_base_addr = mvport->port_crqb_dmamap->dm_segs[0].ds_addr &
 2707             (EDMA_REQQP_ERQQBAP_MASK | EDMA_REQQP_ERQQBA_MASK);
 2708         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQBAH, (crqb_base_addr >> 16) >> 16);
 2709         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQIP,
 2710             crqb_base_addr | (erqqip << EDMA_REQQP_ERQQP_SHIFT));
 2711 
 2712         return 0;
 2713 }
 2714 
 2715 static int
 2716 mvsata_edma_handle(struct mvsata_port *mvport, struct ata_xfer *xfer1)
 2717 {
 2718         struct ata_channel *chp = &mvport->port_ata_channel;
 2719         struct crpb *crpb;
 2720         struct ata_bio *ata_bio;
 2721         struct ata_xfer *xfer;
 2722         uint32_t reg;
 2723         int erqqop, erpqip, erpqop, prev_erpqop, quetag, handled = 0, n;
 2724         int st, dmaerr;
 2725 
 2726         /* First, Sync for Request Queue buffer */
 2727         reg = MVSATA_EDMA_READ_4(mvport, EDMA_REQQOP);
 2728         erqqop = (reg & EDMA_REQQP_ERQQP_MASK) >> EDMA_REQQP_ERQQP_SHIFT;
 2729         if (mvport->port_prev_erqqop != erqqop) {
 2730                 const int s = sizeof(union mvsata_crqb);
 2731 
 2732                 if (mvport->port_prev_erqqop < erqqop)
 2733                         n = erqqop - mvport->port_prev_erqqop;
 2734                 else {
 2735                         if (erqqop > 0)
 2736                                 bus_dmamap_sync(mvport->port_dmat,
 2737                                     mvport->port_crqb_dmamap, 0, erqqop * s,
 2738                                     BUS_DMASYNC_POSTWRITE);
 2739                         n = MVSATA_EDMAQ_LEN - mvport->port_prev_erqqop;
 2740                 }
 2741                 if (n > 0)
 2742                         bus_dmamap_sync(mvport->port_dmat,
 2743                             mvport->port_crqb_dmamap,
 2744                             mvport->port_prev_erqqop * s, n * s,
 2745                             BUS_DMASYNC_POSTWRITE);
 2746                 mvport->port_prev_erqqop = erqqop;
 2747         }
 2748 
 2749         reg = MVSATA_EDMA_READ_4(mvport, EDMA_RESQIP);
 2750         erpqip = (reg & EDMA_RESQP_ERPQP_MASK) >> EDMA_RESQP_ERPQP_SHIFT;
 2751         reg = MVSATA_EDMA_READ_4(mvport, EDMA_RESQOP);
 2752         erpqop = (reg & EDMA_RESQP_ERPQP_MASK) >> EDMA_RESQP_ERPQP_SHIFT;
 2753 
 2754         DPRINTF(DEBUG_XFERS,
 2755             ("%s:%d:%d: mvsata_edma_handle: erpqip=%d, erpqop=%d\n",
 2756             device_xname(MVSATA_DEV2(mvport)), mvport->port_hc->hc,
 2757             mvport->port, erpqip, erpqop));
 2758 
 2759         if (erpqop == erpqip)
 2760                 return 0;
 2761 
 2762         if (erpqop < erpqip)
 2763                 n = erpqip - erpqop;
 2764         else {
 2765                 if (erpqip > 0)
 2766                         bus_dmamap_sync(mvport->port_dmat,
 2767                             mvport->port_crpb_dmamap,
 2768                             0, erpqip * sizeof(struct crpb),
 2769                             BUS_DMASYNC_POSTREAD);
 2770                 n = MVSATA_EDMAQ_LEN - erpqop;
 2771         }
 2772         if (n > 0)
 2773                 bus_dmamap_sync(mvport->port_dmat, mvport->port_crpb_dmamap,
 2774                     erpqop * sizeof(struct crpb),
 2775                     n * sizeof(struct crpb), BUS_DMASYNC_POSTREAD);
 2776 
 2777         uint32_t aslots = ata_queue_active(chp);
 2778 
 2779         prev_erpqop = erpqop;
 2780         while (erpqop != erpqip) {
 2781 #ifdef MVSATA_DEBUG
 2782                 if (mvsata_debug >= 3)
 2783                         mvsata_print_crpb(mvport, erpqop);
 2784 #endif
 2785                 crpb = mvport->port_crpb + erpqop;
 2786                 MVSATA_EDMAQ_INC(erpqop);
 2787 
 2788                 quetag = CRPB_CHOSTQUETAG(le16toh(crpb->id));
 2789 
 2790                 if ((aslots & __BIT(quetag)) == 0) {
 2791                         /* not actually executing */
 2792                         continue;
 2793                 }
 2794 
 2795                 xfer = ata_queue_hwslot_to_xfer(chp, quetag);
 2796 
 2797                 bus_dmamap_sync(mvport->port_dmat, mvport->port_eprd_dmamap,
 2798                     mvport->port_reqtbl[xfer->c_slot].eprd_offset,
 2799                     MVSATA_EPRD_MAX_SIZE, BUS_DMASYNC_POSTWRITE);
 2800 
 2801                 st = CRPB_CDEVSTS(le16toh(crpb->rspflg));
 2802                 dmaerr = CRPB_CEDMASTS(le16toh(crpb->rspflg));
 2803 
 2804                 ata_bio = &xfer->c_bio;
 2805                 ata_bio->error = NOERROR;
 2806                 if (dmaerr != 0)
 2807                         ata_bio->error = ERR_DMA;
 2808 
 2809                 mvsata_dma_bufunload(mvport, quetag, ata_bio->flags);
 2810 
 2811                 KASSERT(xfer->c_flags & C_DMA);
 2812                 mvsata_bio_intr(chp, xfer, ATACH_ERR_ST(0, st));
 2813 
 2814                 if (xfer1 == NULL)
 2815                         handled++;
 2816                 else if (xfer == xfer1) {
 2817                         handled = 1;
 2818                         break;
 2819                 }
 2820         }
 2821         if (prev_erpqop < erpqop)
 2822                 n = erpqop - prev_erpqop;
 2823         else {
 2824                 if (erpqop > 0)
 2825                         bus_dmamap_sync(mvport->port_dmat,
 2826                             mvport->port_crpb_dmamap, 0,
 2827                             erpqop * sizeof(struct crpb), BUS_DMASYNC_PREREAD);
 2828                 n = MVSATA_EDMAQ_LEN - prev_erpqop;
 2829         }
 2830         if (n > 0)
 2831                 bus_dmamap_sync(mvport->port_dmat, mvport->port_crpb_dmamap,
 2832                     prev_erpqop * sizeof(struct crpb),
 2833                     n * sizeof(struct crpb), BUS_DMASYNC_PREREAD);
 2834 
 2835         reg &= ~EDMA_RESQP_ERPQP_MASK;
 2836         reg |= (erpqop << EDMA_RESQP_ERPQP_SHIFT);
 2837         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQOP, reg);
 2838 
 2839         return handled;
 2840 }
 2841 
 2842 static int
 2843 mvsata_edma_wait(struct mvsata_port *mvport, struct ata_xfer *xfer, int timeout)
 2844 {
 2845         int xtime;
 2846 
 2847         for (xtime = 0;  xtime < timeout * 10; xtime++) {
 2848                 if (mvsata_edma_handle(mvport, xfer))
 2849                         return 0;
 2850                 DELAY(100);
 2851         }
 2852 
 2853         DPRINTF(DEBUG_FUNCS, ("%s: timeout: %p\n", __func__, xfer));
 2854         mvsata_edma_rqq_remove(mvport, xfer);
 2855         xfer->c_flags |= C_TIMEOU;
 2856         return 1;
 2857 }
 2858 
 2859 static void
 2860 mvsata_edma_rqq_remove(struct mvsata_port *mvport, struct ata_xfer *xfer)
 2861 {
 2862         struct ata_channel *chp = &mvport->port_ata_channel;
 2863         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 2864         bus_addr_t crqb_base_addr;
 2865         int erqqip, i;
 2866 
 2867         /* First, hardware reset, stop EDMA */
 2868         mvsata_hreset_port(mvport);
 2869 
 2870         /* cleanup completed EDMA safely */
 2871         mvsata_edma_handle(mvport, NULL);
 2872 
 2873         bus_dmamap_sync(mvport->port_dmat, mvport->port_crqb_dmamap, 0,
 2874             sizeof(union mvsata_crqb) * MVSATA_EDMAQ_LEN, BUS_DMASYNC_PREWRITE);
 2875 
 2876         uint32_t aslots = ata_queue_active(chp);
 2877 
 2878         for (i = 0, erqqip = 0; i < MVSATA_EDMAQ_LEN; i++) {
 2879                 struct ata_xfer *rqxfer;
 2880 
 2881                 if ((aslots & __BIT(i)) == 0)
 2882                         continue;
 2883 
 2884                 if (i == xfer->c_slot) {
 2885                         /* remove xfer from EDMA request queue */
 2886                         bus_dmamap_sync(mvport->port_dmat,
 2887                             mvport->port_eprd_dmamap,
 2888                             mvport->port_reqtbl[i].eprd_offset,
 2889                             MVSATA_EPRD_MAX_SIZE, BUS_DMASYNC_POSTWRITE);
 2890                         mvsata_dma_bufunload(mvport, i, xfer->c_bio.flags);
 2891                         /* quetag freed by caller later */
 2892                         continue;
 2893                 }
 2894 
 2895                 rqxfer = ata_queue_hwslot_to_xfer(chp, i);
 2896                 sc->sc_edma_setup_crqb(mvport, erqqip, rqxfer);
 2897                 erqqip++;
 2898         }
 2899         bus_dmamap_sync(mvport->port_dmat, mvport->port_crqb_dmamap, 0,
 2900             sizeof(union mvsata_crqb) * MVSATA_EDMAQ_LEN,
 2901             BUS_DMASYNC_POSTWRITE);
 2902 
 2903         mvsata_edma_config(mvport, mvport->port_edmamode_curr);
 2904         mvsata_edma_reset_qptr(mvport);
 2905         mvsata_edma_enable(mvport);
 2906 
 2907         crqb_base_addr = mvport->port_crqb_dmamap->dm_segs[0].ds_addr &
 2908             (EDMA_REQQP_ERQQBAP_MASK | EDMA_REQQP_ERQQBA_MASK);
 2909         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQBAH, (crqb_base_addr >> 16) >> 16);
 2910         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQIP,
 2911             crqb_base_addr | (erqqip << EDMA_REQQP_ERQQP_SHIFT));
 2912 }
 2913 
 2914 #if NATAPIBUS > 0
 2915 static int
 2916 mvsata_bdma_init(struct mvsata_port *mvport, struct ata_xfer *xfer)
 2917 {
 2918         struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
 2919         struct eprd *eprd;
 2920         bus_dmamap_t data_dmamap;
 2921         bus_addr_t eprd_addr;
 2922         int i, rv;
 2923         void *databuf = (uint8_t *)xfer->c_databuf + xfer->c_skip;
 2924 
 2925         DPRINTF(DEBUG_FUNCS|DEBUG_XFERS,
 2926             ("%s:%d:%d: mvsata_bdma_init: datalen=%d, xs_control=0x%x\n",
 2927             device_xname(MVSATA_DEV2(mvport)), mvport->port_hc->hc,
 2928             mvport->port, sc_xfer->datalen, sc_xfer->xs_control));
 2929 
 2930         rv = mvsata_dma_bufload(mvport, xfer->c_slot, databuf,
 2931             sc_xfer->datalen,
 2932             sc_xfer->xs_control & XS_CTL_DATA_IN ? ATA_READ : 0);
 2933         if (rv != 0)
 2934                 return rv;
 2935 
 2936         /* setup EDMA Physical Region Descriptors (ePRD) Table Data */
 2937         data_dmamap = mvport->port_reqtbl[xfer->c_slot].data_dmamap;
 2938         eprd = mvport->port_reqtbl[xfer->c_slot].eprd;
 2939         for (i = 0; i < data_dmamap->dm_nsegs; i++) {
 2940                 bus_addr_t ds_addr = data_dmamap->dm_segs[i].ds_addr;
 2941                 bus_size_t ds_len = data_dmamap->dm_segs[i].ds_len;
 2942 
 2943                 eprd->prdbal = htole32(ds_addr & EPRD_PRDBAL_MASK);
 2944                 eprd->bytecount = htole32(EPRD_BYTECOUNT(ds_len));
 2945                 eprd->eot = htole16(0);
 2946                 eprd->prdbah = htole32((ds_addr >> 16) >> 16);
 2947                 eprd++;
 2948         }
 2949         (eprd - 1)->eot |= htole16(EPRD_EOT);
 2950 #ifdef MVSATA_DEBUG
 2951         if (mvsata_debug >= 3)
 2952                 mvsata_print_eprd(mvport, xfer->c_slot);
 2953 #endif
 2954         bus_dmamap_sync(mvport->port_dmat, mvport->port_eprd_dmamap,
 2955             mvport->port_reqtbl[xfer->c_slot].eprd_offset,
 2956             MVSATA_EPRD_MAX_SIZE, BUS_DMASYNC_PREWRITE);
 2957         eprd_addr = mvport->port_eprd_dmamap->dm_segs[0].ds_addr +
 2958             mvport->port_reqtbl[xfer->c_slot].eprd_offset;
 2959 
 2960         MVSATA_EDMA_WRITE_4(mvport, DMA_DTLBA, eprd_addr & DMA_DTLBA_MASK);
 2961         MVSATA_EDMA_WRITE_4(mvport, DMA_DTHBA, (eprd_addr >> 16) >> 16);
 2962 
 2963         if (sc_xfer->xs_control & XS_CTL_DATA_IN)
 2964                 MVSATA_EDMA_WRITE_4(mvport, DMA_C, DMA_C_READ);
 2965         else
 2966                 MVSATA_EDMA_WRITE_4(mvport, DMA_C, 0);
 2967 
 2968         return 0;
 2969 }
 2970 
 2971 static void
 2972 mvsata_bdma_start(struct mvsata_port *mvport)
 2973 {
 2974 
 2975 #ifdef MVSATA_DEBUG
 2976         if (mvsata_debug >= 3)
 2977                 mvsata_print_eprd(mvport, 0);
 2978 #endif
 2979 
 2980         MVSATA_EDMA_WRITE_4(mvport, DMA_C,
 2981             MVSATA_EDMA_READ_4(mvport, DMA_C) | DMA_C_START);
 2982 }
 2983 #endif
 2984 #endif
 2985 
 2986 
 2987 static int
 2988 mvsata_port_init(struct mvsata_hc *mvhc, int port)
 2989 {
 2990         struct mvsata_softc *sc = mvhc->hc_sc;
 2991         struct mvsata_port *mvport;
 2992         struct ata_channel *chp;
 2993         int channel, rv, i;
 2994         const int crqbq_size = sizeof(union mvsata_crqb) * MVSATA_EDMAQ_LEN;
 2995         const int crpbq_size = sizeof(struct crpb) * MVSATA_EDMAQ_LEN;
 2996         const int eprd_buf_size = MVSATA_EPRD_MAX_SIZE * MVSATA_EDMAQ_LEN;
 2997 
 2998         mvport = malloc(sizeof(struct mvsata_port), M_DEVBUF,
 2999             M_ZERO | M_WAITOK);
 3000         mvport->port = port;
 3001         mvport->port_hc = mvhc;
 3002         mvport->port_edmamode_negotiated = nodma;
 3003 
 3004         rv = bus_space_subregion(mvhc->hc_iot, mvhc->hc_ioh,
 3005             EDMA_REGISTERS_OFFSET + port * EDMA_REGISTERS_SIZE,
 3006             EDMA_REGISTERS_SIZE, &mvport->port_ioh);
 3007         if (rv != 0) {
 3008                 aprint_error("%s:%d: can't subregion EDMA %d registers\n",
 3009                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port);
 3010                 goto fail0;
 3011         }
 3012         mvport->port_iot = mvhc->hc_iot;
 3013         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh, SATA_SS, 4,
 3014             &mvport->port_sata_sstatus);
 3015         if (rv != 0) {
 3016                 aprint_error("%s:%d:%d: couldn't subregion sstatus regs\n",
 3017                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port);
 3018                 goto fail0;
 3019         }
 3020         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh, SATA_SE, 4,
 3021             &mvport->port_sata_serror);
 3022         if (rv != 0) {
 3023                 aprint_error("%s:%d:%d: couldn't subregion serror regs\n",
 3024                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port);
 3025                 goto fail0;
 3026         }
 3027         if (sc->sc_rev == gen1)
 3028                 rv = bus_space_subregion(mvhc->hc_iot, mvhc->hc_ioh,
 3029                     SATAHC_I_R02(port), 4, &mvport->port_sata_scontrol);
 3030         else
 3031                 rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3032                     SATA_SC, 4, &mvport->port_sata_scontrol);
 3033         if (rv != 0) {
 3034                 aprint_error("%s:%d:%d: couldn't subregion scontrol regs\n",
 3035                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port);
 3036                 goto fail0;
 3037         }
 3038         mvport->port_dmat = sc->sc_dmat;
 3039         mvhc->hc_ports[port] = mvport;
 3040 
 3041         channel = mvhc->hc * sc->sc_port + port;
 3042         chp = &mvport->port_ata_channel;
 3043         chp->ch_channel = channel;
 3044         chp->ch_atac = &sc->sc_wdcdev.sc_atac;
 3045         chp->ch_queue = ata_queue_alloc(MVSATA_EDMAQ_LEN);
 3046         sc->sc_ata_channels[channel] = chp;
 3047 
 3048         rv = mvsata_wdc_reg_init(mvport, sc->sc_wdcdev.regs + channel);
 3049         if (rv != 0)
 3050                 goto fail0;
 3051 
 3052         rv = bus_dmamap_create(mvport->port_dmat, crqbq_size, 1, crqbq_size, 0,
 3053             BUS_DMA_NOWAIT, &mvport->port_crqb_dmamap);
 3054         if (rv != 0) {
 3055                 aprint_error(
 3056                     "%s:%d:%d: EDMA CRQB map create failed: error=%d\n",
 3057                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port, rv);
 3058                 goto fail0;
 3059         }
 3060         rv = bus_dmamap_create(mvport->port_dmat, crpbq_size, 1, crpbq_size, 0,
 3061             BUS_DMA_NOWAIT, &mvport->port_crpb_dmamap);
 3062         if (rv != 0) {
 3063                 aprint_error(
 3064                     "%s:%d:%d: EDMA CRPB map create failed: error=%d\n",
 3065                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port, rv);
 3066                 goto fail1;
 3067         }
 3068         rv = bus_dmamap_create(mvport->port_dmat, eprd_buf_size, 1,
 3069             eprd_buf_size, 0, BUS_DMA_NOWAIT, &mvport->port_eprd_dmamap);
 3070         if (rv != 0) {
 3071                 aprint_error(
 3072                     "%s:%d:%d: EDMA ePRD buffer map create failed: error=%d\n",
 3073                     device_xname(MVSATA_DEV(sc)), mvhc->hc, port, rv);
 3074                 goto fail2;
 3075         }
 3076         for (i = 0; i < MVSATA_EDMAQ_LEN; i++) {
 3077                 rv = bus_dmamap_create(mvport->port_dmat, MAXPHYS,
 3078                     MVSATA_MAX_SEGS, MAXPHYS, 0, BUS_DMA_NOWAIT,
 3079                     &mvport->port_reqtbl[i].data_dmamap);
 3080                 if (rv != 0) {
 3081                         aprint_error("%s:%d:%d:"
 3082                             " EDMA data map(%d) create failed: error=%d\n",
 3083                             device_xname(MVSATA_DEV(sc)), mvhc->hc, port, i,
 3084                             rv);
 3085                         goto fail3;
 3086                 }
 3087         }
 3088 
 3089         return 0;
 3090 
 3091 fail3:
 3092         for (i--; i >= 0; i--)
 3093                 bus_dmamap_destroy(mvport->port_dmat,
 3094                     mvport->port_reqtbl[i].data_dmamap);
 3095         bus_dmamap_destroy(mvport->port_dmat, mvport->port_eprd_dmamap);
 3096 fail2:
 3097         bus_dmamap_destroy(mvport->port_dmat, mvport->port_crpb_dmamap);
 3098 fail1:
 3099         bus_dmamap_destroy(mvport->port_dmat, mvport->port_crqb_dmamap);
 3100 fail0:
 3101         return rv;
 3102 }
 3103 
 3104 static int
 3105 mvsata_wdc_reg_init(struct mvsata_port *mvport, struct wdc_regs *wdr)
 3106 {
 3107         int hc, port, rv, i;
 3108 
 3109         hc = mvport->port_hc->hc;
 3110         port = mvport->port;
 3111 
 3112         /* Create subregion for Shadow Registers Map */
 3113         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3114             SHADOW_REG_BLOCK_OFFSET, SHADOW_REG_BLOCK_SIZE, &wdr->cmd_baseioh);
 3115         if (rv != 0) {
 3116                 aprint_error("%s:%d:%d: couldn't subregion shadow block regs\n",
 3117                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3118                 return rv;
 3119         }
 3120         wdr->cmd_iot = mvport->port_iot;
 3121 
 3122         /* Once create subregion for each command registers */
 3123         for (i = 0; i < WDC_NREG; i++) {
 3124                 rv = bus_space_subregion(wdr->cmd_iot, wdr->cmd_baseioh,
 3125                     i * 4, sizeof(uint32_t), &wdr->cmd_iohs[i]);
 3126                 if (rv != 0) {
 3127                         aprint_error("%s:%d:%d: couldn't subregion cmd regs\n",
 3128                             device_xname(MVSATA_DEV2(mvport)), hc, port);
 3129                         return rv;
 3130                 }
 3131         }
 3132         /* Create subregion for Alternate Status register */
 3133         rv = bus_space_subregion(wdr->cmd_iot, wdr->cmd_baseioh,
 3134             i * 4, sizeof(uint32_t), &wdr->ctl_ioh);
 3135         if (rv != 0) {
 3136                 aprint_error("%s:%d:%d: couldn't subregion cmd regs\n",
 3137                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3138                 return rv;
 3139         }
 3140         wdr->ctl_iot = mvport->port_iot;
 3141 
 3142         wdc_init_shadow_regs(wdr);
 3143 
 3144         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3145             SATA_SS, sizeof(uint32_t) * 3, &wdr->sata_baseioh);
 3146         if (rv != 0) {
 3147                 aprint_error("%s:%d:%d: couldn't subregion SATA regs\n",
 3148                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3149                 return rv;
 3150         }
 3151         wdr->sata_iot = mvport->port_iot;
 3152         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3153             SATA_SC, sizeof(uint32_t), &wdr->sata_control);
 3154         if (rv != 0) {
 3155                 aprint_error("%s:%d:%d: couldn't subregion SControl\n",
 3156                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3157                 return rv;
 3158         }
 3159         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3160             SATA_SS, sizeof(uint32_t), &wdr->sata_status);
 3161         if (rv != 0) {
 3162                 aprint_error("%s:%d:%d: couldn't subregion SStatus\n",
 3163                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3164                 return rv;
 3165         }
 3166         rv = bus_space_subregion(mvport->port_iot, mvport->port_ioh,
 3167             SATA_SE, sizeof(uint32_t), &wdr->sata_error);
 3168         if (rv != 0) {
 3169                 aprint_error("%s:%d:%d: couldn't subregion SError\n",
 3170                     device_xname(MVSATA_DEV2(mvport)), hc, port);
 3171                 return rv;
 3172         }
 3173 
 3174         return 0;
 3175 }
 3176 
 3177 
 3178 #ifndef MVSATA_WITHOUTDMA
 3179 static void *
 3180 mvsata_edma_resource_prepare(struct mvsata_port *mvport, bus_dma_tag_t dmat,
 3181                              bus_dmamap_t *dmamap, size_t size, int write)
 3182 {
 3183         bus_dma_segment_t seg;
 3184         int nseg, rv;
 3185         void *kva;
 3186 
 3187         rv = bus_dmamem_alloc(dmat, size, PAGE_SIZE, 0, &seg, 1, &nseg,
 3188             BUS_DMA_NOWAIT);
 3189         if (rv != 0) {
 3190                 aprint_error("%s:%d:%d: DMA memory alloc failed: error=%d\n",
 3191                     device_xname(MVSATA_DEV2(mvport)),
 3192                     mvport->port_hc->hc, mvport->port, rv);
 3193                 goto fail;
 3194         }
 3195 
 3196         rv = bus_dmamem_map(dmat, &seg, nseg, size, &kva, BUS_DMA_NOWAIT);
 3197         if (rv != 0) {
 3198                 aprint_error("%s:%d:%d: DMA memory map failed: error=%d\n",
 3199                     device_xname(MVSATA_DEV2(mvport)),
 3200                     mvport->port_hc->hc, mvport->port, rv);
 3201                 goto free;
 3202         }
 3203 
 3204         rv = bus_dmamap_load(dmat, *dmamap, kva, size, NULL,
 3205             BUS_DMA_NOWAIT | (write ? BUS_DMA_WRITE : BUS_DMA_READ));
 3206         if (rv != 0) {
 3207                 aprint_error("%s:%d:%d: DMA map load failed: error=%d\n",
 3208                     device_xname(MVSATA_DEV2(mvport)),
 3209                     mvport->port_hc->hc, mvport->port, rv);
 3210                 goto unmap;
 3211         }
 3212 
 3213         if (!write)
 3214                 bus_dmamap_sync(dmat, *dmamap, 0, size, BUS_DMASYNC_PREREAD);
 3215 
 3216         return kva;
 3217 
 3218 unmap:
 3219         bus_dmamem_unmap(dmat, kva, size);
 3220 free:
 3221         bus_dmamem_free(dmat, &seg, nseg);
 3222 fail:
 3223         return NULL;
 3224 }
 3225 
 3226 /* ARGSUSED */
 3227 static void
 3228 mvsata_edma_resource_purge(struct mvsata_port *mvport, bus_dma_tag_t dmat,
 3229                            bus_dmamap_t dmamap, void *kva)
 3230 {
 3231 
 3232         bus_dmamap_unload(dmat, dmamap);
 3233         bus_dmamem_unmap(dmat, kva, dmamap->dm_mapsize);
 3234         bus_dmamem_free(dmat, dmamap->dm_segs, dmamap->dm_nsegs);
 3235 }
 3236 
 3237 static int
 3238 mvsata_dma_bufload(struct mvsata_port *mvport, int index, void *databuf,
 3239                    size_t datalen, int flags)
 3240 {
 3241         int rv, lop, sop;
 3242         bus_dmamap_t data_dmamap = mvport->port_reqtbl[index].data_dmamap;
 3243 
 3244         lop = (flags & ATA_READ) ? BUS_DMA_READ : BUS_DMA_WRITE;
 3245         sop = (flags & ATA_READ) ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE;
 3246 
 3247         rv = bus_dmamap_load(mvport->port_dmat, data_dmamap, databuf, datalen,
 3248             NULL, BUS_DMA_NOWAIT | lop);
 3249         if (rv) {
 3250                 aprint_error("%s:%d:%d: buffer load failed: error=%d\n",
 3251                     device_xname(MVSATA_DEV2(mvport)), mvport->port_hc->hc,
 3252                     mvport->port, rv);
 3253                 return rv;
 3254         }
 3255         bus_dmamap_sync(mvport->port_dmat, data_dmamap, 0,
 3256             data_dmamap->dm_mapsize, sop);
 3257 
 3258         return 0;
 3259 }
 3260 
 3261 static inline void
 3262 mvsata_dma_bufunload(struct mvsata_port *mvport, int index, int flags)
 3263 {
 3264         bus_dmamap_t data_dmamap = mvport->port_reqtbl[index].data_dmamap;
 3265 
 3266         bus_dmamap_sync(mvport->port_dmat, data_dmamap, 0,
 3267             data_dmamap->dm_mapsize,
 3268             (flags & ATA_READ) ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
 3269         bus_dmamap_unload(mvport->port_dmat, data_dmamap);
 3270 }
 3271 #endif
 3272 
 3273 static void
 3274 mvsata_hreset_port(struct mvsata_port *mvport)
 3275 {
 3276         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 3277 
 3278         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, EDMA_CMD_EATARST);
 3279 
 3280         delay(25);              /* allow reset propagation */
 3281 
 3282         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, 0);
 3283 
 3284         mvport->_fix_phy_param._fix_phy(mvport);
 3285 
 3286         if (sc->sc_gen == gen1)
 3287                 delay(1000);
 3288 }
 3289 
 3290 static void
 3291 mvsata_reset_port(struct mvsata_port *mvport)
 3292 {
 3293         device_t parent = device_parent(MVSATA_DEV2(mvport));
 3294 
 3295         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, EDMA_CMD_EDSEDMA);
 3296 
 3297         mvsata_hreset_port(mvport);
 3298 
 3299         if (device_is_a(parent, "pci"))
 3300                 MVSATA_EDMA_WRITE_4(mvport, EDMA_CFG,
 3301                     EDMA_CFG_RESERVED | EDMA_CFG_ERDBSZ);
 3302         else    /* SoC */
 3303                 MVSATA_EDMA_WRITE_4(mvport, EDMA_CFG,
 3304                     EDMA_CFG_RESERVED | EDMA_CFG_RESERVED2);
 3305         MVSATA_EDMA_WRITE_4(mvport, EDMA_T, 0);
 3306         MVSATA_EDMA_WRITE_4(mvport, SATA_SEIM, 0x019c0000);
 3307         MVSATA_EDMA_WRITE_4(mvport, SATA_SE, ~0);
 3308         MVSATA_EDMA_WRITE_4(mvport, SATA_FISIC, 0);
 3309         MVSATA_EDMA_WRITE_4(mvport, EDMA_IEC, 0);
 3310         MVSATA_EDMA_WRITE_4(mvport, EDMA_IEM, 0);
 3311         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQBAH, 0);
 3312         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQIP, 0);
 3313         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQOP, 0);
 3314         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQBAH, 0);
 3315         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQIP, 0);
 3316         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQOP, 0);
 3317         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, 0);
 3318         MVSATA_EDMA_WRITE_4(mvport, EDMA_TC, 0);
 3319         MVSATA_EDMA_WRITE_4(mvport, EDMA_IORT, 0xbc);
 3320 }
 3321 
 3322 static void
 3323 mvsata_reset_hc(struct mvsata_hc *mvhc)
 3324 {
 3325 #if 0
 3326         uint32_t val;
 3327 #endif
 3328 
 3329         MVSATA_HC_WRITE_4(mvhc, SATAHC_ICT, 0);
 3330         MVSATA_HC_WRITE_4(mvhc, SATAHC_ITT, 0);
 3331         MVSATA_HC_WRITE_4(mvhc, SATAHC_IC, 0);
 3332 
 3333 #if 0   /* XXXX needs? */
 3334         MVSATA_HC_WRITE_4(mvhc, 0x01c, 0);
 3335 
 3336         /*
 3337          * Keep the SS during power on and the reference clock bits (reset
 3338          * sample)
 3339          */
 3340         val = MVSATA_HC_READ_4(mvhc, 0x020);
 3341         val &= 0x1c1c1c1c;
 3342         val |= 0x03030303;
 3343         MVSATA_HC_READ_4(mvhc, 0x020, 0);
 3344 #endif
 3345 }
 3346 
 3347 static uint32_t
 3348 mvsata_softreset(struct mvsata_port *mvport, int flags)
 3349 {
 3350         struct ata_channel *chp = &mvport->port_ata_channel;
 3351         uint32_t sig0 = ~0;
 3352         int timeout;
 3353         uint8_t st0;
 3354 
 3355         ata_channel_lock_owned(chp);
 3356 
 3357         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_RST | WDCTL_IDS | WDCTL_4BIT);
 3358         delay(10);
 3359         (void) MVSATA_WDC_READ_1(mvport, SRB_FE);
 3360         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_IDS | WDCTL_4BIT);
 3361         delay(10);
 3362 
 3363         /* wait for BSY to deassert */
 3364         for (timeout = 0; timeout < WDC_RESET_WAIT / 10; timeout++) {
 3365                 st0 = MVSATA_WDC_READ_1(mvport, SRB_CS);
 3366 
 3367                 if ((st0 & WDCS_BSY) == 0) {
 3368                         sig0 = MVSATA_WDC_READ_1(mvport, SRB_SC) << 0;
 3369                         sig0 |= MVSATA_WDC_READ_1(mvport, SRB_LBAL) << 8;
 3370                         sig0 |= MVSATA_WDC_READ_1(mvport, SRB_LBAM) << 16;
 3371                         sig0 |= MVSATA_WDC_READ_1(mvport, SRB_LBAH) << 24;
 3372                         goto out;
 3373                 }
 3374                 ata_delay(chp, 10, "atarst", flags);
 3375         }
 3376 
 3377         aprint_error("%s:%d:%d: %s: timeout\n",
 3378             device_xname(MVSATA_DEV2(mvport)),
 3379             mvport->port_hc->hc, mvport->port, __func__);
 3380 
 3381 out:
 3382         MVSATA_WDC_WRITE_1(mvport, SRB_CAS, WDCTL_4BIT);
 3383         return sig0;
 3384 }
 3385 
 3386 #ifndef MVSATA_WITHOUTDMA
 3387 static void
 3388 mvsata_edma_reset_qptr(struct mvsata_port *mvport)
 3389 {
 3390         const bus_addr_t crpb_addr =
 3391             mvport->port_crpb_dmamap->dm_segs[0].ds_addr;
 3392         const uint32_t crpb_addr_mask =
 3393             EDMA_RESQP_ERPQBAP_MASK | EDMA_RESQP_ERPQBA_MASK;
 3394 
 3395         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQBAH, 0);
 3396         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQIP, 0);
 3397         MVSATA_EDMA_WRITE_4(mvport, EDMA_REQQOP, 0);
 3398         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQBAH, (crpb_addr >> 16) >> 16);
 3399         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQIP, 0);
 3400         MVSATA_EDMA_WRITE_4(mvport, EDMA_RESQOP, (crpb_addr & crpb_addr_mask));
 3401 }
 3402 
 3403 static inline void
 3404 mvsata_edma_enable(struct mvsata_port *mvport)
 3405 {
 3406 
 3407         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, EDMA_CMD_EENEDMA);
 3408 }
 3409 
 3410 static void
 3411 mvsata_edma_disable(struct mvsata_port *mvport, int timeout, int wflags)
 3412 {
 3413         struct ata_channel *chp = &mvport->port_ata_channel;
 3414         uint32_t command;
 3415         int t;
 3416 
 3417         ata_channel_lock_owned(chp);
 3418 
 3419         /* The disable bit (eDsEDMA) is self negated. */
 3420         MVSATA_EDMA_WRITE_4(mvport, EDMA_CMD, EDMA_CMD_EDSEDMA);
 3421 
 3422         timeout = mstohz(timeout + hztoms(1) - 1);
 3423 
 3424         for (t = 0; ; ++t) {
 3425                 command = MVSATA_EDMA_READ_4(mvport, EDMA_CMD);
 3426                 if (!(command & EDMA_CMD_EENEDMA))
 3427                         return;
 3428                 if (t >= timeout)
 3429                         break;
 3430                 ata_delay(chp, hztoms(1), "mvsata_edma2", wflags);
 3431         }
 3432 
 3433         aprint_error("%s:%d:%d: unable to disable EDMA\n",
 3434             device_xname(MVSATA_DEV2(mvport)),
 3435             mvport->port_hc->hc, mvport->port);
 3436 }
 3437 
 3438 /*
 3439  * Set EDMA registers according to mode.
 3440  *       ex. NCQ/TCQ(queued)/non queued.
 3441  */
 3442 static void
 3443 mvsata_edma_config(struct mvsata_port *mvport, enum mvsata_edmamode mode)
 3444 {
 3445         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 3446         uint32_t reg;
 3447 
 3448         reg = MVSATA_EDMA_READ_4(mvport, EDMA_CFG);
 3449         reg |= EDMA_CFG_RESERVED;
 3450 
 3451         if (mode == ncq) {
 3452                 if (sc->sc_gen == gen1) {
 3453                         aprint_error_dev(MVSATA_DEV2(mvport),
 3454                             "GenI not support NCQ\n");
 3455                         return;
 3456                 } else if (sc->sc_gen == gen2)
 3457                         reg |= EDMA_CFG_EDEVERR;
 3458                 reg |= EDMA_CFG_ESATANATVCMDQUE;
 3459         } else if (mode == queued) {
 3460                 reg &= ~EDMA_CFG_ESATANATVCMDQUE;
 3461                 reg |= EDMA_CFG_EQUE;
 3462         } else
 3463                 reg &= ~(EDMA_CFG_ESATANATVCMDQUE | EDMA_CFG_EQUE);
 3464 
 3465         if (sc->sc_gen == gen1)
 3466                 reg |= EDMA_CFG_ERDBSZ;
 3467         else if (sc->sc_gen == gen2)
 3468                 reg |= (EDMA_CFG_ERDBSZEXT | EDMA_CFG_EWRBUFFERLEN);
 3469         else if (sc->sc_gen == gen2e) {
 3470                 device_t parent = device_parent(MVSATA_DEV(sc));
 3471 
 3472                 reg |= (EDMA_CFG_EMASKRXPM | EDMA_CFG_EHOSTQUEUECACHEEN);
 3473                 reg &= ~(EDMA_CFG_EEDMAFBS | EDMA_CFG_EEDMAQUELEN);
 3474 
 3475                 if (device_is_a(parent, "pci"))
 3476                         reg |= (
 3477 #if NATAPIBUS > 0
 3478                             EDMA_CFG_EEARLYCOMPLETIONEN |
 3479 #endif
 3480                             EDMA_CFG_ECUTTHROUGHEN |
 3481                             EDMA_CFG_EWRBUFFERLEN |
 3482                             EDMA_CFG_ERDBSZEXT);
 3483         }
 3484         MVSATA_EDMA_WRITE_4(mvport, EDMA_CFG, reg);
 3485 
 3486         reg = (
 3487             EDMA_IE_EIORDYERR |
 3488             EDMA_IE_ETRANSINT |
 3489             EDMA_IE_EDEVCON |
 3490             EDMA_IE_EDEVDIS);
 3491         if (sc->sc_gen != gen1)
 3492                 reg |= (
 3493                     EDMA_IE_TRANSPROTERR |
 3494                     EDMA_IE_LINKDATATXERR(EDMA_IE_LINKTXERR_FISTXABORTED) |
 3495                     EDMA_IE_LINKDATATXERR(EDMA_IE_LINKXERR_OTHERERRORS) |
 3496                     EDMA_IE_LINKDATATXERR(EDMA_IE_LINKXERR_LINKLAYERRESET) |
 3497                     EDMA_IE_LINKDATATXERR(EDMA_IE_LINKXERR_INTERNALFIFO) |
 3498                     EDMA_IE_LINKDATATXERR(EDMA_IE_LINKXERR_SATACRC) |
 3499                     EDMA_IE_LINKCTLTXERR(EDMA_IE_LINKXERR_OTHERERRORS) |
 3500                     EDMA_IE_LINKCTLTXERR(EDMA_IE_LINKXERR_LINKLAYERRESET) |
 3501                     EDMA_IE_LINKCTLTXERR(EDMA_IE_LINKXERR_INTERNALFIFO) |
 3502                     EDMA_IE_LINKDATARXERR(EDMA_IE_LINKXERR_OTHERERRORS) |
 3503                     EDMA_IE_LINKDATARXERR(EDMA_IE_LINKXERR_LINKLAYERRESET) |
 3504                     EDMA_IE_LINKDATARXERR(EDMA_IE_LINKXERR_INTERNALFIFO) |
 3505                     EDMA_IE_LINKDATARXERR(EDMA_IE_LINKXERR_SATACRC) |
 3506                     EDMA_IE_LINKCTLRXERR(EDMA_IE_LINKXERR_OTHERERRORS) |
 3507                     EDMA_IE_LINKCTLRXERR(EDMA_IE_LINKXERR_LINKLAYERRESET) |
 3508                     EDMA_IE_LINKCTLRXERR(EDMA_IE_LINKXERR_INTERNALFIFO) |
 3509                     EDMA_IE_LINKCTLRXERR(EDMA_IE_LINKXERR_SATACRC) |
 3510                     EDMA_IE_ESELFDIS);
 3511 
 3512         if (mode == ncq)
 3513             reg |= EDMA_IE_EDEVERR;
 3514         MVSATA_EDMA_WRITE_4(mvport, EDMA_IEM, reg);
 3515         reg = MVSATA_EDMA_READ_4(mvport, EDMA_HC);
 3516         reg &= ~EDMA_IE_EDEVERR;
 3517         if (mode != ncq)
 3518             reg |= EDMA_IE_EDEVERR;
 3519         MVSATA_EDMA_WRITE_4(mvport, EDMA_HC, reg);
 3520         if (sc->sc_gen == gen2e) {
 3521                 /*
 3522                  * Clear FISWait4HostRdyEn[0] and [2].
 3523                  *   [0]: Device to Host FIS with <ERR> or <DF> bit set to 1.
 3524                  *   [2]: SDB FIS is received with <ERR> bit set to 1.
 3525                  */
 3526                 reg = MVSATA_EDMA_READ_4(mvport, SATA_FISC);
 3527                 reg &= ~(SATA_FISC_FISWAIT4HOSTRDYEN_B0 |
 3528                     SATA_FISC_FISWAIT4HOSTRDYEN_B2);
 3529                 MVSATA_EDMA_WRITE_4(mvport, SATA_FISC, reg);
 3530         }
 3531 
 3532         mvport->port_edmamode_curr = mode;
 3533 }
 3534 
 3535 
 3536 /*
 3537  * Generation dependent functions
 3538  */
 3539 
 3540 static void
 3541 mvsata_edma_setup_crqb(struct mvsata_port *mvport, int erqqip,
 3542                        struct ata_xfer  *xfer)
 3543 {
 3544         struct crqb *crqb;
 3545         bus_addr_t eprd_addr;
 3546         daddr_t blkno;
 3547         uint32_t rw;
 3548         uint8_t cmd, head;
 3549         int i;
 3550         struct ata_bio *ata_bio = &xfer->c_bio;
 3551 
 3552         eprd_addr = mvport->port_eprd_dmamap->dm_segs[0].ds_addr +
 3553             mvport->port_reqtbl[xfer->c_slot].eprd_offset;
 3554         rw = (ata_bio->flags & ATA_READ) ? CRQB_CDIR_READ : CRQB_CDIR_WRITE;
 3555         cmd = (ata_bio->flags & ATA_READ) ? WDCC_READDMA : WDCC_WRITEDMA;
 3556         if (ata_bio->flags & (ATA_LBA|ATA_LBA48)) {
 3557                 head = WDSD_LBA;
 3558         } else {
 3559                 head = 0;
 3560         }
 3561         blkno = ata_bio->blkno;
 3562         if (ata_bio->flags & ATA_LBA48)
 3563                 cmd = atacmd_to48(cmd);
 3564         else {
 3565                 head |= ((ata_bio->blkno >> 24) & 0xf);
 3566                 blkno &= 0xffffff;
 3567         }
 3568         crqb = &mvport->port_crqb->crqb + erqqip;
 3569         crqb->cprdbl = htole32(eprd_addr & CRQB_CRQBL_EPRD_MASK);
 3570         crqb->cprdbh = htole32((eprd_addr >> 16) >> 16);
 3571         crqb->ctrlflg =
 3572             htole16(rw | CRQB_CHOSTQUETAG(xfer->c_slot) |
 3573                 CRQB_CPMPORT(xfer->c_drive));
 3574         i = 0;
 3575         if (mvport->port_edmamode_curr == dma) {
 3576                 if (ata_bio->flags & ATA_LBA48)
 3577                         crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3578                             CRQB_ATACOMMAND_SECTORCOUNT, ata_bio->nblks >> 8));
 3579                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3580                     CRQB_ATACOMMAND_SECTORCOUNT, ata_bio->nblks));
 3581         } else { /* ncq/queued */
 3582 
 3583                 /*
 3584                  * XXXX: Oops, ata command is not correct.  And, atabus layer
 3585                  * has not been supported yet now.
 3586                  *   Queued DMA read/write.
 3587                  *   read/write FPDMAQueued.
 3588                  */
 3589 
 3590                 if (ata_bio->flags & ATA_LBA48)
 3591                         crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3592                             CRQB_ATACOMMAND_FEATURES, ata_bio->nblks >> 8));
 3593                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3594                     CRQB_ATACOMMAND_FEATURES, ata_bio->nblks));
 3595                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3596                     CRQB_ATACOMMAND_SECTORCOUNT, xfer->c_slot << 3));
 3597         }
 3598         if (ata_bio->flags & ATA_LBA48) {
 3599                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3600                     CRQB_ATACOMMAND_LBALOW, blkno >> 24));
 3601                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3602                     CRQB_ATACOMMAND_LBAMID, blkno >> 32));
 3603                 crqb->atacommand[i++] = htole16(CRQB_ATACOMMAND(
 3604                     CRQB_ATACOMMAND_LBAHIGH, blkno >> 40));
 3605         }
 3606         crqb->atacommand[i++] =
 3607             htole16(CRQB_ATACOMMAND(CRQB_ATACOMMAND_LBALOW, blkno));
 3608         crqb->atacommand[i++] =
 3609             htole16(CRQB_ATACOMMAND(CRQB_ATACOMMAND_LBAMID, blkno >> 8));
 3610         crqb->atacommand[i++] =
 3611             htole16(CRQB_ATACOMMAND(CRQB_ATACOMMAND_LBAHIGH, blkno >> 16));
 3612         crqb->atacommand[i++] =
 3613             htole16(CRQB_ATACOMMAND(CRQB_ATACOMMAND_DEVICE, head));
 3614         crqb->atacommand[i++] = htole16(
 3615             CRQB_ATACOMMAND(CRQB_ATACOMMAND_COMMAND, cmd) |
 3616             CRQB_ATACOMMAND_LAST);
 3617 }
 3618 #endif
 3619 
 3620 static uint32_t
 3621 mvsata_read_preamps_gen1(struct mvsata_port *mvport)
 3622 {
 3623         struct mvsata_hc *hc = mvport->port_hc;
 3624         uint32_t reg;
 3625 
 3626         reg = MVSATA_HC_READ_4(hc, SATAHC_I_PHYMODE(mvport->port));
 3627         /*
 3628          * [12:11] : pre
 3629          * [7:5]   : amps
 3630          */
 3631         return reg & 0x000018e0;
 3632 }
 3633 
 3634 static void
 3635 mvsata_fix_phy_gen1(struct mvsata_port *mvport)
 3636 {
 3637         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 3638         struct mvsata_hc *mvhc = mvport->port_hc;
 3639         uint32_t reg;
 3640         int port = mvport->port, fix_apm_sq = 0;
 3641 
 3642         if (sc->sc_model == PCI_PRODUCT_MARVELL_88SX5080) {
 3643                 if (sc->sc_rev == 0x01)
 3644                         fix_apm_sq = 1;
 3645         } else {
 3646                 if (sc->sc_rev == 0x00)
 3647                         fix_apm_sq = 1;
 3648         }
 3649 
 3650         if (fix_apm_sq) {
 3651                 /*
 3652                  * Disable auto-power management
 3653                  *   88SX50xx FEr SATA#12
 3654                  */
 3655                 reg = MVSATA_HC_READ_4(mvhc, SATAHC_I_LTMODE(port));
 3656                 reg |= (1 << 19);
 3657                 MVSATA_HC_WRITE_4(mvhc, SATAHC_I_LTMODE(port), reg);
 3658 
 3659                 /*
 3660                  * Fix squelch threshold
 3661                  *   88SX50xx FEr SATA#9
 3662                  */
 3663                 reg = MVSATA_HC_READ_4(mvhc, SATAHC_I_PHYCONTROL(port));
 3664                 reg &= ~0x3;
 3665                 reg |= 0x1;
 3666                 MVSATA_HC_WRITE_4(mvhc, SATAHC_I_PHYCONTROL(port), reg);
 3667         }
 3668 
 3669         /* Revert values of pre-emphasis and signal amps to the saved ones */
 3670         reg = MVSATA_HC_READ_4(mvhc, SATAHC_I_PHYMODE(port));
 3671         reg &= ~0x000018e0;     /* pre and amps mask */
 3672         reg |= mvport->_fix_phy_param.pre_amps;
 3673         MVSATA_HC_WRITE_4(mvhc, SATAHC_I_PHYMODE(port), reg);
 3674 }
 3675 
 3676 static void
 3677 mvsata_devconn_gen1(struct mvsata_port *mvport)
 3678 {
 3679         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 3680 
 3681         /* Fix for 88SX50xx FEr SATA#2 */
 3682         mvport->_fix_phy_param._fix_phy(mvport);
 3683 
 3684         /* If disk is connected, then enable the activity LED */
 3685         if (sc->sc_rev == 0x03) {
 3686                 /* XXXXX */
 3687         }
 3688 }
 3689 
 3690 static uint32_t
 3691 mvsata_read_preamps_gen2(struct mvsata_port *mvport)
 3692 {
 3693         uint32_t reg;
 3694 
 3695         reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM2);
 3696         /*
 3697          * [10:8] : amps
 3698          * [7:5]  : pre
 3699          */
 3700         return reg & 0x000007e0;
 3701 }
 3702 
 3703 static void
 3704 mvsata_fix_phy_gen2(struct mvsata_port *mvport)
 3705 {
 3706         struct mvsata_softc *sc = device_private(MVSATA_DEV2(mvport));
 3707         uint32_t reg;
 3708 
 3709         if ((sc->sc_gen == gen2 && sc->sc_rev == 0x07) ||
 3710             sc->sc_gen == gen2e) {
 3711                 /*
 3712                  * Fix for
 3713                  *   88SX60X1 FEr SATA #23
 3714                  *   88SX6042/88SX7042 FEr SATA #23
 3715                  *   88F5182 FEr #SATA-S13
 3716                  *   88F5082 FEr #SATA-S13
 3717                  */
 3718                 reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM2);
 3719                 reg &= ~(1 << 16);
 3720                 reg |= (1 << 31);
 3721                 MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM2, reg);
 3722 
 3723                 delay(200);
 3724 
 3725                 reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM2);
 3726                 reg &= ~((1 << 16) | (1 << 31));
 3727                 MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM2, reg);
 3728 
 3729                 delay(200);
 3730         }
 3731 
 3732         /* Fix values in PHY Mode 3 Register.*/
 3733         reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM3);
 3734         reg &= ~0x7F900000;
 3735         reg |= 0x2A800000;
 3736         /* Implement Guidline 88F5182, 88F5082, 88F6082 (GL# SATA-S11) */
 3737         if (sc->sc_model == PCI_PRODUCT_MARVELL_88F5082 ||
 3738             sc->sc_model == PCI_PRODUCT_MARVELL_88F5182 ||
 3739             sc->sc_model == PCI_PRODUCT_MARVELL_88F6082)
 3740                 reg &= ~0x0000001c;
 3741         MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM3, reg);
 3742 
 3743         /*
 3744          * Fix values in PHY Mode 4 Register.
 3745          *   88SX60x1 FEr SATA#10
 3746          *   88F5182 GL #SATA-S10
 3747          *   88F5082 GL #SATA-S10
 3748          */
 3749         if ((sc->sc_gen == gen2 && sc->sc_rev == 0x07) ||
 3750             sc->sc_gen == gen2e) {
 3751                 uint32_t tmp = 0;
 3752 
 3753                 /* 88SX60x1 FEr SATA #13 */
 3754                 if (sc->sc_gen == 2 && sc->sc_rev == 0x07)
 3755                         tmp = MVSATA_EDMA_READ_4(mvport, SATA_PHYM3);
 3756 
 3757                 reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM4);
 3758                 reg |= (1 << 0);
 3759                 reg &= ~(1 << 1);
 3760                 /* PHY Mode 4 Register of Gen IIE has some restriction */
 3761                 if (sc->sc_gen == gen2e) {
 3762                         reg &= ~0x5de3fffc;
 3763                         reg |= (1 << 2);
 3764                 }
 3765                 MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM4, reg);
 3766 
 3767                 /* 88SX60x1 FEr SATA #13 */
 3768                 if (sc->sc_gen == 2 && sc->sc_rev == 0x07)
 3769                         MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM3, tmp);
 3770         }
 3771 
 3772         /* Revert values of pre-emphasis and signal amps to the saved ones */
 3773         reg = MVSATA_EDMA_READ_4(mvport, SATA_PHYM2);
 3774         reg &= ~0x000007e0;     /* pre and amps mask */
 3775         reg |= mvport->_fix_phy_param.pre_amps;
 3776         reg &= ~(1 << 16);
 3777         if (sc->sc_gen == gen2e) {
 3778                 /*
 3779                  * according to mvSata 3.6.1, some IIE values are fixed.
 3780                  * some reserved fields must be written with fixed values.
 3781                  */
 3782                 reg &= ~0xC30FF01F;
 3783                 reg |= 0x0000900F;
 3784         }
 3785         MVSATA_EDMA_WRITE_4(mvport, SATA_PHYM2, reg);
 3786 }
 3787 
 3788 #ifndef MVSATA_WITHOUTDMA
 3789 static void
 3790 mvsata_edma_setup_crqb_gen2e(struct mvsata_port *mvport, int erqqip,
 3791                              struct ata_xfer  *xfer)
 3792 {
 3793         struct crqb_gen2e *crqb;
 3794         bus_addr_t eprd_addr;
 3795         uint32_t ctrlflg, rw;
 3796         uint8_t fis[RHD_FISLEN];
 3797 
 3798         eprd_addr = mvport->port_eprd_dmamap->dm_segs[0].ds_addr +
 3799             mvport->port_reqtbl[xfer->c_slot].eprd_offset;
 3800         rw = (xfer->c_bio.flags & ATA_READ) ? CRQB_CDIR_READ : CRQB_CDIR_WRITE;
 3801         ctrlflg = (rw | CRQB_CDEVICEQUETAG(xfer->c_slot) |
 3802             CRQB_CPMPORT(xfer->c_drive) |
 3803             CRQB_CPRDMODE_EPRD | CRQB_CHOSTQUETAG_GEN2(xfer->c_slot));
 3804 
 3805         crqb = &mvport->port_crqb->crqb_gen2e + erqqip;
 3806         crqb->cprdbl = htole32(eprd_addr & CRQB_CRQBL_EPRD_MASK);
 3807         crqb->cprdbh = htole32((eprd_addr >> 16) >> 16);
 3808         crqb->ctrlflg = htole32(ctrlflg);
 3809 
 3810         satafis_rhd_construct_bio(xfer, fis);
 3811 
 3812         crqb->atacommand[0] = 0;
 3813         crqb->atacommand[1] = 0;
 3814         /* copy over the ATA command part of the fis */
 3815         memcpy(&crqb->atacommand[2], &fis[rhd_command],
 3816             MIN(sizeof(crqb->atacommand) - 2, RHD_FISLEN - rhd_command));
 3817 }
 3818 
 3819 #ifdef MVSATA_DEBUG
 3820 #define MVSATA_DEBUG_PRINT(type, size, n, p)            \
 3821         do {                                            \
 3822                 int _i;                                 \
 3823                 u_char *_p = (p);                       \
 3824                                                         \
 3825                 printf(#type "(%d)", (n));              \
 3826                 for (_i = 0; _i < (size); _i++, _p++) { \
 3827                         if (_i % 16 == 0)               \
 3828                                 printf("\n   ");        \
 3829                         printf(" %02x", *_p);           \
 3830                 }                                       \
 3831                 printf("\n");                           \
 3832         } while (0 /* CONSTCOND */)
 3833 
 3834 static void
 3835 mvsata_print_crqb(struct mvsata_port *mvport, int n)
 3836 {
 3837 
 3838         MVSATA_DEBUG_PRINT(crqb, sizeof(union mvsata_crqb),
 3839             n, (u_char *)(mvport->port_crqb + n));
 3840 }
 3841 
 3842 static void
 3843 mvsata_print_crpb(struct mvsata_port *mvport, int n)
 3844 {
 3845 
 3846         MVSATA_DEBUG_PRINT(crpb, sizeof(struct crpb),
 3847             n, (u_char *)(mvport->port_crpb + n));
 3848 }
 3849 
 3850 static void
 3851 mvsata_print_eprd(struct mvsata_port *mvport, int n)
 3852 {
 3853         struct eprd *eprd;
 3854         int i = 0;
 3855 
 3856         eprd = mvport->port_reqtbl[n].eprd;
 3857         while (1 /*CONSTCOND*/) {
 3858                 MVSATA_DEBUG_PRINT(eprd, sizeof(struct eprd),
 3859                     i, (u_char *)eprd);
 3860                 if (eprd->eot & EPRD_EOT)
 3861                         break;
 3862                 eprd++;
 3863                 i++;
 3864         }
 3865 }
 3866 #endif
 3867 #endif
Cache object: 449d279df11b6509b987c1b817144a86
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/ic/mvsata.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/ic/mvsata.c
