nvme.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (C) 2012-2014 Intel Corporation
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  */
   26 
   27 #include <sys/cdefs.h>
   28 __FBSDID("$FreeBSD$");
   29 
   30 #include <sys/param.h>
   31 #include <sys/bus.h>
   32 #include <sys/conf.h>
   33 #include <sys/module.h>
   34 
   35 #include <vm/uma.h>
   36 
   37 #include <dev/pci/pcireg.h>
   38 #include <dev/pci/pcivar.h>
   39 
   40 #include "nvme_private.h"
   41 
   42 struct nvme_consumer {
   43         uint32_t                id;
   44         nvme_cons_ns_fn_t       ns_fn;
   45         nvme_cons_ctrlr_fn_t    ctrlr_fn;
   46         nvme_cons_async_fn_t    async_fn;
   47         nvme_cons_fail_fn_t     fail_fn;
   48 };
   49 
   50 struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
   51 #define INVALID_CONSUMER_ID     0xFFFF
   52 
   53 uma_zone_t      nvme_request_zone;
   54 int32_t         nvme_retry_count;
   55 
   56 MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
   57 
   58 static int    nvme_probe(device_t);
   59 static int    nvme_attach(device_t);
   60 static int    nvme_detach(device_t);
   61 static int    nvme_shutdown(device_t);
   62 
   63 static devclass_t nvme_devclass;
   64 
   65 static device_method_t nvme_pci_methods[] = {
   66         /* Device interface */
   67         DEVMETHOD(device_probe,     nvme_probe),
   68         DEVMETHOD(device_attach,    nvme_attach),
   69         DEVMETHOD(device_detach,    nvme_detach),
   70         DEVMETHOD(device_shutdown,  nvme_shutdown),
   71         { 0, 0 }
   72 };
   73 
   74 static driver_t nvme_pci_driver = {
   75         "nvme",
   76         nvme_pci_methods,
   77         sizeof(struct nvme_controller),
   78 };
   79 
   80 DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, NULL, NULL);
   81 MODULE_VERSION(nvme, 1);
   82 MODULE_DEPEND(nvme, cam, 1, 1, 1);
   83 
   84 static struct _pcsid
   85 {
   86         uint32_t        devid;
   87         int             match_subdevice;
   88         uint16_t        subdevice;
   89         const char      *desc;
   90         uint32_t        quirks;
   91 } pci_ids[] = {
   92         { 0x01118086,           0, 0, "NVMe Controller"  },
   93         { IDT32_PCI_ID,         0, 0, "IDT NVMe Controller (32 channel)"  },
   94         { IDT8_PCI_ID,          0, 0, "IDT NVMe Controller (8 channel)" },
   95         { 0x09538086,           1, 0x3702, "DC P3700 SSD" },
   96         { 0x09538086,           1, 0x3703, "DC P3700 SSD [2.5\" SFF]" },
   97         { 0x09538086,           1, 0x3704, "DC P3500 SSD [Add-in Card]" },
   98         { 0x09538086,           1, 0x3705, "DC P3500 SSD [2.5\" SFF]" },
   99         { 0x09538086,           1, 0x3709, "DC P3600 SSD [Add-in Card]" },
  100         { 0x09538086,           1, 0x370a, "DC P3600 SSD [2.5\" SFF]" },
  101         { 0x00031c58,           0, 0, "HGST SN100",     QUIRK_DELAY_B4_CHK_RDY },
  102         { 0x00231c58,           0, 0, "WDC SN200",      QUIRK_DELAY_B4_CHK_RDY },
  103         { 0x05401c5f,           0, 0, "Memblaze Pblaze4", QUIRK_DELAY_B4_CHK_RDY },
  104         { 0xa821144d,           0, 0, "Samsung PM1725", QUIRK_DELAY_B4_CHK_RDY },
  105         { 0xa822144d,           0, 0, "Samsung PM1725a", QUIRK_DELAY_B4_CHK_RDY },
  106         { 0x01161179,           0, 0, "Toshiba XG5", QUIRK_DISABLE_TIMEOUT },
  107         { 0x00000000,           0, 0, NULL  }
  108 };
  109 
  110 static int
  111 nvme_match(uint32_t devid, uint16_t subdevice, struct _pcsid *ep)
  112 {
  113         if (devid != ep->devid)
  114                 return 0;
  115 
  116         if (!ep->match_subdevice)
  117                 return 1;
  118 
  119         if (subdevice == ep->subdevice)
  120                 return 1;
  121         else
  122                 return 0;
  123 }
  124 
  125 static int
  126 nvme_probe (device_t device)
  127 {
  128         struct _pcsid   *ep;
  129         uint32_t        devid;
  130         uint16_t        subdevice;
  131 
  132         devid = pci_get_devid(device);
  133         subdevice = pci_get_subdevice(device);
  134         ep = pci_ids;
  135 
  136         while (ep->devid) {
  137                 if (nvme_match(devid, subdevice, ep))
  138                         break;
  139                 ++ep;
  140         }
  141 
  142         if (ep->desc) {
  143                 device_set_desc(device, ep->desc);
  144                 return (BUS_PROBE_DEFAULT);
  145         }
  146 
  147 #if defined(PCIS_STORAGE_NVM)
  148         if (pci_get_class(device)    == PCIC_STORAGE &&
  149             pci_get_subclass(device) == PCIS_STORAGE_NVM &&
  150             pci_get_progif(device)   == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) {
  151                 device_set_desc(device, "Generic NVMe Device");
  152                 return (BUS_PROBE_GENERIC);
  153         }
  154 #endif
  155 
  156         return (ENXIO);
  157 }
  158 
  159 static void
  160 nvme_init(void)
  161 {
  162         uint32_t        i;
  163 
  164         nvme_request_zone = uma_zcreate("nvme_request",
  165             sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0);
  166 
  167         for (i = 0; i < NVME_MAX_CONSUMERS; i++)
  168                 nvme_consumer[i].id = INVALID_CONSUMER_ID;
  169 }
  170 
  171 SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL);
  172 
  173 static void
  174 nvme_uninit(void)
  175 {
  176         uma_zdestroy(nvme_request_zone);
  177 }
  178 
  179 SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL);
  180 
  181 static int
  182 nvme_shutdown(device_t dev)
  183 {
  184         struct nvme_controller  *ctrlr;
  185 
  186         ctrlr = DEVICE2SOFTC(dev);
  187         nvme_ctrlr_shutdown(ctrlr);
  188 
  189         return (0);
  190 }
  191 
  192 void
  193 nvme_dump_command(struct nvme_command *cmd)
  194 {
  195         printf(
  196 "opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
  197             cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
  198             cmd->rsvd2, cmd->rsvd3,
  199             (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2,
  200             cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
  201             cmd->cdw15);
  202 }
  203 
  204 void
  205 nvme_dump_completion(struct nvme_completion *cpl)
  206 {
  207         printf("cdw0:%08x sqhd:%04x sqid:%04x "
  208             "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
  209             cpl->cdw0, cpl->sqhd, cpl->sqid,
  210             cpl->cid, cpl->status.p, cpl->status.sc, cpl->status.sct,
  211             cpl->status.m, cpl->status.dnr);
  212 }
  213 
  214 static int
  215 nvme_attach(device_t dev)
  216 {
  217         struct nvme_controller  *ctrlr = DEVICE2SOFTC(dev);
  218         int                     status;
  219         struct _pcsid           *ep;
  220         uint32_t                devid;
  221         uint16_t                subdevice;
  222 
  223         devid = pci_get_devid(dev);
  224         subdevice = pci_get_subdevice(dev);
  225         ep = pci_ids;
  226         while (ep->devid) {
  227                 if (nvme_match(devid, subdevice, ep))
  228                         break;
  229                 ++ep;
  230         }
  231         ctrlr->quirks = ep->quirks;
  232 
  233         status = nvme_ctrlr_construct(ctrlr, dev);
  234 
  235         if (status != 0) {
  236                 nvme_ctrlr_destruct(ctrlr, dev);
  237                 return (status);
  238         }
  239 
  240         /*
  241          * Some drives do not implement the completion timeout feature
  242          * correctly. There's a WAR from the manufacturer to just disable it.
  243          * The driver wouldn't respond correctly to a timeout anyway.
  244          */
  245         if (ep->quirks & QUIRK_DISABLE_TIMEOUT) {
  246                 int ptr;
  247                 uint16_t devctl2;
  248 
  249                 status = pci_find_cap(dev, PCIY_EXPRESS, &ptr);
  250                 if (status) {
  251                         device_printf(dev, "Can't locate PCIe capability?");
  252                         return (status);
  253                 }
  254                 devctl2 = pci_read_config(dev, ptr + PCIER_DEVICE_CTL2, sizeof(devctl2));
  255                 devctl2 |= PCIEM_CTL2_COMP_TIMO_DISABLE;
  256                 pci_write_config(dev, ptr + PCIER_DEVICE_CTL2, devctl2, sizeof(devctl2));
  257         }
  258 
  259         /*
  260          * Enable busmastering so the completion status messages can
  261          * be busmastered back to the host.
  262          */
  263         pci_enable_busmaster(dev);
  264 
  265         /*
  266          * Reset controller twice to ensure we do a transition from cc.en==1
  267          *  to cc.en==0.  This is because we don't really know what status
  268          *  the controller was left in when boot handed off to OS.
  269          */
  270         status = nvme_ctrlr_hw_reset(ctrlr);
  271         if (status != 0) {
  272                 nvme_ctrlr_destruct(ctrlr, dev);
  273                 return (status);
  274         }
  275 
  276         status = nvme_ctrlr_hw_reset(ctrlr);
  277         if (status != 0) {
  278                 nvme_ctrlr_destruct(ctrlr, dev);
  279                 return (status);
  280         }
  281 
  282         ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook;
  283         ctrlr->config_hook.ich_arg = ctrlr;
  284 
  285         config_intrhook_establish(&ctrlr->config_hook);
  286 
  287         return (0);
  288 }
  289 
  290 static int
  291 nvme_detach (device_t dev)
  292 {
  293         struct nvme_controller  *ctrlr = DEVICE2SOFTC(dev);
  294 
  295         nvme_ctrlr_destruct(ctrlr, dev);
  296         pci_disable_busmaster(dev);
  297         return (0);
  298 }
  299 
  300 static void
  301 nvme_notify(struct nvme_consumer *cons,
  302             struct nvme_controller *ctrlr)
  303 {
  304         struct nvme_namespace   *ns;
  305         void                    *ctrlr_cookie;
  306         int                     cmpset, ns_idx;
  307 
  308         /*
  309          * The consumer may register itself after the nvme devices
  310          *  have registered with the kernel, but before the
  311          *  driver has completed initialization.  In that case,
  312          *  return here, and when initialization completes, the
  313          *  controller will make sure the consumer gets notified.
  314          */
  315         if (!ctrlr->is_initialized)
  316                 return;
  317 
  318         cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1);
  319         if (cmpset == 0)
  320                 return;
  321 
  322         if (cons->ctrlr_fn != NULL)
  323                 ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr);
  324         else
  325                 ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de;
  326         ctrlr->cons_cookie[cons->id] = ctrlr_cookie;
  327 
  328         /* ctrlr_fn has failed.  Nothing to notify here any more. */
  329         if (ctrlr_cookie == NULL)
  330                 return;
  331 
  332         if (ctrlr->is_failed) {
  333                 ctrlr->cons_cookie[cons->id] = NULL;
  334                 if (cons->fail_fn != NULL)
  335                         (*cons->fail_fn)(ctrlr_cookie);
  336                 /*
  337                  * Do not notify consumers about the namespaces of a
  338                  *  failed controller.
  339                  */
  340                 return;
  341         }
  342         for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) {
  343                 ns = &ctrlr->ns[ns_idx];
  344                 if (ns->data.nsze == 0)
  345                         continue;
  346                 if (cons->ns_fn != NULL)
  347                         ns->cons_cookie[cons->id] =
  348                             (*cons->ns_fn)(ns, ctrlr_cookie);
  349         }
  350 }
  351 
  352 void
  353 nvme_notify_new_controller(struct nvme_controller *ctrlr)
  354 {
  355         int i;
  356 
  357         for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
  358                 if (nvme_consumer[i].id != INVALID_CONSUMER_ID) {
  359                         nvme_notify(&nvme_consumer[i], ctrlr);
  360                 }
  361         }
  362 }
  363 
  364 static void
  365 nvme_notify_new_consumer(struct nvme_consumer *cons)
  366 {
  367         device_t                *devlist;
  368         struct nvme_controller  *ctrlr;
  369         int                     dev_idx, devcount;
  370 
  371         if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
  372                 return;
  373 
  374         for (dev_idx = 0; dev_idx < devcount; dev_idx++) {
  375                 ctrlr = DEVICE2SOFTC(devlist[dev_idx]);
  376                 nvme_notify(cons, ctrlr);
  377         }
  378 
  379         free(devlist, M_TEMP);
  380 }
  381 
  382 void
  383 nvme_notify_async_consumers(struct nvme_controller *ctrlr,
  384                             const struct nvme_completion *async_cpl,
  385                             uint32_t log_page_id, void *log_page_buffer,
  386                             uint32_t log_page_size)
  387 {
  388         struct nvme_consumer    *cons;
  389         void                    *ctrlr_cookie;
  390         uint32_t                i;
  391 
  392         for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
  393                 cons = &nvme_consumer[i];
  394                 if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL &&
  395                     (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
  396                         (*cons->async_fn)(ctrlr_cookie, async_cpl,
  397                             log_page_id, log_page_buffer, log_page_size);
  398                 }
  399         }
  400 }
  401 
  402 void
  403 nvme_notify_fail_consumers(struct nvme_controller *ctrlr)
  404 {
  405         struct nvme_consumer    *cons;
  406         void                    *ctrlr_cookie;
  407         uint32_t                i;
  408 
  409         /*
  410          * This controller failed during initialization (i.e. IDENTIFY
  411          *  command failed or timed out).  Do not notify any nvme
  412          *  consumers of the failure here, since the consumer does not
  413          *  even know about the controller yet.
  414          */
  415         if (!ctrlr->is_initialized)
  416                 return;
  417 
  418         for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
  419                 cons = &nvme_consumer[i];
  420                 if (cons->id != INVALID_CONSUMER_ID &&
  421                     (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
  422                         ctrlr->cons_cookie[i] = NULL;
  423                         if (cons->fail_fn != NULL)
  424                                 cons->fail_fn(ctrlr_cookie);
  425                 }
  426         }
  427 }
  428 
  429 struct nvme_consumer *
  430 nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn,
  431                        nvme_cons_async_fn_t async_fn,
  432                        nvme_cons_fail_fn_t fail_fn)
  433 {
  434         int i;
  435 
  436         /*
  437          * TODO: add locking around consumer registration.  Not an issue
  438          *  right now since we only have one nvme consumer - nvd(4).
  439          */
  440         for (i = 0; i < NVME_MAX_CONSUMERS; i++)
  441                 if (nvme_consumer[i].id == INVALID_CONSUMER_ID) {
  442                         nvme_consumer[i].id = i;
  443                         nvme_consumer[i].ns_fn = ns_fn;
  444                         nvme_consumer[i].ctrlr_fn = ctrlr_fn;
  445                         nvme_consumer[i].async_fn = async_fn;
  446                         nvme_consumer[i].fail_fn = fail_fn;
  447 
  448                         nvme_notify_new_consumer(&nvme_consumer[i]);
  449                         return (&nvme_consumer[i]);
  450                 }
  451 
  452         printf("nvme(4): consumer not registered - no slots available\n");
  453         return (NULL);
  454 }
  455 
  456 void
  457 nvme_unregister_consumer(struct nvme_consumer *consumer)
  458 {
  459 
  460         consumer->id = INVALID_CONSUMER_ID;
  461 }
  462 
  463 void
  464 nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
  465 {
  466         struct nvme_completion_poll_status      *status = arg;
  467 
  468         /*
  469          * Copy status into the argument passed by the caller, so that
  470          *  the caller can check the status to determine if the
  471          *  the request passed or failed.
  472          */
  473         memcpy(&status->cpl, cpl, sizeof(*cpl));
  474         atomic_store_rel_int(&status->done, 1);
  475 }
Cache object: 530e4943e0f18140ca85008727c4ea7c
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/nvme/nvme.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/nvme/nvme.c
