The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/common/io/scsi/adapters/mpt_sas/mptsas.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or http://www.opensolaris.org/os/licensing.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 
   22 /*
   23  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
   24  */
   25 
   26 /*
   27  * Copyright (c) 2000 to 2010, LSI Corporation.
   28  * All rights reserved.
   29  *
   30  * Redistribution and use in source and binary forms of all code within
   31  * this file that is exclusively owned by LSI, with or without
   32  * modification, is permitted provided that, in addition to the CDDL 1.0
   33  * License requirements, the following conditions are met:
   34  *
   35  *    Neither the name of the author nor the names of its contributors may be
   36  *    used to endorse or promote products derived from this software without
   37  *    specific prior written permission.
   38  *
   39  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   40  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   41  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   42  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
   43  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
   44  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   45  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
   46  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   47  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   48  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   49  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   50  * DAMAGE.
   51  */
   52 
   53 /*
   54  * mptsas - This is a driver based on LSI Logic's MPT2.0 interface.
   55  *
   56  */
   57 
   58 #if defined(lint) || defined(DEBUG)
   59 #define MPTSAS_DEBUG
   60 #endif
   61 
   62 /*
   63  * standard header files.
   64  */
   65 #include <sys/note.h>
   66 #include <sys/scsi/scsi.h>
   67 #include <sys/pci.h>
   68 #include <sys/file.h>
   69 #include <sys/cpuvar.h>
   70 #include <sys/policy.h>
   71 #include <sys/sysevent.h>
   72 #include <sys/sysevent/eventdefs.h>
   73 #include <sys/sysevent/dr.h>
   74 #include <sys/sata/sata_defs.h>
   75 #include <sys/scsi/generic/sas.h>
   76 #include <sys/scsi/impl/scsi_sas.h>
   77 
   78 #pragma pack(1)
   79 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_type.h>
   80 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2.h>
   81 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_cnfg.h>
   82 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_init.h>
   83 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_ioc.h>
   84 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_sas.h>
   85 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_tool.h>
   86 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_raid.h>
   87 #pragma pack()
   88 
   89 /*
   90  * private header files.
   91  *
   92  */
   93 #include <sys/scsi/impl/scsi_reset_notify.h>
   94 #include <sys/scsi/adapters/mpt_sas/mptsas_var.h>
   95 #include <sys/scsi/adapters/mpt_sas/mptsas_ioctl.h>
   96 #include <sys/scsi/adapters/mpt_sas/mptsas_smhba.h>
   97 
   98 #include <sys/raidioctl.h>
   99 
  100 #include <sys/fs/dv_node.h>     /* devfs_clean */
  101 
  102 /*
  103  * FMA header files
  104  */
  105 #include <sys/ddifm.h>
  106 #include <sys/fm/protocol.h>
  107 #include <sys/fm/util.h>
  108 #include <sys/fm/io/ddi.h>
  109 
  110 /*
  111  * For anyone who would modify the code in mptsas_driver, it must be awared
  112  * that from snv_145 where CR6910752(mpt_sas driver performance can be
  113  * improved) is integrated, the per_instance mutex m_mutex is not hold
  114  * in the key IO code path, including mptsas_scsi_start(), mptsas_intr()
  115  * and all of the recursive functions called in them, so don't
  116  * make it for granted that all operations are sync/exclude correctly. Before
  117  * doing any modification in key code path, and even other code path such as
  118  * DR, watchsubr, ioctl, passthrough etc, make sure the elements modified have
  119  * no releationship to elements shown in the fastpath
  120  * (function mptsas_handle_io_fastpath()) in ISR and its recursive functions.
  121  * otherwise, you have to use the new introduced mutex to protect them.
  122  * As to how to do correctly, refer to the comments in mptsas_intr().
  123  */
  124 
  125 /*
  126  * autoconfiguration data and routines.
  127  */
  128 static int mptsas_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
  129 static int mptsas_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
  130 static int mptsas_power(dev_info_t *dip, int component, int level);
  131 
  132 /*
  133  * cb_ops function
  134  */
  135 static int mptsas_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
  136         cred_t *credp, int *rval);
  137 #ifdef __sparc
  138 static int mptsas_reset(dev_info_t *devi, ddi_reset_cmd_t cmd);
  139 #else  /* __sparc */
  140 static int mptsas_quiesce(dev_info_t *devi);
  141 #endif  /* __sparc */
  142 
  143 /*
  144  * Resource initilaization for hardware
  145  */
  146 static void mptsas_setup_cmd_reg(mptsas_t *mpt);
  147 static void mptsas_disable_bus_master(mptsas_t *mpt);
  148 static void mptsas_hba_fini(mptsas_t *mpt);
  149 static void mptsas_cfg_fini(mptsas_t *mptsas_blkp);
  150 static int mptsas_hba_setup(mptsas_t *mpt);
  151 static void mptsas_hba_teardown(mptsas_t *mpt);
  152 static int mptsas_config_space_init(mptsas_t *mpt);
  153 static void mptsas_config_space_fini(mptsas_t *mpt);
  154 static void mptsas_iport_register(mptsas_t *mpt);
  155 static int mptsas_smp_setup(mptsas_t *mpt);
  156 static void mptsas_smp_teardown(mptsas_t *mpt);
  157 static int mptsas_cache_create(mptsas_t *mpt);
  158 static void mptsas_cache_destroy(mptsas_t *mpt);
  159 static int mptsas_alloc_request_frames(mptsas_t *mpt);
  160 static int mptsas_alloc_reply_frames(mptsas_t *mpt);
  161 static int mptsas_alloc_free_queue(mptsas_t *mpt);
  162 static int mptsas_alloc_post_queue(mptsas_t *mpt);
  163 static void mptsas_alloc_reply_args(mptsas_t *mpt);
  164 static int mptsas_alloc_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd);
  165 static void mptsas_free_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd);
  166 static int mptsas_init_chip(mptsas_t *mpt, int first_time);
  167 
  168 /*
  169  * SCSA function prototypes
  170  */
  171 static int mptsas_scsi_start(struct scsi_address *ap, struct scsi_pkt *pkt);
  172 static int mptsas_scsi_reset(struct scsi_address *ap, int level);
  173 static int mptsas_scsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
  174 static int mptsas_scsi_getcap(struct scsi_address *ap, char *cap, int tgtonly);
  175 static int mptsas_scsi_setcap(struct scsi_address *ap, char *cap, int value,
  176     int tgtonly);
  177 static void mptsas_scsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt);
  178 static struct scsi_pkt *mptsas_scsi_init_pkt(struct scsi_address *ap,
  179     struct scsi_pkt *pkt, struct buf *bp, int cmdlen, int statuslen,
  180         int tgtlen, int flags, int (*callback)(), caddr_t arg);
  181 static void mptsas_scsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt);
  182 static void mptsas_scsi_destroy_pkt(struct scsi_address *ap,
  183     struct scsi_pkt *pkt);
  184 static int mptsas_scsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
  185     scsi_hba_tran_t *hba_tran, struct scsi_device *sd);
  186 static void mptsas_scsi_tgt_free(dev_info_t *hba_dip, dev_info_t *tgt_dip,
  187     scsi_hba_tran_t *hba_tran, struct scsi_device *sd);
  188 static int mptsas_scsi_reset_notify(struct scsi_address *ap, int flag,
  189     void (*callback)(caddr_t), caddr_t arg);
  190 static int mptsas_get_name(struct scsi_device *sd, char *name, int len);
  191 static int mptsas_get_bus_addr(struct scsi_device *sd, char *name, int len);
  192 static int mptsas_scsi_quiesce(dev_info_t *dip);
  193 static int mptsas_scsi_unquiesce(dev_info_t *dip);
  194 static int mptsas_bus_config(dev_info_t *pdip, uint_t flags,
  195     ddi_bus_config_op_t op, void *arg, dev_info_t **childp);
  196 
  197 /*
  198  * SMP functions
  199  */
  200 static int mptsas_smp_start(struct smp_pkt *smp_pkt);
  201 
  202 /*
  203  * internal function prototypes.
  204  */
  205 static void mptsas_list_add(mptsas_t *mpt);
  206 static void mptsas_list_del(mptsas_t *mpt);
  207 
  208 static int mptsas_quiesce_bus(mptsas_t *mpt);
  209 static int mptsas_unquiesce_bus(mptsas_t *mpt);
  210 
  211 static int mptsas_alloc_handshake_msg(mptsas_t *mpt, size_t alloc_size);
  212 static void mptsas_free_handshake_msg(mptsas_t *mpt);
  213 
  214 static void mptsas_ncmds_checkdrain(void *arg);
  215 
  216 static int mptsas_prepare_pkt(mptsas_cmd_t *cmd);
  217 static int mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *sp);
  218 
  219 static int mptsas_do_detach(dev_info_t *dev);
  220 static int mptsas_do_scsi_reset(mptsas_t *mpt, uint16_t devhdl);
  221 static int mptsas_do_scsi_abort(mptsas_t *mpt, int target, int lun,
  222     struct scsi_pkt *pkt);
  223 static int mptsas_scsi_capchk(char *cap, int tgtonly, int *cidxp);
  224 
  225 static void mptsas_handle_qfull(mptsas_t *mpt, mptsas_cmd_t *cmd);
  226 static void mptsas_handle_event(void *args);
  227 static int mptsas_handle_event_sync(void *args);
  228 static void mptsas_handle_dr(void *args);
  229 static void mptsas_handle_topo_change(mptsas_topo_change_list_t *topo_node,
  230     dev_info_t *pdip);
  231 
  232 static void mptsas_restart_cmd(void *);
  233 
  234 static void mptsas_flush_hba(mptsas_t *mpt);
  235 static void mptsas_flush_target(mptsas_t *mpt, ushort_t target, int lun,
  236         uint8_t tasktype);
  237 static void mptsas_set_pkt_reason(mptsas_t *mpt, mptsas_cmd_t *cmd,
  238     uchar_t reason, uint_t stat);
  239 
  240 static uint_t mptsas_intr(caddr_t arg1, caddr_t arg2);
  241 static void mptsas_process_intr(mptsas_t *mpt,
  242     pMpi2ReplyDescriptorsUnion_t reply_desc_union);
  243 static int mptsas_handle_io_fastpath(mptsas_t *mpt, uint16_t SMID);
  244 static void mptsas_handle_scsi_io_success(mptsas_t *mpt,
  245     pMpi2ReplyDescriptorsUnion_t reply_desc);
  246 static void mptsas_handle_address_reply(mptsas_t *mpt,
  247     pMpi2ReplyDescriptorsUnion_t reply_desc);
  248 static int mptsas_wait_intr(mptsas_t *mpt, int polltime);
  249 static void mptsas_sge_setup(mptsas_t *mpt, mptsas_cmd_t *cmd,
  250     uint32_t *control, pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl);
  251 
  252 static void mptsas_watch(void *arg);
  253 static void mptsas_watchsubr(mptsas_t *mpt);
  254 static void mptsas_cmd_timeout(mptsas_t *mpt, uint16_t devhdl);
  255 
  256 static void mptsas_start_passthru(mptsas_t *mpt, mptsas_cmd_t *cmd);
  257 static int mptsas_do_passthru(mptsas_t *mpt, uint8_t *request, uint8_t *reply,
  258     uint8_t *data, uint32_t request_size, uint32_t reply_size,
  259     uint32_t data_size, uint32_t direction, uint8_t *dataout,
  260     uint32_t dataout_size, short timeout, int mode);
  261 static int mptsas_free_devhdl(mptsas_t *mpt, uint16_t devhdl);
  262 
  263 static uint8_t mptsas_get_fw_diag_buffer_number(mptsas_t *mpt,
  264     uint32_t unique_id);
  265 static void mptsas_start_diag(mptsas_t *mpt, mptsas_cmd_t *cmd);
  266 static int mptsas_post_fw_diag_buffer(mptsas_t *mpt,
  267     mptsas_fw_diagnostic_buffer_t *pBuffer, uint32_t *return_code);
  268 static int mptsas_release_fw_diag_buffer(mptsas_t *mpt,
  269     mptsas_fw_diagnostic_buffer_t *pBuffer, uint32_t *return_code,
  270     uint32_t diag_type);
  271 static int mptsas_diag_register(mptsas_t *mpt,
  272     mptsas_fw_diag_register_t *diag_register, uint32_t *return_code);
  273 static int mptsas_diag_unregister(mptsas_t *mpt,
  274     mptsas_fw_diag_unregister_t *diag_unregister, uint32_t *return_code);
  275 static int mptsas_diag_query(mptsas_t *mpt, mptsas_fw_diag_query_t *diag_query,
  276     uint32_t *return_code);
  277 static int mptsas_diag_read_buffer(mptsas_t *mpt,
  278     mptsas_diag_read_buffer_t *diag_read_buffer, uint8_t *ioctl_buf,
  279     uint32_t *return_code, int ioctl_mode);
  280 static int mptsas_diag_release(mptsas_t *mpt,
  281     mptsas_fw_diag_release_t *diag_release, uint32_t *return_code);
  282 static int mptsas_do_diag_action(mptsas_t *mpt, uint32_t action,
  283     uint8_t *diag_action, uint32_t length, uint32_t *return_code,
  284     int ioctl_mode);
  285 static int mptsas_diag_action(mptsas_t *mpt, mptsas_diag_action_t *data,
  286     int mode);
  287 
  288 static int mptsas_pkt_alloc_extern(mptsas_t *mpt, mptsas_cmd_t *cmd,
  289     int cmdlen, int tgtlen, int statuslen, int kf);
  290 static void mptsas_pkt_destroy_extern(mptsas_t *mpt, mptsas_cmd_t *cmd);
  291 
  292 static int mptsas_kmem_cache_constructor(void *buf, void *cdrarg, int kmflags);
  293 static void mptsas_kmem_cache_destructor(void *buf, void *cdrarg);
  294 
  295 static int mptsas_cache_frames_constructor(void *buf, void *cdrarg,
  296     int kmflags);
  297 static void mptsas_cache_frames_destructor(void *buf, void *cdrarg);
  298 
  299 static void mptsas_check_scsi_io_error(mptsas_t *mpt, pMpi2SCSIIOReply_t reply,
  300     mptsas_cmd_t *cmd);
  301 static void mptsas_check_task_mgt(mptsas_t *mpt,
  302     pMpi2SCSIManagementReply_t reply, mptsas_cmd_t *cmd);
  303 static int mptsas_send_scsi_cmd(mptsas_t *mpt, struct scsi_address *ap,
  304     mptsas_target_t *ptgt, uchar_t *cdb, int cdblen, struct buf *data_bp,
  305     int *resid);
  306 
  307 static int mptsas_alloc_active_slots(mptsas_t *mpt, int flag);
  308 static void mptsas_free_active_slots(mptsas_t *mpt);
  309 static int mptsas_start_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd);
  310 static int mptsas_start_cmd0(mptsas_t *mpt, mptsas_cmd_t *cmd);
  311 
  312 static void mptsas_restart_hba(mptsas_t *mpt);
  313 
  314 static void mptsas_deliver_doneq_thread(mptsas_t *mpt);
  315 static void mptsas_doneq_add(mptsas_t *mpt, mptsas_cmd_t *cmd);
  316 static inline void mptsas_doneq_add0(mptsas_t *mpt, mptsas_cmd_t *cmd);
  317 static void mptsas_doneq_mv(mptsas_t *mpt, uint64_t t);
  318 
  319 static mptsas_cmd_t *mptsas_doneq_thread_rm(mptsas_t *mpt, uint64_t t);
  320 static void mptsas_doneq_empty(mptsas_t *mpt);
  321 static void mptsas_doneq_thread(mptsas_doneq_thread_arg_t *arg);
  322 
  323 static mptsas_cmd_t *mptsas_waitq_rm(mptsas_t *mpt);
  324 static void mptsas_waitq_delete(mptsas_t *mpt, mptsas_cmd_t *cmd);
  325 
  326 static void mptsas_start_watch_reset_delay();
  327 static void mptsas_setup_bus_reset_delay(mptsas_t *mpt);
  328 static void mptsas_watch_reset_delay(void *arg);
  329 static int mptsas_watch_reset_delay_subr(mptsas_t *mpt);
  330 
  331 static int mptsas_outstanding_cmds_n(mptsas_t *mpt);
  332 /*
  333  * helper functions
  334  */
  335 static void mptsas_dump_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd);
  336 
  337 static dev_info_t *mptsas_find_child(dev_info_t *pdip, char *name);
  338 static dev_info_t *mptsas_find_child_phy(dev_info_t *pdip, uint8_t phy);
  339 static dev_info_t *mptsas_find_child_addr(dev_info_t *pdip, uint64_t sasaddr,
  340     int lun);
  341 static mdi_pathinfo_t *mptsas_find_path_addr(dev_info_t *pdip, uint64_t sasaddr,
  342     int lun);
  343 static mdi_pathinfo_t *mptsas_find_path_phy(dev_info_t *pdip, uint8_t phy);
  344 static dev_info_t *mptsas_find_smp_child(dev_info_t *pdip, char *str_wwn);
  345 
  346 static int mptsas_parse_address(char *name, uint64_t *wwid, uint8_t *phy,
  347     int *lun);
  348 static int mptsas_parse_smp_name(char *name, uint64_t *wwn);
  349 
  350 static mptsas_target_t *mptsas_phy_to_tgt(mptsas_t *mpt, int phymask,
  351     uint8_t phy);
  352 static mptsas_target_t *mptsas_wwid_to_ptgt(mptsas_t *mpt, int phymask,
  353     uint64_t wwid);
  354 static mptsas_smp_t *mptsas_wwid_to_psmp(mptsas_t *mpt, int phymask,
  355     uint64_t wwid);
  356 
  357 static int mptsas_inquiry(mptsas_t *mpt, mptsas_target_t *ptgt, int lun,
  358     uchar_t page, unsigned char *buf, int len, int *rlen, uchar_t evpd);
  359 
  360 static int mptsas_get_target_device_info(mptsas_t *mpt, uint32_t page_address,
  361     uint16_t *handle, mptsas_target_t **pptgt);
  362 static void mptsas_update_phymask(mptsas_t *mpt);
  363 static inline void mptsas_remove_cmd0(mptsas_t *mpt, mptsas_cmd_t *cmd);
  364 
  365 static int mptsas_send_sep(mptsas_t *mpt, mptsas_target_t *ptgt,
  366     uint32_t *status, uint8_t cmd);
  367 static dev_info_t *mptsas_get_dip_from_dev(dev_t dev,
  368     mptsas_phymask_t *phymask);
  369 static mptsas_target_t *mptsas_addr_to_ptgt(mptsas_t *mpt, char *addr,
  370     mptsas_phymask_t phymask);
  371 static int mptsas_set_led_status(mptsas_t *mpt, mptsas_target_t *ptgt,
  372     uint32_t slotstatus);
  373 
  374 
  375 /*
  376  * Enumeration / DR functions
  377  */
  378 static void mptsas_config_all(dev_info_t *pdip);
  379 static int mptsas_config_one_addr(dev_info_t *pdip, uint64_t sasaddr, int lun,
  380     dev_info_t **lundip);
  381 static int mptsas_config_one_phy(dev_info_t *pdip, uint8_t phy, int lun,
  382     dev_info_t **lundip);
  383 
  384 static int mptsas_config_target(dev_info_t *pdip, mptsas_target_t *ptgt);
  385 static int mptsas_offline_target(dev_info_t *pdip, char *name);
  386 
  387 static int mptsas_config_raid(dev_info_t *pdip, uint16_t target,
  388     dev_info_t **dip);
  389 
  390 static int mptsas_config_luns(dev_info_t *pdip, mptsas_target_t *ptgt);
  391 static int mptsas_probe_lun(dev_info_t *pdip, int lun,
  392     dev_info_t **dip, mptsas_target_t *ptgt);
  393 
  394 static int mptsas_create_lun(dev_info_t *pdip, struct scsi_inquiry *sd_inq,
  395     dev_info_t **dip, mptsas_target_t *ptgt, int lun);
  396 
  397 static int mptsas_create_phys_lun(dev_info_t *pdip, struct scsi_inquiry *sd,
  398     char *guid, dev_info_t **dip, mptsas_target_t *ptgt, int lun);
  399 static int mptsas_create_virt_lun(dev_info_t *pdip, struct scsi_inquiry *sd,
  400     char *guid, dev_info_t **dip, mdi_pathinfo_t **pip, mptsas_target_t *ptgt,
  401     int lun);
  402 
  403 static void mptsas_offline_missed_luns(dev_info_t *pdip,
  404     uint16_t *repluns, int lun_cnt, mptsas_target_t *ptgt);
  405 static int mptsas_offline_lun(dev_info_t *pdip, dev_info_t *rdip,
  406     mdi_pathinfo_t *rpip, uint_t flags);
  407 
  408 static int mptsas_config_smp(dev_info_t *pdip, uint64_t sas_wwn,
  409     dev_info_t **smp_dip);
  410 static int mptsas_offline_smp(dev_info_t *pdip, mptsas_smp_t *smp_node,
  411     uint_t flags);
  412 
  413 static int mptsas_event_query(mptsas_t *mpt, mptsas_event_query_t *data,
  414     int mode, int *rval);
  415 static int mptsas_event_enable(mptsas_t *mpt, mptsas_event_enable_t *data,
  416     int mode, int *rval);
  417 static int mptsas_event_report(mptsas_t *mpt, mptsas_event_report_t *data,
  418     int mode, int *rval);
  419 static void mptsas_record_event(void *args);
  420 static int mptsas_reg_access(mptsas_t *mpt, mptsas_reg_access_t *data,
  421     int mode);
  422 
  423 static void mptsas_hash_init(mptsas_hash_table_t *hashtab);
  424 static void mptsas_hash_uninit(mptsas_hash_table_t *hashtab, size_t datalen);
  425 static void mptsas_hash_add(mptsas_hash_table_t *hashtab, void *data);
  426 static void * mptsas_hash_rem(mptsas_hash_table_t *hashtab, uint64_t key1,
  427     mptsas_phymask_t key2);
  428 static void * mptsas_hash_search(mptsas_hash_table_t *hashtab, uint64_t key1,
  429     mptsas_phymask_t key2);
  430 static void * mptsas_hash_traverse(mptsas_hash_table_t *hashtab, int pos);
  431 
  432 mptsas_target_t *mptsas_tgt_alloc(mptsas_hash_table_t *, uint16_t, uint64_t,
  433     uint32_t, mptsas_phymask_t, uint8_t, mptsas_t *);
  434 static mptsas_smp_t *mptsas_smp_alloc(mptsas_hash_table_t *hashtab,
  435     mptsas_smp_t *data);
  436 static void mptsas_smp_free(mptsas_hash_table_t *hashtab, uint64_t wwid,
  437     mptsas_phymask_t phymask);
  438 static void mptsas_tgt_free(mptsas_hash_table_t *, uint64_t, mptsas_phymask_t);
  439 static void * mptsas_search_by_devhdl(mptsas_hash_table_t *, uint16_t);
  440 static int mptsas_online_smp(dev_info_t *pdip, mptsas_smp_t *smp_node,
  441     dev_info_t **smp_dip);
  442 
  443 /*
  444  * Power management functions
  445  */
  446 static int mptsas_get_pci_cap(mptsas_t *mpt);
  447 static int mptsas_init_pm(mptsas_t *mpt);
  448 
  449 /*
  450  * MPT MSI tunable:
  451  *
  452  * By default MSI is enabled on all supported platforms.
  453  */
  454 boolean_t mptsas_enable_msi = B_TRUE;
  455 boolean_t mptsas_physical_bind_failed_page_83 = B_FALSE;
  456 
  457 static int mptsas_register_intrs(mptsas_t *);
  458 static void mptsas_unregister_intrs(mptsas_t *);
  459 static int mptsas_add_intrs(mptsas_t *, int);
  460 static void mptsas_rem_intrs(mptsas_t *);
  461 
  462 /*
  463  * FMA Prototypes
  464  */
  465 static void mptsas_fm_init(mptsas_t *mpt);
  466 static void mptsas_fm_fini(mptsas_t *mpt);
  467 static int mptsas_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void *);
  468 
  469 extern pri_t minclsyspri, maxclsyspri;
  470 
  471 /*
  472  * This device is created by the SCSI pseudo nexus driver (SCSI vHCI).  It is
  473  * under this device that the paths to a physical device are created when
  474  * MPxIO is used.
  475  */
  476 extern dev_info_t       *scsi_vhci_dip;
  477 
  478 /*
  479  * Tunable timeout value for Inquiry VPD page 0x83
  480  * By default the value is 30 seconds.
  481  */
  482 int mptsas_inq83_retry_timeout = 30;
  483 
  484 /*
  485  * This is used to allocate memory for message frame storage, not for
  486  * data I/O DMA. All message frames must be stored in the first 4G of
  487  * physical memory.
  488  */
  489 ddi_dma_attr_t mptsas_dma_attrs = {
  490         DMA_ATTR_V0,    /* attribute layout version             */
  491         0x0ull,         /* address low - should be 0 (longlong) */
  492         0xffffffffull,  /* address high - 32-bit max range      */
  493         0x00ffffffull,  /* count max - max DMA object size      */
  494         4,              /* allocation alignment requirements    */
  495         0x78,           /* burstsizes - binary encoded values   */
  496         1,              /* minxfer - gran. of DMA engine        */
  497         0x00ffffffull,  /* maxxfer - gran. of DMA engine        */
  498         0xffffffffull,  /* max segment size (DMA boundary)      */
  499         MPTSAS_MAX_DMA_SEGS, /* scatter/gather list length      */
  500         512,            /* granularity - device transfer size   */
  501         0               /* flags, set to 0                      */
  502 };
  503 
  504 /*
  505  * This is used for data I/O DMA memory allocation. (full 64-bit DMA
  506  * physical addresses are supported.)
  507  */
  508 ddi_dma_attr_t mptsas_dma_attrs64 = {
  509         DMA_ATTR_V0,    /* attribute layout version             */
  510         0x0ull,         /* address low - should be 0 (longlong) */
  511         0xffffffffffffffffull,  /* address high - 64-bit max    */
  512         0x00ffffffull,  /* count max - max DMA object size      */
  513         4,              /* allocation alignment requirements    */
  514         0x78,           /* burstsizes - binary encoded values   */
  515         1,              /* minxfer - gran. of DMA engine        */
  516         0x00ffffffull,  /* maxxfer - gran. of DMA engine        */
  517         0xffffffffull,  /* max segment size (DMA boundary)      */
  518         MPTSAS_MAX_DMA_SEGS, /* scatter/gather list length      */
  519         512,            /* granularity - device transfer size   */
  520         DDI_DMA_RELAXED_ORDERING        /* flags, enable relaxed ordering */
  521 };
  522 
  523 ddi_device_acc_attr_t mptsas_dev_attr = {
  524         DDI_DEVICE_ATTR_V1,
  525         DDI_STRUCTURE_LE_ACC,
  526         DDI_STRICTORDER_ACC,
  527         DDI_DEFAULT_ACC
  528 };
  529 
  530 static struct cb_ops mptsas_cb_ops = {
  531         scsi_hba_open,          /* open */
  532         scsi_hba_close,         /* close */
  533         nodev,                  /* strategy */
  534         nodev,                  /* print */
  535         nodev,                  /* dump */
  536         nodev,                  /* read */
  537         nodev,                  /* write */
  538         mptsas_ioctl,           /* ioctl */
  539         nodev,                  /* devmap */
  540         nodev,                  /* mmap */
  541         nodev,                  /* segmap */
  542         nochpoll,               /* chpoll */
  543         ddi_prop_op,            /* cb_prop_op */
  544         NULL,                   /* streamtab */
  545         D_MP,                   /* cb_flag */
  546         CB_REV,                 /* rev */
  547         nodev,                  /* aread */
  548         nodev                   /* awrite */
  549 };
  550 
  551 static struct dev_ops mptsas_ops = {
  552         DEVO_REV,               /* devo_rev, */
  553         0,                      /* refcnt  */
  554         ddi_no_info,            /* info */
  555         nulldev,                /* identify */
  556         nulldev,                /* probe */
  557         mptsas_attach,          /* attach */
  558         mptsas_detach,          /* detach */
  559 #ifdef  __sparc
  560         mptsas_reset,
  561 #else
  562         nodev,                  /* reset */
  563 #endif  /* __sparc */
  564         &mptsas_cb_ops,         /* driver operations */
  565         NULL,                   /* bus operations */
  566         mptsas_power,           /* power management */
  567 #ifdef  __sparc
  568         ddi_quiesce_not_needed
  569 #else
  570         mptsas_quiesce          /* quiesce */
  571 #endif  /* __sparc */
  572 };
  573 
  574 
  575 #define MPTSAS_MOD_STRING "MPTSAS HBA Driver 00.00.00.24"
  576 
  577 static struct modldrv modldrv = {
  578         &mod_driverops, /* Type of module. This one is a driver */
  579         MPTSAS_MOD_STRING, /* Name of the module. */
  580         &mptsas_ops,    /* driver ops */
  581 };
  582 
  583 static struct modlinkage modlinkage = {
  584         MODREV_1, &modldrv, NULL
  585 };
  586 #define TARGET_PROP     "target"
  587 #define LUN_PROP        "lun"
  588 #define LUN64_PROP      "lun64"
  589 #define SAS_PROP        "sas-mpt"
  590 #define MDI_GUID        "wwn"
  591 #define NDI_GUID        "guid"
  592 #define MPTSAS_DEV_GONE "mptsas_dev_gone"
  593 
  594 /*
  595  * Local static data
  596  */
  597 #if defined(MPTSAS_DEBUG)
  598 uint32_t mptsas_debug_flags = 0;
  599 #endif  /* defined(MPTSAS_DEBUG) */
  600 uint32_t mptsas_debug_resets = 0;
  601 
  602 static kmutex_t         mptsas_global_mutex;
  603 static void             *mptsas_state;          /* soft state ptr */
  604 static krwlock_t        mptsas_global_rwlock;
  605 
  606 static kmutex_t         mptsas_log_mutex;
  607 static char             mptsas_log_buf[256];
  608 _NOTE(MUTEX_PROTECTS_DATA(mptsas_log_mutex, mptsas_log_buf))
  609 
  610 static mptsas_t *mptsas_head, *mptsas_tail;
  611 static clock_t mptsas_scsi_watchdog_tick;
  612 static clock_t mptsas_tick;
  613 static timeout_id_t mptsas_reset_watch;
  614 static timeout_id_t mptsas_timeout_id;
  615 static int mptsas_timeouts_enabled = 0;
  616 /*
  617  * warlock directives
  618  */
  619 _NOTE(SCHEME_PROTECTS_DATA("unique per pkt", scsi_pkt \
  620         mptsas_cmd NcrTableIndirect buf scsi_cdb scsi_status))
  621 _NOTE(SCHEME_PROTECTS_DATA("unique per pkt", smp_pkt))
  622 _NOTE(SCHEME_PROTECTS_DATA("stable data", scsi_device scsi_address))
  623 _NOTE(SCHEME_PROTECTS_DATA("No Mutex Needed", mptsas_tgt_private))
  624 _NOTE(SCHEME_PROTECTS_DATA("No Mutex Needed", scsi_hba_tran::tran_tgt_private))
  625 
  626 /*
  627  * SM - HBA statics
  628  */
  629 char    *mptsas_driver_rev = MPTSAS_MOD_STRING;
  630 
  631 #ifdef MPTSAS_DEBUG
  632 void debug_enter(char *);
  633 #endif
  634 
  635 /*
  636  * Notes:
  637  *      - scsi_hba_init(9F) initializes SCSI HBA modules
  638  *      - must call scsi_hba_fini(9F) if modload() fails
  639  */
  640 int
  641 _init(void)
  642 {
  643         int status;
  644         /* CONSTCOND */
  645         ASSERT(NO_COMPETING_THREADS);
  646 
  647         NDBG0(("_init"));
  648 
  649         status = ddi_soft_state_init(&mptsas_state, MPTSAS_SIZE,
  650             MPTSAS_INITIAL_SOFT_SPACE);
  651         if (status != 0) {
  652                 return (status);
  653         }
  654 
  655         if ((status = scsi_hba_init(&modlinkage)) != 0) {
  656                 ddi_soft_state_fini(&mptsas_state);
  657                 return (status);
  658         }
  659 
  660         mutex_init(&mptsas_global_mutex, NULL, MUTEX_DRIVER, NULL);
  661         rw_init(&mptsas_global_rwlock, NULL, RW_DRIVER, NULL);
  662         mutex_init(&mptsas_log_mutex, NULL, MUTEX_DRIVER, NULL);
  663 
  664         if ((status = mod_install(&modlinkage)) != 0) {
  665                 mutex_destroy(&mptsas_log_mutex);
  666                 rw_destroy(&mptsas_global_rwlock);
  667                 mutex_destroy(&mptsas_global_mutex);
  668                 ddi_soft_state_fini(&mptsas_state);
  669                 scsi_hba_fini(&modlinkage);
  670         }
  671 
  672         return (status);
  673 }
  674 
  675 /*
  676  * Notes:
  677  *      - scsi_hba_fini(9F) uninitializes SCSI HBA modules
  678  */
  679 int
  680 _fini(void)
  681 {
  682         int     status;
  683         /* CONSTCOND */
  684         ASSERT(NO_COMPETING_THREADS);
  685 
  686         NDBG0(("_fini"));
  687 
  688         if ((status = mod_remove(&modlinkage)) == 0) {
  689                 ddi_soft_state_fini(&mptsas_state);
  690                 scsi_hba_fini(&modlinkage);
  691                 mutex_destroy(&mptsas_global_mutex);
  692                 rw_destroy(&mptsas_global_rwlock);
  693                 mutex_destroy(&mptsas_log_mutex);
  694         }
  695         return (status);
  696 }
  697 
  698 /*
  699  * The loadable-module _info(9E) entry point
  700  */
  701 int
  702 _info(struct modinfo *modinfop)
  703 {
  704         /* CONSTCOND */
  705         ASSERT(NO_COMPETING_THREADS);
  706         NDBG0(("mptsas _info"));
  707 
  708         return (mod_info(&modlinkage, modinfop));
  709 }
  710 
  711 
  712 static int
  713 mptsas_iport_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
  714 {
  715         dev_info_t              *pdip;
  716         mptsas_t                *mpt;
  717         scsi_hba_tran_t         *hba_tran;
  718         char                    *iport = NULL;
  719         char                    phymask[MPTSAS_MAX_PHYS];
  720         mptsas_phymask_t        phy_mask = 0;
  721         int                     dynamic_port = 0;
  722         uint32_t                page_address;
  723         char                    initiator_wwnstr[MPTSAS_WWN_STRLEN];
  724         int                     rval = DDI_FAILURE;
  725         int                     i = 0;
  726         uint8_t                 numphys = 0;
  727         uint8_t                 phy_id;
  728         uint8_t                 phy_port = 0;
  729         uint16_t                attached_devhdl = 0;
  730         uint32_t                dev_info;
  731         uint64_t                attached_sas_wwn;
  732         uint16_t                dev_hdl;
  733         uint16_t                pdev_hdl;
  734         uint16_t                bay_num, enclosure;
  735         char                    attached_wwnstr[MPTSAS_WWN_STRLEN];
  736 
  737         /* CONSTCOND */
  738         ASSERT(NO_COMPETING_THREADS);
  739 
  740         switch (cmd) {
  741         case DDI_ATTACH:
  742                 break;
  743 
  744         case DDI_RESUME:
  745                 /*
  746                  * If this a scsi-iport node, nothing to do here.
  747                  */
  748                 return (DDI_SUCCESS);
  749 
  750         default:
  751                 return (DDI_FAILURE);
  752         }
  753 
  754         pdip = ddi_get_parent(dip);
  755 
  756         if ((hba_tran = ndi_flavorv_get(pdip, SCSA_FLAVOR_SCSI_DEVICE)) ==
  757             NULL) {
  758                 cmn_err(CE_WARN, "Failed attach iport because fail to "
  759                     "get tran vector for the HBA node");
  760                 return (DDI_FAILURE);
  761         }
  762 
  763         mpt = TRAN2MPT(hba_tran);
  764         ASSERT(mpt != NULL);
  765         if (mpt == NULL)
  766                 return (DDI_FAILURE);
  767 
  768         if ((hba_tran = ndi_flavorv_get(dip, SCSA_FLAVOR_SCSI_DEVICE)) ==
  769             NULL) {
  770                 mptsas_log(mpt, CE_WARN, "Failed attach iport because fail to "
  771                     "get tran vector for the iport node");
  772                 return (DDI_FAILURE);
  773         }
  774 
  775         /*
  776          * Overwrite parent's tran_hba_private to iport's tran vector
  777          */
  778         hba_tran->tran_hba_private = mpt;
  779 
  780         ddi_report_dev(dip);
  781 
  782         /*
  783          * Get SAS address for initiator port according dev_handle
  784          */
  785         iport = ddi_get_name_addr(dip);
  786         if (iport && strncmp(iport, "v0", 2) == 0) {
  787                 if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
  788                     MPTSAS_VIRTUAL_PORT, 1) !=
  789                     DDI_PROP_SUCCESS) {
  790                         (void) ddi_prop_remove(DDI_DEV_T_NONE, dip,
  791                             MPTSAS_VIRTUAL_PORT);
  792                         mptsas_log(mpt, CE_WARN, "mptsas virtual port "
  793                             "prop update failed");
  794                         return (DDI_FAILURE);
  795                 }
  796                 return (DDI_SUCCESS);
  797         }
  798 
  799         mutex_enter(&mpt->m_mutex);
  800         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
  801                 bzero(phymask, sizeof (phymask));
  802                 (void) sprintf(phymask,
  803                     "%x", mpt->m_phy_info[i].phy_mask);
  804                 if (strcmp(phymask, iport) == 0) {
  805                         break;
  806                 }
  807         }
  808 
  809         if (i == MPTSAS_MAX_PHYS) {
  810                 mptsas_log(mpt, CE_WARN, "Failed attach port %s because port"
  811                     "seems not exist", iport);
  812                 mutex_exit(&mpt->m_mutex);
  813                 return (DDI_FAILURE);
  814         }
  815 
  816         phy_mask = mpt->m_phy_info[i].phy_mask;
  817 
  818         if (mpt->m_phy_info[i].port_flags & AUTO_PORT_CONFIGURATION)
  819                 dynamic_port = 1;
  820         else
  821                 dynamic_port = 0;
  822 
  823         /*
  824          * Update PHY info for smhba
  825          */
  826         if (mptsas_smhba_phy_init(mpt)) {
  827                 mutex_exit(&mpt->m_mutex);
  828                 mptsas_log(mpt, CE_WARN, "mptsas phy update "
  829                     "failed");
  830                 return (DDI_FAILURE);
  831         }
  832 
  833         mutex_exit(&mpt->m_mutex);
  834 
  835         numphys = 0;
  836         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
  837                 if ((phy_mask >> i) & 0x01) {
  838                         numphys++;
  839                 }
  840         }
  841 
  842         bzero(initiator_wwnstr, sizeof (initiator_wwnstr));
  843         (void) sprintf(initiator_wwnstr, "w%016"PRIx64,
  844             mpt->un.m_base_wwid);
  845 
  846         if (ddi_prop_update_string(DDI_DEV_T_NONE, dip,
  847             SCSI_ADDR_PROP_INITIATOR_PORT, initiator_wwnstr) !=
  848             DDI_PROP_SUCCESS) {
  849                 (void) ddi_prop_remove(DDI_DEV_T_NONE,
  850                     dip, SCSI_ADDR_PROP_INITIATOR_PORT);
  851                 mptsas_log(mpt, CE_WARN, "mptsas Initiator port "
  852                     "prop update failed");
  853                 return (DDI_FAILURE);
  854         }
  855         if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
  856             MPTSAS_NUM_PHYS, numphys) !=
  857             DDI_PROP_SUCCESS) {
  858                 (void) ddi_prop_remove(DDI_DEV_T_NONE, dip, MPTSAS_NUM_PHYS);
  859                 return (DDI_FAILURE);
  860         }
  861 
  862         if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
  863             "phymask", phy_mask) !=
  864             DDI_PROP_SUCCESS) {
  865                 (void) ddi_prop_remove(DDI_DEV_T_NONE, dip, "phymask");
  866                 mptsas_log(mpt, CE_WARN, "mptsas phy mask "
  867                     "prop update failed");
  868                 return (DDI_FAILURE);
  869         }
  870 
  871         if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
  872             "dynamic-port", dynamic_port) !=
  873             DDI_PROP_SUCCESS) {
  874                 (void) ddi_prop_remove(DDI_DEV_T_NONE, dip, "dynamic-port");
  875                 mptsas_log(mpt, CE_WARN, "mptsas dynamic port "
  876                     "prop update failed");
  877                 return (DDI_FAILURE);
  878         }
  879         if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
  880             MPTSAS_VIRTUAL_PORT, 0) !=
  881             DDI_PROP_SUCCESS) {
  882                 (void) ddi_prop_remove(DDI_DEV_T_NONE, dip,
  883                     MPTSAS_VIRTUAL_PORT);
  884                 mptsas_log(mpt, CE_WARN, "mptsas virtual port "
  885                     "prop update failed");
  886                 return (DDI_FAILURE);
  887         }
  888         mptsas_smhba_set_phy_props(mpt,
  889             iport, dip, numphys, &attached_devhdl);
  890 
  891         mutex_enter(&mpt->m_mutex);
  892         page_address = (MPI2_SAS_DEVICE_PGAD_FORM_HANDLE &
  893             MPI2_SAS_DEVICE_PGAD_FORM_MASK) | (uint32_t)attached_devhdl;
  894         rval = mptsas_get_sas_device_page0(mpt, page_address, &dev_hdl,
  895             &attached_sas_wwn, &dev_info, &phy_port, &phy_id,
  896             &pdev_hdl, &bay_num, &enclosure);
  897         if (rval != DDI_SUCCESS) {
  898                 mptsas_log(mpt, CE_WARN,
  899                     "Failed to get device page0 for handle:%d",
  900                     attached_devhdl);
  901                 mutex_exit(&mpt->m_mutex);
  902                 return (DDI_FAILURE);
  903         }
  904 
  905         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
  906                 bzero(phymask, sizeof (phymask));
  907                 (void) sprintf(phymask, "%x", mpt->m_phy_info[i].phy_mask);
  908                 if (strcmp(phymask, iport) == 0) {
  909                         (void) sprintf(&mpt->m_phy_info[i].smhba_info.path[0],
  910                             "%x",
  911                             mpt->m_phy_info[i].phy_mask);
  912                 }
  913         }
  914         mutex_exit(&mpt->m_mutex);
  915 
  916         bzero(attached_wwnstr, sizeof (attached_wwnstr));
  917         (void) sprintf(attached_wwnstr, "w%016"PRIx64,
  918             attached_sas_wwn);
  919         if (ddi_prop_update_string(DDI_DEV_T_NONE, dip,
  920             SCSI_ADDR_PROP_ATTACHED_PORT, attached_wwnstr) !=
  921             DDI_PROP_SUCCESS) {
  922                 (void) ddi_prop_remove(DDI_DEV_T_NONE,
  923                     dip, SCSI_ADDR_PROP_ATTACHED_PORT);
  924                 return (DDI_FAILURE);
  925         }
  926 
  927         /* Create kstats for each phy on this iport */
  928 
  929         mptsas_create_phy_stats(mpt, iport, dip);
  930 
  931         /*
  932          * register sas hba iport with mdi (MPxIO/vhci)
  933          */
  934         if (mdi_phci_register(MDI_HCI_CLASS_SCSI,
  935             dip, 0) == MDI_SUCCESS) {
  936                 mpt->m_mpxio_enable = TRUE;
  937         }
  938         return (DDI_SUCCESS);
  939 }
  940 
  941 /*
  942  * Notes:
  943  *      Set up all device state and allocate data structures,
  944  *      mutexes, condition variables, etc. for device operation.
  945  *      Add interrupts needed.
  946  *      Return DDI_SUCCESS if device is ready, else return DDI_FAILURE.
  947  */
  948 static int
  949 mptsas_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
  950 {
  951         mptsas_t                *mpt = NULL;
  952         int                     instance, i, j;
  953         int                     doneq_thread_num;
  954         char                    intr_added = 0;
  955         char                    map_setup = 0;
  956         char                    config_setup = 0;
  957         char                    hba_attach_setup = 0;
  958         char                    smp_attach_setup = 0;
  959         char                    mutex_init_done = 0;
  960         char                    event_taskq_create = 0;
  961         char                    dr_taskq_create = 0;
  962         char                    doneq_thread_create = 0;
  963         scsi_hba_tran_t         *hba_tran;
  964         uint_t                  mem_bar = MEM_SPACE;
  965         int                     rval = DDI_FAILURE;
  966 
  967         /* CONSTCOND */
  968         ASSERT(NO_COMPETING_THREADS);
  969 
  970         if (scsi_hba_iport_unit_address(dip)) {
  971                 return (mptsas_iport_attach(dip, cmd));
  972         }
  973 
  974         switch (cmd) {
  975         case DDI_ATTACH:
  976                 break;
  977 
  978         case DDI_RESUME:
  979                 if ((hba_tran = ddi_get_driver_private(dip)) == NULL)
  980                         return (DDI_FAILURE);
  981 
  982                 mpt = TRAN2MPT(hba_tran);
  983 
  984                 if (!mpt) {
  985                         return (DDI_FAILURE);
  986                 }
  987 
  988                 /*
  989                  * Reset hardware and softc to "no outstanding commands"
  990                  * Note that a check condition can result on first command
  991                  * to a target.
  992                  */
  993                 mutex_enter(&mpt->m_mutex);
  994 
  995                 /*
  996                  * raise power.
  997                  */
  998                 if (mpt->m_options & MPTSAS_OPT_PM) {
  999                         mutex_exit(&mpt->m_mutex);
 1000                         (void) pm_busy_component(dip, 0);
 1001                         rval = pm_power_has_changed(dip, 0, PM_LEVEL_D0);
 1002                         if (rval == DDI_SUCCESS) {
 1003                                 mutex_enter(&mpt->m_mutex);
 1004                         } else {
 1005                                 /*
 1006                                  * The pm_raise_power() call above failed,
 1007                                  * and that can only occur if we were unable
 1008                                  * to reset the hardware.  This is probably
 1009                                  * due to unhealty hardware, and because
 1010                                  * important filesystems(such as the root
 1011                                  * filesystem) could be on the attached disks,
 1012                                  * it would not be a good idea to continue,
 1013                                  * as we won't be entirely certain we are
 1014                                  * writing correct data.  So we panic() here
 1015                                  * to not only prevent possible data corruption,
 1016                                  * but to give developers or end users a hope
 1017                                  * of identifying and correcting any problems.
 1018                                  */
 1019                                 fm_panic("mptsas could not reset hardware "
 1020                                     "during resume");
 1021                         }
 1022                 }
 1023 
 1024                 mpt->m_suspended = 0;
 1025 
 1026                 /*
 1027                  * Reinitialize ioc
 1028                  */
 1029                 mpt->m_softstate |= MPTSAS_SS_MSG_UNIT_RESET;
 1030                 if (mptsas_init_chip(mpt, FALSE) == DDI_FAILURE) {
 1031                         mutex_exit(&mpt->m_mutex);
 1032                         if (mpt->m_options & MPTSAS_OPT_PM) {
 1033                                 (void) pm_idle_component(dip, 0);
 1034                         }
 1035                         fm_panic("mptsas init chip fail during resume");
 1036                 }
 1037                 /*
 1038                  * mptsas_update_driver_data needs interrupts so enable them
 1039                  * first.
 1040                  */
 1041                 MPTSAS_ENABLE_INTR(mpt);
 1042                 mptsas_update_driver_data(mpt);
 1043 
 1044                 /* start requests, if possible */
 1045                 mptsas_restart_hba(mpt);
 1046 
 1047                 mutex_exit(&mpt->m_mutex);
 1048 
 1049                 /*
 1050                  * Restart watch thread
 1051                  */
 1052                 mutex_enter(&mptsas_global_mutex);
 1053                 if (mptsas_timeout_id == 0) {
 1054                         mptsas_timeout_id = timeout(mptsas_watch, NULL,
 1055                             mptsas_tick);
 1056                         mptsas_timeouts_enabled = 1;
 1057                 }
 1058                 mutex_exit(&mptsas_global_mutex);
 1059 
 1060                 /* report idle status to pm framework */
 1061                 if (mpt->m_options & MPTSAS_OPT_PM) {
 1062                         (void) pm_idle_component(dip, 0);
 1063                 }
 1064 
 1065                 return (DDI_SUCCESS);
 1066 
 1067         default:
 1068                 return (DDI_FAILURE);
 1069 
 1070         }
 1071 
 1072         instance = ddi_get_instance(dip);
 1073 
 1074         /*
 1075          * Allocate softc information.
 1076          */
 1077         if (ddi_soft_state_zalloc(mptsas_state, instance) != DDI_SUCCESS) {
 1078                 mptsas_log(NULL, CE_WARN,
 1079                     "mptsas%d: cannot allocate soft state", instance);
 1080                 goto fail;
 1081         }
 1082 
 1083         mpt = ddi_get_soft_state(mptsas_state, instance);
 1084 
 1085         if (mpt == NULL) {
 1086                 mptsas_log(NULL, CE_WARN,
 1087                     "mptsas%d: cannot get soft state", instance);
 1088                 goto fail;
 1089         }
 1090 
 1091         /* Indicate that we are 'sizeof (scsi_*(9S))' clean. */
 1092         scsi_size_clean(dip);
 1093 
 1094         mpt->m_dip = dip;
 1095         mpt->m_instance = instance;
 1096 
 1097         /* Make a per-instance copy of the structures */
 1098         mpt->m_io_dma_attr = mptsas_dma_attrs64;
 1099         mpt->m_msg_dma_attr = mptsas_dma_attrs;
 1100         mpt->m_reg_acc_attr = mptsas_dev_attr;
 1101         mpt->m_dev_acc_attr = mptsas_dev_attr;
 1102 
 1103         /*
 1104          * Initialize FMA
 1105          */
 1106         mpt->m_fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, mpt->m_dip,
 1107             DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
 1108             DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
 1109             DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
 1110 
 1111         mptsas_fm_init(mpt);
 1112 
 1113         if (mptsas_alloc_handshake_msg(mpt,
 1114             sizeof (Mpi2SCSITaskManagementRequest_t)) == DDI_FAILURE) {
 1115                 mptsas_log(mpt, CE_WARN, "cannot initialize handshake msg.");
 1116                 goto fail;
 1117         }
 1118 
 1119         /*
 1120          * Setup configuration space
 1121          */
 1122         if (mptsas_config_space_init(mpt) == FALSE) {
 1123                 mptsas_log(mpt, CE_WARN, "mptsas_config_space_init failed");
 1124                 goto fail;
 1125         }
 1126         config_setup++;
 1127 
 1128         if (ddi_regs_map_setup(dip, mem_bar, (caddr_t *)&mpt->m_reg,
 1129             0, 0, &mpt->m_reg_acc_attr, &mpt->m_datap) != DDI_SUCCESS) {
 1130                 mptsas_log(mpt, CE_WARN, "map setup failed");
 1131                 goto fail;
 1132         }
 1133         map_setup++;
 1134 
 1135         /*
 1136          * A taskq is created for dealing with the event handler
 1137          */
 1138         if ((mpt->m_event_taskq = ddi_taskq_create(dip, "mptsas_event_taskq",
 1139             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 1140                 mptsas_log(mpt, CE_NOTE, "ddi_taskq_create failed");
 1141                 goto fail;
 1142         }
 1143         event_taskq_create++;
 1144 
 1145         /*
 1146          * A taskq is created for dealing with dr events
 1147          */
 1148         if ((mpt->m_dr_taskq = ddi_taskq_create(dip,
 1149             "mptsas_dr_taskq",
 1150             1, TASKQ_DEFAULTPRI, 0)) == NULL) {
 1151                 mptsas_log(mpt, CE_NOTE, "ddi_taskq_create for discovery "
 1152                     "failed");
 1153                 goto fail;
 1154         }
 1155         dr_taskq_create++;
 1156 
 1157         mpt->m_doneq_thread_threshold = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 1158             0, "mptsas_doneq_thread_threshold_prop", 10);
 1159         mpt->m_doneq_length_threshold = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 1160             0, "mptsas_doneq_length_threshold_prop", 8);
 1161         mpt->m_doneq_thread_n = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 1162             0, "mptsas_doneq_thread_n_prop", 8);
 1163 
 1164         if (mpt->m_doneq_thread_n) {
 1165                 cv_init(&mpt->m_doneq_thread_cv, NULL, CV_DRIVER, NULL);
 1166                 mutex_init(&mpt->m_doneq_mutex, NULL, MUTEX_DRIVER, NULL);
 1167 
 1168                 mutex_enter(&mpt->m_doneq_mutex);
 1169                 mpt->m_doneq_thread_id =
 1170                     kmem_zalloc(sizeof (mptsas_doneq_thread_list_t)
 1171                     * mpt->m_doneq_thread_n, KM_SLEEP);
 1172 
 1173                 for (j = 0; j < mpt->m_doneq_thread_n; j++) {
 1174                         cv_init(&mpt->m_doneq_thread_id[j].cv, NULL,
 1175                             CV_DRIVER, NULL);
 1176                         mutex_init(&mpt->m_doneq_thread_id[j].mutex, NULL,
 1177                             MUTEX_DRIVER, NULL);
 1178                         mutex_enter(&mpt->m_doneq_thread_id[j].mutex);
 1179                         mpt->m_doneq_thread_id[j].flag |=
 1180                             MPTSAS_DONEQ_THREAD_ACTIVE;
 1181                         mpt->m_doneq_thread_id[j].arg.mpt = mpt;
 1182                         mpt->m_doneq_thread_id[j].arg.t = j;
 1183                         mpt->m_doneq_thread_id[j].threadp =
 1184                             thread_create(NULL, 0, mptsas_doneq_thread,
 1185                             &mpt->m_doneq_thread_id[j].arg,
 1186                             0, &p0, TS_RUN, minclsyspri);
 1187                         mpt->m_doneq_thread_id[j].donetail =
 1188                             &mpt->m_doneq_thread_id[j].doneq;
 1189                         mutex_exit(&mpt->m_doneq_thread_id[j].mutex);
 1190                 }
 1191                 mutex_exit(&mpt->m_doneq_mutex);
 1192                 doneq_thread_create++;
 1193         }
 1194 
 1195         /* Initialize mutex used in interrupt handler */
 1196         mutex_init(&mpt->m_mutex, NULL, MUTEX_DRIVER,
 1197             DDI_INTR_PRI(mpt->m_intr_pri));
 1198         mutex_init(&mpt->m_passthru_mutex, NULL, MUTEX_DRIVER, NULL);
 1199         mutex_init(&mpt->m_intr_mutex, NULL, MUTEX_DRIVER,
 1200             DDI_INTR_PRI(mpt->m_intr_pri));
 1201         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
 1202                 mutex_init(&mpt->m_phy_info[i].smhba_info.phy_mutex,
 1203                     NULL, MUTEX_DRIVER,
 1204                     DDI_INTR_PRI(mpt->m_intr_pri));
 1205         }
 1206 
 1207         cv_init(&mpt->m_cv, NULL, CV_DRIVER, NULL);
 1208         cv_init(&mpt->m_passthru_cv, NULL, CV_DRIVER, NULL);
 1209         cv_init(&mpt->m_fw_cv, NULL, CV_DRIVER, NULL);
 1210         cv_init(&mpt->m_config_cv, NULL, CV_DRIVER, NULL);
 1211         cv_init(&mpt->m_fw_diag_cv, NULL, CV_DRIVER, NULL);
 1212         mutex_init_done++;
 1213 
 1214         /*
 1215          * Disable hardware interrupt since we're not ready to
 1216          * handle it yet.
 1217          */
 1218         MPTSAS_DISABLE_INTR(mpt);
 1219         if (mptsas_register_intrs(mpt) == FALSE)
 1220                 goto fail;
 1221         intr_added++;
 1222 
 1223         mutex_enter(&mpt->m_mutex);
 1224         /*
 1225          * Initialize power management component
 1226          */
 1227         if (mpt->m_options & MPTSAS_OPT_PM) {
 1228                 if (mptsas_init_pm(mpt)) {
 1229                         mutex_exit(&mpt->m_mutex);
 1230                         mptsas_log(mpt, CE_WARN, "mptsas pm initialization "
 1231                             "failed");
 1232                         goto fail;
 1233                 }
 1234         }
 1235 
 1236         /*
 1237          * Initialize chip using Message Unit Reset, if allowed
 1238          */
 1239         mpt->m_softstate |= MPTSAS_SS_MSG_UNIT_RESET;
 1240         if (mptsas_init_chip(mpt, TRUE) == DDI_FAILURE) {
 1241                 mutex_exit(&mpt->m_mutex);
 1242                 mptsas_log(mpt, CE_WARN, "mptsas chip initialization failed");
 1243                 goto fail;
 1244         }
 1245 
 1246         /*
 1247          * Fill in the phy_info structure and get the base WWID
 1248          */
 1249         if (mptsas_get_manufacture_page5(mpt) == DDI_FAILURE) {
 1250                 mptsas_log(mpt, CE_WARN,
 1251                     "mptsas_get_manufacture_page5 failed!");
 1252                 goto fail;
 1253         }
 1254 
 1255         if (mptsas_get_sas_io_unit_page_hndshk(mpt)) {
 1256                 mptsas_log(mpt, CE_WARN,
 1257                     "mptsas_get_sas_io_unit_page_hndshk failed!");
 1258                 goto fail;
 1259         }
 1260 
 1261         if (mptsas_get_manufacture_page0(mpt) == DDI_FAILURE) {
 1262                 mptsas_log(mpt, CE_WARN,
 1263                     "mptsas_get_manufacture_page0 failed!");
 1264                 goto fail;
 1265         }
 1266 
 1267         mutex_exit(&mpt->m_mutex);
 1268 
 1269         /*
 1270          * Register the iport for multiple port HBA
 1271          */
 1272         mptsas_iport_register(mpt);
 1273 
 1274         /*
 1275          * initialize SCSI HBA transport structure
 1276          */
 1277         if (mptsas_hba_setup(mpt) == FALSE)
 1278                 goto fail;
 1279         hba_attach_setup++;
 1280 
 1281         if (mptsas_smp_setup(mpt) == FALSE)
 1282                 goto fail;
 1283         smp_attach_setup++;
 1284 
 1285         if (mptsas_cache_create(mpt) == FALSE)
 1286                 goto fail;
 1287 
 1288         mpt->m_scsi_reset_delay = ddi_prop_get_int(DDI_DEV_T_ANY,
 1289             dip, 0, "scsi-reset-delay", SCSI_DEFAULT_RESET_DELAY);
 1290         if (mpt->m_scsi_reset_delay == 0) {
 1291                 mptsas_log(mpt, CE_NOTE,
 1292                     "scsi_reset_delay of 0 is not recommended,"
 1293                     " resetting to SCSI_DEFAULT_RESET_DELAY\n");
 1294                 mpt->m_scsi_reset_delay = SCSI_DEFAULT_RESET_DELAY;
 1295         }
 1296 
 1297         /*
 1298          * Initialize the wait and done FIFO queue
 1299          */
 1300         mpt->m_donetail = &mpt->m_doneq;
 1301         mpt->m_waitqtail = &mpt->m_waitq;
 1302 
 1303         /*
 1304          * ioc cmd queue initialize
 1305          */
 1306         mpt->m_ioc_event_cmdtail = &mpt->m_ioc_event_cmdq;
 1307         mpt->m_dev_handle = 0xFFFF;
 1308 
 1309         MPTSAS_ENABLE_INTR(mpt);
 1310 
 1311         /*
 1312          * enable event notification
 1313          */
 1314         mutex_enter(&mpt->m_mutex);
 1315         if (mptsas_ioc_enable_event_notification(mpt)) {
 1316                 mutex_exit(&mpt->m_mutex);
 1317                 goto fail;
 1318         }
 1319         mutex_exit(&mpt->m_mutex);
 1320 
 1321         /*
 1322          * Initialize PHY info for smhba
 1323          */
 1324         if (mptsas_smhba_setup(mpt)) {
 1325                 mptsas_log(mpt, CE_WARN, "mptsas phy initialization "
 1326                     "failed");
 1327                 goto fail;
 1328         }
 1329 
 1330         /* Check all dma handles allocated in attach */
 1331         if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl)
 1332             != DDI_SUCCESS) ||
 1333             (mptsas_check_dma_handle(mpt->m_dma_reply_frame_hdl)
 1334             != DDI_SUCCESS) ||
 1335             (mptsas_check_dma_handle(mpt->m_dma_free_queue_hdl)
 1336             != DDI_SUCCESS) ||
 1337             (mptsas_check_dma_handle(mpt->m_dma_post_queue_hdl)
 1338             != DDI_SUCCESS) ||
 1339             (mptsas_check_dma_handle(mpt->m_hshk_dma_hdl)
 1340             != DDI_SUCCESS)) {
 1341                 goto fail;
 1342         }
 1343 
 1344         /* Check all acc handles allocated in attach */
 1345         if ((mptsas_check_acc_handle(mpt->m_datap) != DDI_SUCCESS) ||
 1346             (mptsas_check_acc_handle(mpt->m_acc_req_frame_hdl)
 1347             != DDI_SUCCESS) ||
 1348             (mptsas_check_acc_handle(mpt->m_acc_reply_frame_hdl)
 1349             != DDI_SUCCESS) ||
 1350             (mptsas_check_acc_handle(mpt->m_acc_free_queue_hdl)
 1351             != DDI_SUCCESS) ||
 1352             (mptsas_check_acc_handle(mpt->m_acc_post_queue_hdl)
 1353             != DDI_SUCCESS) ||
 1354             (mptsas_check_acc_handle(mpt->m_hshk_acc_hdl)
 1355             != DDI_SUCCESS) ||
 1356             (mptsas_check_acc_handle(mpt->m_config_handle)
 1357             != DDI_SUCCESS)) {
 1358                 goto fail;
 1359         }
 1360 
 1361         /*
 1362          * After this point, we are not going to fail the attach.
 1363          */
 1364         /*
 1365          * used for mptsas_watch
 1366          */
 1367         mptsas_list_add(mpt);
 1368 
 1369         mutex_enter(&mptsas_global_mutex);
 1370         if (mptsas_timeouts_enabled == 0) {
 1371                 mptsas_scsi_watchdog_tick = ddi_prop_get_int(DDI_DEV_T_ANY,
 1372                     dip, 0, "scsi-watchdog-tick", DEFAULT_WD_TICK);
 1373 
 1374                 mptsas_tick = mptsas_scsi_watchdog_tick *
 1375                     drv_usectohz((clock_t)1000000);
 1376 
 1377                 mptsas_timeout_id = timeout(mptsas_watch, NULL, mptsas_tick);
 1378                 mptsas_timeouts_enabled = 1;
 1379         }
 1380         mutex_exit(&mptsas_global_mutex);
 1381 
 1382         /* Print message of HBA present */
 1383         ddi_report_dev(dip);
 1384 
 1385         /* report idle status to pm framework */
 1386         if (mpt->m_options & MPTSAS_OPT_PM) {
 1387                 (void) pm_idle_component(dip, 0);
 1388         }
 1389 
 1390         return (DDI_SUCCESS);
 1391 
 1392 fail:
 1393         mptsas_log(mpt, CE_WARN, "attach failed");
 1394         mptsas_fm_ereport(mpt, DDI_FM_DEVICE_NO_RESPONSE);
 1395         ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_LOST);
 1396         if (mpt) {
 1397                 mutex_enter(&mptsas_global_mutex);
 1398 
 1399                 if (mptsas_timeout_id && (mptsas_head == NULL)) {
 1400                         timeout_id_t tid = mptsas_timeout_id;
 1401                         mptsas_timeouts_enabled = 0;
 1402                         mptsas_timeout_id = 0;
 1403                         mutex_exit(&mptsas_global_mutex);
 1404                         (void) untimeout(tid);
 1405                         mutex_enter(&mptsas_global_mutex);
 1406                 }
 1407                 mutex_exit(&mptsas_global_mutex);
 1408                 /* deallocate in reverse order */
 1409                 mptsas_cache_destroy(mpt);
 1410 
 1411                 if (smp_attach_setup) {
 1412                         mptsas_smp_teardown(mpt);
 1413                 }
 1414                 if (hba_attach_setup) {
 1415                         mptsas_hba_teardown(mpt);
 1416                 }
 1417 
 1418                 if (mpt->m_active) {
 1419                         mptsas_hash_uninit(&mpt->m_active->m_smptbl,
 1420                             sizeof (mptsas_smp_t));
 1421                         mptsas_hash_uninit(&mpt->m_active->m_tgttbl,
 1422                             sizeof (mptsas_target_t));
 1423                         mptsas_free_active_slots(mpt);
 1424                 }
 1425                 if (intr_added) {
 1426                         mptsas_unregister_intrs(mpt);
 1427                 }
 1428 
 1429                 if (doneq_thread_create) {
 1430                         mutex_enter(&mpt->m_doneq_mutex);
 1431                         doneq_thread_num = mpt->m_doneq_thread_n;
 1432                         for (j = 0; j < mpt->m_doneq_thread_n; j++) {
 1433                                 mutex_enter(&mpt->m_doneq_thread_id[j].mutex);
 1434                                 mpt->m_doneq_thread_id[j].flag &=
 1435                                     (~MPTSAS_DONEQ_THREAD_ACTIVE);
 1436                                 cv_signal(&mpt->m_doneq_thread_id[j].cv);
 1437                                 mutex_exit(&mpt->m_doneq_thread_id[j].mutex);
 1438                         }
 1439                         while (mpt->m_doneq_thread_n) {
 1440                                 cv_wait(&mpt->m_doneq_thread_cv,
 1441                                     &mpt->m_doneq_mutex);
 1442                         }
 1443                         for (j = 0; j < doneq_thread_num; j++) {
 1444                                 cv_destroy(&mpt->m_doneq_thread_id[j].cv);
 1445                                 mutex_destroy(&mpt->m_doneq_thread_id[j].mutex);
 1446                         }
 1447                         kmem_free(mpt->m_doneq_thread_id,
 1448                             sizeof (mptsas_doneq_thread_list_t)
 1449                             * doneq_thread_num);
 1450                         mutex_exit(&mpt->m_doneq_mutex);
 1451                         cv_destroy(&mpt->m_doneq_thread_cv);
 1452                         mutex_destroy(&mpt->m_doneq_mutex);
 1453                 }
 1454                 if (event_taskq_create) {
 1455                         ddi_taskq_destroy(mpt->m_event_taskq);
 1456                 }
 1457                 if (dr_taskq_create) {
 1458                         ddi_taskq_destroy(mpt->m_dr_taskq);
 1459                 }
 1460                 if (mutex_init_done) {
 1461                         mutex_destroy(&mpt->m_intr_mutex);
 1462                         mutex_destroy(&mpt->m_passthru_mutex);
 1463                         mutex_destroy(&mpt->m_mutex);
 1464                         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
 1465                                 mutex_destroy(
 1466                                     &mpt->m_phy_info[i].smhba_info.phy_mutex);
 1467                         }
 1468                         cv_destroy(&mpt->m_cv);
 1469                         cv_destroy(&mpt->m_passthru_cv);
 1470                         cv_destroy(&mpt->m_fw_cv);
 1471                         cv_destroy(&mpt->m_config_cv);
 1472                         cv_destroy(&mpt->m_fw_diag_cv);
 1473                 }
 1474 
 1475                 if (map_setup) {
 1476                         mptsas_cfg_fini(mpt);
 1477                 }
 1478                 if (config_setup) {
 1479                         mptsas_config_space_fini(mpt);
 1480                 }
 1481                 mptsas_free_handshake_msg(mpt);
 1482                 mptsas_hba_fini(mpt);
 1483 
 1484                 mptsas_fm_fini(mpt);
 1485                 ddi_soft_state_free(mptsas_state, instance);
 1486                 ddi_prop_remove_all(dip);
 1487         }
 1488         return (DDI_FAILURE);
 1489 }
 1490 
 1491 static int
 1492 mptsas_suspend(dev_info_t *devi)
 1493 {
 1494         mptsas_t        *mpt, *g;
 1495         scsi_hba_tran_t *tran;
 1496 
 1497         if (scsi_hba_iport_unit_address(devi)) {
 1498                 return (DDI_SUCCESS);
 1499         }
 1500 
 1501         if ((tran = ddi_get_driver_private(devi)) == NULL)
 1502                 return (DDI_SUCCESS);
 1503 
 1504         mpt = TRAN2MPT(tran);
 1505         if (!mpt) {
 1506                 return (DDI_SUCCESS);
 1507         }
 1508 
 1509         mutex_enter(&mpt->m_mutex);
 1510 
 1511         if (mpt->m_suspended++) {
 1512                 mutex_exit(&mpt->m_mutex);
 1513                 return (DDI_SUCCESS);
 1514         }
 1515 
 1516         /*
 1517          * Cancel timeout threads for this mpt
 1518          */
 1519         if (mpt->m_quiesce_timeid) {
 1520                 timeout_id_t tid = mpt->m_quiesce_timeid;
 1521                 mpt->m_quiesce_timeid = 0;
 1522                 mutex_exit(&mpt->m_mutex);
 1523                 (void) untimeout(tid);
 1524                 mutex_enter(&mpt->m_mutex);
 1525         }
 1526 
 1527         if (mpt->m_restart_cmd_timeid) {
 1528                 timeout_id_t tid = mpt->m_restart_cmd_timeid;
 1529                 mpt->m_restart_cmd_timeid = 0;
 1530                 mutex_exit(&mpt->m_mutex);
 1531                 (void) untimeout(tid);
 1532                 mutex_enter(&mpt->m_mutex);
 1533         }
 1534 
 1535         mutex_exit(&mpt->m_mutex);
 1536 
 1537         (void) pm_idle_component(mpt->m_dip, 0);
 1538 
 1539         /*
 1540          * Cancel watch threads if all mpts suspended
 1541          */
 1542         rw_enter(&mptsas_global_rwlock, RW_WRITER);
 1543         for (g = mptsas_head; g != NULL; g = g->m_next) {
 1544                 if (!g->m_suspended)
 1545                         break;
 1546         }
 1547         rw_exit(&mptsas_global_rwlock);
 1548 
 1549         mutex_enter(&mptsas_global_mutex);
 1550         if (g == NULL) {
 1551                 timeout_id_t tid;
 1552 
 1553                 mptsas_timeouts_enabled = 0;
 1554                 if (mptsas_timeout_id) {
 1555                         tid = mptsas_timeout_id;
 1556                         mptsas_timeout_id = 0;
 1557                         mutex_exit(&mptsas_global_mutex);
 1558                         (void) untimeout(tid);
 1559                         mutex_enter(&mptsas_global_mutex);
 1560                 }
 1561                 if (mptsas_reset_watch) {
 1562                         tid = mptsas_reset_watch;
 1563                         mptsas_reset_watch = 0;
 1564                         mutex_exit(&mptsas_global_mutex);
 1565                         (void) untimeout(tid);
 1566                         mutex_enter(&mptsas_global_mutex);
 1567                 }
 1568         }
 1569         mutex_exit(&mptsas_global_mutex);
 1570 
 1571         mutex_enter(&mpt->m_mutex);
 1572 
 1573         /*
 1574          * If this mpt is not in full power(PM_LEVEL_D0), just return.
 1575          */
 1576         if ((mpt->m_options & MPTSAS_OPT_PM) &&
 1577             (mpt->m_power_level != PM_LEVEL_D0)) {
 1578                 mutex_exit(&mpt->m_mutex);
 1579                 return (DDI_SUCCESS);
 1580         }
 1581 
 1582         /* Disable HBA interrupts in hardware */
 1583         MPTSAS_DISABLE_INTR(mpt);
 1584         /*
 1585          * Send RAID action system shutdown to sync IR
 1586          */
 1587         mptsas_raid_action_system_shutdown(mpt);
 1588 
 1589         mutex_exit(&mpt->m_mutex);
 1590 
 1591         /* drain the taskq */
 1592         ddi_taskq_wait(mpt->m_event_taskq);
 1593         ddi_taskq_wait(mpt->m_dr_taskq);
 1594 
 1595         return (DDI_SUCCESS);
 1596 }
 1597 
 1598 #ifdef  __sparc
 1599 /*ARGSUSED*/
 1600 static int
 1601 mptsas_reset(dev_info_t *devi, ddi_reset_cmd_t cmd)
 1602 {
 1603         mptsas_t        *mpt;
 1604         scsi_hba_tran_t *tran;
 1605 
 1606         /*
 1607          * If this call is for iport, just return.
 1608          */
 1609         if (scsi_hba_iport_unit_address(devi))
 1610                 return (DDI_SUCCESS);
 1611 
 1612         if ((tran = ddi_get_driver_private(devi)) == NULL)
 1613                 return (DDI_SUCCESS);
 1614 
 1615         if ((mpt = TRAN2MPT(tran)) == NULL)
 1616                 return (DDI_SUCCESS);
 1617 
 1618         /*
 1619          * Send RAID action system shutdown to sync IR.  Disable HBA
 1620          * interrupts in hardware first.
 1621          */
 1622         MPTSAS_DISABLE_INTR(mpt);
 1623         mptsas_raid_action_system_shutdown(mpt);
 1624 
 1625         return (DDI_SUCCESS);
 1626 }
 1627 #else /* __sparc */
 1628 /*
 1629  * quiesce(9E) entry point.
 1630  *
 1631  * This function is called when the system is single-threaded at high
 1632  * PIL with preemption disabled. Therefore, this function must not be
 1633  * blocked.
 1634  *
 1635  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 1636  * DDI_FAILURE indicates an error condition and should almost never happen.
 1637  */
 1638 static int
 1639 mptsas_quiesce(dev_info_t *devi)
 1640 {
 1641         mptsas_t        *mpt;
 1642         scsi_hba_tran_t *tran;
 1643 
 1644         /*
 1645          * If this call is for iport, just return.
 1646          */
 1647         if (scsi_hba_iport_unit_address(devi))
 1648                 return (DDI_SUCCESS);
 1649 
 1650         if ((tran = ddi_get_driver_private(devi)) == NULL)
 1651                 return (DDI_SUCCESS);
 1652 
 1653         if ((mpt = TRAN2MPT(tran)) == NULL)
 1654                 return (DDI_SUCCESS);
 1655 
 1656         /* Disable HBA interrupts in hardware */
 1657         MPTSAS_DISABLE_INTR(mpt);
 1658         /* Send RAID action system shutdonw to sync IR */
 1659         mptsas_raid_action_system_shutdown(mpt);
 1660 
 1661         return (DDI_SUCCESS);
 1662 }
 1663 #endif  /* __sparc */
 1664 
 1665 /*
 1666  * detach(9E).  Remove all device allocations and system resources;
 1667  * disable device interrupts.
 1668  * Return DDI_SUCCESS if done; DDI_FAILURE if there's a problem.
 1669  */
 1670 static int
 1671 mptsas_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
 1672 {
 1673         /* CONSTCOND */
 1674         ASSERT(NO_COMPETING_THREADS);
 1675         NDBG0(("mptsas_detach: dip=0x%p cmd=0x%p", (void *)devi, (void *)cmd));
 1676 
 1677         switch (cmd) {
 1678         case DDI_DETACH:
 1679                 return (mptsas_do_detach(devi));
 1680 
 1681         case DDI_SUSPEND:
 1682                 return (mptsas_suspend(devi));
 1683 
 1684         default:
 1685                 return (DDI_FAILURE);
 1686         }
 1687         /* NOTREACHED */
 1688 }
 1689 
 1690 static int
 1691 mptsas_do_detach(dev_info_t *dip)
 1692 {
 1693         mptsas_t        *mpt;
 1694         scsi_hba_tran_t *tran;
 1695         int             circ = 0;
 1696         int             circ1 = 0;
 1697         mdi_pathinfo_t  *pip = NULL;
 1698         int             i;
 1699         int             doneq_thread_num = 0;
 1700 
 1701         NDBG0(("mptsas_do_detach: dip=0x%p", (void *)dip));
 1702 
 1703         if ((tran = ndi_flavorv_get(dip, SCSA_FLAVOR_SCSI_DEVICE)) == NULL)
 1704                 return (DDI_FAILURE);
 1705 
 1706         mpt = TRAN2MPT(tran);
 1707         if (!mpt) {
 1708                 return (DDI_FAILURE);
 1709         }
 1710         /*
 1711          * Still have pathinfo child, should not detach mpt driver
 1712          */
 1713         if (scsi_hba_iport_unit_address(dip)) {
 1714                 if (mpt->m_mpxio_enable) {
 1715                         /*
 1716                          * MPxIO enabled for the iport
 1717                          */
 1718                         ndi_devi_enter(scsi_vhci_dip, &circ1);
 1719                         ndi_devi_enter(dip, &circ);
 1720                         while (pip = mdi_get_next_client_path(dip, NULL)) {
 1721                                 if (mdi_pi_free(pip, 0) == MDI_SUCCESS) {
 1722                                         continue;
 1723                                 }
 1724                                 ndi_devi_exit(dip, circ);
 1725                                 ndi_devi_exit(scsi_vhci_dip, circ1);
 1726                                 NDBG12(("detach failed because of "
 1727                                     "outstanding path info"));
 1728                                 return (DDI_FAILURE);
 1729                         }
 1730                         ndi_devi_exit(dip, circ);
 1731                         ndi_devi_exit(scsi_vhci_dip, circ1);
 1732                         (void) mdi_phci_unregister(dip, 0);
 1733                 }
 1734 
 1735                 ddi_prop_remove_all(dip);
 1736 
 1737                 return (DDI_SUCCESS);
 1738         }
 1739 
 1740         /* Make sure power level is D0 before accessing registers */
 1741         if (mpt->m_options & MPTSAS_OPT_PM) {
 1742                 (void) pm_busy_component(dip, 0);
 1743                 if (mpt->m_power_level != PM_LEVEL_D0) {
 1744                         if (pm_raise_power(dip, 0, PM_LEVEL_D0) !=
 1745                             DDI_SUCCESS) {
 1746                                 mptsas_log(mpt, CE_WARN,
 1747                                     "mptsas%d: Raise power request failed.",
 1748                                     mpt->m_instance);
 1749                                 (void) pm_idle_component(dip, 0);
 1750                                 return (DDI_FAILURE);
 1751                         }
 1752                 }
 1753         }
 1754 
 1755         /*
 1756          * Send RAID action system shutdown to sync IR.  After action, send a
 1757          * Message Unit Reset. Since after that DMA resource will be freed,
 1758          * set ioc to READY state will avoid HBA initiated DMA operation.
 1759          */
 1760         mutex_enter(&mpt->m_mutex);
 1761         MPTSAS_DISABLE_INTR(mpt);
 1762         mptsas_raid_action_system_shutdown(mpt);
 1763         mpt->m_softstate |= MPTSAS_SS_MSG_UNIT_RESET;
 1764         (void) mptsas_ioc_reset(mpt);
 1765         mutex_exit(&mpt->m_mutex);
 1766         mptsas_rem_intrs(mpt);
 1767         ddi_taskq_destroy(mpt->m_event_taskq);
 1768         ddi_taskq_destroy(mpt->m_dr_taskq);
 1769 
 1770         if (mpt->m_doneq_thread_n) {
 1771                 mutex_enter(&mpt->m_doneq_mutex);
 1772                 doneq_thread_num = mpt->m_doneq_thread_n;
 1773                 for (i = 0; i < mpt->m_doneq_thread_n; i++) {
 1774                         mutex_enter(&mpt->m_doneq_thread_id[i].mutex);
 1775                         mpt->m_doneq_thread_id[i].flag &=
 1776                             (~MPTSAS_DONEQ_THREAD_ACTIVE);
 1777                         cv_signal(&mpt->m_doneq_thread_id[i].cv);
 1778                         mutex_exit(&mpt->m_doneq_thread_id[i].mutex);
 1779                 }
 1780                 while (mpt->m_doneq_thread_n) {
 1781                         cv_wait(&mpt->m_doneq_thread_cv,
 1782                             &mpt->m_doneq_mutex);
 1783                 }
 1784                 for (i = 0;  i < doneq_thread_num; i++) {
 1785                         cv_destroy(&mpt->m_doneq_thread_id[i].cv);
 1786                         mutex_destroy(&mpt->m_doneq_thread_id[i].mutex);
 1787                 }
 1788                 kmem_free(mpt->m_doneq_thread_id,
 1789                     sizeof (mptsas_doneq_thread_list_t)
 1790                     * doneq_thread_num);
 1791                 mutex_exit(&mpt->m_doneq_mutex);
 1792                 cv_destroy(&mpt->m_doneq_thread_cv);
 1793                 mutex_destroy(&mpt->m_doneq_mutex);
 1794         }
 1795 
 1796         scsi_hba_reset_notify_tear_down(mpt->m_reset_notify_listf);
 1797 
 1798         mptsas_list_del(mpt);
 1799 
 1800         /*
 1801          * Cancel timeout threads for this mpt
 1802          */
 1803         mutex_enter(&mpt->m_mutex);
 1804         if (mpt->m_quiesce_timeid) {
 1805                 timeout_id_t tid = mpt->m_quiesce_timeid;
 1806                 mpt->m_quiesce_timeid = 0;
 1807                 mutex_exit(&mpt->m_mutex);
 1808                 (void) untimeout(tid);
 1809                 mutex_enter(&mpt->m_mutex);
 1810         }
 1811 
 1812         if (mpt->m_restart_cmd_timeid) {
 1813                 timeout_id_t tid = mpt->m_restart_cmd_timeid;
 1814                 mpt->m_restart_cmd_timeid = 0;
 1815                 mutex_exit(&mpt->m_mutex);
 1816                 (void) untimeout(tid);
 1817                 mutex_enter(&mpt->m_mutex);
 1818         }
 1819 
 1820         mutex_exit(&mpt->m_mutex);
 1821 
 1822         /*
 1823          * last mpt? ... if active, CANCEL watch threads.
 1824          */
 1825         mutex_enter(&mptsas_global_mutex);
 1826         if (mptsas_head == NULL) {
 1827                 timeout_id_t tid;
 1828                 /*
 1829                  * Clear mptsas_timeouts_enable so that the watch thread
 1830                  * gets restarted on DDI_ATTACH
 1831                  */
 1832                 mptsas_timeouts_enabled = 0;
 1833                 if (mptsas_timeout_id) {
 1834                         tid = mptsas_timeout_id;
 1835                         mptsas_timeout_id = 0;
 1836                         mutex_exit(&mptsas_global_mutex);
 1837                         (void) untimeout(tid);
 1838                         mutex_enter(&mptsas_global_mutex);
 1839                 }
 1840                 if (mptsas_reset_watch) {
 1841                         tid = mptsas_reset_watch;
 1842                         mptsas_reset_watch = 0;
 1843                         mutex_exit(&mptsas_global_mutex);
 1844                         (void) untimeout(tid);
 1845                         mutex_enter(&mptsas_global_mutex);
 1846                 }
 1847         }
 1848         mutex_exit(&mptsas_global_mutex);
 1849 
 1850         /*
 1851          * Delete Phy stats
 1852          */
 1853         mptsas_destroy_phy_stats(mpt);
 1854 
 1855         /*
 1856          * Delete nt_active.
 1857          */
 1858         mutex_enter(&mpt->m_mutex);
 1859         mptsas_hash_uninit(&mpt->m_active->m_tgttbl, sizeof (mptsas_target_t));
 1860         mptsas_hash_uninit(&mpt->m_active->m_smptbl, sizeof (mptsas_smp_t));
 1861         mptsas_free_active_slots(mpt);
 1862         mutex_exit(&mpt->m_mutex);
 1863 
 1864         /* deallocate everything that was allocated in mptsas_attach */
 1865         mptsas_cache_destroy(mpt);
 1866 
 1867         mptsas_hba_fini(mpt);
 1868         mptsas_cfg_fini(mpt);
 1869 
 1870         /* Lower the power informing PM Framework */
 1871         if (mpt->m_options & MPTSAS_OPT_PM) {
 1872                 if (pm_lower_power(dip, 0, PM_LEVEL_D3) != DDI_SUCCESS)
 1873                         mptsas_log(mpt, CE_WARN,
 1874                             "!mptsas%d: Lower power request failed "
 1875                             "during detach, ignoring.",
 1876                             mpt->m_instance);
 1877         }
 1878 
 1879         mutex_destroy(&mpt->m_intr_mutex);
 1880         mutex_destroy(&mpt->m_passthru_mutex);
 1881         mutex_destroy(&mpt->m_mutex);
 1882         for (i = 0; i < MPTSAS_MAX_PHYS; i++) {
 1883                 mutex_destroy(&mpt->m_phy_info[i].smhba_info.phy_mutex);
 1884         }
 1885         cv_destroy(&mpt->m_cv);
 1886         cv_destroy(&mpt->m_passthru_cv);
 1887         cv_destroy(&mpt->m_fw_cv);
 1888         cv_destroy(&mpt->m_config_cv);
 1889         cv_destroy(&mpt->m_fw_diag_cv);
 1890 
 1891 
 1892         mptsas_smp_teardown(mpt);
 1893         mptsas_hba_teardown(mpt);
 1894 
 1895         mptsas_config_space_fini(mpt);
 1896 
 1897         mptsas_free_handshake_msg(mpt);
 1898 
 1899         mptsas_fm_fini(mpt);
 1900         ddi_soft_state_free(mptsas_state, ddi_get_instance(dip));
 1901         ddi_prop_remove_all(dip);
 1902 
 1903         return (DDI_SUCCESS);
 1904 }
 1905 
 1906 static void
 1907 mptsas_list_add(mptsas_t *mpt)
 1908 {
 1909         rw_enter(&mptsas_global_rwlock, RW_WRITER);
 1910 
 1911         if (mptsas_head == NULL) {
 1912                 mptsas_head = mpt;
 1913         } else {
 1914                 mptsas_tail->m_next = mpt;
 1915         }
 1916         mptsas_tail = mpt;
 1917         rw_exit(&mptsas_global_rwlock);
 1918 }
 1919 
 1920 static void
 1921 mptsas_list_del(mptsas_t *mpt)
 1922 {
 1923         mptsas_t *m;
 1924         /*
 1925          * Remove device instance from the global linked list
 1926          */
 1927         rw_enter(&mptsas_global_rwlock, RW_WRITER);
 1928         if (mptsas_head == mpt) {
 1929                 m = mptsas_head = mpt->m_next;
 1930         } else {
 1931                 for (m = mptsas_head; m != NULL; m = m->m_next) {
 1932                         if (m->m_next == mpt) {
 1933                                 m->m_next = mpt->m_next;
 1934                                 break;
 1935                         }
 1936                 }
 1937                 if (m == NULL) {
 1938                         mptsas_log(mpt, CE_PANIC, "Not in softc list!");
 1939                 }
 1940         }
 1941 
 1942         if (mptsas_tail == mpt) {
 1943                 mptsas_tail = m;
 1944         }
 1945         rw_exit(&mptsas_global_rwlock);
 1946 }
 1947 
 1948 static int
 1949 mptsas_alloc_handshake_msg(mptsas_t *mpt, size_t alloc_size)
 1950 {
 1951         ddi_dma_attr_t  task_dma_attrs;
 1952 
 1953         task_dma_attrs = mpt->m_msg_dma_attr;
 1954         task_dma_attrs.dma_attr_sgllen = 1;
 1955         task_dma_attrs.dma_attr_granular = (uint32_t)(alloc_size);
 1956 
 1957         /* allocate Task Management ddi_dma resources */
 1958         if (mptsas_dma_addr_create(mpt, task_dma_attrs,
 1959             &mpt->m_hshk_dma_hdl, &mpt->m_hshk_acc_hdl, &mpt->m_hshk_memp,
 1960             alloc_size, NULL) == FALSE) {
 1961                 return (DDI_FAILURE);
 1962         }
 1963         mpt->m_hshk_dma_size = alloc_size;
 1964 
 1965         return (DDI_SUCCESS);
 1966 }
 1967 
 1968 static void
 1969 mptsas_free_handshake_msg(mptsas_t *mpt)
 1970 {
 1971         mptsas_dma_addr_destroy(&mpt->m_hshk_dma_hdl, &mpt->m_hshk_acc_hdl);
 1972         mpt->m_hshk_dma_size = 0;
 1973 }
 1974 
 1975 static int
 1976 mptsas_hba_setup(mptsas_t *mpt)
 1977 {
 1978         scsi_hba_tran_t         *hba_tran;
 1979         int                     tran_flags;
 1980 
 1981         /* Allocate a transport structure */
 1982         hba_tran = mpt->m_tran = scsi_hba_tran_alloc(mpt->m_dip,
 1983             SCSI_HBA_CANSLEEP);
 1984         ASSERT(mpt->m_tran != NULL);
 1985 
 1986         hba_tran->tran_hba_private      = mpt;
 1987         hba_tran->tran_tgt_private      = NULL;
 1988 
 1989         hba_tran->tran_tgt_init         = mptsas_scsi_tgt_init;
 1990         hba_tran->tran_tgt_free         = mptsas_scsi_tgt_free;
 1991 
 1992         hba_tran->tran_start            = mptsas_scsi_start;
 1993         hba_tran->tran_reset            = mptsas_scsi_reset;
 1994         hba_tran->tran_abort            = mptsas_scsi_abort;
 1995         hba_tran->tran_getcap           = mptsas_scsi_getcap;
 1996         hba_tran->tran_setcap           = mptsas_scsi_setcap;
 1997         hba_tran->tran_init_pkt         = mptsas_scsi_init_pkt;
 1998         hba_tran->tran_destroy_pkt      = mptsas_scsi_destroy_pkt;
 1999 
 2000         hba_tran->tran_dmafree          = mptsas_scsi_dmafree;
 2001         hba_tran->tran_sync_pkt         = mptsas_scsi_sync_pkt;
 2002         hba_tran->tran_reset_notify     = mptsas_scsi_reset_notify;
 2003 
 2004         hba_tran->tran_get_bus_addr     = mptsas_get_bus_addr;
 2005         hba_tran->tran_get_name         = mptsas_get_name;
 2006 
 2007         hba_tran->tran_quiesce          = mptsas_scsi_quiesce;
 2008         hba_tran->tran_unquiesce        = mptsas_scsi_unquiesce;
 2009         hba_tran->tran_bus_reset        = NULL;
 2010 
 2011         hba_tran->tran_add_eventcall    = NULL;
 2012         hba_tran->tran_get_eventcookie  = NULL;
 2013         hba_tran->tran_post_event       = NULL;
 2014         hba_tran->tran_remove_eventcall = NULL;
 2015 
 2016         hba_tran->tran_bus_config       = mptsas_bus_config;
 2017 
 2018         hba_tran->tran_interconnect_type = INTERCONNECT_SAS;
 2019 
 2020         /*
 2021          * All children of the HBA are iports. We need tran was cloned.
 2022          * So we pass the flags to SCSA. SCSI_HBA_TRAN_CLONE will be
 2023          * inherited to iport's tran vector.
 2024          */
 2025         tran_flags = (SCSI_HBA_HBA | SCSI_HBA_TRAN_CLONE);
 2026 
 2027         if (scsi_hba_attach_setup(mpt->m_dip, &mpt->m_msg_dma_attr,
 2028             hba_tran, tran_flags) != DDI_SUCCESS) {
 2029                 mptsas_log(mpt, CE_WARN, "hba attach setup failed");
 2030                 scsi_hba_tran_free(hba_tran);
 2031                 mpt->m_tran = NULL;
 2032                 return (FALSE);
 2033         }
 2034         return (TRUE);
 2035 }
 2036 
 2037 static void
 2038 mptsas_hba_teardown(mptsas_t *mpt)
 2039 {
 2040         (void) scsi_hba_detach(mpt->m_dip);
 2041         if (mpt->m_tran != NULL) {
 2042                 scsi_hba_tran_free(mpt->m_tran);
 2043                 mpt->m_tran = NULL;
 2044         }
 2045 }
 2046 
 2047 static void
 2048 mptsas_iport_register(mptsas_t *mpt)
 2049 {
 2050         int i, j;
 2051         mptsas_phymask_t        mask = 0x0;
 2052         /*
 2053          * initial value of mask is 0
 2054          */
 2055         mutex_enter(&mpt->m_mutex);
 2056         for (i = 0; i < mpt->m_num_phys; i++) {
 2057                 mptsas_phymask_t phy_mask = 0x0;
 2058                 char phy_mask_name[MPTSAS_MAX_PHYS];
 2059                 uint8_t current_port;
 2060 
 2061                 if (mpt->m_phy_info[i].attached_devhdl == 0)
 2062                         continue;
 2063 
 2064                 bzero(phy_mask_name, sizeof (phy_mask_name));
 2065 
 2066                 current_port = mpt->m_phy_info[i].port_num;
 2067 
 2068                 if ((mask & (1 << i)) != 0)
 2069                         continue;
 2070 
 2071                 for (j = 0; j < mpt->m_num_phys; j++) {
 2072                         if (mpt->m_phy_info[j].attached_devhdl &&
 2073                             (mpt->m_phy_info[j].port_num == current_port)) {
 2074                                 phy_mask |= (1 << j);
 2075                         }
 2076                 }
 2077                 mask = mask | phy_mask;
 2078 
 2079                 for (j = 0; j < mpt->m_num_phys; j++) {
 2080                         if ((phy_mask >> j) & 0x01) {
 2081                                 mpt->m_phy_info[j].phy_mask = phy_mask;
 2082                         }
 2083                 }
 2084 
 2085                 (void) sprintf(phy_mask_name, "%x", phy_mask);
 2086 
 2087                 mutex_exit(&mpt->m_mutex);
 2088                 /*
 2089                  * register a iport
 2090                  */
 2091                 (void) scsi_hba_iport_register(mpt->m_dip, phy_mask_name);
 2092                 mutex_enter(&mpt->m_mutex);
 2093         }
 2094         mutex_exit(&mpt->m_mutex);
 2095         /*
 2096          * register a virtual port for RAID volume always
 2097          */
 2098         (void) scsi_hba_iport_register(mpt->m_dip, "v0");
 2099 
 2100 }
 2101 
 2102 static int
 2103 mptsas_smp_setup(mptsas_t *mpt)
 2104 {
 2105         mpt->m_smptran = smp_hba_tran_alloc(mpt->m_dip);
 2106         ASSERT(mpt->m_smptran != NULL);
 2107         mpt->m_smptran->smp_tran_hba_private = mpt;
 2108         mpt->m_smptran->smp_tran_start = mptsas_smp_start;
 2109         if (smp_hba_attach_setup(mpt->m_dip, mpt->m_smptran) != DDI_SUCCESS) {
 2110                 mptsas_log(mpt, CE_WARN, "smp attach setup failed");
 2111                 smp_hba_tran_free(mpt->m_smptran);
 2112                 mpt->m_smptran = NULL;
 2113                 return (FALSE);
 2114         }
 2115         /*
 2116          * Initialize smp hash table
 2117          */
 2118         mptsas_hash_init(&mpt->m_active->m_smptbl);
 2119         mpt->m_smp_devhdl = 0xFFFF;
 2120 
 2121         return (TRUE);
 2122 }
 2123 
 2124 static void
 2125 mptsas_smp_teardown(mptsas_t *mpt)
 2126 {
 2127         (void) smp_hba_detach(mpt->m_dip);
 2128         if (mpt->m_smptran != NULL) {
 2129                 smp_hba_tran_free(mpt->m_smptran);
 2130                 mpt->m_smptran = NULL;
 2131         }
 2132         mpt->m_smp_devhdl = 0;
 2133 }
 2134 
 2135 static int
 2136 mptsas_cache_create(mptsas_t *mpt)
 2137 {
 2138         int instance = mpt->m_instance;
 2139         char buf[64];
 2140 
 2141         /*
 2142          * create kmem cache for packets
 2143          */
 2144         (void) sprintf(buf, "mptsas%d_cache", instance);
 2145         mpt->m_kmem_cache = kmem_cache_create(buf,
 2146             sizeof (struct mptsas_cmd) + scsi_pkt_size(), 8,
 2147             mptsas_kmem_cache_constructor, mptsas_kmem_cache_destructor,
 2148             NULL, (void *)mpt, NULL, 0);
 2149 
 2150         if (mpt->m_kmem_cache == NULL) {
 2151                 mptsas_log(mpt, CE_WARN, "creating kmem cache failed");
 2152                 return (FALSE);
 2153         }
 2154 
 2155         /*
 2156          * create kmem cache for extra SGL frames if SGL cannot
 2157          * be accomodated into main request frame.
 2158          */
 2159         (void) sprintf(buf, "mptsas%d_cache_frames", instance);
 2160         mpt->m_cache_frames = kmem_cache_create(buf,
 2161             sizeof (mptsas_cache_frames_t), 8,
 2162             mptsas_cache_frames_constructor, mptsas_cache_frames_destructor,
 2163             NULL, (void *)mpt, NULL, 0);
 2164 
 2165         if (mpt->m_cache_frames == NULL) {
 2166                 mptsas_log(mpt, CE_WARN, "creating cache for frames failed");
 2167                 return (FALSE);
 2168         }
 2169 
 2170         return (TRUE);
 2171 }
 2172 
 2173 static void
 2174 mptsas_cache_destroy(mptsas_t *mpt)
 2175 {
 2176         /* deallocate in reverse order */
 2177         if (mpt->m_cache_frames) {
 2178                 kmem_cache_destroy(mpt->m_cache_frames);
 2179                 mpt->m_cache_frames = NULL;
 2180         }
 2181         if (mpt->m_kmem_cache) {
 2182                 kmem_cache_destroy(mpt->m_kmem_cache);
 2183                 mpt->m_kmem_cache = NULL;
 2184         }
 2185 }
 2186 
 2187 static int
 2188 mptsas_power(dev_info_t *dip, int component, int level)
 2189 {
 2190 #ifndef __lock_lint
 2191         _NOTE(ARGUNUSED(component))
 2192 #endif
 2193         mptsas_t        *mpt;
 2194         int             rval = DDI_SUCCESS;
 2195         int             polls = 0;
 2196         uint32_t        ioc_status;
 2197 
 2198         if (scsi_hba_iport_unit_address(dip) != 0)
 2199                 return (DDI_SUCCESS);
 2200 
 2201         mpt = ddi_get_soft_state(mptsas_state, ddi_get_instance(dip));
 2202         if (mpt == NULL) {
 2203                 return (DDI_FAILURE);
 2204         }
 2205 
 2206         mutex_enter(&mpt->m_mutex);
 2207 
 2208         /*
 2209          * If the device is busy, don't lower its power level
 2210          */
 2211         if (mpt->m_busy && (mpt->m_power_level > level)) {
 2212                 mutex_exit(&mpt->m_mutex);
 2213                 return (DDI_FAILURE);
 2214         }
 2215         switch (level) {
 2216         case PM_LEVEL_D0:
 2217                 NDBG11(("mptsas%d: turning power ON.", mpt->m_instance));
 2218                 MPTSAS_POWER_ON(mpt);
 2219                 /*
 2220                  * Wait up to 30 seconds for IOC to come out of reset.
 2221                  */
 2222                 while (((ioc_status = ddi_get32(mpt->m_datap,
 2223                     &mpt->m_reg->Doorbell)) &
 2224                     MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
 2225                         if (polls++ > 3000) {
 2226                                 break;
 2227                         }
 2228                         delay(drv_usectohz(10000));
 2229                 }
 2230                 /*
 2231                  * If IOC is not in operational state, try to hard reset it.
 2232                  */
 2233                 if ((ioc_status & MPI2_IOC_STATE_MASK) !=
 2234                     MPI2_IOC_STATE_OPERATIONAL) {
 2235                         mpt->m_softstate &= ~MPTSAS_SS_MSG_UNIT_RESET;
 2236                         if (mptsas_restart_ioc(mpt) == DDI_FAILURE) {
 2237                                 mptsas_log(mpt, CE_WARN,
 2238                                     "mptsas_power: hard reset failed");
 2239                                 mutex_exit(&mpt->m_mutex);
 2240                                 return (DDI_FAILURE);
 2241                         }
 2242                 }
 2243                 mutex_enter(&mpt->m_intr_mutex);
 2244                 mpt->m_power_level = PM_LEVEL_D0;
 2245                 mutex_exit(&mpt->m_intr_mutex);
 2246                 break;
 2247         case PM_LEVEL_D3:
 2248                 NDBG11(("mptsas%d: turning power OFF.", mpt->m_instance));
 2249                 MPTSAS_POWER_OFF(mpt);
 2250                 break;
 2251         default:
 2252                 mptsas_log(mpt, CE_WARN, "mptsas%d: unknown power level <%x>.",
 2253                     mpt->m_instance, level);
 2254                 rval = DDI_FAILURE;
 2255                 break;
 2256         }
 2257         mutex_exit(&mpt->m_mutex);
 2258         return (rval);
 2259 }
 2260 
 2261 /*
 2262  * Initialize configuration space and figure out which
 2263  * chip and revison of the chip the mpt driver is using.
 2264  */
 2265 static int
 2266 mptsas_config_space_init(mptsas_t *mpt)
 2267 {
 2268         NDBG0(("mptsas_config_space_init"));
 2269 
 2270         if (mpt->m_config_handle != NULL)
 2271                 return (TRUE);
 2272 
 2273         if (pci_config_setup(mpt->m_dip,
 2274             &mpt->m_config_handle) != DDI_SUCCESS) {
 2275                 mptsas_log(mpt, CE_WARN, "cannot map configuration space.");
 2276                 return (FALSE);
 2277         }
 2278 
 2279         /*
 2280          * This is a workaround for a XMITS ASIC bug which does not
 2281          * drive the CBE upper bits.
 2282          */
 2283         if (pci_config_get16(mpt->m_config_handle, PCI_CONF_STAT) &
 2284             PCI_STAT_PERROR) {
 2285                 pci_config_put16(mpt->m_config_handle, PCI_CONF_STAT,
 2286                     PCI_STAT_PERROR);
 2287         }
 2288 
 2289         mptsas_setup_cmd_reg(mpt);
 2290 
 2291         /*
 2292          * Get the chip device id:
 2293          */
 2294         mpt->m_devid = pci_config_get16(mpt->m_config_handle, PCI_CONF_DEVID);
 2295 
 2296         /*
 2297          * Save the revision.
 2298          */
 2299         mpt->m_revid = pci_config_get8(mpt->m_config_handle, PCI_CONF_REVID);
 2300 
 2301         /*
 2302          * Save the SubSystem Vendor and Device IDs
 2303          */
 2304         mpt->m_svid = pci_config_get16(mpt->m_config_handle, PCI_CONF_SUBVENID);
 2305         mpt->m_ssid = pci_config_get16(mpt->m_config_handle, PCI_CONF_SUBSYSID);
 2306 
 2307         /*
 2308          * Set the latency timer to 0x40 as specified by the upa -> pci
 2309          * bridge chip design team.  This may be done by the sparc pci
 2310          * bus nexus driver, but the driver should make sure the latency
 2311          * timer is correct for performance reasons.
 2312          */
 2313         pci_config_put8(mpt->m_config_handle, PCI_CONF_LATENCY_TIMER,
 2314             MPTSAS_LATENCY_TIMER);
 2315 
 2316         (void) mptsas_get_pci_cap(mpt);
 2317         return (TRUE);
 2318 }
 2319 
 2320 static void
 2321 mptsas_config_space_fini(mptsas_t *mpt)
 2322 {
 2323         if (mpt->m_config_handle != NULL) {
 2324                 mptsas_disable_bus_master(mpt);
 2325                 pci_config_teardown(&mpt->m_config_handle);
 2326                 mpt->m_config_handle = NULL;
 2327         }
 2328 }
 2329 
 2330 static void
 2331 mptsas_setup_cmd_reg(mptsas_t *mpt)
 2332 {
 2333         ushort_t        cmdreg;
 2334 
 2335         /*
 2336          * Set the command register to the needed values.
 2337          */
 2338         cmdreg = pci_config_get16(mpt->m_config_handle, PCI_CONF_COMM);
 2339         cmdreg |= (PCI_COMM_ME | PCI_COMM_SERR_ENABLE |
 2340             PCI_COMM_PARITY_DETECT | PCI_COMM_MAE);
 2341         cmdreg &= ~PCI_COMM_IO;
 2342         pci_config_put16(mpt->m_config_handle, PCI_CONF_COMM, cmdreg);
 2343 }
 2344 
 2345 static void
 2346 mptsas_disable_bus_master(mptsas_t *mpt)
 2347 {
 2348         ushort_t        cmdreg;
 2349 
 2350         /*
 2351          * Clear the master enable bit in the PCI command register.
 2352          * This prevents any bus mastering activity like DMA.
 2353          */
 2354         cmdreg = pci_config_get16(mpt->m_config_handle, PCI_CONF_COMM);
 2355         cmdreg &= ~PCI_COMM_ME;
 2356         pci_config_put16(mpt->m_config_handle, PCI_CONF_COMM, cmdreg);
 2357 }
 2358 
 2359 int
 2360 mptsas_dma_alloc(mptsas_t *mpt, mptsas_dma_alloc_state_t *dma_statep)
 2361 {
 2362         ddi_dma_attr_t  attrs;
 2363 
 2364         attrs = mpt->m_io_dma_attr;
 2365         attrs.dma_attr_sgllen = 1;
 2366 
 2367         ASSERT(dma_statep != NULL);
 2368 
 2369         if (mptsas_dma_addr_create(mpt, attrs, &dma_statep->handle,
 2370             &dma_statep->accessp, &dma_statep->memp, dma_statep->size,
 2371             &dma_statep->cookie) == FALSE) {
 2372                 return (DDI_FAILURE);
 2373         }
 2374 
 2375         return (DDI_SUCCESS);
 2376 }
 2377 
 2378 void
 2379 mptsas_dma_free(mptsas_dma_alloc_state_t *dma_statep)
 2380 {
 2381         ASSERT(dma_statep != NULL);
 2382         mptsas_dma_addr_destroy(&dma_statep->handle, &dma_statep->accessp);
 2383         dma_statep->size = 0;
 2384 }
 2385 
 2386 int
 2387 mptsas_do_dma(mptsas_t *mpt, uint32_t size, int var, int (*callback)())
 2388 {
 2389         ddi_dma_attr_t          attrs;
 2390         ddi_dma_handle_t        dma_handle;
 2391         caddr_t                 memp;
 2392         ddi_acc_handle_t        accessp;
 2393         int                     rval;
 2394 
 2395         ASSERT(mutex_owned(&mpt->m_mutex));
 2396 
 2397         attrs = mpt->m_msg_dma_attr;
 2398         attrs.dma_attr_sgllen = 1;
 2399         attrs.dma_attr_granular = size;
 2400 
 2401         if (mptsas_dma_addr_create(mpt, attrs, &dma_handle,
 2402             &accessp, &memp, size, NULL) == FALSE) {
 2403                 return (DDI_FAILURE);
 2404         }
 2405 
 2406         rval = (*callback) (mpt, memp, var, accessp);
 2407 
 2408         if ((mptsas_check_dma_handle(dma_handle) != DDI_SUCCESS) ||
 2409             (mptsas_check_acc_handle(accessp) != DDI_SUCCESS)) {
 2410                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 2411                 rval = DDI_FAILURE;
 2412         }
 2413 
 2414         mptsas_dma_addr_destroy(&dma_handle, &accessp);
 2415         return (rval);
 2416 
 2417 }
 2418 
 2419 static int
 2420 mptsas_alloc_request_frames(mptsas_t *mpt)
 2421 {
 2422         ddi_dma_attr_t          frame_dma_attrs;
 2423         caddr_t                 memp;
 2424         ddi_dma_cookie_t        cookie;
 2425         size_t                  mem_size;
 2426 
 2427         /*
 2428          * re-alloc when it has already alloced
 2429          */
 2430         mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
 2431             &mpt->m_acc_req_frame_hdl);
 2432 
 2433         /*
 2434          * The size of the request frame pool is:
 2435          *   Number of Request Frames * Request Frame Size
 2436          */
 2437         mem_size = mpt->m_max_requests * mpt->m_req_frame_size;
 2438 
 2439         /*
 2440          * set the DMA attributes.  System Request Message Frames must be
 2441          * aligned on a 16-byte boundry.
 2442          */
 2443         frame_dma_attrs = mpt->m_msg_dma_attr;
 2444         frame_dma_attrs.dma_attr_align = 16;
 2445         frame_dma_attrs.dma_attr_sgllen = 1;
 2446 
 2447         /*
 2448          * allocate the request frame pool.
 2449          */
 2450         if (mptsas_dma_addr_create(mpt, frame_dma_attrs,
 2451             &mpt->m_dma_req_frame_hdl, &mpt->m_acc_req_frame_hdl, &memp,
 2452             mem_size, &cookie) == FALSE) {
 2453                 return (DDI_FAILURE);
 2454         }
 2455 
 2456         /*
 2457          * Store the request frame memory address.  This chip uses this
 2458          * address to dma to and from the driver's frame.  The second
 2459          * address is the address mpt uses to fill in the frame.
 2460          */
 2461         mpt->m_req_frame_dma_addr = cookie.dmac_laddress;
 2462         mpt->m_req_frame = memp;
 2463 
 2464         /*
 2465          * Clear the request frame pool.
 2466          */
 2467         bzero(mpt->m_req_frame, mem_size);
 2468 
 2469         return (DDI_SUCCESS);
 2470 }
 2471 
 2472 static int
 2473 mptsas_alloc_reply_frames(mptsas_t *mpt)
 2474 {
 2475         ddi_dma_attr_t          frame_dma_attrs;
 2476         caddr_t                 memp;
 2477         ddi_dma_cookie_t        cookie;
 2478         size_t                  mem_size;
 2479 
 2480         /*
 2481          * re-alloc when it has already alloced
 2482          */
 2483         mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
 2484             &mpt->m_acc_reply_frame_hdl);
 2485 
 2486         /*
 2487          * The size of the reply frame pool is:
 2488          *   Number of Reply Frames * Reply Frame Size
 2489          */
 2490         mem_size = mpt->m_max_replies * mpt->m_reply_frame_size;
 2491 
 2492         /*
 2493          * set the DMA attributes.   System Reply Message Frames must be
 2494          * aligned on a 4-byte boundry.  This is the default.
 2495          */
 2496         frame_dma_attrs = mpt->m_msg_dma_attr;
 2497         frame_dma_attrs.dma_attr_sgllen = 1;
 2498 
 2499         /*
 2500          * allocate the reply frame pool
 2501          */
 2502         if (mptsas_dma_addr_create(mpt, frame_dma_attrs,
 2503             &mpt->m_dma_reply_frame_hdl, &mpt->m_acc_reply_frame_hdl, &memp,
 2504             mem_size, &cookie) == FALSE) {
 2505                 return (DDI_FAILURE);
 2506         }
 2507 
 2508         /*
 2509          * Store the reply frame memory address.  This chip uses this
 2510          * address to dma to and from the driver's frame.  The second
 2511          * address is the address mpt uses to process the frame.
 2512          */
 2513         mpt->m_reply_frame_dma_addr = cookie.dmac_laddress;
 2514         mpt->m_reply_frame = memp;
 2515 
 2516         /*
 2517          * Clear the reply frame pool.
 2518          */
 2519         bzero(mpt->m_reply_frame, mem_size);
 2520 
 2521         return (DDI_SUCCESS);
 2522 }
 2523 
 2524 static int
 2525 mptsas_alloc_free_queue(mptsas_t *mpt)
 2526 {
 2527         ddi_dma_attr_t          frame_dma_attrs;
 2528         caddr_t                 memp;
 2529         ddi_dma_cookie_t        cookie;
 2530         size_t                  mem_size;
 2531 
 2532         /*
 2533          * re-alloc when it has already alloced
 2534          */
 2535         mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
 2536             &mpt->m_acc_free_queue_hdl);
 2537 
 2538         /*
 2539          * The reply free queue size is:
 2540          *   Reply Free Queue Depth * 4
 2541          * The "4" is the size of one 32 bit address (low part of 64-bit
 2542          *   address)
 2543          */
 2544         mem_size = mpt->m_free_queue_depth * 4;
 2545 
 2546         /*
 2547          * set the DMA attributes  The Reply Free Queue must be aligned on a
 2548          * 16-byte boundry.
 2549          */
 2550         frame_dma_attrs = mpt->m_msg_dma_attr;
 2551         frame_dma_attrs.dma_attr_align = 16;
 2552         frame_dma_attrs.dma_attr_sgllen = 1;
 2553 
 2554         /*
 2555          * allocate the reply free queue
 2556          */
 2557         if (mptsas_dma_addr_create(mpt, frame_dma_attrs,
 2558             &mpt->m_dma_free_queue_hdl, &mpt->m_acc_free_queue_hdl, &memp,
 2559             mem_size, &cookie) == FALSE) {
 2560                 return (DDI_FAILURE);
 2561         }
 2562 
 2563         /*
 2564          * Store the reply free queue memory address.  This chip uses this
 2565          * address to read from the reply free queue.  The second address
 2566          * is the address mpt uses to manage the queue.
 2567          */
 2568         mpt->m_free_queue_dma_addr = cookie.dmac_laddress;
 2569         mpt->m_free_queue = memp;
 2570 
 2571         /*
 2572          * Clear the reply free queue memory.
 2573          */
 2574         bzero(mpt->m_free_queue, mem_size);
 2575 
 2576         return (DDI_SUCCESS);
 2577 }
 2578 
 2579 static int
 2580 mptsas_alloc_post_queue(mptsas_t *mpt)
 2581 {
 2582         ddi_dma_attr_t          frame_dma_attrs;
 2583         caddr_t                 memp;
 2584         ddi_dma_cookie_t        cookie;
 2585         size_t                  mem_size;
 2586 
 2587         /*
 2588          * re-alloc when it has already alloced
 2589          */
 2590         mptsas_dma_addr_destroy(&mpt->m_dma_post_queue_hdl,
 2591             &mpt->m_acc_post_queue_hdl);
 2592 
 2593         /*
 2594          * The reply descriptor post queue size is:
 2595          *   Reply Descriptor Post Queue Depth * 8
 2596          * The "8" is the size of each descriptor (8 bytes or 64 bits).
 2597          */
 2598         mem_size = mpt->m_post_queue_depth * 8;
 2599 
 2600         /*
 2601          * set the DMA attributes.  The Reply Descriptor Post Queue must be
 2602          * aligned on a 16-byte boundry.
 2603          */
 2604         frame_dma_attrs = mpt->m_msg_dma_attr;
 2605         frame_dma_attrs.dma_attr_align = 16;
 2606         frame_dma_attrs.dma_attr_sgllen = 1;
 2607 
 2608         /*
 2609          * allocate the reply post queue
 2610          */
 2611         if (mptsas_dma_addr_create(mpt, frame_dma_attrs,
 2612             &mpt->m_dma_post_queue_hdl, &mpt->m_acc_post_queue_hdl, &memp,
 2613             mem_size, &cookie) == FALSE) {
 2614                 return (DDI_FAILURE);
 2615         }
 2616 
 2617         /*
 2618          * Store the reply descriptor post queue memory address.  This chip
 2619          * uses this address to write to the reply descriptor post queue.  The
 2620          * second address is the address mpt uses to manage the queue.
 2621          */
 2622         mpt->m_post_queue_dma_addr = cookie.dmac_laddress;
 2623         mpt->m_post_queue = memp;
 2624 
 2625         /*
 2626          * Clear the reply post queue memory.
 2627          */
 2628         bzero(mpt->m_post_queue, mem_size);
 2629 
 2630         return (DDI_SUCCESS);
 2631 }
 2632 
 2633 static void
 2634 mptsas_alloc_reply_args(mptsas_t *mpt)
 2635 {
 2636         if (mpt->m_replyh_args != NULL) {
 2637                 kmem_free(mpt->m_replyh_args, sizeof (m_replyh_arg_t)
 2638                     * mpt->m_max_replies);
 2639                 mpt->m_replyh_args = NULL;
 2640         }
 2641         mpt->m_replyh_args = kmem_zalloc(sizeof (m_replyh_arg_t) *
 2642             mpt->m_max_replies, KM_SLEEP);
 2643 }
 2644 
 2645 static int
 2646 mptsas_alloc_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd)
 2647 {
 2648         mptsas_cache_frames_t   *frames = NULL;
 2649         if (cmd->cmd_extra_frames == NULL) {
 2650                 frames = kmem_cache_alloc(mpt->m_cache_frames, KM_NOSLEEP);
 2651                 if (frames == NULL) {
 2652                         return (DDI_FAILURE);
 2653                 }
 2654                 cmd->cmd_extra_frames = frames;
 2655         }
 2656         return (DDI_SUCCESS);
 2657 }
 2658 
 2659 static void
 2660 mptsas_free_extra_sgl_frame(mptsas_t *mpt, mptsas_cmd_t *cmd)
 2661 {
 2662         if (cmd->cmd_extra_frames) {
 2663                 kmem_cache_free(mpt->m_cache_frames,
 2664                     (void *)cmd->cmd_extra_frames);
 2665                 cmd->cmd_extra_frames = NULL;
 2666         }
 2667 }
 2668 
 2669 static void
 2670 mptsas_cfg_fini(mptsas_t *mpt)
 2671 {
 2672         NDBG0(("mptsas_cfg_fini"));
 2673         ddi_regs_map_free(&mpt->m_datap);
 2674 }
 2675 
 2676 static void
 2677 mptsas_hba_fini(mptsas_t *mpt)
 2678 {
 2679         NDBG0(("mptsas_hba_fini"));
 2680 
 2681         /*
 2682          * Free up any allocated memory
 2683          */
 2684         mptsas_dma_addr_destroy(&mpt->m_dma_req_frame_hdl,
 2685             &mpt->m_acc_req_frame_hdl);
 2686 
 2687         mptsas_dma_addr_destroy(&mpt->m_dma_reply_frame_hdl,
 2688             &mpt->m_acc_reply_frame_hdl);
 2689 
 2690         mptsas_dma_addr_destroy(&mpt->m_dma_free_queue_hdl,
 2691             &mpt->m_acc_free_queue_hdl);
 2692 
 2693         mptsas_dma_addr_destroy(&mpt->m_dma_post_queue_hdl,
 2694             &mpt->m_acc_post_queue_hdl);
 2695 
 2696         if (mpt->m_replyh_args != NULL) {
 2697                 kmem_free(mpt->m_replyh_args, sizeof (m_replyh_arg_t)
 2698                     * mpt->m_max_replies);
 2699         }
 2700 }
 2701 
 2702 static int
 2703 mptsas_name_child(dev_info_t *lun_dip, char *name, int len)
 2704 {
 2705         int             lun = 0;
 2706         char            *sas_wwn = NULL;
 2707         int             phynum = -1;
 2708         int             reallen = 0;
 2709 
 2710         /* Get the target num */
 2711         lun = ddi_prop_get_int(DDI_DEV_T_ANY, lun_dip, DDI_PROP_DONTPASS,
 2712             LUN_PROP, 0);
 2713 
 2714         if ((phynum = ddi_prop_get_int(DDI_DEV_T_ANY, lun_dip,
 2715             DDI_PROP_DONTPASS, "sata-phy", -1)) != -1) {
 2716                 /*
 2717                  * Stick in the address of form "pPHY,LUN"
 2718                  */
 2719                 reallen = snprintf(name, len, "p%x,%x", phynum, lun);
 2720         } else if (ddi_prop_lookup_string(DDI_DEV_T_ANY, lun_dip,
 2721             DDI_PROP_DONTPASS, SCSI_ADDR_PROP_TARGET_PORT, &sas_wwn)
 2722             == DDI_PROP_SUCCESS) {
 2723                 /*
 2724                  * Stick in the address of the form "wWWN,LUN"
 2725                  */
 2726                 reallen = snprintf(name, len, "%s,%x", sas_wwn, lun);
 2727                 ddi_prop_free(sas_wwn);
 2728         } else {
 2729                 return (DDI_FAILURE);
 2730         }
 2731 
 2732         ASSERT(reallen < len);
 2733         if (reallen >= len) {
 2734                 mptsas_log(0, CE_WARN, "!mptsas_get_name: name parameter "
 2735                     "length too small, it needs to be %d bytes", reallen + 1);
 2736         }
 2737         return (DDI_SUCCESS);
 2738 }
 2739 
 2740 /*
 2741  * tran_tgt_init(9E) - target device instance initialization
 2742  */
 2743 static int
 2744 mptsas_scsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
 2745     scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
 2746 {
 2747 #ifndef __lock_lint
 2748         _NOTE(ARGUNUSED(hba_tran))
 2749 #endif
 2750 
 2751         /*
 2752          * At this point, the scsi_device structure already exists
 2753          * and has been initialized.
 2754          *
 2755          * Use this function to allocate target-private data structures,
 2756          * if needed by this HBA.  Add revised flow-control and queue
 2757          * properties for child here, if desired and if you can tell they
 2758          * support tagged queueing by now.
 2759          */
 2760         mptsas_t                *mpt;
 2761         int                     lun = sd->sd_address.a_lun;
 2762         mdi_pathinfo_t          *pip = NULL;
 2763         mptsas_tgt_private_t    *tgt_private = NULL;
 2764         mptsas_target_t         *ptgt = NULL;
 2765         char                    *psas_wwn = NULL;
 2766         int                     phymask = 0;
 2767         uint64_t                sas_wwn = 0;
 2768         mpt = SDEV2MPT(sd);
 2769 
 2770         ASSERT(scsi_hba_iport_unit_address(hba_dip) != 0);
 2771 
 2772         NDBG0(("mptsas_scsi_tgt_init: hbadip=0x%p tgtdip=0x%p lun=%d",
 2773             (void *)hba_dip, (void *)tgt_dip, lun));
 2774 
 2775         if (ndi_dev_is_persistent_node(tgt_dip) == 0) {
 2776                 (void) ndi_merge_node(tgt_dip, mptsas_name_child);
 2777                 ddi_set_name_addr(tgt_dip, NULL);
 2778                 return (DDI_FAILURE);
 2779         }
 2780         /*
 2781          * phymask is 0 means the virtual port for RAID
 2782          */
 2783         phymask = ddi_prop_get_int(DDI_DEV_T_ANY, hba_dip, 0,
 2784             "phymask", 0);
 2785         if (mdi_component_is_client(tgt_dip, NULL) == MDI_SUCCESS) {
 2786                 if ((pip = (void *)(sd->sd_private)) == NULL) {
 2787                         /*
 2788                          * Very bad news if this occurs. Somehow scsi_vhci has
 2789                          * lost the pathinfo node for this target.
 2790                          */
 2791                         return (DDI_NOT_WELL_FORMED);
 2792                 }
 2793 
 2794                 if (mdi_prop_lookup_int(pip, LUN_PROP, &lun) !=
 2795                     DDI_PROP_SUCCESS) {
 2796                         mptsas_log(mpt, CE_WARN, "Get lun property failed\n");
 2797                         return (DDI_FAILURE);
 2798                 }
 2799 
 2800                 if (mdi_prop_lookup_string(pip, SCSI_ADDR_PROP_TARGET_PORT,
 2801                     &psas_wwn) == MDI_SUCCESS) {
 2802                         if (scsi_wwnstr_to_wwn(psas_wwn, &sas_wwn)) {
 2803                                 sas_wwn = 0;
 2804                         }
 2805                         (void) mdi_prop_free(psas_wwn);
 2806                 }
 2807         } else {
 2808                 lun = ddi_prop_get_int(DDI_DEV_T_ANY, tgt_dip,
 2809                     DDI_PROP_DONTPASS, LUN_PROP, 0);
 2810                 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, tgt_dip,
 2811                     DDI_PROP_DONTPASS, SCSI_ADDR_PROP_TARGET_PORT, &psas_wwn) ==
 2812                     DDI_PROP_SUCCESS) {
 2813                         if (scsi_wwnstr_to_wwn(psas_wwn, &sas_wwn)) {
 2814                                 sas_wwn = 0;
 2815                         }
 2816                         ddi_prop_free(psas_wwn);
 2817                 } else {
 2818                         sas_wwn = 0;
 2819                 }
 2820         }
 2821         ASSERT((sas_wwn != 0) || (phymask != 0));
 2822         mutex_enter(&mpt->m_mutex);
 2823         ptgt = mptsas_hash_search(&mpt->m_active->m_tgttbl, sas_wwn, phymask);
 2824         mutex_exit(&mpt->m_mutex);
 2825         if (ptgt == NULL) {
 2826                 mptsas_log(mpt, CE_WARN, "!tgt_init: target doesn't exist or "
 2827                     "gone already! phymask:%x, saswwn %"PRIx64, phymask,
 2828                     sas_wwn);
 2829                 return (DDI_FAILURE);
 2830         }
 2831         if (hba_tran->tran_tgt_private == NULL) {
 2832                 tgt_private = kmem_zalloc(sizeof (mptsas_tgt_private_t),
 2833                     KM_SLEEP);
 2834                 tgt_private->t_lun = lun;
 2835                 tgt_private->t_private = ptgt;
 2836                 hba_tran->tran_tgt_private = tgt_private;
 2837         }
 2838 
 2839         if (mdi_component_is_client(tgt_dip, NULL) == MDI_SUCCESS) {
 2840                 return (DDI_SUCCESS);
 2841         }
 2842         mutex_enter(&mpt->m_mutex);
 2843 
 2844         if (ptgt->m_deviceinfo &
 2845             (MPI2_SAS_DEVICE_INFO_SATA_DEVICE |
 2846             MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE)) {
 2847                 uchar_t *inq89 = NULL;
 2848                 int inq89_len = 0x238;
 2849                 int reallen = 0;
 2850                 int rval = 0;
 2851                 struct sata_id *sid = NULL;
 2852                 char model[SATA_ID_MODEL_LEN + 1];
 2853                 char fw[SATA_ID_FW_LEN + 1];
 2854                 char *vid, *pid;
 2855                 int i;
 2856 
 2857                 mutex_exit(&mpt->m_mutex);
 2858                 /*
 2859                  * According SCSI/ATA Translation -2 (SAT-2) revision 01a
 2860                  * chapter 12.4.2 VPD page 89h includes 512 bytes ATA IDENTIFY
 2861                  * DEVICE data or ATA IDENTIFY PACKET DEVICE data.
 2862                  */
 2863                 inq89 = kmem_zalloc(inq89_len, KM_SLEEP);
 2864                 rval = mptsas_inquiry(mpt, ptgt, 0, 0x89,
 2865                     inq89, inq89_len, &reallen, 1);
 2866 
 2867                 if (rval != 0) {
 2868                         if (inq89 != NULL) {
 2869                                 kmem_free(inq89, inq89_len);
 2870                         }
 2871 
 2872                         mptsas_log(mpt, CE_WARN, "!mptsas request inquiry page "
 2873                             "0x89 for SATA target:%x failed!", ptgt->m_devhdl);
 2874                         return (DDI_SUCCESS);
 2875                 }
 2876                 sid = (void *)(&inq89[60]);
 2877 
 2878                 swab(sid->ai_model, model, SATA_ID_MODEL_LEN);
 2879                 swab(sid->ai_fw, fw, SATA_ID_FW_LEN);
 2880 
 2881                 model[SATA_ID_MODEL_LEN] = 0;
 2882                 fw[SATA_ID_FW_LEN] = 0;
 2883 
 2884                 /*
 2885                  * split model into into vid/pid
 2886                  */
 2887                 for (i = 0, pid = model; i < SATA_ID_MODEL_LEN; i++, pid++)
 2888                         if ((*pid == ' ') || (*pid == '\t'))
 2889                                 break;
 2890                 if (i < SATA_ID_MODEL_LEN) {
 2891                         vid = model;
 2892                         /*
 2893                          * terminate vid, establish pid
 2894                          */
 2895                         *pid++ = 0;
 2896                 } else {
 2897                         /*
 2898                          * vid will stay "ATA     ", the rule is same
 2899                          * as sata framework implementation.
 2900                          */
 2901                         vid = NULL;
 2902                         /*
 2903                          * model is all pid
 2904                          */
 2905                         pid = model;
 2906                 }
 2907 
 2908                 /*
 2909                  * override SCSA "inquiry-*" properties
 2910                  */
 2911                 if (vid)
 2912                         (void) scsi_device_prop_update_inqstring(sd,
 2913                             INQUIRY_VENDOR_ID, vid, strlen(vid));
 2914                 if (pid)
 2915                         (void) scsi_device_prop_update_inqstring(sd,
 2916                             INQUIRY_PRODUCT_ID, pid, strlen(pid));
 2917                 (void) scsi_device_prop_update_inqstring(sd,
 2918                     INQUIRY_REVISION_ID, fw, strlen(fw));
 2919 
 2920                 if (inq89 != NULL) {
 2921                         kmem_free(inq89, inq89_len);
 2922                 }
 2923         } else {
 2924                 mutex_exit(&mpt->m_mutex);
 2925         }
 2926 
 2927         return (DDI_SUCCESS);
 2928 }
 2929 /*
 2930  * tran_tgt_free(9E) - target device instance deallocation
 2931  */
 2932 static void
 2933 mptsas_scsi_tgt_free(dev_info_t *hba_dip, dev_info_t *tgt_dip,
 2934     scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
 2935 {
 2936 #ifndef __lock_lint
 2937         _NOTE(ARGUNUSED(hba_dip, tgt_dip, hba_tran, sd))
 2938 #endif
 2939 
 2940         mptsas_tgt_private_t    *tgt_private = hba_tran->tran_tgt_private;
 2941 
 2942         if (tgt_private != NULL) {
 2943                 kmem_free(tgt_private, sizeof (mptsas_tgt_private_t));
 2944                 hba_tran->tran_tgt_private = NULL;
 2945         }
 2946 }
 2947 
 2948 /*
 2949  * scsi_pkt handling
 2950  *
 2951  * Visible to the external world via the transport structure.
 2952  */
 2953 
 2954 /*
 2955  * Notes:
 2956  *      - transport the command to the addressed SCSI target/lun device
 2957  *      - normal operation is to schedule the command to be transported,
 2958  *        and return TRAN_ACCEPT if this is successful.
 2959  *      - if NO_INTR, tran_start must poll device for command completion
 2960  */
 2961 static int
 2962 mptsas_scsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
 2963 {
 2964 #ifndef __lock_lint
 2965         _NOTE(ARGUNUSED(ap))
 2966 #endif
 2967         mptsas_t        *mpt = PKT2MPT(pkt);
 2968         mptsas_cmd_t    *cmd = PKT2CMD(pkt);
 2969         int             rval;
 2970         mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
 2971 
 2972         NDBG1(("mptsas_scsi_start: pkt=0x%p", (void *)pkt));
 2973         ASSERT(ptgt);
 2974         if (ptgt == NULL)
 2975                 return (TRAN_FATAL_ERROR);
 2976 
 2977         /*
 2978          * prepare the pkt before taking mutex.
 2979          */
 2980         rval = mptsas_prepare_pkt(cmd);
 2981         if (rval != TRAN_ACCEPT) {
 2982                 return (rval);
 2983         }
 2984 
 2985         /*
 2986          * Send the command to target/lun, however your HBA requires it.
 2987          * If busy, return TRAN_BUSY; if there's some other formatting error
 2988          * in the packet, return TRAN_BADPKT; otherwise, fall through to the
 2989          * return of TRAN_ACCEPT.
 2990          *
 2991          * Remember that access to shared resources, including the mptsas_t
 2992          * data structure and the HBA hardware registers, must be protected
 2993          * with mutexes, here and everywhere.
 2994          *
 2995          * Also remember that at interrupt time, you'll get an argument
 2996          * to the interrupt handler which is a pointer to your mptsas_t
 2997          * structure; you'll have to remember which commands are outstanding
 2998          * and which scsi_pkt is the currently-running command so the
 2999          * interrupt handler can refer to the pkt to set completion
 3000          * status, call the target driver back through pkt_comp, etc.
 3001          */
 3002 
 3003         mutex_enter(&ptgt->m_tgt_intr_mutex);
 3004         if (ptgt->m_dr_flag == MPTSAS_DR_INTRANSITION) {
 3005                 if (cmd->cmd_pkt_flags & FLAG_NOQUEUE) {
 3006                         /*
 3007                          * commands should be allowed to retry by
 3008                          * returning TRAN_BUSY to stall the I/O's
 3009                          * which come from scsi_vhci since the device/
 3010                          * path is in unstable state now.
 3011                          */
 3012                         mutex_exit(&ptgt->m_tgt_intr_mutex);
 3013                         return (TRAN_BUSY);
 3014                 } else {
 3015                         /*
 3016                          * The device is offline, just fail the
 3017                          * command by returning TRAN_FATAL_ERROR.
 3018                          */
 3019                         mutex_exit(&ptgt->m_tgt_intr_mutex);
 3020                         return (TRAN_FATAL_ERROR);
 3021                 }
 3022         }
 3023         mutex_exit(&ptgt->m_tgt_intr_mutex);
 3024         rval = mptsas_accept_pkt(mpt, cmd);
 3025 
 3026         return (rval);
 3027 }
 3028 
 3029 static int
 3030 mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd)
 3031 {
 3032         int             rval = TRAN_ACCEPT;
 3033         mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
 3034 
 3035         NDBG1(("mptsas_accept_pkt: cmd=0x%p", (void *)cmd));
 3036 
 3037         if ((cmd->cmd_flags & CFLAG_PREPARED) == 0) {
 3038                 rval = mptsas_prepare_pkt(cmd);
 3039                 if (rval != TRAN_ACCEPT) {
 3040                         cmd->cmd_flags &= ~CFLAG_TRANFLAG;
 3041                         return (rval);
 3042                 }
 3043         }
 3044 
 3045         /*
 3046          * reset the throttle if we were draining
 3047          */
 3048         mutex_enter(&ptgt->m_tgt_intr_mutex);
 3049         if ((ptgt->m_t_ncmds == 0) &&
 3050             (ptgt->m_t_throttle == DRAIN_THROTTLE)) {
 3051                 NDBG23(("reset throttle"));
 3052                 ASSERT(ptgt->m_reset_delay == 0);
 3053                 mptsas_set_throttle(mpt, ptgt, MAX_THROTTLE);
 3054         }
 3055 
 3056         /*
 3057          * If device handle has already been invalidated, just
 3058          * fail the command. In theory, command from scsi_vhci
 3059          * client is impossible send down command with invalid
 3060          * devhdl since devhdl is set after path offline, target
 3061          * driver is not suppose to select a offlined path.
 3062          */
 3063         if (ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) {
 3064                 NDBG20(("rejecting command, it might because invalid devhdl "
 3065                     "request."));
 3066                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 3067                 mutex_enter(&mpt->m_mutex);
 3068                 /*
 3069                  * If HBA is being reset, the DevHandles are being
 3070                  * re-initialized, which means that they could be invalid
 3071                  * even if the target is still attached. Check if being reset
 3072                  * and if DevHandle is being re-initialized. If this is the
 3073                  * case, return BUSY so the I/O can be retried later.
 3074                  */
 3075                 if (mpt->m_in_reset) {
 3076                         mptsas_set_pkt_reason(mpt, cmd, CMD_RESET,
 3077                             STAT_BUS_RESET);
 3078                         if (cmd->cmd_flags & CFLAG_TXQ) {
 3079                                 mptsas_doneq_add(mpt, cmd);
 3080                                 mptsas_doneq_empty(mpt);
 3081                                 mutex_exit(&mpt->m_mutex);
 3082                                 return (rval);
 3083                         } else {
 3084                                 mutex_exit(&mpt->m_mutex);
 3085                                 return (TRAN_BUSY);
 3086                         }
 3087                 }
 3088                 mptsas_set_pkt_reason(mpt, cmd, CMD_DEV_GONE, STAT_TERMINATED);
 3089                 if (cmd->cmd_flags & CFLAG_TXQ) {
 3090                         mptsas_doneq_add(mpt, cmd);
 3091                         mptsas_doneq_empty(mpt);
 3092                         mutex_exit(&mpt->m_mutex);
 3093                         return (rval);
 3094                 } else {
 3095                         mutex_exit(&mpt->m_mutex);
 3096                         return (TRAN_FATAL_ERROR);
 3097                 }
 3098         }
 3099         mutex_exit(&ptgt->m_tgt_intr_mutex);
 3100         /*
 3101          * The first case is the normal case.  mpt gets a command from the
 3102          * target driver and starts it.
 3103          * Since SMID 0 is reserved and the TM slot is reserved, the actual max
 3104          * commands is m_max_requests - 2.
 3105          */
 3106         mutex_enter(&ptgt->m_tgt_intr_mutex);
 3107         if ((ptgt->m_t_throttle > HOLD_THROTTLE) &&
 3108             (ptgt->m_t_ncmds < ptgt->m_t_throttle) &&
 3109             (ptgt->m_reset_delay == 0) &&
 3110             (ptgt->m_t_nwait == 0) &&
 3111             ((cmd->cmd_pkt_flags & FLAG_NOINTR) == 0)) {
 3112                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 3113                 if (mptsas_save_cmd(mpt, cmd) == TRUE) {
 3114                         (void) mptsas_start_cmd0(mpt, cmd);
 3115                 } else {
 3116                         mutex_enter(&mpt->m_mutex);
 3117                         mptsas_waitq_add(mpt, cmd);
 3118                         mutex_exit(&mpt->m_mutex);
 3119                 }
 3120         } else {
 3121                 /*
 3122                  * Add this pkt to the work queue
 3123                  */
 3124                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 3125                 mutex_enter(&mpt->m_mutex);
 3126                 mptsas_waitq_add(mpt, cmd);
 3127 
 3128                 if (cmd->cmd_pkt_flags & FLAG_NOINTR) {
 3129                         (void) mptsas_poll(mpt, cmd, MPTSAS_POLL_TIME);
 3130 
 3131                         /*
 3132                          * Only flush the doneq if this is not a TM
 3133                          * cmd.  For TM cmds the flushing of the
 3134                          * doneq will be done in those routines.
 3135                          */
 3136                         if ((cmd->cmd_flags & CFLAG_TM_CMD) == 0) {
 3137                                 mptsas_doneq_empty(mpt);
 3138                         }
 3139                 }
 3140                 mutex_exit(&mpt->m_mutex);
 3141         }
 3142         return (rval);
 3143 }
 3144 
 3145 int
 3146 mptsas_save_cmd(mptsas_t *mpt, mptsas_cmd_t *cmd)
 3147 {
 3148         mptsas_slots_t  *slots;
 3149         int             slot;
 3150         mptsas_target_t *ptgt = cmd->cmd_tgt_addr;
 3151         mptsas_slot_free_e_t    *pe;
 3152         int             qn, qn_first;
 3153 
 3154         slots = mpt->m_active;
 3155 
 3156         /*
 3157          * Account for reserved TM request slot and reserved SMID of 0.
 3158          */
 3159         ASSERT(slots->m_n_slots == (mpt->m_max_requests - 2));
 3160 
 3161         qn = qn_first = CPU->cpu_seqid & (mpt->m_slot_freeq_pair_n - 1);
 3162 
 3163 qpair_retry:
 3164         ASSERT(qn < mpt->m_slot_freeq_pair_n);
 3165         mutex_enter(&mpt->m_slot_freeq_pairp[qn].m_slot_allocq.s.m_fq_mutex);
 3166         pe = list_head(&mpt->m_slot_freeq_pairp[qn].m_slot_allocq.
 3167             s.m_fq_list);
 3168         if (!pe) { /* switch the allocq and releq */
 3169                 mutex_enter(&mpt->m_slot_freeq_pairp[qn].m_slot_releq.
 3170                     s.m_fq_mutex);
 3171                 if (mpt->m_slot_freeq_pairp[qn].m_slot_releq.s.m_fq_n) {
 3172                         mpt->m_slot_freeq_pairp[qn].
 3173                             m_slot_allocq.s.m_fq_n =
 3174                             mpt->m_slot_freeq_pairp[qn].
 3175                             m_slot_releq.s.m_fq_n;
 3176                         mpt->m_slot_freeq_pairp[qn].
 3177                             m_slot_allocq.s.m_fq_list.list_head.list_next =
 3178                             mpt->m_slot_freeq_pairp[qn].
 3179                             m_slot_releq.s.m_fq_list.list_head.list_next;
 3180                         mpt->m_slot_freeq_pairp[qn].
 3181                             m_slot_allocq.s.m_fq_list.list_head.list_prev =
 3182                             mpt->m_slot_freeq_pairp[qn].
 3183                             m_slot_releq.s.m_fq_list.list_head.list_prev;
 3184                         mpt->m_slot_freeq_pairp[qn].
 3185                             m_slot_releq.s.m_fq_list.list_head.list_prev->
 3186                             list_next =
 3187                             &mpt->m_slot_freeq_pairp[qn].
 3188                             m_slot_allocq.s.m_fq_list.list_head;
 3189                         mpt->m_slot_freeq_pairp[qn].
 3190                             m_slot_releq.s.m_fq_list.list_head.list_next->
 3191                             list_prev =
 3192                             &mpt->m_slot_freeq_pairp[qn].
 3193                             m_slot_allocq.s.m_fq_list.list_head;
 3194 
 3195                         mpt->m_slot_freeq_pairp[qn].
 3196                             m_slot_releq.s.m_fq_list.list_head.list_next =
 3197                             mpt->m_slot_freeq_pairp[qn].
 3198                             m_slot_releq.s.m_fq_list.list_head.list_prev =
 3199                             &mpt->m_slot_freeq_pairp[qn].
 3200                             m_slot_releq.s.m_fq_list.list_head;
 3201                         mpt->m_slot_freeq_pairp[qn].
 3202                             m_slot_releq.s.m_fq_n = 0;
 3203                 } else {
 3204                         mutex_exit(&mpt->m_slot_freeq_pairp[qn].
 3205                             m_slot_releq.s.m_fq_mutex);
 3206                         mutex_exit(&mpt->m_slot_freeq_pairp[qn].
 3207                             m_slot_allocq.s.m_fq_mutex);
 3208                         qn = (qn + 1) & (mpt->m_slot_freeq_pair_n - 1);
 3209                         if (qn == qn_first)
 3210                                 return (FALSE);
 3211                         else
 3212                                 goto qpair_retry;
 3213                 }
 3214                 mutex_exit(&mpt->m_slot_freeq_pairp[qn].
 3215                     m_slot_releq.s.m_fq_mutex);
 3216                 pe = list_head(&mpt->m_slot_freeq_pairp[qn].
 3217                     m_slot_allocq.s.m_fq_list);
 3218                 ASSERT(pe);
 3219         }
 3220         list_remove(&mpt->m_slot_freeq_pairp[qn].
 3221             m_slot_allocq.s.m_fq_list, pe);
 3222         slot = pe->slot;
 3223         /*
 3224          * Make sure SMID is not using reserved value of 0
 3225          * and the TM request slot.
 3226          */
 3227         ASSERT((slot > 0) && (slot <= slots->m_n_slots) &&
 3228             mpt->m_slot_freeq_pairp[qn].m_slot_allocq.s.m_fq_n > 0);
 3229         cmd->cmd_slot = slot;
 3230         mpt->m_slot_freeq_pairp[qn].m_slot_allocq.s.m_fq_n--;
 3231         ASSERT(mpt->m_slot_freeq_pairp[qn].m_slot_allocq.s.m_fq_n >= 0);
 3232 
 3233         mutex_exit(&mpt->m_slot_freeq_pairp[qn].m_slot_allocq.s.m_fq_mutex);
 3234         /*
 3235          * only increment per target ncmds if this is not a
 3236          * command that has no target associated with it (i.e. a
 3237          * event acknoledgment)
 3238          */
 3239         if ((cmd->cmd_flags & CFLAG_CMDIOC) == 0) {
 3240                 mutex_enter(&ptgt->m_tgt_intr_mutex);
 3241                 ptgt->m_t_ncmds++;
 3242                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 3243         }
 3244         cmd->cmd_active_timeout = cmd->cmd_pkt->pkt_time;
 3245 
 3246         /*
 3247          * If initial timout is less than or equal to one tick, bump
 3248          * the timeout by a tick so that command doesn't timeout before
 3249          * its allotted time.
 3250          */
 3251         if (cmd->cmd_active_timeout <= mptsas_scsi_watchdog_tick) {
 3252                 cmd->cmd_active_timeout += mptsas_scsi_watchdog_tick;
 3253         }
 3254         return (TRUE);
 3255 }
 3256 
 3257 /*
 3258  * prepare the pkt:
 3259  * the pkt may have been resubmitted or just reused so
 3260  * initialize some fields and do some checks.
 3261  */
 3262 static int
 3263 mptsas_prepare_pkt(mptsas_cmd_t *cmd)
 3264 {
 3265         struct scsi_pkt *pkt = CMD2PKT(cmd);
 3266 
 3267         NDBG1(("mptsas_prepare_pkt: cmd=0x%p", (void *)cmd));
 3268 
 3269         /*
 3270          * Reinitialize some fields that need it; the packet may
 3271          * have been resubmitted
 3272          */
 3273         pkt->pkt_reason = CMD_CMPLT;
 3274         pkt->pkt_state = 0;
 3275         pkt->pkt_statistics = 0;
 3276         pkt->pkt_resid = 0;
 3277         cmd->cmd_age = 0;
 3278         cmd->cmd_pkt_flags = pkt->pkt_flags;
 3279 
 3280         /*
 3281          * zero status byte.
 3282          */
 3283         *(pkt->pkt_scbp) = 0;
 3284 
 3285         if (cmd->cmd_flags & CFLAG_DMAVALID) {
 3286                 pkt->pkt_resid = cmd->cmd_dmacount;
 3287 
 3288                 /*
 3289                  * consistent packets need to be sync'ed first
 3290                  * (only for data going out)
 3291                  */
 3292                 if ((cmd->cmd_flags & CFLAG_CMDIOPB) &&
 3293                     (cmd->cmd_flags & CFLAG_DMASEND)) {
 3294                         (void) ddi_dma_sync(cmd->cmd_dmahandle, 0, 0,
 3295                             DDI_DMA_SYNC_FORDEV);
 3296                 }
 3297         }
 3298 
 3299         cmd->cmd_flags =
 3300             (cmd->cmd_flags & ~(CFLAG_TRANFLAG)) |
 3301             CFLAG_PREPARED | CFLAG_IN_TRANSPORT;
 3302 
 3303         return (TRAN_ACCEPT);
 3304 }
 3305 
 3306 /*
 3307  * tran_init_pkt(9E) - allocate scsi_pkt(9S) for command
 3308  *
 3309  * One of three possibilities:
 3310  *      - allocate scsi_pkt
 3311  *      - allocate scsi_pkt and DMA resources
 3312  *      - allocate DMA resources to an already-allocated pkt
 3313  */
 3314 static struct scsi_pkt *
 3315 mptsas_scsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt,
 3316     struct buf *bp, int cmdlen, int statuslen, int tgtlen, int flags,
 3317     int (*callback)(), caddr_t arg)
 3318 {
 3319         mptsas_cmd_t            *cmd, *new_cmd;
 3320         mptsas_t                *mpt = ADDR2MPT(ap);
 3321         int                     failure = 1;
 3322 #ifndef __sparc
 3323         uint_t                  oldcookiec;
 3324 #endif  /* __sparc */
 3325         mptsas_target_t         *ptgt = NULL;
 3326         int                     rval;
 3327         mptsas_tgt_private_t    *tgt_private;
 3328         int                     kf;
 3329 
 3330         kf = (callback == SLEEP_FUNC)? KM_SLEEP: KM_NOSLEEP;
 3331 
 3332         tgt_private = (mptsas_tgt_private_t *)ap->a_hba_tran->
 3333             tran_tgt_private;
 3334         ASSERT(tgt_private != NULL);
 3335         if (tgt_private == NULL) {
 3336                 return (NULL);
 3337         }
 3338         ptgt = tgt_private->t_private;
 3339         ASSERT(ptgt != NULL);
 3340         if (ptgt == NULL)
 3341                 return (NULL);
 3342         ap->a_target = ptgt->m_devhdl;
 3343         ap->a_lun = tgt_private->t_lun;
 3344 
 3345         ASSERT(callback == NULL_FUNC || callback == SLEEP_FUNC);
 3346 #ifdef MPTSAS_TEST_EXTRN_ALLOC
 3347         statuslen *= 100; tgtlen *= 4;
 3348 #endif
 3349         NDBG3(("mptsas_scsi_init_pkt:\n"
 3350             "\ttgt=%d in=0x%p bp=0x%p clen=%d slen=%d tlen=%d flags=%x",
 3351             ap->a_target, (void *)pkt, (void *)bp,
 3352             cmdlen, statuslen, tgtlen, flags));
 3353 
 3354         /*
 3355          * Allocate the new packet.
 3356          */
 3357         if (pkt == NULL) {
 3358                 ddi_dma_handle_t        save_dma_handle;
 3359                 ddi_dma_handle_t        save_arq_dma_handle;
 3360                 struct buf              *save_arq_bp;
 3361                 ddi_dma_cookie_t        save_arqcookie;
 3362 #ifdef  __sparc
 3363                 mptti_t                 *save_sg;
 3364 #endif  /* __sparc */
 3365 
 3366                 cmd = kmem_cache_alloc(mpt->m_kmem_cache, kf);
 3367 
 3368                 if (cmd) {
 3369                         save_dma_handle = cmd->cmd_dmahandle;
 3370                         save_arq_dma_handle = cmd->cmd_arqhandle;
 3371                         save_arq_bp = cmd->cmd_arq_buf;
 3372                         save_arqcookie = cmd->cmd_arqcookie;
 3373 #ifdef  __sparc
 3374                         save_sg = cmd->cmd_sg;
 3375 #endif  /* __sparc */
 3376                         bzero(cmd, sizeof (*cmd) + scsi_pkt_size());
 3377                         cmd->cmd_dmahandle = save_dma_handle;
 3378                         cmd->cmd_arqhandle = save_arq_dma_handle;
 3379                         cmd->cmd_arq_buf = save_arq_bp;
 3380                         cmd->cmd_arqcookie = save_arqcookie;
 3381 #ifdef  __sparc
 3382                         cmd->cmd_sg = save_sg;
 3383 #endif  /* __sparc */
 3384                         pkt = (void *)((uchar_t *)cmd +
 3385                             sizeof (struct mptsas_cmd));
 3386                         pkt->pkt_ha_private = (opaque_t)cmd;
 3387                         pkt->pkt_address = *ap;
 3388                         pkt->pkt_private = (opaque_t)cmd->cmd_pkt_private;
 3389                         pkt->pkt_scbp = (opaque_t)&cmd->cmd_scb;
 3390                         pkt->pkt_cdbp = (opaque_t)&cmd->cmd_cdb;
 3391                         cmd->cmd_pkt = (struct scsi_pkt *)pkt;
 3392                         cmd->cmd_cdblen = (uchar_t)cmdlen;
 3393                         cmd->cmd_scblen = statuslen;
 3394                         cmd->cmd_rqslen = SENSE_LENGTH;
 3395                         cmd->cmd_tgt_addr = ptgt;
 3396                         failure = 0;
 3397                 }
 3398 
 3399                 if (failure || (cmdlen > sizeof (cmd->cmd_cdb)) ||
 3400                     (tgtlen > PKT_PRIV_LEN) ||
 3401                     (statuslen > EXTCMDS_STATUS_SIZE)) {
 3402                         if (failure == 0) {
 3403                                 /*
 3404                                  * if extern alloc fails, all will be
 3405                                  * deallocated, including cmd
 3406                                  */
 3407                                 failure = mptsas_pkt_alloc_extern(mpt, cmd,
 3408                                     cmdlen, tgtlen, statuslen, kf);
 3409                         }
 3410                         if (failure) {
 3411                                 /*
 3412                                  * if extern allocation fails, it will
 3413                                  * deallocate the new pkt as well
 3414                                  */
 3415                                 return (NULL);
 3416                         }
 3417                 }
 3418                 new_cmd = cmd;
 3419 
 3420         } else {
 3421                 cmd = PKT2CMD(pkt);
 3422                 new_cmd = NULL;
 3423         }
 3424 
 3425 
 3426 #ifndef __sparc
 3427         /* grab cmd->cmd_cookiec here as oldcookiec */
 3428 
 3429         oldcookiec = cmd->cmd_cookiec;
 3430 #endif  /* __sparc */
 3431 
 3432         /*
 3433          * If the dma was broken up into PARTIAL transfers cmd_nwin will be
 3434          * greater than 0 and we'll need to grab the next dma window
 3435          */
 3436         /*
 3437          * SLM-not doing extra command frame right now; may add later
 3438          */
 3439 
 3440         if (cmd->cmd_nwin > 0) {
 3441 
 3442                 /*
 3443                  * Make sure we havn't gone past the the total number
 3444                  * of windows
 3445                  */
 3446                 if (++cmd->cmd_winindex >= cmd->cmd_nwin) {
 3447                         return (NULL);
 3448                 }
 3449                 if (ddi_dma_getwin(cmd->cmd_dmahandle, cmd->cmd_winindex,
 3450                     &cmd->cmd_dma_offset, &cmd->cmd_dma_len,
 3451                     &cmd->cmd_cookie, &cmd->cmd_cookiec) == DDI_FAILURE) {
 3452                         return (NULL);
 3453                 }
 3454                 goto get_dma_cookies;
 3455         }
 3456 
 3457 
 3458         if (flags & PKT_XARQ) {
 3459                 cmd->cmd_flags |= CFLAG_XARQ;
 3460         }
 3461 
 3462         /*
 3463          * DMA resource allocation.  This version assumes your
 3464          * HBA has some sort of bus-mastering or onboard DMA capability, with a
 3465          * scatter-gather list of length MPTSAS_MAX_DMA_SEGS, as given in the
 3466          * ddi_dma_attr_t structure and passed to scsi_impl_dmaget.
 3467          */
 3468         if (bp && (bp->b_bcount != 0) &&
 3469             (cmd->cmd_flags & CFLAG_DMAVALID) == 0) {
 3470 
 3471                 int     cnt, dma_flags;
 3472                 mptti_t *dmap;          /* ptr to the S/G list */
 3473 
 3474                 /*
 3475                  * Set up DMA memory and position to the next DMA segment.
 3476                  */
 3477                 ASSERT(cmd->cmd_dmahandle != NULL);
 3478 
 3479                 if (bp->b_flags & B_READ) {
 3480                         dma_flags = DDI_DMA_READ;
 3481                         cmd->cmd_flags &= ~CFLAG_DMASEND;
 3482                 } else {
 3483                         dma_flags = DDI_DMA_WRITE;
 3484                         cmd->cmd_flags |= CFLAG_DMASEND;
 3485                 }
 3486                 if (flags & PKT_CONSISTENT) {
 3487                         cmd->cmd_flags |= CFLAG_CMDIOPB;
 3488                         dma_flags |= DDI_DMA_CONSISTENT;
 3489                 }
 3490 
 3491                 if (flags & PKT_DMA_PARTIAL) {
 3492                         dma_flags |= DDI_DMA_PARTIAL;
 3493                 }
 3494 
 3495                 /*
 3496                  * workaround for byte hole issue on psycho and
 3497                  * schizo pre 2.1
 3498                  */
 3499                 if ((bp->b_flags & B_READ) && ((bp->b_flags &
 3500                     (B_PAGEIO|B_REMAPPED)) != B_PAGEIO) &&
 3501                     ((uintptr_t)bp->b_un.b_addr & 0x7)) {
 3502                         dma_flags |= DDI_DMA_CONSISTENT;
 3503                 }
 3504 
 3505                 rval = ddi_dma_buf_bind_handle(cmd->cmd_dmahandle, bp,
 3506                     dma_flags, callback, arg,
 3507                     &cmd->cmd_cookie, &cmd->cmd_cookiec);
 3508                 if (rval == DDI_DMA_PARTIAL_MAP) {
 3509                         (void) ddi_dma_numwin(cmd->cmd_dmahandle,
 3510                             &cmd->cmd_nwin);
 3511                         cmd->cmd_winindex = 0;
 3512                         (void) ddi_dma_getwin(cmd->cmd_dmahandle,
 3513                             cmd->cmd_winindex, &cmd->cmd_dma_offset,
 3514                             &cmd->cmd_dma_len, &cmd->cmd_cookie,
 3515                             &cmd->cmd_cookiec);
 3516                 } else if (rval && (rval != DDI_DMA_MAPPED)) {
 3517                         switch (rval) {
 3518                         case DDI_DMA_NORESOURCES:
 3519                                 bioerror(bp, 0);
 3520                                 break;
 3521                         case DDI_DMA_BADATTR:
 3522                         case DDI_DMA_NOMAPPING:
 3523                                 bioerror(bp, EFAULT);
 3524                                 break;
 3525                         case DDI_DMA_TOOBIG:
 3526                         default:
 3527                                 bioerror(bp, EINVAL);
 3528                                 break;
 3529                         }
 3530                         cmd->cmd_flags &= ~CFLAG_DMAVALID;
 3531                         if (new_cmd) {
 3532                                 mptsas_scsi_destroy_pkt(ap, pkt);
 3533                         }
 3534                         return ((struct scsi_pkt *)NULL);
 3535                 }
 3536 
 3537 get_dma_cookies:
 3538                 cmd->cmd_flags |= CFLAG_DMAVALID;
 3539                 ASSERT(cmd->cmd_cookiec > 0);
 3540 
 3541                 if (cmd->cmd_cookiec > MPTSAS_MAX_CMD_SEGS) {
 3542                         mptsas_log(mpt, CE_NOTE, "large cookiec received %d\n",
 3543                             cmd->cmd_cookiec);
 3544                         bioerror(bp, EINVAL);
 3545                         if (new_cmd) {
 3546                                 mptsas_scsi_destroy_pkt(ap, pkt);
 3547                         }
 3548                         return ((struct scsi_pkt *)NULL);
 3549                 }
 3550 
 3551                 /*
 3552                  * Allocate extra SGL buffer if needed.
 3553                  */
 3554                 if ((cmd->cmd_cookiec > MPTSAS_MAX_FRAME_SGES64(mpt)) &&
 3555                     (cmd->cmd_extra_frames == NULL)) {
 3556                         if (mptsas_alloc_extra_sgl_frame(mpt, cmd) ==
 3557                             DDI_FAILURE) {
 3558                                 mptsas_log(mpt, CE_WARN, "MPT SGL mem alloc "
 3559                                     "failed");
 3560                                 bioerror(bp, ENOMEM);
 3561                                 if (new_cmd) {
 3562                                         mptsas_scsi_destroy_pkt(ap, pkt);
 3563                                 }
 3564                                 return ((struct scsi_pkt *)NULL);
 3565                         }
 3566                 }
 3567 
 3568                 /*
 3569                  * Always use scatter-gather transfer
 3570                  * Use the loop below to store physical addresses of
 3571                  * DMA segments, from the DMA cookies, into your HBA's
 3572                  * scatter-gather list.
 3573                  * We need to ensure we have enough kmem alloc'd
 3574                  * for the sg entries since we are no longer using an
 3575                  * array inside mptsas_cmd_t.
 3576                  *
 3577                  * We check cmd->cmd_cookiec against oldcookiec so
 3578                  * the scatter-gather list is correctly allocated
 3579                  */
 3580 #ifndef __sparc
 3581                 if (oldcookiec != cmd->cmd_cookiec) {
 3582                         if (cmd->cmd_sg != (mptti_t *)NULL) {
 3583                                 kmem_free(cmd->cmd_sg, sizeof (mptti_t) *
 3584                                     oldcookiec);
 3585                                 cmd->cmd_sg = NULL;
 3586                         }
 3587                 }
 3588 
 3589                 if (cmd->cmd_sg == (mptti_t *)NULL) {
 3590                         cmd->cmd_sg = kmem_alloc((size_t)(sizeof (mptti_t)*
 3591                             cmd->cmd_cookiec), kf);
 3592 
 3593                         if (cmd->cmd_sg == (mptti_t *)NULL) {
 3594                                 mptsas_log(mpt, CE_WARN,
 3595                                     "unable to kmem_alloc enough memory "
 3596                                     "for scatter/gather list");
 3597                 /*
 3598                  * if we have an ENOMEM condition we need to behave
 3599                  * the same way as the rest of this routine
 3600                  */
 3601 
 3602                                 bioerror(bp, ENOMEM);
 3603                                 if (new_cmd) {
 3604                                         mptsas_scsi_destroy_pkt(ap, pkt);
 3605                                 }
 3606                                 return ((struct scsi_pkt *)NULL);
 3607                         }
 3608                 }
 3609 #endif  /* __sparc */
 3610                 dmap = cmd->cmd_sg;
 3611 
 3612                 ASSERT(cmd->cmd_cookie.dmac_size != 0);
 3613 
 3614                 /*
 3615                  * store the first segment into the S/G list
 3616                  */
 3617                 dmap->count = cmd->cmd_cookie.dmac_size;
 3618                 dmap->addr.address64.Low = (uint32_t)
 3619                     (cmd->cmd_cookie.dmac_laddress & 0xffffffffull);
 3620                 dmap->addr.address64.High = (uint32_t)
 3621                     (cmd->cmd_cookie.dmac_laddress >> 32);
 3622 
 3623                 /*
 3624                  * dmacount counts the size of the dma for this window
 3625                  * (if partial dma is being used).  totaldmacount
 3626                  * keeps track of the total amount of dma we have
 3627                  * transferred for all the windows (needed to calculate
 3628                  * the resid value below).
 3629                  */
 3630                 cmd->cmd_dmacount = cmd->cmd_cookie.dmac_size;
 3631                 cmd->cmd_totaldmacount += cmd->cmd_cookie.dmac_size;
 3632 
 3633                 /*
 3634                  * We already stored the first DMA scatter gather segment,
 3635                  * start at 1 if we need to store more.
 3636                  */
 3637                 for (cnt = 1; cnt < cmd->cmd_cookiec; cnt++) {
 3638                         /*
 3639                          * Get next DMA cookie
 3640                          */
 3641                         ddi_dma_nextcookie(cmd->cmd_dmahandle,
 3642                             &cmd->cmd_cookie);
 3643                         dmap++;
 3644 
 3645                         cmd->cmd_dmacount += cmd->cmd_cookie.dmac_size;
 3646                         cmd->cmd_totaldmacount += cmd->cmd_cookie.dmac_size;
 3647 
 3648                         /*
 3649                          * store the segment parms into the S/G list
 3650                          */
 3651                         dmap->count = cmd->cmd_cookie.dmac_size;
 3652                         dmap->addr.address64.Low = (uint32_t)
 3653                             (cmd->cmd_cookie.dmac_laddress & 0xffffffffull);
 3654                         dmap->addr.address64.High = (uint32_t)
 3655                             (cmd->cmd_cookie.dmac_laddress >> 32);
 3656                 }
 3657 
 3658                 /*
 3659                  * If this was partially allocated we set the resid
 3660                  * the amount of data NOT transferred in this window
 3661                  * If there is only one window, the resid will be 0
 3662                  */
 3663                 pkt->pkt_resid = (bp->b_bcount - cmd->cmd_totaldmacount);
 3664                 NDBG16(("mptsas_dmaget: cmd_dmacount=%d.", cmd->cmd_dmacount));
 3665         }
 3666         return (pkt);
 3667 }
 3668 
 3669 /*
 3670  * tran_destroy_pkt(9E) - scsi_pkt(9s) deallocation
 3671  *
 3672  * Notes:
 3673  *      - also frees DMA resources if allocated
 3674  *      - implicit DMA synchonization
 3675  */
 3676 static void
 3677 mptsas_scsi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
 3678 {
 3679         mptsas_cmd_t    *cmd = PKT2CMD(pkt);
 3680         mptsas_t        *mpt = ADDR2MPT(ap);
 3681 
 3682         NDBG3(("mptsas_scsi_destroy_pkt: target=%d pkt=0x%p",
 3683             ap->a_target, (void *)pkt));
 3684 
 3685         if (cmd->cmd_flags & CFLAG_DMAVALID) {
 3686                 (void) ddi_dma_unbind_handle(cmd->cmd_dmahandle);
 3687                 cmd->cmd_flags &= ~CFLAG_DMAVALID;
 3688         }
 3689 #ifndef __sparc
 3690         if (cmd->cmd_sg) {
 3691                 kmem_free(cmd->cmd_sg, sizeof (mptti_t) * cmd->cmd_cookiec);
 3692                 cmd->cmd_sg = NULL;
 3693         }
 3694 #endif  /* __sparc */
 3695         mptsas_free_extra_sgl_frame(mpt, cmd);
 3696 
 3697         if ((cmd->cmd_flags &
 3698             (CFLAG_FREE | CFLAG_CDBEXTERN | CFLAG_PRIVEXTERN |
 3699             CFLAG_SCBEXTERN)) == 0) {
 3700                 cmd->cmd_flags = CFLAG_FREE;
 3701                 kmem_cache_free(mpt->m_kmem_cache, (void *)cmd);
 3702         } else {
 3703                 mptsas_pkt_destroy_extern(mpt, cmd);
 3704         }
 3705 }
 3706 
 3707 /*
 3708  * kmem cache constructor and destructor:
 3709  * When constructing, we bzero the cmd and allocate the dma handle
 3710  * When destructing, just free the dma handle
 3711  */
 3712 static int
 3713 mptsas_kmem_cache_constructor(void *buf, void *cdrarg, int kmflags)
 3714 {
 3715         mptsas_cmd_t            *cmd = buf;
 3716         mptsas_t                *mpt  = cdrarg;
 3717         struct scsi_address     ap;
 3718         uint_t                  cookiec;
 3719         ddi_dma_attr_t          arq_dma_attr;
 3720         int                     (*callback)(caddr_t);
 3721 
 3722         callback = (kmflags == KM_SLEEP)? DDI_DMA_SLEEP: DDI_DMA_DONTWAIT;
 3723 
 3724         NDBG4(("mptsas_kmem_cache_constructor"));
 3725 
 3726         ap.a_hba_tran = mpt->m_tran;
 3727         ap.a_target = 0;
 3728         ap.a_lun = 0;
 3729 
 3730         /*
 3731          * allocate a dma handle
 3732          */
 3733         if ((ddi_dma_alloc_handle(mpt->m_dip, &mpt->m_io_dma_attr, callback,
 3734             NULL, &cmd->cmd_dmahandle)) != DDI_SUCCESS) {
 3735                 cmd->cmd_dmahandle = NULL;
 3736                 return (-1);
 3737         }
 3738 
 3739         cmd->cmd_arq_buf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
 3740             SENSE_LENGTH, B_READ, callback, NULL);
 3741         if (cmd->cmd_arq_buf == NULL) {
 3742                 ddi_dma_free_handle(&cmd->cmd_dmahandle);
 3743                 cmd->cmd_dmahandle = NULL;
 3744                 return (-1);
 3745         }
 3746 
 3747         /*
 3748          * allocate a arq handle
 3749          */
 3750         arq_dma_attr = mpt->m_msg_dma_attr;
 3751         arq_dma_attr.dma_attr_sgllen = 1;
 3752         if ((ddi_dma_alloc_handle(mpt->m_dip, &arq_dma_attr, callback,
 3753             NULL, &cmd->cmd_arqhandle)) != DDI_SUCCESS) {
 3754                 ddi_dma_free_handle(&cmd->cmd_dmahandle);
 3755                 scsi_free_consistent_buf(cmd->cmd_arq_buf);
 3756                 cmd->cmd_dmahandle = NULL;
 3757                 cmd->cmd_arqhandle = NULL;
 3758                 return (-1);
 3759         }
 3760 
 3761         if (ddi_dma_buf_bind_handle(cmd->cmd_arqhandle,
 3762             cmd->cmd_arq_buf, (DDI_DMA_READ | DDI_DMA_CONSISTENT),
 3763             callback, NULL, &cmd->cmd_arqcookie, &cookiec) != DDI_SUCCESS) {
 3764                 ddi_dma_free_handle(&cmd->cmd_dmahandle);
 3765                 ddi_dma_free_handle(&cmd->cmd_arqhandle);
 3766                 scsi_free_consistent_buf(cmd->cmd_arq_buf);
 3767                 cmd->cmd_dmahandle = NULL;
 3768                 cmd->cmd_arqhandle = NULL;
 3769                 cmd->cmd_arq_buf = NULL;
 3770                 return (-1);
 3771         }
 3772         /*
 3773          * In sparc, the sgl length in most of the cases would be 1, so we
 3774          * pre-allocate it in cache. On x86, the max number would be 256,
 3775          * pre-allocate a maximum would waste a lot of memory especially
 3776          * when many cmds are put onto waitq.
 3777          */
 3778 #ifdef  __sparc
 3779         cmd->cmd_sg = kmem_alloc((size_t)(sizeof (mptti_t)*
 3780             MPTSAS_MAX_CMD_SEGS), KM_SLEEP);
 3781 #endif  /* __sparc */
 3782 
 3783         return (0);
 3784 }
 3785 
 3786 static void
 3787 mptsas_kmem_cache_destructor(void *buf, void *cdrarg)
 3788 {
 3789 #ifndef __lock_lint
 3790         _NOTE(ARGUNUSED(cdrarg))
 3791 #endif
 3792         mptsas_cmd_t    *cmd = buf;
 3793 
 3794         NDBG4(("mptsas_kmem_cache_destructor"));
 3795 
 3796         if (cmd->cmd_arqhandle) {
 3797                 (void) ddi_dma_unbind_handle(cmd->cmd_arqhandle);
 3798                 ddi_dma_free_handle(&cmd->cmd_arqhandle);
 3799                 cmd->cmd_arqhandle = NULL;
 3800         }
 3801         if (cmd->cmd_arq_buf) {
 3802                 scsi_free_consistent_buf(cmd->cmd_arq_buf);
 3803                 cmd->cmd_arq_buf = NULL;
 3804         }
 3805         if (cmd->cmd_dmahandle) {
 3806                 ddi_dma_free_handle(&cmd->cmd_dmahandle);
 3807                 cmd->cmd_dmahandle = NULL;
 3808         }
 3809 #ifdef  __sparc
 3810         if (cmd->cmd_sg) {
 3811                 kmem_free(cmd->cmd_sg, sizeof (mptti_t)* MPTSAS_MAX_CMD_SEGS);
 3812                 cmd->cmd_sg = NULL;
 3813         }
 3814 #endif  /* __sparc */
 3815 }
 3816 
 3817 static int
 3818 mptsas_cache_frames_constructor(void *buf, void *cdrarg, int kmflags)
 3819 {
 3820         mptsas_cache_frames_t   *p = buf;
 3821         mptsas_t                *mpt = cdrarg;
 3822         ddi_dma_attr_t          frame_dma_attr;
 3823         size_t                  mem_size, alloc_len;
 3824         ddi_dma_cookie_t        cookie;
 3825         uint_t                  ncookie;
 3826         int (*callback)(caddr_t) = (kmflags == KM_SLEEP)
 3827             ? DDI_DMA_SLEEP: DDI_DMA_DONTWAIT;
 3828 
 3829         frame_dma_attr = mpt->m_msg_dma_attr;
 3830         frame_dma_attr.dma_attr_align = 0x10;
 3831         frame_dma_attr.dma_attr_sgllen = 1;
 3832 
 3833         if (ddi_dma_alloc_handle(mpt->m_dip, &frame_dma_attr, callback, NULL,
 3834             &p->m_dma_hdl) != DDI_SUCCESS) {
 3835                 mptsas_log(mpt, CE_WARN, "Unable to allocate dma handle for"
 3836                     " extra SGL.");
 3837                 return (DDI_FAILURE);
 3838         }
 3839 
 3840         mem_size = (mpt->m_max_request_frames - 1) * mpt->m_req_frame_size;
 3841 
 3842         if (ddi_dma_mem_alloc(p->m_dma_hdl, mem_size, &mpt->m_dev_acc_attr,
 3843             DDI_DMA_CONSISTENT, callback, NULL, (caddr_t *)&p->m_frames_addr,
 3844             &alloc_len, &p->m_acc_hdl) != DDI_SUCCESS) {
 3845                 ddi_dma_free_handle(&p->m_dma_hdl);
 3846                 p->m_dma_hdl = NULL;
 3847                 mptsas_log(mpt, CE_WARN, "Unable to allocate dma memory for"
 3848                     " extra SGL.");
 3849                 return (DDI_FAILURE);
 3850         }
 3851 
 3852         if (ddi_dma_addr_bind_handle(p->m_dma_hdl, NULL, p->m_frames_addr,
 3853             alloc_len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
 3854             &cookie, &ncookie) != DDI_DMA_MAPPED) {
 3855                 (void) ddi_dma_mem_free(&p->m_acc_hdl);
 3856                 ddi_dma_free_handle(&p->m_dma_hdl);
 3857                 p->m_dma_hdl = NULL;
 3858                 mptsas_log(mpt, CE_WARN, "Unable to bind DMA resources for"
 3859                     " extra SGL");
 3860                 return (DDI_FAILURE);
 3861         }
 3862 
 3863         /*
 3864          * Store the SGL memory address.  This chip uses this
 3865          * address to dma to and from the driver.  The second
 3866          * address is the address mpt uses to fill in the SGL.
 3867          */
 3868         p->m_phys_addr = cookie.dmac_address;
 3869 
 3870         return (DDI_SUCCESS);
 3871 }
 3872 
 3873 static void
 3874 mptsas_cache_frames_destructor(void *buf, void *cdrarg)
 3875 {
 3876 #ifndef __lock_lint
 3877         _NOTE(ARGUNUSED(cdrarg))
 3878 #endif
 3879         mptsas_cache_frames_t   *p = buf;
 3880         if (p->m_dma_hdl != NULL) {
 3881                 (void) ddi_dma_unbind_handle(p->m_dma_hdl);
 3882                 (void) ddi_dma_mem_free(&p->m_acc_hdl);
 3883                 ddi_dma_free_handle(&p->m_dma_hdl);
 3884                 p->m_phys_addr = NULL;
 3885                 p->m_frames_addr = NULL;
 3886                 p->m_dma_hdl = NULL;
 3887                 p->m_acc_hdl = NULL;
 3888         }
 3889 
 3890 }
 3891 
 3892 /*
 3893  * allocate and deallocate external pkt space (ie. not part of mptsas_cmd)
 3894  * for non-standard length cdb, pkt_private, status areas
 3895  * if allocation fails, then deallocate all external space and the pkt
 3896  */
 3897 /* ARGSUSED */
 3898 static int
 3899 mptsas_pkt_alloc_extern(mptsas_t *mpt, mptsas_cmd_t *cmd,
 3900     int cmdlen, int tgtlen, int statuslen, int kf)
 3901 {
 3902         caddr_t                 cdbp, scbp, tgt;
 3903         int                     (*callback)(caddr_t) = (kf == KM_SLEEP) ?
 3904             DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
 3905         struct scsi_address     ap;
 3906         size_t                  senselength;
 3907         ddi_dma_attr_t          ext_arq_dma_attr;
 3908         uint_t                  cookiec;
 3909 
 3910         NDBG3(("mptsas_pkt_alloc_extern: "
 3911             "cmd=0x%p cmdlen=%d tgtlen=%d statuslen=%d kf=%x",
 3912             (void *)cmd, cmdlen, tgtlen, statuslen, kf));
 3913 
 3914         tgt = cdbp = scbp = NULL;
 3915         cmd->cmd_scblen         = statuslen;
 3916         cmd->cmd_privlen        = (uchar_t)tgtlen;
 3917 
 3918         if (cmdlen > sizeof (cmd->cmd_cdb)) {
 3919                 if ((cdbp = kmem_zalloc((size_t)cmdlen, kf)) == NULL) {
 3920                         goto fail;
 3921                 }
 3922                 cmd->cmd_pkt->pkt_cdbp = (opaque_t)cdbp;
 3923                 cmd->cmd_flags |= CFLAG_CDBEXTERN;
 3924         }
 3925         if (tgtlen > PKT_PRIV_LEN) {
 3926                 if ((tgt = kmem_zalloc((size_t)tgtlen, kf)) == NULL) {
 3927                         goto fail;
 3928                 }
 3929                 cmd->cmd_flags |= CFLAG_PRIVEXTERN;
 3930                 cmd->cmd_pkt->pkt_private = tgt;
 3931         }
 3932         if (statuslen > EXTCMDS_STATUS_SIZE) {
 3933                 if ((scbp = kmem_zalloc((size_t)statuslen, kf)) == NULL) {
 3934                         goto fail;
 3935                 }
 3936                 cmd->cmd_flags |= CFLAG_SCBEXTERN;
 3937                 cmd->cmd_pkt->pkt_scbp = (opaque_t)scbp;
 3938 
 3939                 /* allocate sense data buf for DMA */
 3940 
 3941                 senselength = statuslen - MPTSAS_GET_ITEM_OFF(
 3942                     struct scsi_arq_status, sts_sensedata);
 3943                 cmd->cmd_rqslen = (uchar_t)senselength;
 3944 
 3945                 ap.a_hba_tran = mpt->m_tran;
 3946                 ap.a_target = 0;
 3947                 ap.a_lun = 0;
 3948 
 3949                 cmd->cmd_ext_arq_buf = scsi_alloc_consistent_buf(&ap,
 3950                     (struct buf *)NULL, senselength, B_READ,
 3951                     callback, NULL);
 3952 
 3953                 if (cmd->cmd_ext_arq_buf == NULL) {
 3954                         goto fail;
 3955                 }
 3956                 /*
 3957                  * allocate a extern arq handle and bind the buf
 3958                  */
 3959                 ext_arq_dma_attr = mpt->m_msg_dma_attr;
 3960                 ext_arq_dma_attr.dma_attr_sgllen = 1;
 3961                 if ((ddi_dma_alloc_handle(mpt->m_dip,
 3962                     &ext_arq_dma_attr, callback,
 3963                     NULL, &cmd->cmd_ext_arqhandle)) != DDI_SUCCESS) {
 3964                         goto fail;
 3965                 }
 3966 
 3967                 if (ddi_dma_buf_bind_handle(cmd->cmd_ext_arqhandle,
 3968                     cmd->cmd_ext_arq_buf, (DDI_DMA_READ | DDI_DMA_CONSISTENT),
 3969                     callback, NULL, &cmd->cmd_ext_arqcookie,
 3970                     &cookiec)
 3971                     != DDI_SUCCESS) {
 3972                         goto fail;
 3973                 }
 3974                 cmd->cmd_flags |= CFLAG_EXTARQBUFVALID;
 3975         }
 3976         return (0);
 3977 fail:
 3978         mptsas_pkt_destroy_extern(mpt, cmd);
 3979         return (1);
 3980 }
 3981 
 3982 /*
 3983  * deallocate external pkt space and deallocate the pkt
 3984  */
 3985 static void
 3986 mptsas_pkt_destroy_extern(mptsas_t *mpt, mptsas_cmd_t *cmd)
 3987 {
 3988         NDBG3(("mptsas_pkt_destroy_extern: cmd=0x%p", (void *)cmd));
 3989 
 3990         if (cmd->cmd_flags & CFLAG_FREE) {
 3991                 mptsas_log(mpt, CE_PANIC,
 3992                     "mptsas_pkt_destroy_extern: freeing free packet");
 3993                 _NOTE(NOT_REACHED)
 3994                 /* NOTREACHED */
 3995         }
 3996         if (cmd->cmd_flags & CFLAG_CDBEXTERN) {
 3997                 kmem_free(cmd->cmd_pkt->pkt_cdbp, (size_t)cmd->cmd_cdblen);
 3998         }
 3999         if (cmd->cmd_flags & CFLAG_SCBEXTERN) {
 4000                 kmem_free(cmd->cmd_pkt->pkt_scbp, (size_t)cmd->cmd_scblen);
 4001                 if (cmd->cmd_flags & CFLAG_EXTARQBUFVALID) {
 4002                         (void) ddi_dma_unbind_handle(cmd->cmd_ext_arqhandle);
 4003                 }
 4004                 if (cmd->cmd_ext_arqhandle) {
 4005                         ddi_dma_free_handle(&cmd->cmd_ext_arqhandle);
 4006                         cmd->cmd_ext_arqhandle = NULL;
 4007                 }
 4008                 if (cmd->cmd_ext_arq_buf)
 4009                         scsi_free_consistent_buf(cmd->cmd_ext_arq_buf);
 4010         }
 4011         if (cmd->cmd_flags & CFLAG_PRIVEXTERN) {
 4012                 kmem_free(cmd->cmd_pkt->pkt_private, (size_t)cmd->cmd_privlen);
 4013         }
 4014         cmd->cmd_flags = CFLAG_FREE;
 4015         kmem_cache_free(mpt->m_kmem_cache, (void *)cmd);
 4016 }
 4017 
 4018 /*
 4019  * tran_sync_pkt(9E) - explicit DMA synchronization
 4020  */
 4021 /*ARGSUSED*/
 4022 static void
 4023 mptsas_scsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
 4024 {
 4025         mptsas_cmd_t    *cmd = PKT2CMD(pkt);
 4026 
 4027         NDBG3(("mptsas_scsi_sync_pkt: target=%d, pkt=0x%p",
 4028             ap->a_target, (void *)pkt));
 4029 
 4030         if (cmd->cmd_dmahandle) {
 4031                 (void) ddi_dma_sync(cmd->cmd_dmahandle, 0, 0,
 4032                     (cmd->cmd_flags & CFLAG_DMASEND) ?
 4033                     DDI_DMA_SYNC_FORDEV : DDI_DMA_SYNC_FORCPU);
 4034         }
 4035 }
 4036 
 4037 /*
 4038  * tran_dmafree(9E) - deallocate DMA resources allocated for command
 4039  */
 4040 /*ARGSUSED*/
 4041 static void
 4042 mptsas_scsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt)
 4043 {
 4044         mptsas_cmd_t    *cmd = PKT2CMD(pkt);
 4045         mptsas_t        *mpt = ADDR2MPT(ap);
 4046 
 4047         NDBG3(("mptsas_scsi_dmafree: target=%d pkt=0x%p",
 4048             ap->a_target, (void *)pkt));
 4049 
 4050         if (cmd->cmd_flags & CFLAG_DMAVALID) {
 4051                 (void) ddi_dma_unbind_handle(cmd->cmd_dmahandle);
 4052                 cmd->cmd_flags &= ~CFLAG_DMAVALID;
 4053         }
 4054 
 4055         if (cmd->cmd_flags & CFLAG_EXTARQBUFVALID) {
 4056                 (void) ddi_dma_unbind_handle(cmd->cmd_ext_arqhandle);
 4057                 cmd->cmd_flags &= ~CFLAG_EXTARQBUFVALID;
 4058         }
 4059 
 4060         mptsas_free_extra_sgl_frame(mpt, cmd);
 4061 }
 4062 
 4063 static void
 4064 mptsas_pkt_comp(struct scsi_pkt *pkt, mptsas_cmd_t *cmd)
 4065 {
 4066         if ((cmd->cmd_flags & CFLAG_CMDIOPB) &&
 4067             (!(cmd->cmd_flags & CFLAG_DMASEND))) {
 4068                 (void) ddi_dma_sync(cmd->cmd_dmahandle, 0, 0,
 4069                     DDI_DMA_SYNC_FORCPU);
 4070         }
 4071         (*pkt->pkt_comp)(pkt);
 4072 }
 4073 
 4074 static void
 4075 mptsas_sge_setup(mptsas_t *mpt, mptsas_cmd_t *cmd, uint32_t *control,
 4076         pMpi2SCSIIORequest_t frame, ddi_acc_handle_t acc_hdl)
 4077 {
 4078         uint_t                  cookiec;
 4079         mptti_t                 *dmap;
 4080         uint32_t                flags;
 4081         pMpi2SGESimple64_t      sge;
 4082         pMpi2SGEChain64_t       sgechain;
 4083         ASSERT(cmd->cmd_flags & CFLAG_DMAVALID);
 4084 
 4085         /*
 4086          * Save the number of entries in the DMA
 4087          * Scatter/Gather list
 4088          */
 4089         cookiec = cmd->cmd_cookiec;
 4090 
 4091         NDBG1(("mptsas_sge_setup: cookiec=%d", cookiec));
 4092 
 4093         /*
 4094          * Set read/write bit in control.
 4095          */
 4096         if (cmd->cmd_flags & CFLAG_DMASEND) {
 4097                 *control |= MPI2_SCSIIO_CONTROL_WRITE;
 4098         } else {
 4099                 *control |= MPI2_SCSIIO_CONTROL_READ;
 4100         }
 4101 
 4102         ddi_put32(acc_hdl, &frame->DataLength, cmd->cmd_dmacount);
 4103 
 4104         /*
 4105          * We have 2 cases here.  First where we can fit all the
 4106          * SG elements into the main frame, and the case
 4107          * where we can't.
 4108          * If we have more cookies than we can attach to a frame
 4109          * we will need to use a chain element to point
 4110          * a location of memory where the rest of the S/G
 4111          * elements reside.
 4112          */
 4113         if (cookiec <= MPTSAS_MAX_FRAME_SGES64(mpt)) {
 4114                 dmap = cmd->cmd_sg;
 4115                 sge = (pMpi2SGESimple64_t)(&frame->SGL);
 4116                 while (cookiec--) {
 4117                         ddi_put32(acc_hdl,
 4118                             &sge->Address.Low, dmap->addr.address64.Low);
 4119                         ddi_put32(acc_hdl,
 4120                             &sge->Address.High, dmap->addr.address64.High);
 4121                         ddi_put32(acc_hdl, &sge->FlagsLength,
 4122                             dmap->count);
 4123                         flags = ddi_get32(acc_hdl, &sge->FlagsLength);
 4124                         flags |= ((uint32_t)
 4125                             (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
 4126                             MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
 4127                             MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
 4128                             MPI2_SGE_FLAGS_SHIFT);
 4129 
 4130                         /*
 4131                          * If this is the last cookie, we set the flags
 4132                          * to indicate so
 4133                          */
 4134                         if (cookiec == 0) {
 4135                                 flags |=
 4136                                     ((uint32_t)(MPI2_SGE_FLAGS_LAST_ELEMENT
 4137                                     | MPI2_SGE_FLAGS_END_OF_BUFFER
 4138                                     | MPI2_SGE_FLAGS_END_OF_LIST) <<
 4139                                     MPI2_SGE_FLAGS_SHIFT);
 4140                         }
 4141                         if (cmd->cmd_flags & CFLAG_DMASEND) {
 4142                                 flags |= (MPI2_SGE_FLAGS_HOST_TO_IOC <<
 4143                                     MPI2_SGE_FLAGS_SHIFT);
 4144                         } else {
 4145                                 flags |= (MPI2_SGE_FLAGS_IOC_TO_HOST <<
 4146                                     MPI2_SGE_FLAGS_SHIFT);
 4147                         }
 4148                         ddi_put32(acc_hdl, &sge->FlagsLength, flags);
 4149                         dmap++;
 4150                         sge++;
 4151                 }
 4152         } else {
 4153                 /*
 4154                  * Hereby we start to deal with multiple frames.
 4155                  * The process is as follows:
 4156                  * 1. Determine how many frames are needed for SGL element
 4157                  *    storage; Note that all frames are stored in contiguous
 4158                  *    memory space and in 64-bit DMA mode each element is
 4159                  *    3 double-words (12 bytes) long.
 4160                  * 2. Fill up the main frame. We need to do this separately
 4161                  *    since it contains the SCSI IO request header and needs
 4162                  *    dedicated processing. Note that the last 4 double-words
 4163                  *    of the SCSI IO header is for SGL element storage
 4164                  *    (MPI2_SGE_IO_UNION).
 4165                  * 3. Fill the chain element in the main frame, so the DMA
 4166                  *    engine can use the following frames.
 4167                  * 4. Enter a loop to fill the remaining frames. Note that the
 4168                  *    last frame contains no chain element.  The remaining
 4169                  *    frames go into the mpt SGL buffer allocated on the fly,
 4170                  *    not immediately following the main message frame, as in
 4171                  *    Gen1.
 4172                  * Some restrictions:
 4173                  * 1. For 64-bit DMA, the simple element and chain element
 4174                  *    are both of 3 double-words (12 bytes) in size, even
 4175                  *    though all frames are stored in the first 4G of mem
 4176                  *    range and the higher 32-bits of the address are always 0.
 4177                  * 2. On some controllers (like the 1064/1068), a frame can
 4178                  *    hold SGL elements with the last 1 or 2 double-words
 4179                  *    (4 or 8 bytes) un-used. On these controllers, we should
 4180                  *    recognize that there's not enough room for another SGL
 4181                  *    element and move the sge pointer to the next frame.
 4182                  */
 4183                 int             i, j, k, l, frames, sgemax;
 4184                 int             temp;
 4185                 uint8_t         chainflags;
 4186                 uint16_t        chainlength;
 4187                 mptsas_cache_frames_t *p;
 4188 
 4189                 /*
 4190                  * Sgemax is the number of SGE's that will fit
 4191                  * each extra frame and frames is total
 4192                  * number of frames we'll need.  1 sge entry per
 4193                  * frame is reseverd for the chain element thus the -1 below.
 4194                  */
 4195                 sgemax = ((mpt->m_req_frame_size / sizeof (MPI2_SGE_SIMPLE64))
 4196                     - 1);
 4197                 temp = (cookiec - (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) / sgemax;
 4198 
 4199                 /*
 4200                  * A little check to see if we need to round up the number
 4201                  * of frames we need
 4202                  */
 4203                 if ((cookiec - (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) - (temp *
 4204                     sgemax) > 1) {
 4205                         frames = (temp + 1);
 4206                 } else {
 4207                         frames = temp;
 4208                 }
 4209                 dmap = cmd->cmd_sg;
 4210                 sge = (pMpi2SGESimple64_t)(&frame->SGL);
 4211 
 4212                 /*
 4213                  * First fill in the main frame
 4214                  */
 4215                 for (j = 1; j < MPTSAS_MAX_FRAME_SGES64(mpt); j++) {
 4216                         ddi_put32(acc_hdl, &sge->Address.Low,
 4217                             dmap->addr.address64.Low);
 4218                         ddi_put32(acc_hdl, &sge->Address.High,
 4219                             dmap->addr.address64.High);
 4220                         ddi_put32(acc_hdl, &sge->FlagsLength, dmap->count);
 4221                         flags = ddi_get32(acc_hdl, &sge->FlagsLength);
 4222                         flags |= ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
 4223                             MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
 4224                             MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
 4225                             MPI2_SGE_FLAGS_SHIFT);
 4226 
 4227                         /*
 4228                          * If this is the last SGE of this frame
 4229                          * we set the end of list flag
 4230                          */
 4231                         if (j == (MPTSAS_MAX_FRAME_SGES64(mpt) - 1)) {
 4232                                 flags |= ((uint32_t)
 4233                                     (MPI2_SGE_FLAGS_LAST_ELEMENT) <<
 4234                                     MPI2_SGE_FLAGS_SHIFT);
 4235                         }
 4236                         if (cmd->cmd_flags & CFLAG_DMASEND) {
 4237                                 flags |=
 4238                                     (MPI2_SGE_FLAGS_HOST_TO_IOC <<
 4239                                     MPI2_SGE_FLAGS_SHIFT);
 4240                         } else {
 4241                                 flags |=
 4242                                     (MPI2_SGE_FLAGS_IOC_TO_HOST <<
 4243                                     MPI2_SGE_FLAGS_SHIFT);
 4244                         }
 4245                         ddi_put32(acc_hdl, &sge->FlagsLength, flags);
 4246                         dmap++;
 4247                         sge++;
 4248                 }
 4249 
 4250                 /*
 4251                  * Fill in the chain element in the main frame.
 4252                  * About calculation on ChainOffset:
 4253                  * 1. Struct msg_scsi_io_request has 4 double-words (16 bytes)
 4254                  *    in the end reserved for SGL element storage
 4255                  *    (MPI2_SGE_IO_UNION); we should count it in our
 4256                  *    calculation.  See its definition in the header file.
 4257                  * 2. Constant j is the counter of the current SGL element
 4258                  *    that will be processed, and (j - 1) is the number of
 4259                  *    SGL elements that have been processed (stored in the
 4260                  *    main frame).
 4261                  * 3. ChainOffset value should be in units of double-words (4
 4262                  *    bytes) so the last value should be divided by 4.
 4263                  */
 4264                 ddi_put8(acc_hdl, &frame->ChainOffset,
 4265                     (sizeof (MPI2_SCSI_IO_REQUEST) -
 4266                     sizeof (MPI2_SGE_IO_UNION) +
 4267                     (j - 1) * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
 4268                 sgechain = (pMpi2SGEChain64_t)sge;
 4269                 chainflags = (MPI2_SGE_FLAGS_CHAIN_ELEMENT |
 4270                     MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
 4271                     MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
 4272                 ddi_put8(acc_hdl, &sgechain->Flags, chainflags);
 4273 
 4274                 /*
 4275                  * The size of the next frame is the accurate size of space
 4276                  * (in bytes) used to store the SGL elements. j is the counter
 4277                  * of SGL elements. (j - 1) is the number of SGL elements that
 4278                  * have been processed (stored in frames).
 4279                  */
 4280                 if (frames >= 2) {
 4281                         chainlength = mpt->m_req_frame_size /
 4282                             sizeof (MPI2_SGE_SIMPLE64) *
 4283                             sizeof (MPI2_SGE_SIMPLE64);
 4284                 } else {
 4285                         chainlength = ((cookiec - (j - 1)) *
 4286                             sizeof (MPI2_SGE_SIMPLE64));
 4287                 }
 4288 
 4289                 p = cmd->cmd_extra_frames;
 4290 
 4291                 ddi_put16(acc_hdl, &sgechain->Length, chainlength);
 4292                 ddi_put32(acc_hdl, &sgechain->Address.Low,
 4293                     p->m_phys_addr);
 4294                 /* SGL is allocated in the first 4G mem range */
 4295                 ddi_put32(acc_hdl, &sgechain->Address.High, 0);
 4296 
 4297                 /*
 4298                  * If there are more than 2 frames left we have to
 4299                  * fill in the next chain offset to the location of
 4300                  * the chain element in the next frame.
 4301                  * sgemax is the number of simple elements in an extra
 4302                  * frame. Note that the value NextChainOffset should be
 4303                  * in double-words (4 bytes).
 4304                  */
 4305                 if (frames >= 2) {
 4306                         ddi_put8(acc_hdl, &sgechain->NextChainOffset,
 4307                             (sgemax * sizeof (MPI2_SGE_SIMPLE64)) >> 2);
 4308                 } else {
 4309                         ddi_put8(acc_hdl, &sgechain->NextChainOffset, 0);
 4310                 }
 4311 
 4312                 /*
 4313                  * Jump to next frame;
 4314                  * Starting here, chain buffers go into the per command SGL.
 4315                  * This buffer is allocated when chain buffers are needed.
 4316                  */
 4317                 sge = (pMpi2SGESimple64_t)p->m_frames_addr;
 4318                 i = cookiec;
 4319 
 4320                 /*
 4321                  * Start filling in frames with SGE's.  If we
 4322                  * reach the end of frame and still have SGE's
 4323                  * to fill we need to add a chain element and
 4324                  * use another frame.  j will be our counter
 4325                  * for what cookie we are at and i will be
 4326                  * the total cookiec. k is the current frame
 4327                  */
 4328                 for (k = 1; k <= frames; k++) {
 4329                         for (l = 1; (l <= (sgemax + 1)) && (j <= i); j++, l++) {
 4330 
 4331                                 /*
 4332                                  * If we have reached the end of frame
 4333                                  * and we have more SGE's to fill in
 4334                                  * we have to fill the final entry
 4335                                  * with a chain element and then
 4336                                  * continue to the next frame
 4337                                  */
 4338                                 if ((l == (sgemax + 1)) && (k != frames)) {
 4339                                         sgechain = (pMpi2SGEChain64_t)sge;
 4340                                         j--;
 4341                                         chainflags = (
 4342                                             MPI2_SGE_FLAGS_CHAIN_ELEMENT |
 4343                                             MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
 4344                                             MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
 4345                                         ddi_put8(p->m_acc_hdl,
 4346                                             &sgechain->Flags, chainflags);
 4347                                         /*
 4348                                          * k is the frame counter and (k + 1)
 4349                                          * is the number of the next frame.
 4350                                          * Note that frames are in contiguous
 4351                                          * memory space.
 4352                                          */
 4353                                         ddi_put32(p->m_acc_hdl,
 4354                                             &sgechain->Address.Low,
 4355                                             (p->m_phys_addr +
 4356                                             (mpt->m_req_frame_size * k)));
 4357                                         ddi_put32(p->m_acc_hdl,
 4358                                             &sgechain->Address.High, 0);
 4359 
 4360                                         /*
 4361                                          * If there are more than 2 frames left
 4362                                          * we have to next chain offset to
 4363                                          * the location of the chain element
 4364                                          * in the next frame and fill in the
 4365                                          * length of the next chain
 4366                                          */
 4367                                         if ((frames - k) >= 2) {
 4368                                                 ddi_put8(p->m_acc_hdl,
 4369                                                     &sgechain->NextChainOffset,
 4370                                                     (sgemax *
 4371                                                     sizeof (MPI2_SGE_SIMPLE64))
 4372                                                     >> 2);
 4373                                                 ddi_put16(p->m_acc_hdl,
 4374                                                     &sgechain->Length,
 4375                                                     mpt->m_req_frame_size /
 4376                                                     sizeof (MPI2_SGE_SIMPLE64) *
 4377                                                     sizeof (MPI2_SGE_SIMPLE64));
 4378                                         } else {
 4379                                                 /*
 4380                                                  * This is the last frame. Set
 4381                                                  * the NextChainOffset to 0 and
 4382                                                  * Length is the total size of
 4383                                                  * all remaining simple elements
 4384                                                  */
 4385                                                 ddi_put8(p->m_acc_hdl,
 4386                                                     &sgechain->NextChainOffset,
 4387                                                     0);
 4388                                                 ddi_put16(p->m_acc_hdl,
 4389                                                     &sgechain->Length,
 4390                                                     (cookiec - j) *
 4391                                                     sizeof (MPI2_SGE_SIMPLE64));
 4392                                         }
 4393 
 4394                                         /* Jump to the next frame */
 4395                                         sge = (pMpi2SGESimple64_t)
 4396                                             ((char *)p->m_frames_addr +
 4397                                             (int)mpt->m_req_frame_size * k);
 4398 
 4399                                         continue;
 4400                                 }
 4401 
 4402                                 ddi_put32(p->m_acc_hdl,
 4403                                     &sge->Address.Low,
 4404                                     dmap->addr.address64.Low);
 4405                                 ddi_put32(p->m_acc_hdl,
 4406                                     &sge->Address.High,
 4407                                     dmap->addr.address64.High);
 4408                                 ddi_put32(p->m_acc_hdl,
 4409                                     &sge->FlagsLength, dmap->count);
 4410                                 flags = ddi_get32(p->m_acc_hdl,
 4411                                     &sge->FlagsLength);
 4412                                 flags |= ((uint32_t)(
 4413                                     MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
 4414                                     MPI2_SGE_FLAGS_SYSTEM_ADDRESS |
 4415                                     MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
 4416                                     MPI2_SGE_FLAGS_SHIFT);
 4417 
 4418                                 /*
 4419                                  * If we are at the end of the frame and
 4420                                  * there is another frame to fill in
 4421                                  * we set the last simple element as last
 4422                                  * element
 4423                                  */
 4424                                 if ((l == sgemax) && (k != frames)) {
 4425                                         flags |= ((uint32_t)
 4426                                             (MPI2_SGE_FLAGS_LAST_ELEMENT) <<
 4427                                             MPI2_SGE_FLAGS_SHIFT);
 4428                                 }
 4429 
 4430                                 /*
 4431                                  * If this is the final cookie we
 4432                                  * indicate it by setting the flags
 4433                                  */
 4434                                 if (j == i) {
 4435                                         flags |= ((uint32_t)
 4436                                             (MPI2_SGE_FLAGS_LAST_ELEMENT |
 4437                                             MPI2_SGE_FLAGS_END_OF_BUFFER |
 4438                                             MPI2_SGE_FLAGS_END_OF_LIST) <<
 4439                                             MPI2_SGE_FLAGS_SHIFT);
 4440                                 }
 4441                                 if (cmd->cmd_flags & CFLAG_DMASEND) {
 4442                                         flags |=
 4443                                             (MPI2_SGE_FLAGS_HOST_TO_IOC <<
 4444                                             MPI2_SGE_FLAGS_SHIFT);
 4445                                 } else {
 4446                                         flags |=
 4447                                             (MPI2_SGE_FLAGS_IOC_TO_HOST <<
 4448                                             MPI2_SGE_FLAGS_SHIFT);
 4449                                 }
 4450                                 ddi_put32(p->m_acc_hdl,
 4451                                     &sge->FlagsLength, flags);
 4452                                 dmap++;
 4453                                 sge++;
 4454                         }
 4455                 }
 4456 
 4457                 /*
 4458                  * Sync DMA with the chain buffers that were just created
 4459                  */
 4460                 (void) ddi_dma_sync(p->m_dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
 4461         }
 4462 }
 4463 
 4464 /*
 4465  * Interrupt handling
 4466  * Utility routine.  Poll for status of a command sent to HBA
 4467  * without interrupts (a FLAG_NOINTR command).
 4468  */
 4469 int
 4470 mptsas_poll(mptsas_t *mpt, mptsas_cmd_t *poll_cmd, int polltime)
 4471 {
 4472         int     rval = TRUE;
 4473 
 4474         NDBG5(("mptsas_poll: cmd=0x%p", (void *)poll_cmd));
 4475 
 4476         /*
 4477          * In order to avoid using m_mutex in ISR(a new separate mutex
 4478          * m_intr_mutex is introduced) and keep the same lock logic,
 4479          * the m_intr_mutex should be used to protect the getting and
 4480          * setting of the ReplyDescriptorIndex.
 4481          *
 4482          * Since the m_intr_mutex would be released during processing the poll
 4483          * cmd, so we should set the poll flag earlier here to make sure the
 4484          * polled cmd be handled in this thread/context. A side effect is other
 4485          * cmds during the period between the flag set and reset are also
 4486          * handled in this thread and not the ISR. Since the poll cmd is not
 4487          * so common, so the performance degradation in this case is not a big
 4488          * issue.
 4489          */
 4490         mutex_enter(&mpt->m_intr_mutex);
 4491         mpt->m_polled_intr = 1;
 4492         mutex_exit(&mpt->m_intr_mutex);
 4493 
 4494         if ((poll_cmd->cmd_flags & CFLAG_TM_CMD) == 0) {
 4495                 mptsas_restart_hba(mpt);
 4496         }
 4497 
 4498         /*
 4499          * Wait, using drv_usecwait(), long enough for the command to
 4500          * reasonably return from the target if the target isn't
 4501          * "dead".  A polled command may well be sent from scsi_poll, and
 4502          * there are retries built in to scsi_poll if the transport
 4503          * accepted the packet (TRAN_ACCEPT).  scsi_poll waits 1 second
 4504          * and retries the transport up to scsi_poll_busycnt times
 4505          * (currently 60) if
 4506          * 1. pkt_reason is CMD_INCOMPLETE and pkt_state is 0, or
 4507          * 2. pkt_reason is CMD_CMPLT and *pkt_scbp has STATUS_BUSY
 4508          *
 4509          * limit the waiting to avoid a hang in the event that the
 4510          * cmd never gets started but we are still receiving interrupts
 4511          */
 4512         while (!(poll_cmd->cmd_flags & CFLAG_FINISHED)) {
 4513                 if (mptsas_wait_intr(mpt, polltime) == FALSE) {
 4514                         NDBG5(("mptsas_poll: command incomplete"));
 4515                         rval = FALSE;
 4516                         break;
 4517                 }
 4518         }
 4519 
 4520         mutex_enter(&mpt->m_intr_mutex);
 4521         mpt->m_polled_intr = 0;
 4522         mutex_exit(&mpt->m_intr_mutex);
 4523 
 4524         if (rval == FALSE) {
 4525 
 4526                 /*
 4527                  * this isn't supposed to happen, the hba must be wedged
 4528                  * Mark this cmd as a timeout.
 4529                  */
 4530                 mptsas_set_pkt_reason(mpt, poll_cmd, CMD_TIMEOUT,
 4531                     (STAT_TIMEOUT|STAT_ABORTED));
 4532 
 4533                 if (poll_cmd->cmd_queued == FALSE) {
 4534 
 4535                         NDBG5(("mptsas_poll: not on waitq"));
 4536 
 4537                         poll_cmd->cmd_pkt->pkt_state |=
 4538                             (STATE_GOT_BUS|STATE_GOT_TARGET|STATE_SENT_CMD);
 4539                 } else {
 4540 
 4541                         /* find and remove it from the waitq */
 4542                         NDBG5(("mptsas_poll: delete from waitq"));
 4543                         mptsas_waitq_delete(mpt, poll_cmd);
 4544                 }
 4545 
 4546         }
 4547         mptsas_fma_check(mpt, poll_cmd);
 4548         NDBG5(("mptsas_poll: done"));
 4549         return (rval);
 4550 }
 4551 
 4552 /*
 4553  * Used for polling cmds and TM function
 4554  */
 4555 static int
 4556 mptsas_wait_intr(mptsas_t *mpt, int polltime)
 4557 {
 4558         int                             cnt;
 4559         pMpi2ReplyDescriptorsUnion_t    reply_desc_union;
 4560         Mpi2ReplyDescriptorsUnion_t     reply_desc_union_v;
 4561         uint32_t                        int_mask;
 4562         uint8_t reply_type;
 4563 
 4564         NDBG5(("mptsas_wait_intr"));
 4565 
 4566 
 4567         /*
 4568          * Get the current interrupt mask and disable interrupts.  When
 4569          * re-enabling ints, set mask to saved value.
 4570          */
 4571         int_mask = ddi_get32(mpt->m_datap, &mpt->m_reg->HostInterruptMask);
 4572         MPTSAS_DISABLE_INTR(mpt);
 4573 
 4574         /*
 4575          * Keep polling for at least (polltime * 1000) seconds
 4576          */
 4577         for (cnt = 0; cnt < polltime; cnt++) {
 4578                 mutex_enter(&mpt->m_intr_mutex);
 4579                 (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
 4580                     DDI_DMA_SYNC_FORCPU);
 4581 
 4582                 reply_desc_union = (pMpi2ReplyDescriptorsUnion_t)
 4583                     MPTSAS_GET_NEXT_REPLY(mpt, mpt->m_post_index);
 4584 
 4585                 if (ddi_get32(mpt->m_acc_post_queue_hdl,
 4586                     &reply_desc_union->Words.Low) == 0xFFFFFFFF ||
 4587                     ddi_get32(mpt->m_acc_post_queue_hdl,
 4588                     &reply_desc_union->Words.High) == 0xFFFFFFFF) {
 4589                         mutex_exit(&mpt->m_intr_mutex);
 4590                         drv_usecwait(1000);
 4591                         continue;
 4592                 }
 4593 
 4594                 reply_type = ddi_get8(mpt->m_acc_post_queue_hdl,
 4595                     &reply_desc_union->Default.ReplyFlags);
 4596                 reply_type &= MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
 4597                 reply_desc_union_v.Default.ReplyFlags = reply_type;
 4598                 if (reply_type == MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS) {
 4599                         reply_desc_union_v.SCSIIOSuccess.SMID =
 4600                             ddi_get16(mpt->m_acc_post_queue_hdl,
 4601                             &reply_desc_union->SCSIIOSuccess.SMID);
 4602                 } else if (reply_type ==
 4603                     MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
 4604                         reply_desc_union_v.AddressReply.ReplyFrameAddress =
 4605                             ddi_get32(mpt->m_acc_post_queue_hdl,
 4606                             &reply_desc_union->AddressReply.ReplyFrameAddress);
 4607                         reply_desc_union_v.AddressReply.SMID =
 4608                             ddi_get16(mpt->m_acc_post_queue_hdl,
 4609                             &reply_desc_union->AddressReply.SMID);
 4610                 }
 4611                 /*
 4612                  * Clear the reply descriptor for re-use and increment
 4613                  * index.
 4614                  */
 4615                 ddi_put64(mpt->m_acc_post_queue_hdl,
 4616                     &((uint64_t *)(void *)mpt->m_post_queue)[mpt->m_post_index],
 4617                     0xFFFFFFFFFFFFFFFF);
 4618                 (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
 4619                     DDI_DMA_SYNC_FORDEV);
 4620 
 4621                 if (++mpt->m_post_index == mpt->m_post_queue_depth) {
 4622                         mpt->m_post_index = 0;
 4623                 }
 4624 
 4625                 /*
 4626                  * Update the global reply index
 4627                  */
 4628                 ddi_put32(mpt->m_datap,
 4629                     &mpt->m_reg->ReplyPostHostIndex, mpt->m_post_index);
 4630                 mutex_exit(&mpt->m_intr_mutex);
 4631 
 4632                 /*
 4633                  * The reply is valid, process it according to its
 4634                  * type.
 4635                  */
 4636                 mptsas_process_intr(mpt, &reply_desc_union_v);
 4637 
 4638 
 4639                 /*
 4640                  * Re-enable interrupts and quit.
 4641                  */
 4642                 ddi_put32(mpt->m_datap, &mpt->m_reg->HostInterruptMask,
 4643                     int_mask);
 4644                 return (TRUE);
 4645 
 4646         }
 4647 
 4648         /*
 4649          * Clear polling flag, re-enable interrupts and quit.
 4650          */
 4651         ddi_put32(mpt->m_datap, &mpt->m_reg->HostInterruptMask, int_mask);
 4652         return (FALSE);
 4653 }
 4654 
 4655 /*
 4656  * For fastpath, the m_intr_mutex should be held from the begining to the end,
 4657  * so we only treat those cmds that need not release m_intr_mutex(even just for
 4658  * a moment) as candidate for fast processing. otherwise, we don't handle them
 4659  * and just return, then in ISR, those cmds would be handled later with m_mutex
 4660  * held and m_intr_mutex not held.
 4661  */
 4662 static int
 4663 mptsas_handle_io_fastpath(mptsas_t *mpt,
 4664     uint16_t SMID)
 4665 {
 4666         mptsas_slots_t                          *slots = mpt->m_active;
 4667         mptsas_cmd_t                            *cmd = NULL;
 4668         struct scsi_pkt                         *pkt;
 4669 
 4670         /*
 4671          * This is a success reply so just complete the IO.  First, do a sanity
 4672          * check on the SMID.  The final slot is used for TM requests, which
 4673          * would not come into this reply handler.
 4674          */
 4675         if ((SMID == 0) || (SMID > slots->m_n_slots)) {
 4676                 mptsas_log(mpt, CE_WARN, "?Received invalid SMID of %d\n",
 4677                     SMID);
 4678                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 4679                 return (TRUE);
 4680         }
 4681 
 4682         cmd = slots->m_slot[SMID];
 4683 
 4684         /*
 4685          * print warning and return if the slot is empty
 4686          */
 4687         if (cmd == NULL) {
 4688                 mptsas_log(mpt, CE_WARN, "?NULL command for successful SCSI IO "
 4689                     "in slot %d", SMID);
 4690                 return (TRUE);
 4691         }
 4692 
 4693         pkt = CMD2PKT(cmd);
 4694         pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
 4695             STATE_GOT_STATUS);
 4696         if (cmd->cmd_flags & CFLAG_DMAVALID) {
 4697                 pkt->pkt_state |= STATE_XFERRED_DATA;
 4698         }
 4699         pkt->pkt_resid = 0;
 4700 
 4701         /*
 4702          * If the cmd is a IOC, or a passthrough, then we don't process it in
 4703          * fastpath, and later it would be handled by mptsas_process_intr()
 4704          * with m_mutex protected.
 4705          */
 4706         if (cmd->cmd_flags & (CFLAG_PASSTHRU | CFLAG_CMDIOC)) {
 4707                 return (FALSE);
 4708         } else {
 4709                 mptsas_remove_cmd0(mpt, cmd);
 4710         }
 4711 
 4712         if (cmd->cmd_flags & CFLAG_RETRY) {
 4713                 /*
 4714                  * The target returned QFULL or busy, do not add tihs
 4715                  * pkt to the doneq since the hba will retry
 4716                  * this cmd.
 4717                  *
 4718                  * The pkt has already been resubmitted in
 4719                  * mptsas_handle_qfull() or in mptsas_check_scsi_io_error().
 4720                  * Remove this cmd_flag here.
 4721                  */
 4722                 cmd->cmd_flags &= ~CFLAG_RETRY;
 4723         } else {
 4724                 mptsas_doneq_add0(mpt, cmd);
 4725         }
 4726 
 4727         /*
 4728          * In fastpath, the cmd should only be a context reply, so just check
 4729          * the post queue of the reply descriptor and the dmahandle of the cmd
 4730          * is enough. No sense data in this case and no need to check the dma
 4731          * handle where sense data dma info is saved, the dma handle of the
 4732          * reply frame, and the dma handle of the reply free queue.
 4733          * For the dma handle of the request queue. Check fma here since we
 4734          * are sure the request must have already been sent/DMAed correctly.
 4735          * otherwise checking in mptsas_scsi_start() is not correct since
 4736          * at that time the dma may not start.
 4737          */
 4738         if ((mptsas_check_dma_handle(mpt->m_dma_req_frame_hdl) !=
 4739             DDI_SUCCESS) ||
 4740             (mptsas_check_dma_handle(mpt->m_dma_post_queue_hdl) !=
 4741             DDI_SUCCESS)) {
 4742                 ddi_fm_service_impact(mpt->m_dip,
 4743                     DDI_SERVICE_UNAFFECTED);
 4744                 pkt->pkt_reason = CMD_TRAN_ERR;
 4745                 pkt->pkt_statistics = 0;
 4746         }
 4747         if (cmd->cmd_dmahandle &&
 4748             (mptsas_check_dma_handle(cmd->cmd_dmahandle) != DDI_SUCCESS)) {
 4749                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 4750                 pkt->pkt_reason = CMD_TRAN_ERR;
 4751                 pkt->pkt_statistics = 0;
 4752         }
 4753         if ((cmd->cmd_extra_frames &&
 4754             ((mptsas_check_dma_handle(cmd->cmd_extra_frames->m_dma_hdl) !=
 4755             DDI_SUCCESS) ||
 4756             (mptsas_check_acc_handle(cmd->cmd_extra_frames->m_acc_hdl) !=
 4757             DDI_SUCCESS)))) {
 4758                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 4759                 pkt->pkt_reason = CMD_TRAN_ERR;
 4760                 pkt->pkt_statistics = 0;
 4761         }
 4762 
 4763         return (TRUE);
 4764 }
 4765 
 4766 static void
 4767 mptsas_handle_scsi_io_success(mptsas_t *mpt,
 4768     pMpi2ReplyDescriptorsUnion_t reply_desc)
 4769 {
 4770         pMpi2SCSIIOSuccessReplyDescriptor_t     scsi_io_success;
 4771         uint16_t                                SMID;
 4772         mptsas_slots_t                          *slots = mpt->m_active;
 4773         mptsas_cmd_t                            *cmd = NULL;
 4774         struct scsi_pkt                         *pkt;
 4775 
 4776         scsi_io_success = (pMpi2SCSIIOSuccessReplyDescriptor_t)reply_desc;
 4777         SMID = scsi_io_success->SMID;
 4778 
 4779         /*
 4780          * This is a success reply so just complete the IO.  First, do a sanity
 4781          * check on the SMID.  The final slot is used for TM requests, which
 4782          * would not come into this reply handler.
 4783          */
 4784         if ((SMID == 0) || (SMID > slots->m_n_slots)) {
 4785                 mptsas_log(mpt, CE_WARN, "?Received invalid SMID of %d\n",
 4786                     SMID);
 4787                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 4788                 return;
 4789         }
 4790 
 4791         cmd = slots->m_slot[SMID];
 4792 
 4793         /*
 4794          * print warning and return if the slot is empty
 4795          */
 4796         if (cmd == NULL) {
 4797                 mptsas_log(mpt, CE_WARN, "?NULL command for successful SCSI IO "
 4798                     "in slot %d", SMID);
 4799                 return;
 4800         }
 4801 
 4802         pkt = CMD2PKT(cmd);
 4803         pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
 4804             STATE_GOT_STATUS);
 4805         if (cmd->cmd_flags & CFLAG_DMAVALID) {
 4806                 pkt->pkt_state |= STATE_XFERRED_DATA;
 4807         }
 4808         pkt->pkt_resid = 0;
 4809 
 4810         if (cmd->cmd_flags & CFLAG_PASSTHRU) {
 4811                 cmd->cmd_flags |= CFLAG_FINISHED;
 4812                 cv_broadcast(&mpt->m_passthru_cv);
 4813                 return;
 4814         } else {
 4815                 mptsas_remove_cmd(mpt, cmd);
 4816         }
 4817 
 4818         if (cmd->cmd_flags & CFLAG_RETRY) {
 4819                 /*
 4820                  * The target returned QFULL or busy, do not add tihs
 4821                  * pkt to the doneq since the hba will retry
 4822                  * this cmd.
 4823                  *
 4824                  * The pkt has already been resubmitted in
 4825                  * mptsas_handle_qfull() or in mptsas_check_scsi_io_error().
 4826                  * Remove this cmd_flag here.
 4827                  */
 4828                 cmd->cmd_flags &= ~CFLAG_RETRY;
 4829         } else {
 4830                 mptsas_doneq_add(mpt, cmd);
 4831         }
 4832 }
 4833 
 4834 static void
 4835 mptsas_handle_address_reply(mptsas_t *mpt,
 4836     pMpi2ReplyDescriptorsUnion_t reply_desc)
 4837 {
 4838         pMpi2AddressReplyDescriptor_t   address_reply;
 4839         pMPI2DefaultReply_t             reply;
 4840         mptsas_fw_diagnostic_buffer_t   *pBuffer;
 4841         uint32_t                        reply_addr;
 4842         uint16_t                        SMID, iocstatus;
 4843         mptsas_slots_t                  *slots = mpt->m_active;
 4844         mptsas_cmd_t                    *cmd = NULL;
 4845         uint8_t                         function, buffer_type;
 4846         m_replyh_arg_t                  *args;
 4847         int                             reply_frame_no;
 4848 
 4849         ASSERT(mutex_owned(&mpt->m_mutex));
 4850 
 4851         address_reply = (pMpi2AddressReplyDescriptor_t)reply_desc;
 4852 
 4853         reply_addr = address_reply->ReplyFrameAddress;
 4854         SMID = address_reply->SMID;
 4855         /*
 4856          * If reply frame is not in the proper range we should ignore this
 4857          * message and exit the interrupt handler.
 4858          */
 4859         if ((reply_addr < mpt->m_reply_frame_dma_addr) ||
 4860             (reply_addr >= (mpt->m_reply_frame_dma_addr +
 4861             (mpt->m_reply_frame_size * mpt->m_max_replies))) ||
 4862             ((reply_addr - mpt->m_reply_frame_dma_addr) %
 4863             mpt->m_reply_frame_size != 0)) {
 4864                 mptsas_log(mpt, CE_WARN, "?Received invalid reply frame "
 4865                     "address 0x%x\n", reply_addr);
 4866                 ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 4867                 return;
 4868         }
 4869 
 4870         (void) ddi_dma_sync(mpt->m_dma_reply_frame_hdl, 0, 0,
 4871             DDI_DMA_SYNC_FORCPU);
 4872         reply = (pMPI2DefaultReply_t)(mpt->m_reply_frame + (reply_addr -
 4873             mpt->m_reply_frame_dma_addr));
 4874         function = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->Function);
 4875 
 4876         /*
 4877          * don't get slot information and command for events since these values
 4878          * don't exist
 4879          */
 4880         if ((function != MPI2_FUNCTION_EVENT_NOTIFICATION) &&
 4881             (function != MPI2_FUNCTION_DIAG_BUFFER_POST)) {
 4882                 /*
 4883                  * This could be a TM reply, which use the last allocated SMID,
 4884                  * so allow for that.
 4885                  */
 4886                 if ((SMID == 0) || (SMID > (slots->m_n_slots + 1))) {
 4887                         mptsas_log(mpt, CE_WARN, "?Received invalid SMID of "
 4888                             "%d\n", SMID);
 4889                         ddi_fm_service_impact(mpt->m_dip,
 4890                             DDI_SERVICE_UNAFFECTED);
 4891                         return;
 4892                 }
 4893 
 4894                 cmd = slots->m_slot[SMID];
 4895 
 4896                 /*
 4897                  * print warning and return if the slot is empty
 4898                  */
 4899                 if (cmd == NULL) {
 4900                         mptsas_log(mpt, CE_WARN, "?NULL command for address "
 4901                             "reply in slot %d", SMID);
 4902                         return;
 4903                 }
 4904                 if ((cmd->cmd_flags & CFLAG_PASSTHRU) ||
 4905                     (cmd->cmd_flags & CFLAG_CONFIG) ||
 4906                     (cmd->cmd_flags & CFLAG_FW_DIAG)) {
 4907                         cmd->cmd_rfm = reply_addr;
 4908                         cmd->cmd_flags |= CFLAG_FINISHED;
 4909                         cv_broadcast(&mpt->m_passthru_cv);
 4910                         cv_broadcast(&mpt->m_config_cv);
 4911                         cv_broadcast(&mpt->m_fw_diag_cv);
 4912                         return;
 4913                 } else if (!(cmd->cmd_flags & CFLAG_FW_CMD)) {
 4914                         mptsas_remove_cmd(mpt, cmd);
 4915                 }
 4916                 NDBG31(("\t\tmptsas_process_intr: slot=%d", SMID));
 4917         }
 4918         /*
 4919          * Depending on the function, we need to handle
 4920          * the reply frame (and cmd) differently.
 4921          */
 4922         switch (function) {
 4923         case MPI2_FUNCTION_SCSI_IO_REQUEST:
 4924                 mptsas_check_scsi_io_error(mpt, (pMpi2SCSIIOReply_t)reply, cmd);
 4925                 break;
 4926         case MPI2_FUNCTION_SCSI_TASK_MGMT:
 4927                 cmd->cmd_rfm = reply_addr;
 4928                 mptsas_check_task_mgt(mpt, (pMpi2SCSIManagementReply_t)reply,
 4929                     cmd);
 4930                 break;
 4931         case MPI2_FUNCTION_FW_DOWNLOAD:
 4932                 cmd->cmd_flags |= CFLAG_FINISHED;
 4933                 cv_signal(&mpt->m_fw_cv);
 4934                 break;
 4935         case MPI2_FUNCTION_EVENT_NOTIFICATION:
 4936                 reply_frame_no = (reply_addr - mpt->m_reply_frame_dma_addr) /
 4937                     mpt->m_reply_frame_size;
 4938                 args = &mpt->m_replyh_args[reply_frame_no];
 4939                 args->mpt = (void *)mpt;
 4940                 args->rfm = reply_addr;
 4941 
 4942                 /*
 4943                  * Record the event if its type is enabled in
 4944                  * this mpt instance by ioctl.
 4945                  */
 4946                 mptsas_record_event(args);
 4947 
 4948                 /*
 4949                  * Handle time critical events
 4950                  * NOT_RESPONDING/ADDED only now
 4951                  */
 4952                 if (mptsas_handle_event_sync(args) == DDI_SUCCESS) {
 4953                         /*
 4954                          * Would not return main process,
 4955                          * just let taskq resolve ack action
 4956                          * and ack would be sent in taskq thread
 4957                          */
 4958                         NDBG20(("send mptsas_handle_event_sync success"));
 4959                 }
 4960                 if ((ddi_taskq_dispatch(mpt->m_event_taskq, mptsas_handle_event,
 4961                     (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
 4962                         mptsas_log(mpt, CE_WARN, "No memory available"
 4963                         "for dispatch taskq");
 4964                         /*
 4965                          * Return the reply frame to the free queue.
 4966                          */
 4967                         ddi_put32(mpt->m_acc_free_queue_hdl,
 4968                             &((uint32_t *)(void *)
 4969                             mpt->m_free_queue)[mpt->m_free_index], reply_addr);
 4970                         (void) ddi_dma_sync(mpt->m_dma_free_queue_hdl, 0, 0,
 4971                             DDI_DMA_SYNC_FORDEV);
 4972                         if (++mpt->m_free_index == mpt->m_free_queue_depth) {
 4973                                 mpt->m_free_index = 0;
 4974                         }
 4975 
 4976                         ddi_put32(mpt->m_datap,
 4977                             &mpt->m_reg->ReplyFreeHostIndex, mpt->m_free_index);
 4978                 }
 4979                 return;
 4980         case MPI2_FUNCTION_DIAG_BUFFER_POST:
 4981                 /*
 4982                  * If SMID is 0, this implies that the reply is due to a
 4983                  * release function with a status that the buffer has been
 4984                  * released.  Set the buffer flags accordingly.
 4985                  */
 4986                 if (SMID == 0) {
 4987                         iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
 4988                             &reply->IOCStatus);
 4989                         buffer_type = ddi_get8(mpt->m_acc_reply_frame_hdl,
 4990                             &(((pMpi2DiagBufferPostReply_t)reply)->BufferType));
 4991                         if (iocstatus == MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED) {
 4992                                 pBuffer =
 4993                                     &mpt->m_fw_diag_buffer_list[buffer_type];
 4994                                 pBuffer->valid_data = TRUE;
 4995                                 pBuffer->owned_by_firmware = FALSE;
 4996                                 pBuffer->immediate = FALSE;
 4997                         }
 4998                 } else {
 4999                         /*
 5000                          * Normal handling of diag post reply with SMID.
 5001                          */
 5002                         cmd = slots->m_slot[SMID];
 5003 
 5004                         /*
 5005                          * print warning and return if the slot is empty
 5006                          */
 5007                         if (cmd == NULL) {
 5008                                 mptsas_log(mpt, CE_WARN, "?NULL command for "
 5009                                     "address reply in slot %d", SMID);
 5010                                 return;
 5011                         }
 5012                         cmd->cmd_rfm = reply_addr;
 5013                         cmd->cmd_flags |= CFLAG_FINISHED;
 5014                         cv_broadcast(&mpt->m_fw_diag_cv);
 5015                 }
 5016                 return;
 5017         default:
 5018                 mptsas_log(mpt, CE_WARN, "Unknown function 0x%x ", function);
 5019                 break;
 5020         }
 5021 
 5022         /*
 5023          * Return the reply frame to the free queue.
 5024          */
 5025         ddi_put32(mpt->m_acc_free_queue_hdl,
 5026             &((uint32_t *)(void *)mpt->m_free_queue)[mpt->m_free_index],
 5027             reply_addr);
 5028         (void) ddi_dma_sync(mpt->m_dma_free_queue_hdl, 0, 0,
 5029             DDI_DMA_SYNC_FORDEV);
 5030         if (++mpt->m_free_index == mpt->m_free_queue_depth) {
 5031                 mpt->m_free_index = 0;
 5032         }
 5033         ddi_put32(mpt->m_datap, &mpt->m_reg->ReplyFreeHostIndex,
 5034             mpt->m_free_index);
 5035 
 5036         if (cmd->cmd_flags & CFLAG_FW_CMD)
 5037                 return;
 5038 
 5039         if (cmd->cmd_flags & CFLAG_RETRY) {
 5040                 /*
 5041                  * The target returned QFULL or busy, do not add tihs
 5042                  * pkt to the doneq since the hba will retry
 5043                  * this cmd.
 5044                  *
 5045                  * The pkt has already been resubmitted in
 5046                  * mptsas_handle_qfull() or in mptsas_check_scsi_io_error().
 5047                  * Remove this cmd_flag here.
 5048                  */
 5049                 cmd->cmd_flags &= ~CFLAG_RETRY;
 5050         } else {
 5051                 mptsas_doneq_add(mpt, cmd);
 5052         }
 5053 }
 5054 
 5055 static void
 5056 mptsas_check_scsi_io_error(mptsas_t *mpt, pMpi2SCSIIOReply_t reply,
 5057     mptsas_cmd_t *cmd)
 5058 {
 5059         uint8_t                 scsi_status, scsi_state;
 5060         uint16_t                ioc_status;
 5061         uint32_t                xferred, sensecount, responsedata, loginfo = 0;
 5062         struct scsi_pkt         *pkt;
 5063         struct scsi_arq_status  *arqstat;
 5064         struct buf              *bp;
 5065         mptsas_target_t         *ptgt = cmd->cmd_tgt_addr;
 5066         uint8_t                 *sensedata = NULL;
 5067 
 5068         if ((cmd->cmd_flags & (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) ==
 5069             (CFLAG_SCBEXTERN | CFLAG_EXTARQBUFVALID)) {
 5070                 bp = cmd->cmd_ext_arq_buf;
 5071         } else {
 5072                 bp = cmd->cmd_arq_buf;
 5073         }
 5074 
 5075         scsi_status = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->SCSIStatus);
 5076         ioc_status = ddi_get16(mpt->m_acc_reply_frame_hdl, &reply->IOCStatus);
 5077         scsi_state = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->SCSIState);
 5078         xferred = ddi_get32(mpt->m_acc_reply_frame_hdl, &reply->TransferCount);
 5079         sensecount = ddi_get32(mpt->m_acc_reply_frame_hdl, &reply->SenseCount);
 5080         responsedata = ddi_get32(mpt->m_acc_reply_frame_hdl,
 5081             &reply->ResponseInfo);
 5082 
 5083         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
 5084                 loginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
 5085                     &reply->IOCLogInfo);
 5086                 mptsas_log(mpt, CE_NOTE,
 5087                     "?Log info 0x%x received for target %d.\n"
 5088                     "\tscsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
 5089                     loginfo, Tgt(cmd), scsi_status, ioc_status,
 5090                     scsi_state);
 5091         }
 5092 
 5093         NDBG31(("\t\tscsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
 5094             scsi_status, ioc_status, scsi_state));
 5095 
 5096         pkt = CMD2PKT(cmd);
 5097         *(pkt->pkt_scbp) = scsi_status;
 5098 
 5099         if (loginfo == 0x31170000) {
 5100                 /*
 5101                  * if loginfo PL_LOGINFO_CODE_IO_DEVICE_MISSING_DELAY_RETRY
 5102                  * 0x31170000 comes, that means the device missing delay
 5103                  * is in progressing, the command need retry later.
 5104                  */
 5105                 *(pkt->pkt_scbp) = STATUS_BUSY;
 5106                 return;
 5107         }
 5108 
 5109         if ((scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS) &&
 5110             ((ioc_status & MPI2_IOCSTATUS_MASK) ==
 5111             MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE)) {
 5112                 pkt->pkt_reason = CMD_INCOMPLETE;
 5113                 pkt->pkt_state |= STATE_GOT_BUS;
 5114                 mutex_enter(&ptgt->m_tgt_intr_mutex);
 5115                 if (ptgt->m_reset_delay == 0) {
 5116                         mptsas_set_throttle(mpt, ptgt,
 5117                             DRAIN_THROTTLE);
 5118                 }
 5119                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 5120                 return;
 5121         }
 5122 
 5123         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
 5124                 responsedata &= 0x000000FF;
 5125                 if (responsedata & MPTSAS_SCSI_RESPONSE_CODE_TLR_OFF) {
 5126                         mptsas_log(mpt, CE_NOTE, "Do not support the TLR\n");
 5127                         pkt->pkt_reason = CMD_TLR_OFF;
 5128                         return;
 5129                 }
 5130         }
 5131 
 5132 
 5133         switch (scsi_status) {
 5134         case MPI2_SCSI_STATUS_CHECK_CONDITION:
 5135                 pkt->pkt_resid = (cmd->cmd_dmacount - xferred);
 5136                 arqstat = (void*)(pkt->pkt_scbp);
 5137                 arqstat->sts_rqpkt_status = *((struct scsi_status *)
 5138                     (pkt->pkt_scbp));
 5139                 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
 5140                     STATE_SENT_CMD | STATE_GOT_STATUS | STATE_ARQ_DONE);
 5141                 if (cmd->cmd_flags & CFLAG_XARQ) {
 5142                         pkt->pkt_state |= STATE_XARQ_DONE;
 5143                 }
 5144                 if (pkt->pkt_resid != cmd->cmd_dmacount) {
 5145                         pkt->pkt_state |= STATE_XFERRED_DATA;
 5146                 }
 5147                 arqstat->sts_rqpkt_reason = pkt->pkt_reason;
 5148                 arqstat->sts_rqpkt_state  = pkt->pkt_state;
 5149                 arqstat->sts_rqpkt_state |= STATE_XFERRED_DATA;
 5150                 arqstat->sts_rqpkt_statistics = pkt->pkt_statistics;
 5151                 sensedata = (uint8_t *)&arqstat->sts_sensedata;
 5152 
 5153                 bcopy((uchar_t *)bp->b_un.b_addr, sensedata,
 5154                     ((cmd->cmd_rqslen >= sensecount) ? sensecount :
 5155                     cmd->cmd_rqslen));
 5156                 arqstat->sts_rqpkt_resid = (cmd->cmd_rqslen - sensecount);
 5157                 cmd->cmd_flags |= CFLAG_CMDARQ;
 5158                 /*
 5159                  * Set proper status for pkt if autosense was valid
 5160                  */
 5161                 if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
 5162                         struct scsi_status zero_status = { 0 };
 5163                         arqstat->sts_rqpkt_status = zero_status;
 5164                 }
 5165 
 5166                 /*
 5167                  * ASC=0x47 is parity error
 5168                  * ASC=0x48 is initiator detected error received
 5169                  */
 5170                 if ((scsi_sense_key(sensedata) == KEY_ABORTED_COMMAND) &&
 5171                     ((scsi_sense_asc(sensedata) == 0x47) ||
 5172                     (scsi_sense_asc(sensedata) == 0x48))) {
 5173                         mptsas_log(mpt, CE_NOTE, "Aborted_command!");
 5174                 }
 5175 
 5176                 /*
 5177                  * ASC/ASCQ=0x3F/0x0E means report_luns data changed
 5178                  * ASC/ASCQ=0x25/0x00 means invalid lun
 5179                  */
 5180                 if (((scsi_sense_key(sensedata) == KEY_UNIT_ATTENTION) &&
 5181                     (scsi_sense_asc(sensedata) == 0x3F) &&
 5182                     (scsi_sense_ascq(sensedata) == 0x0E)) ||
 5183                     ((scsi_sense_key(sensedata) == KEY_ILLEGAL_REQUEST) &&
 5184                     (scsi_sense_asc(sensedata) == 0x25) &&
 5185                     (scsi_sense_ascq(sensedata) == 0x00))) {
 5186                         mptsas_topo_change_list_t *topo_node = NULL;
 5187 
 5188                         topo_node = kmem_zalloc(
 5189                             sizeof (mptsas_topo_change_list_t),
 5190                             KM_NOSLEEP);
 5191                         if (topo_node == NULL) {
 5192                                 mptsas_log(mpt, CE_NOTE, "No memory"
 5193                                     "resource for handle SAS dynamic"
 5194                                     "reconfigure.\n");
 5195                                 break;
 5196                         }
 5197                         topo_node->mpt = mpt;
 5198                         topo_node->event = MPTSAS_DR_EVENT_RECONFIG_TARGET;
 5199                         topo_node->un.phymask = ptgt->m_phymask;
 5200                         topo_node->devhdl = ptgt->m_devhdl;
 5201                         topo_node->object = (void *)ptgt;
 5202                         topo_node->flags = MPTSAS_TOPO_FLAG_LUN_ASSOCIATED;
 5203 
 5204                         if ((ddi_taskq_dispatch(mpt->m_dr_taskq,
 5205                             mptsas_handle_dr,
 5206                             (void *)topo_node,
 5207                             DDI_NOSLEEP)) != DDI_SUCCESS) {
 5208                                 mptsas_log(mpt, CE_NOTE, "mptsas start taskq"
 5209                                     "for handle SAS dynamic reconfigure"
 5210                                     "failed. \n");
 5211                         }
 5212                 }
 5213                 break;
 5214         case MPI2_SCSI_STATUS_GOOD:
 5215                 switch (ioc_status & MPI2_IOCSTATUS_MASK) {
 5216                 case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
 5217                         pkt->pkt_reason = CMD_DEV_GONE;
 5218                         pkt->pkt_state |= STATE_GOT_BUS;
 5219                         mutex_enter(&ptgt->m_tgt_intr_mutex);
 5220                         if (ptgt->m_reset_delay == 0) {
 5221                                 mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
 5222                         }
 5223                         mutex_exit(&ptgt->m_tgt_intr_mutex);
 5224                         NDBG31(("lost disk for target%d, command:%x",
 5225                             Tgt(cmd), pkt->pkt_cdbp[0]));
 5226                         break;
 5227                 case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
 5228                         NDBG31(("data overrun: xferred=%d", xferred));
 5229                         NDBG31(("dmacount=%d", cmd->cmd_dmacount));
 5230                         pkt->pkt_reason = CMD_DATA_OVR;
 5231                         pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET
 5232                             | STATE_SENT_CMD | STATE_GOT_STATUS
 5233                             | STATE_XFERRED_DATA);
 5234                         pkt->pkt_resid = 0;
 5235                         break;
 5236                 case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
 5237                 case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
 5238                         NDBG31(("data underrun: xferred=%d", xferred));
 5239                         NDBG31(("dmacount=%d", cmd->cmd_dmacount));
 5240                         pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET
 5241                             | STATE_SENT_CMD | STATE_GOT_STATUS);
 5242                         pkt->pkt_resid = (cmd->cmd_dmacount - xferred);
 5243                         if (pkt->pkt_resid != cmd->cmd_dmacount) {
 5244                                 pkt->pkt_state |= STATE_XFERRED_DATA;
 5245                         }
 5246                         break;
 5247                 case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
 5248                         mptsas_set_pkt_reason(mpt,
 5249                             cmd, CMD_RESET, STAT_BUS_RESET);
 5250                         break;
 5251                 case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
 5252                 case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
 5253                         mptsas_set_pkt_reason(mpt,
 5254                             cmd, CMD_RESET, STAT_DEV_RESET);
 5255                         break;
 5256                 case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
 5257                 case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
 5258                         pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET);
 5259                         mptsas_set_pkt_reason(mpt,
 5260                             cmd, CMD_TERMINATED, STAT_TERMINATED);
 5261                         break;
 5262                 case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
 5263                 case MPI2_IOCSTATUS_BUSY:
 5264                         /*
 5265                          * set throttles to drain
 5266                          */
 5267                         ptgt = (mptsas_target_t *)mptsas_hash_traverse(
 5268                             &mpt->m_active->m_tgttbl, MPTSAS_HASH_FIRST);
 5269                         while (ptgt != NULL) {
 5270                                 mutex_enter(&ptgt->m_tgt_intr_mutex);
 5271                                 mptsas_set_throttle(mpt, ptgt, DRAIN_THROTTLE);
 5272                                 mutex_exit(&ptgt->m_tgt_intr_mutex);
 5273 
 5274                                 ptgt = (mptsas_target_t *)mptsas_hash_traverse(
 5275                                     &mpt->m_active->m_tgttbl, MPTSAS_HASH_NEXT);
 5276                         }
 5277 
 5278                         /*
 5279                          * retry command
 5280                          */
 5281                         cmd->cmd_flags |= CFLAG_RETRY;
 5282                         cmd->cmd_pkt_flags |= FLAG_HEAD;
 5283 
 5284                         mutex_exit(&mpt->m_mutex);
 5285                         (void) mptsas_accept_pkt(mpt, cmd);
 5286                         mutex_enter(&mpt->m_mutex);
 5287                         break;
 5288                 default:
 5289                         mptsas_log(mpt, CE_WARN,
 5290                             "unknown ioc_status = %x\n", ioc_status);
 5291                         mptsas_log(mpt, CE_CONT, "scsi_state = %x, transfer "
 5292                             "count = %x, scsi_status = %x", scsi_state,
 5293                             xferred, scsi_status);
 5294                         break;
 5295                 }
 5296                 break;
 5297         case MPI2_SCSI_STATUS_TASK_SET_FULL:
 5298                 mptsas_handle_qfull(mpt, cmd);
 5299                 break;
 5300         case MPI2_SCSI_STATUS_BUSY:
 5301                 NDBG31(("scsi_status busy received"));
 5302                 break;
 5303         case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
 5304                 NDBG31(("scsi_status reservation conflict received"));
 5305                 break;
 5306         default:
 5307                 mptsas_log(mpt, CE_WARN, "scsi_status=%x, ioc_status=%x\n",
 5308                     scsi_status, ioc_status);
 5309                 mptsas_log(mpt, CE_WARN,
 5310                     "mptsas_process_intr: invalid scsi status\n");
 5311                 break;
 5312         }
 5313 }
 5314 
 5315 static void
 5316 mptsas_check_task_mgt(mptsas_t *mpt, pMpi2SCSIManagementReply_t reply,
 5317         mptsas_cmd_t *cmd)
 5318 {
 5319         uint8_t         task_type;
 5320         uint16_t        ioc_status;
 5321         uint32_t        log_info;
 5322         uint16_t        dev_handle;
 5323         struct scsi_pkt *pkt = CMD2PKT(cmd);
 5324 
 5325         task_type = ddi_get8(mpt->m_acc_reply_frame_hdl, &reply->TaskType);
 5326         ioc_status = ddi_get16(mpt->m_acc_reply_frame_hdl, &reply->IOCStatus);
 5327         log_info = ddi_get32(mpt->m_acc_reply_frame_hdl, &reply->IOCLogInfo);
 5328         dev_handle = ddi_get16(mpt->m_acc_reply_frame_hdl, &reply->DevHandle);
 5329 
 5330         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
 5331                 mptsas_log(mpt, CE_WARN, "mptsas_check_task_mgt: Task 0x%x "
 5332                     "failed. IOCStatus=0x%x IOCLogInfo=0x%x target=%d\n",
 5333                     task_type, ioc_status, log_info, dev_handle);
 5334                 pkt->pkt_reason = CMD_INCOMPLETE;
 5335                 return;
 5336         }
 5337 
 5338         switch (task_type) {
 5339         case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
 5340         case MPI2_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET:
 5341         case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
 5342         case MPI2_SCSITASKMGMT_TASKTYPE_CLR_ACA:
 5343         case MPI2_SCSITASKMGMT_TASKTYPE_QRY_TASK_SET:
 5344         case MPI2_SCSITASKMGMT_TASKTYPE_QRY_UNIT_ATTENTION:
 5345                 break;
 5346         case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
 5347         case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
 5348         case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
 5349                 /*
 5350                  * Check for invalid DevHandle of 0 in case application
 5351                  * sends bad command.  DevHandle of 0 could cause problems.
 5352                  */
 5353                 if (dev_handle == 0) {
 5354                         mptsas_log(mpt, CE_WARN, "!Can't flush target with"
 5355                             " DevHandle of 0.");
 5356                 } else {
 5357                         mptsas_flush_target(mpt, dev_handle, Lun(cmd),
 5358                             task_type);
 5359                 }
 5360                 break;
 5361         default:
 5362                 mptsas_log(mpt, CE_WARN, "Unknown task management type %d.",
 5363                     task_type);
 5364                 mptsas_log(mpt, CE_WARN, "ioc status = %x", ioc_status);
 5365                 break;
 5366         }
 5367 }
 5368 
 5369 static void
 5370 mptsas_doneq_thread(mptsas_doneq_thread_arg_t *arg)
 5371 {
 5372         mptsas_t                        *mpt = arg->mpt;
 5373         uint64_t                        t = arg->t;
 5374         mptsas_cmd_t                    *cmd;
 5375         struct scsi_pkt                 *pkt;
 5376         mptsas_doneq_thread_list_t      *item = &mpt->m_doneq_thread_id[t];
 5377 
 5378         mutex_enter(&item->mutex);
 5379         while (item->flag & MPTSAS_DONEQ_THREAD_ACTIVE) {
 5380                 if (!item->doneq) {
 5381                         cv_wait(&item->cv, &item->mutex);
 5382                 }
 5383                 pkt = NULL;
 5384                 if ((cmd = mptsas_doneq_thread_rm(mpt, t)) != NULL) {
 5385                         cmd->cmd_flags |= CFLAG_COMPLETED;
 5386                         pkt = CMD2PKT(cmd);
 5387                 }
 5388                 mutex_exit(&item->mutex);
 5389                 if (pkt) {
 5390                         mptsas_pkt_comp(pkt, cmd);
 5391                 }
 5392                 mutex_enter(&item->mutex);
 5393         }
 5394         mutex_exit(&item->mutex);
 5395         mutex_enter(&mpt->m_doneq_mutex);
 5396         mpt->m_doneq_thread_n--;
 5397         cv_broadcast(&mpt->m_doneq_thread_cv);
 5398         mutex_exit(&mpt->m_doneq_mutex);
 5399 }
 5400 
 5401 /*
 5402  * mpt interrupt handler.
 5403  */
 5404 static uint_t
 5405 mptsas_intr(caddr_t arg1, caddr_t arg2)
 5406 {
 5407         mptsas_t                        *mpt = (void *)arg1;
 5408         pMpi2ReplyDescriptorsUnion_t    reply_desc_union;
 5409         uchar_t                         did_reply = FALSE;
 5410         int                             i = 0, j;
 5411         uint8_t                         reply_type;
 5412         uint16_t                        SMID;
 5413 
 5414         NDBG1(("mptsas_intr: arg1 0x%p arg2 0x%p", (void *)arg1, (void *)arg2));
 5415 
 5416         /*
 5417          * 1.
 5418          * To avoid using m_mutex in the ISR(ISR referes not only mptsas_intr,
 5419          * but all of the recursive called functions in it. the same below),
 5420          * separate mutexs are introduced to protect the elements shown in ISR.
 5421          * 3 type of mutex are involved here:
 5422          *      a)per instance mutex m_intr_mutex.
 5423          *      b)per target mutex m_tgt_intr_mutex.
 5424          *      c)mutex that protect the free slot.
 5425          *
 5426          * a)per instance mutex m_intr_mutex:
 5427          * used to protect m_options, m_power, m_waitq, etc that would be
 5428          * checked/modified in ISR; protect the getting and setting the reply
 5429          * descriptor index; protect the m_slots[];
 5430          *
 5431          * b)per target mutex m_tgt_intr_mutex:
 5432          * used to protect per target element which has relationship to ISR.
 5433          * contention for the new per target mutex is just as high as it in
 5434          * sd(7d) driver.
 5435          *
 5436          * c)mutexs that protect the free slots:
 5437          * those mutexs are introduced to minimize the mutex contentions
 5438          * between the IO request threads where free slots are allocated
 5439          * for sending cmds and ISR where slots holding outstanding cmds
 5440          * are returned to the free pool.
 5441          * the idea is like this:
 5442          * 1) Partition all of the free slot into NCPU groups. For example,
 5443          * In system where we have 15 slots, and 4 CPU, then slot s1,s5,s9,s13
 5444          * are marked belonging to CPU1, s2,s6,s10,s14 to CPU2, s3,s7,s11,s15
 5445          * to CPU3, and s4,s8,s12 to CPU4.
 5446          * 2) In each of the group, an alloc/release queue pair is created,
 5447          * and both the allocq and the releaseq have a dedicated mutex.
 5448          * 3) When init, all of the slots in a CPU group are inserted into the
 5449          * allocq of its CPU's pair.
 5450          * 4) When doing IO,
 5451          * mptsas_scsi_start()
 5452          * {
 5453          *      cpuid = the cpu NO of the cpu where this thread is running on
 5454          * retry:
 5455          *      mutex_enter(&allocq[cpuid]);
 5456          *      if (get free slot = success) {
 5457          *              remove the slot from the allocq
 5458          *              mutex_exit(&allocq[cpuid]);
 5459          *              return(success);
 5460          *      } else { // exchange allocq and releaseq and try again
 5461          *              mutex_enter(&releq[cpuid]);
 5462          *              exchange the allocq and releaseq of this pair;
 5463          *              mutex_exit(&releq[cpuid]);
 5464          *              if (try to get free slot again = success) {
 5465          *                      remove the slot from the allocq
 5466          *                      mutex_exit(&allocq[cpuid]);
 5467          *                      return(success);
 5468          *              } else {
 5469          *                      MOD(cpuid)++;
 5470          *                      goto retry;
 5471          *                      if (all CPU groups tried)
 5472          *                              mutex_exit(&allocq[cpuid]);
 5473          *                              return(failure);
 5474          *              }
 5475          *      }
 5476          * }
 5477          * ISR()
 5478          * {
 5479          *              cpuid = the CPU group id where the slot sending the
 5480          *              cmd belongs;
 5481          *              mutex_enter(&releq[cpuid]);
 5482          *              remove the slot from the releaseq
 5483          *              mutex_exit(&releq[cpuid]);
 5484          * }
 5485          * This way, only when the queue pair doing exchange have mutex
 5486          * contentions.
 5487          *
 5488          * For mutex m_intr_mutex and m_tgt_intr_mutex, there are 2 scenarios:
 5489          *
 5490          * a)If the elements are only checked but not modified in the ISR, then
 5491          * only the places where those elements are modifed(outside of ISR)
 5492          * need to be protected by the new introduced mutex.
 5493          * For example, data A is only read/checked in ISR, then we need do
 5494          * like this:
 5495          * In ISR:
 5496          * {
 5497          *      mutex_enter(&new_mutex);
 5498          *      read(A);
 5499          *      mutex_exit(&new_mutex);
 5500          *      //the new_mutex here is either the m_tgt_intr_mutex or
 5501          *      //the m_intr_mutex.
 5502          * }
 5503          * In non-ISR
 5504          * {
 5505          *      mutex_enter(&m_mutex); //the stock driver already did this
 5506          *      mutex_enter(&new_mutex);
 5507          *      write(A);
 5508          *      mutex_exit(&new_mutex);
 5509          *      mutex_exit(&m_mutex); //the stock driver already did this
 5510          *
 5511          *      read(A);
 5512          *      // read(A) in non-ISR is not required to be protected by new
 5513          *      // mutex since 'A' has already been protected by m_mutex
 5514          *      // outside of the ISR
 5515          * }
 5516          *
 5517          * Those fields in mptsas_target_t/ptgt which are only read in ISR
 5518          * fall into this catergory. So they, together with the fields which
 5519          * are never read in ISR, are not necessary to be protected by
 5520          * m_tgt_intr_mutex, don't bother.
 5521          * checking of m_waitq also falls into this catergory. so all of the
 5522          * place outside of ISR where the m_waitq is modified, such as in
 5523          * mptsas_waitq_add(), mptsas_waitq_delete(), mptsas_waitq_rm(),
 5524          * m_intr_mutex should be used.
 5525          *
 5526          * b)If the elements are modified in the ISR, then each place where
 5527          * those elements are referred(outside of ISR) need to be protected
 5528          * by the new introduced mutex. Of course, if those elements only
 5529          * appear in the non-key code path, that is, they don't affect
 5530          * performance, then the m_mutex can still be used as before.
 5531          * For example, data B is modified in key code path in ISR, and data C
 5532          * is modified in non-key code path in ISR, then we can do like this:
 5533          * In ISR:
 5534          * {
 5535          *      mutex_enter(&new_mutex);
 5536          *      wirte(B);
 5537          *      mutex_exit(&new_mutex);
 5538          *      if (seldom happen) {
 5539          *              mutex_enter(&m_mutex);
 5540          *              write(C);
 5541          *              mutex_exit(&m_mutex);
 5542          *      }
 5543          *      //the new_mutex here is either the m_tgt_intr_mutex or
 5544          *      //the m_intr_mutex.
 5545          * }
 5546          * In non-ISR
 5547          * {
 5548          *      mutex_enter(&new_mutex);
 5549          *      write(B);
 5550          *      mutex_exit(&new_mutex);
 5551          *
 5552          *      mutex_enter(&new_mutex);
 5553          *      read(B);
 5554          *      mutex_exit(&new_mutex);
 5555          *      // both write(B) and read(B) in non-ISR is required to be
 5556          *      // protected by new mutex outside of the ISR
 5557          *
 5558          *      mutex_enter(&m_mutex); //the stock driver already did this
 5559          *      read(C);
 5560          *      write(C);
 5561          *      mutex_exit(&m_mutex); //the stock driver already did this
 5562          *      // both write(C) and read(C) in non-ISR have been already
 5563          *      // been protected by m_mutex outside of the ISR
 5564          * }
 5565          *
 5566          * For example, ptgt->m_t_ncmds fall into 'B' of this catergory, and
 5567          * elements shown in address reply, restart_hba, passthrough, IOC
 5568          * fall into 'C' of  this catergory.
 5569          *
 5570          * In any case where mutexs are nested, make sure in the following
 5571          * order:
 5572          *      m_mutex -> m_intr_mutex -> m_tgt_intr_mutex
 5573          *      m_intr_mutex -> m_tgt_intr_mutex
 5574          *      m_mutex -> m_intr_mutex
 5575          *      m_mutex -> m_tgt_intr_mutex
 5576          *
 5577          * 2.
 5578          * Make sure at any time, getting the ReplyDescriptor by m_post_index
 5579          * and setting m_post_index to the ReplyDescriptorIndex register are
 5580          * atomic. Since m_mutex is not used for this purpose in ISR, the new
 5581          * mutex m_intr_mutex must play this role. So mptsas_poll(), where this
 5582          * kind of getting/setting is also performed, must use m_intr_mutex.
 5583          * Note, since context reply in ISR/process_intr is the only code path
 5584          * which affect performance, a fast path is introduced to only handle
 5585          * the read/write IO having context reply. For other IOs such as
 5586          * passthrough and IOC with context reply and all address reply, we
 5587          * use the as-is process_intr() to handle them. In order to keep the
 5588          * same semantics in process_intr(), make sure any new mutex is not held
 5589          * before enterring it.
 5590          */
 5591 
 5592         mutex_enter(&mpt->m_intr_mutex);
 5593 
 5594         /*
 5595          * If interrupts are shared by two channels then check whether this
 5596          * interrupt is genuinely for this channel by making sure first the
 5597          * chip is in high power state.
 5598          */
 5599         if ((mpt->m_options & MPTSAS_OPT_PM) &&
 5600             (mpt->m_power_level != PM_LEVEL_D0)) {
 5601                 mutex_exit(&mpt->m_intr_mutex);
 5602                 return (DDI_INTR_UNCLAIMED);
 5603         }
 5604 
 5605         /*
 5606          * If polling, interrupt was triggered by some shared interrupt because
 5607          * IOC interrupts are disabled during polling, so polling routine will
 5608          * handle any replies.  Considering this, if polling is happening,
 5609          * return with interrupt unclaimed.
 5610          */
 5611         if (mpt->m_polled_intr) {
 5612                 mutex_exit(&mpt->m_intr_mutex);
 5613                 mptsas_log(mpt, CE_WARN, "mpt_sas: Unclaimed interrupt");
 5614                 return (DDI_INTR_UNCLAIMED);
 5615         }
 5616 
 5617         /*
 5618          * Read the istat register.
 5619          */
 5620         if ((INTPENDING(mpt)) != 0) {
 5621                 /*
 5622                  * read fifo until empty.
 5623                  */
 5624 #ifndef __lock_lint
 5625                 _NOTE(CONSTCOND)
 5626 #endif
 5627                 while (TRUE) {
 5628                         (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
 5629                             DDI_DMA_SYNC_FORCPU);
 5630                         reply_desc_union = (pMpi2ReplyDescriptorsUnion_t)
 5631                             MPTSAS_GET_NEXT_REPLY(mpt, mpt->m_post_index);
 5632 
 5633                         if (ddi_get32(mpt->m_acc_post_queue_hdl,
 5634                             &reply_desc_union->Words.Low) == 0xFFFFFFFF ||
 5635                             ddi_get32(mpt->m_acc_post_queue_hdl,
 5636                             &reply_desc_union->Words.High) == 0xFFFFFFFF) {
 5637                                 break;
 5638                         }
 5639 
 5640                         /*
 5641                          * The reply is valid, process it according to its
 5642                          * type.  Also, set a flag for updating the reply index
 5643                          * after they've all been processed.
 5644                          */
 5645                         did_reply = TRUE;
 5646 
 5647                         reply_type = ddi_get8(mpt->m_acc_post_queue_hdl,
 5648                             &reply_desc_union->Default.ReplyFlags);
 5649                         reply_type &= MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
 5650                         mpt->m_reply[i].Default.ReplyFlags = reply_type;
 5651                         if (reply_type ==
 5652                             MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS) {
 5653                                 SMID = ddi_get16(mpt->m_acc_post_queue_hdl,
 5654                                     &reply_desc_union->SCSIIOSuccess.SMID);
 5655                                 if (mptsas_handle_io_fastpath(mpt, SMID) !=
 5656                                     TRUE) {
 5657                                         mpt->m_reply[i].SCSIIOSuccess.SMID =
 5658                                             SMID;
 5659                                         i++;
 5660                                 }
 5661                         } else if (reply_type ==
 5662                             MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
 5663                                 mpt->m_reply[i].AddressReply.ReplyFrameAddress =
 5664                                     ddi_get32(mpt->m_acc_post_queue_hdl,
 5665                                     &reply_desc_union->AddressReply.
 5666                                     ReplyFrameAddress);
 5667                                 mpt->m_reply[i].AddressReply.SMID =
 5668                                     ddi_get16(mpt->m_acc_post_queue_hdl,
 5669                                     &reply_desc_union->AddressReply.SMID);
 5670                                 i++;
 5671                         }
 5672                         /*
 5673                          * Clear the reply descriptor for re-use and increment
 5674                          * index.
 5675                          */
 5676                         ddi_put64(mpt->m_acc_post_queue_hdl,
 5677                             &((uint64_t *)(void *)mpt->m_post_queue)
 5678                             [mpt->m_post_index], 0xFFFFFFFFFFFFFFFF);
 5679                         (void) ddi_dma_sync(mpt->m_dma_post_queue_hdl, 0, 0,
 5680                             DDI_DMA_SYNC_FORDEV);
 5681 
 5682                         /*
 5683                          * Increment post index and roll over if needed.
 5684                          */
 5685                         if (++mpt->m_post_index == mpt->m_post_queue_depth) {
 5686                                 mpt->m_post_index = 0;
 5687                         }
 5688                         if (i >= MPI_ADDRESS_COALSCE_MAX)
 5689                                 break;
 5690                 }
 5691 
 5692                 /*
 5693                  * Update the global reply index if at least one reply was
 5694                  * processed.
 5695                  */
 5696                 if (did_reply) {
 5697                         ddi_put32(mpt->m_datap,
 5698                             &mpt->m_reg->ReplyPostHostIndex, mpt->m_post_index);
 5699 
 5700                         /*
 5701                          * For fma, only check the PIO is required and enough
 5702                          * here. Those cases where fastpath is not hit, the
 5703                          * mptsas_fma_check() check all of the types of
 5704                          * fma. That is not necessary and sometimes not
 5705                          * correct. fma check should only be done after
 5706                          * the PIO and/or dma is performed.
 5707                          */
 5708                         if ((mptsas_check_acc_handle(mpt->m_datap) !=
 5709                             DDI_SUCCESS)) {
 5710                                 ddi_fm_service_impact(mpt->m_dip,
 5711                                     DDI_SERVICE_UNAFFECTED);
 5712                         }
 5713 
 5714                 }
 5715         } else {
 5716                 mutex_exit(&mpt->m_intr_mutex);
 5717                 return (DDI_INTR_UNCLAIMED);
 5718         }
 5719         NDBG1(("mptsas_intr complete"));
 5720         mutex_exit(&mpt->m_intr_mutex);
 5721 
 5722         /*
 5723          * Since most of the cmds(read and write IO with success return.)
 5724          * have already been processed in fast path in which the m_mutex
 5725          * is not held, handling here the address reply and other context reply
 5726          * such as passthrough and IOC cmd with m_mutex held should be a big
 5727          * issue for performance.
 5728          * If holding m_mutex to process these cmds was still an obvious issue,
 5729          * we can process them in a taskq.
 5730          */
 5731         for (j = 0; j < i; j++) {
 5732                 mutex_enter(&mpt->m_mutex);
 5733                 mptsas_process_intr(mpt, &mpt->m_reply[j]);
 5734                 mutex_exit(&mpt->m_mutex);
 5735         }
 5736 
 5737         /*
 5738          * If no helper threads are created, process the doneq in ISR. If
 5739          * helpers are created, use the doneq length as a metric to measure the
 5740          * load on the interrupt CPU. If it is long enough, which indicates the
 5741          * load is heavy, then we deliver the IO completions to the helpers.
 5742          * This measurement has some limitations, although it is simple and
 5743          * straightforward and works well for most of the cases at present.
 5744          */
 5745         if (!mpt->m_doneq_thread_n) {
 5746                 mptsas_doneq_empty(mpt);
 5747         } else {
 5748                 int helper = 1;
 5749                 mutex_enter(&mpt->m_intr_mutex);
 5750                 if (mpt->m_doneq_len <= mpt->m_doneq_length_threshold)
 5751                         helper = 0;
 5752                 mutex_exit(&mpt->m_intr_mutex);
 5753                 if (helper) {
 5754                         mptsas_deliver_doneq_thread(mpt);
 5755                 } else {
 5756                         mptsas_doneq_empty(mpt);
 5757                 }
 5758         }
 5759 
 5760         /*
 5761          * If there are queued cmd, start them now.
 5762          */
 5763         mutex_enter(&mpt->m_intr_mutex);
 5764         if (mpt->m_waitq != NULL) {
 5765                 mutex_exit(&mpt->m_intr_mutex);
 5766                 mutex_enter(&mpt->m_mutex);
 5767                 mptsas_restart_hba(mpt);
 5768                 mutex_exit(&mpt->m_mutex);
 5769                 return (DDI_INTR_CLAIMED);
 5770         }
 5771         mutex_exit(&mpt->m_intr_mutex);
 5772         return (DDI_INTR_CLAIMED);
 5773 }
 5774 
 5775 /*
 5776  * In ISR, the successfully completed read and write IO are processed in a