The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/fs/nfsclient/nfs_clstate.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2009 Rick Macklem, University of Guelph
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD$");
   30 
   31 /*
   32  * These functions implement the client side state handling for NFSv4.
   33  * NFSv4 state handling:
   34  * - A lockowner is used to determine lock contention, so it
   35  *   corresponds directly to a Posix pid. (1 to 1 mapping)
   36  * - The correct granularity of an OpenOwner is not nearly so
   37  *   obvious. An OpenOwner does the following:
   38  *   - provides a serial sequencing of Open/Close/Lock-with-new-lockowner
   39  *   - is used to check for Open/Share contention (not applicable to
   40  *     this client, since all Opens are Deny_None)
   41  *   As such, I considered both extreme.
   42  *   1 OpenOwner per ClientID - Simple to manage, but fully serializes
   43  *   all Open, Close and Lock (with a new lockowner) Ops.
   44  *   1 OpenOwner for each Open - This one results in an OpenConfirm for
   45  *   every Open, for most servers.
   46  *   So, I chose to use the same mapping as I did for LockOwnwers.
   47  *   The main concern here is that you can end up with multiple Opens
   48  *   for the same File Handle, but on different OpenOwners (opens
   49  *   inherited from parents, grandparents...) and you do not know
   50  *   which of these the vnodeop close applies to. This is handled by
   51  *   delaying the Close Op(s) until all of the Opens have been closed.
   52  *   (It is not yet obvious if this is the correct granularity.)
   53  * - How the code handles serialization:
   54  *   - For the ClientId, it uses an exclusive lock while getting its
   55  *     SetClientId and during recovery. Otherwise, it uses a shared
   56  *     lock via a reference count.
   57  *   - For the rest of the data structures, it uses an SMP mutex
   58  *     (once the nfs client is SMP safe) and doesn't sleep while
   59  *     manipulating the linked lists.
   60  *   - The serialization of Open/Close/Lock/LockU falls out in the
   61  *     "wash", since OpenOwners and LockOwners are both mapped from
   62  *     Posix pid. In other words, there is only one Posix pid using
   63  *     any given owner, so that owner is serialized. (If you change
   64  *     the granularity of the OpenOwner, then code must be added to
   65  *     serialize Ops on the OpenOwner.)
   66  * - When to get rid of OpenOwners and LockOwners.
   67  *   - The function nfscl_cleanup_common() is executed after a process exits.
   68  *     It goes through the client list looking for all Open and Lock Owners.
   69  *     When one is found, it is marked "defunct" or in the case of
   70  *     an OpenOwner without any Opens, freed.
   71  *     The renew thread scans for defunct Owners and gets rid of them,
   72  *     if it can. The LockOwners will also be deleted when the
   73  *     associated Open is closed.
   74  *   - If the LockU or Close Op(s) fail during close in a way
   75  *     that could be recovered upon retry, they are relinked to the
   76  *     ClientId's defunct open list and retried by the renew thread
   77  *     until they succeed or an unmount/recovery occurs.
   78  *     (Since we are done with them, they do not need to be recovered.)
   79  */
   80 
   81 #ifndef APPLEKEXT
   82 #include <fs/nfs/nfsport.h>
   83 
   84 /*
   85  * Global variables
   86  */
   87 extern struct nfsstats newnfsstats;
   88 extern struct nfsreqhead nfsd_reqq;
   89 NFSREQSPINLOCK;
   90 NFSCLSTATEMUTEX;
   91 int nfscl_inited = 0;
   92 struct nfsclhead nfsclhead;     /* Head of clientid list */
   93 int nfscl_deleghighwater = NFSCLDELEGHIGHWATER;
   94 #endif  /* !APPLEKEXT */
   95 
   96 static int nfscl_delegcnt = 0;
   97 static int nfscl_getopen(struct nfsclownerhead *, u_int8_t *, int, u_int8_t *,
   98     u_int8_t *, u_int32_t, struct nfscllockowner **, struct nfsclopen **);
   99 static void nfscl_clrelease(struct nfsclclient *);
  100 static void nfscl_cleanclient(struct nfsclclient *);
  101 static void nfscl_expireclient(struct nfsclclient *, struct nfsmount *,
  102     struct ucred *, NFSPROC_T *);
  103 static int nfscl_expireopen(struct nfsclclient *, struct nfsclopen *,
  104     struct nfsmount *, struct ucred *, NFSPROC_T *);
  105 static void nfscl_recover(struct nfsclclient *, struct ucred *, NFSPROC_T *);
  106 static void nfscl_insertlock(struct nfscllockowner *, struct nfscllock *,
  107     struct nfscllock *, int);
  108 static int nfscl_updatelock(struct nfscllockowner *, struct nfscllock **,
  109     struct nfscllock **, int);
  110 static void nfscl_delegreturnall(struct nfsclclient *, NFSPROC_T *);
  111 static u_int32_t nfscl_nextcbident(void);
  112 static mount_t nfscl_getmnt(u_int32_t);
  113 static struct nfscldeleg *nfscl_finddeleg(struct nfsclclient *, u_int8_t *,
  114     int);
  115 static int nfscl_checkconflict(struct nfscllockownerhead *, struct nfscllock *,
  116     u_int8_t *, struct nfscllock **);
  117 static void nfscl_freealllocks(struct nfscllockownerhead *, int);
  118 static int nfscl_localconflict(struct nfsclclient *, u_int8_t *, int,
  119     struct nfscllock *, u_int8_t *, struct nfscldeleg *, struct nfscllock **);
  120 static void nfscl_newopen(struct nfsclclient *, struct nfscldeleg *,
  121     struct nfsclowner **, struct nfsclowner **, struct nfsclopen **,
  122     struct nfsclopen **, u_int8_t *, u_int8_t *, int, int *);
  123 static int nfscl_moveopen(vnode_t , struct nfsclclient *,
  124     struct nfsmount *, struct nfsclopen *, struct nfsclowner *,
  125     struct nfscldeleg *, struct ucred *, NFSPROC_T *);
  126 static void nfscl_totalrecall(struct nfsclclient *);
  127 static int nfscl_relock(vnode_t , struct nfsclclient *, struct nfsmount *,
  128     struct nfscllockowner *, struct nfscllock *, struct ucred *, NFSPROC_T *);
  129 static int nfscl_tryopen(struct nfsmount *, vnode_t , u_int8_t *, int,
  130     u_int8_t *, int, u_int32_t, struct nfsclopen *, u_int8_t *, int,
  131     struct nfscldeleg **, int, u_int32_t, struct ucred *, NFSPROC_T *);
  132 static int nfscl_trylock(struct nfsmount *, vnode_t , u_int8_t *,
  133     int, struct nfscllockowner *, int, int, u_int64_t, u_int64_t, short,
  134     struct ucred *, NFSPROC_T *);
  135 static int nfsrpc_reopen(struct nfsmount *, u_int8_t *, int, u_int32_t,
  136     struct nfsclopen *, struct nfscldeleg **, struct ucred *, NFSPROC_T *);
  137 static void nfscl_freedeleg(struct nfscldeleghead *, struct nfscldeleg *);
  138 static int nfscl_errmap(struct nfsrv_descript *);
  139 static void nfscl_cleanup_common(struct nfsclclient *, u_int8_t *);
  140 static int nfscl_recalldeleg(struct nfsclclient *, struct nfsmount *,
  141     struct nfscldeleg *, vnode_t, struct ucred *, NFSPROC_T *, int);
  142 static void nfscl_freeopenowner(struct nfsclowner *, int);
  143 static void nfscl_cleandeleg(struct nfscldeleg *);
  144 static int nfscl_trydelegreturn(struct nfscldeleg *, struct ucred *,
  145     struct nfsmount *, NFSPROC_T *);
  146 static void nfscl_emptylockowner(struct nfscllockowner *,
  147     struct nfscllockownerfhhead *);
  148 
  149 static short nfscberr_null[] = {
  150         0,
  151         0,
  152 };
  153 
  154 static short nfscberr_getattr[] = {
  155         NFSERR_RESOURCE,
  156         NFSERR_BADHANDLE,
  157         NFSERR_BADXDR,
  158         NFSERR_RESOURCE,
  159         NFSERR_SERVERFAULT,
  160         0,
  161 };
  162 
  163 static short nfscberr_recall[] = {
  164         NFSERR_RESOURCE,
  165         NFSERR_BADHANDLE,
  166         NFSERR_BADSTATEID,
  167         NFSERR_BADXDR,
  168         NFSERR_RESOURCE,
  169         NFSERR_SERVERFAULT,
  170         0,
  171 };
  172 
  173 static short *nfscl_cberrmap[] = {
  174         nfscberr_null,
  175         nfscberr_null,
  176         nfscberr_null,
  177         nfscberr_getattr,
  178         nfscberr_recall
  179 };
  180 
  181 #define NETFAMILY(clp) \
  182                 (((clp)->nfsc_flags & NFSCLFLAGS_AFINET6) ? AF_INET6 : AF_INET)
  183 
  184 /*
  185  * Called for an open operation.
  186  * If the nfhp argument is NULL, just get an openowner.
  187  */
  188 APPLESTATIC int
  189 nfscl_open(vnode_t vp, u_int8_t *nfhp, int fhlen, u_int32_t amode, int usedeleg,
  190     struct ucred *cred, NFSPROC_T *p, struct nfsclowner **owpp,
  191     struct nfsclopen **opp, int *newonep, int *retp, int lockit)
  192 {
  193         struct nfsclclient *clp;
  194         struct nfsclowner *owp, *nowp;
  195         struct nfsclopen *op = NULL, *nop = NULL;
  196         struct nfscldeleg *dp;
  197         struct nfsclownerhead *ohp;
  198         u_int8_t own[NFSV4CL_LOCKNAMELEN];
  199         int ret;
  200 
  201         if (newonep != NULL)
  202                 *newonep = 0;
  203         if (opp != NULL)
  204                 *opp = NULL;
  205         if (owpp != NULL)
  206                 *owpp = NULL;
  207 
  208         /*
  209          * Might need one or both of these, so MALLOC them now, to
  210          * avoid a tsleep() in MALLOC later.
  211          */
  212         MALLOC(nowp, struct nfsclowner *, sizeof (struct nfsclowner),
  213             M_NFSCLOWNER, M_WAITOK);
  214         if (nfhp != NULL)
  215             MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
  216                 fhlen - 1, M_NFSCLOPEN, M_WAITOK);
  217         ret = nfscl_getcl(vp, cred, p, &clp);
  218         if (ret != 0) {
  219                 FREE((caddr_t)nowp, M_NFSCLOWNER);
  220                 if (nop != NULL)
  221                         FREE((caddr_t)nop, M_NFSCLOPEN);
  222                 return (ret);
  223         }
  224 
  225         /*
  226          * Get the Open iff it already exists.
  227          * If none found, add the new one or return error, depending upon
  228          * "create".
  229          */
  230         nfscl_filllockowner(p->td_proc, own, F_POSIX);
  231         NFSLOCKCLSTATE();
  232         dp = NULL;
  233         /* First check the delegation list */
  234         if (nfhp != NULL && usedeleg) {
  235                 LIST_FOREACH(dp, NFSCLDELEGHASH(clp, nfhp, fhlen), nfsdl_hash) {
  236                         if (dp->nfsdl_fhlen == fhlen &&
  237                             !NFSBCMP(nfhp, dp->nfsdl_fh, fhlen)) {
  238                                 if (!(amode & NFSV4OPEN_ACCESSWRITE) ||
  239                                     (dp->nfsdl_flags & NFSCLDL_WRITE))
  240                                         break;
  241                                 dp = NULL;
  242                                 break;
  243                         }
  244                 }
  245         }
  246 
  247         if (dp != NULL)
  248                 ohp = &dp->nfsdl_owner;
  249         else
  250                 ohp = &clp->nfsc_owner;
  251         /* Now, search for an openowner */
  252         LIST_FOREACH(owp, ohp, nfsow_list) {
  253                 if (!NFSBCMP(owp->nfsow_owner, own, NFSV4CL_LOCKNAMELEN))
  254                         break;
  255         }
  256 
  257         /*
  258          * Create a new open, as required.
  259          */
  260         nfscl_newopen(clp, dp, &owp, &nowp, &op, &nop, own, nfhp, fhlen,
  261             newonep);
  262 
  263         /*
  264          * Serialize modifications to the open owner for multiple threads
  265          * within the same process using a read/write sleep lock.
  266          */
  267         if (lockit)
  268                 nfscl_lockexcl(&owp->nfsow_rwlock, NFSCLSTATEMUTEXPTR);
  269         NFSUNLOCKCLSTATE();
  270         if (nowp != NULL)
  271                 FREE((caddr_t)nowp, M_NFSCLOWNER);
  272         if (nop != NULL)
  273                 FREE((caddr_t)nop, M_NFSCLOPEN);
  274         if (owpp != NULL)
  275                 *owpp = owp;
  276         if (opp != NULL)
  277                 *opp = op;
  278         if (retp != NULL) {
  279                 if (nfhp != NULL && dp != NULL && nop == NULL)
  280                         /* new local open on delegation */
  281                         *retp = NFSCLOPEN_SETCRED;
  282                 else
  283                         *retp = NFSCLOPEN_OK;
  284         }
  285 
  286         /*
  287          * Now, check the mode on the open and return the appropriate
  288          * value.
  289          */
  290         if (op != NULL && (amode & ~(op->nfso_mode))) {
  291                 op->nfso_mode |= amode;
  292                 if (retp != NULL && dp == NULL)
  293                         *retp = NFSCLOPEN_DOOPEN;
  294         }
  295         return (0);
  296 }
  297 
  298 /*
  299  * Create a new open, as required.
  300  */
  301 static void
  302 nfscl_newopen(struct nfsclclient *clp, struct nfscldeleg *dp,
  303     struct nfsclowner **owpp, struct nfsclowner **nowpp, struct nfsclopen **opp,
  304     struct nfsclopen **nopp, u_int8_t *own, u_int8_t *fhp, int fhlen,
  305     int *newonep)
  306 {
  307         struct nfsclowner *owp = *owpp, *nowp;
  308         struct nfsclopen *op, *nop;
  309 
  310         if (nowpp != NULL)
  311                 nowp = *nowpp;
  312         else
  313                 nowp = NULL;
  314         if (nopp != NULL)
  315                 nop = *nopp;
  316         else
  317                 nop = NULL;
  318         if (owp == NULL && nowp != NULL) {
  319                 NFSBCOPY(own, nowp->nfsow_owner, NFSV4CL_LOCKNAMELEN);
  320                 LIST_INIT(&nowp->nfsow_open);
  321                 nowp->nfsow_clp = clp;
  322                 nowp->nfsow_seqid = 0;
  323                 nowp->nfsow_defunct = 0;
  324                 nfscl_lockinit(&nowp->nfsow_rwlock);
  325                 if (dp != NULL) {
  326                         newnfsstats.cllocalopenowners++;
  327                         LIST_INSERT_HEAD(&dp->nfsdl_owner, nowp, nfsow_list);
  328                 } else {
  329                         newnfsstats.clopenowners++;
  330                         LIST_INSERT_HEAD(&clp->nfsc_owner, nowp, nfsow_list);
  331                 }
  332                 owp = *owpp = nowp;
  333                 *nowpp = NULL;
  334                 if (newonep != NULL)
  335                         *newonep = 1;
  336         }
  337 
  338          /* If an fhp has been specified, create an Open as well. */
  339         if (fhp != NULL) {
  340                 /* and look for the correct open, based upon FH */
  341                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
  342                         if (op->nfso_fhlen == fhlen &&
  343                             !NFSBCMP(op->nfso_fh, fhp, fhlen))
  344                                 break;
  345                 }
  346                 if (op == NULL && nop != NULL) {
  347                         nop->nfso_own = owp;
  348                         nop->nfso_mode = 0;
  349                         nop->nfso_opencnt = 0;
  350                         nop->nfso_posixlock = 1;
  351                         nop->nfso_fhlen = fhlen;
  352                         NFSBCOPY(fhp, nop->nfso_fh, fhlen);
  353                         LIST_INIT(&nop->nfso_lock);
  354                         nop->nfso_stateid.seqid = 0;
  355                         nop->nfso_stateid.other[0] = 0;
  356                         nop->nfso_stateid.other[1] = 0;
  357                         nop->nfso_stateid.other[2] = 0;
  358                         if (dp != NULL) {
  359                                 TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
  360                                 TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp,
  361                                     nfsdl_list);
  362                                 dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
  363                                 newnfsstats.cllocalopens++;
  364                         } else {
  365                                 newnfsstats.clopens++;
  366                         }
  367                         LIST_INSERT_HEAD(&owp->nfsow_open, nop, nfso_list);
  368                         *opp = nop;
  369                         *nopp = NULL;
  370                         if (newonep != NULL)
  371                                 *newonep = 1;
  372                 } else {
  373                         *opp = op;
  374                 }
  375         }
  376 }
  377 
  378 /*
  379  * Called to find/add a delegation to a client.
  380  */
  381 APPLESTATIC int
  382 nfscl_deleg(mount_t mp, struct nfsclclient *clp, u_int8_t *nfhp,
  383     int fhlen, struct ucred *cred, NFSPROC_T *p, struct nfscldeleg **dpp)
  384 {
  385         struct nfscldeleg *dp = *dpp, *tdp;
  386 
  387         /*
  388          * First, if we have received a Read delegation for a file on a
  389          * read/write file system, just return it, because they aren't
  390          * useful, imho.
  391          */
  392         if (mp != NULL && dp != NULL && !NFSMNT_RDONLY(mp) &&
  393             (dp->nfsdl_flags & NFSCLDL_READ)) {
  394                 (void) nfscl_trydelegreturn(dp, cred, VFSTONFS(mp), p);
  395                 FREE((caddr_t)dp, M_NFSCLDELEG);
  396                 *dpp = NULL;
  397                 return (0);
  398         }
  399 
  400         /* Look for the correct deleg, based upon FH */
  401         NFSLOCKCLSTATE();
  402         tdp = nfscl_finddeleg(clp, nfhp, fhlen);
  403         if (tdp == NULL) {
  404                 if (dp == NULL) {
  405                         NFSUNLOCKCLSTATE();
  406                         return (NFSERR_BADSTATEID);
  407                 }
  408                 *dpp = NULL;
  409                 TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp, nfsdl_list);
  410                 LIST_INSERT_HEAD(NFSCLDELEGHASH(clp, nfhp, fhlen), dp,
  411                     nfsdl_hash);
  412                 dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
  413                 newnfsstats.cldelegates++;
  414                 nfscl_delegcnt++;
  415         } else {
  416                 /*
  417                  * Delegation already exists, what do we do if a new one??
  418                  */
  419                 if (dp != NULL) {
  420                         printf("Deleg already exists!\n");
  421                         FREE((caddr_t)dp, M_NFSCLDELEG);
  422                         *dpp = NULL;
  423                 } else {
  424                         *dpp = tdp;
  425                 }
  426         }
  427         NFSUNLOCKCLSTATE();
  428         return (0);
  429 }
  430 
  431 /*
  432  * Find a delegation for this file handle. Return NULL upon failure.
  433  */
  434 static struct nfscldeleg *
  435 nfscl_finddeleg(struct nfsclclient *clp, u_int8_t *fhp, int fhlen)
  436 {
  437         struct nfscldeleg *dp;
  438 
  439         LIST_FOREACH(dp, NFSCLDELEGHASH(clp, fhp, fhlen), nfsdl_hash) {
  440             if (dp->nfsdl_fhlen == fhlen &&
  441                 !NFSBCMP(dp->nfsdl_fh, fhp, fhlen))
  442                 break;
  443         }
  444         return (dp);
  445 }
  446 
  447 /*
  448  * Get a stateid for an I/O operation. First, look for an open and iff
  449  * found, return either a lockowner stateid or the open stateid.
  450  * If no Open is found, just return error and the special stateid of all zeros.
  451  */
  452 APPLESTATIC int
  453 nfscl_getstateid(vnode_t vp, u_int8_t *nfhp, int fhlen, u_int32_t mode,
  454     struct ucred *cred, NFSPROC_T *p, nfsv4stateid_t *stateidp,
  455     void **lckpp)
  456 {
  457         struct nfsclclient *clp;
  458         struct nfsclowner *owp;
  459         struct nfsclopen *op = NULL;
  460         struct nfscllockowner *lp;
  461         struct nfscldeleg *dp;
  462         struct nfsnode *np;
  463         u_int8_t own[NFSV4CL_LOCKNAMELEN];
  464         int error, done;
  465 
  466         *lckpp = NULL;
  467         /*
  468          * Initially, just set the special stateid of all zeros.
  469          */
  470         stateidp->seqid = 0;
  471         stateidp->other[0] = 0;
  472         stateidp->other[1] = 0;
  473         stateidp->other[2] = 0;
  474         if (vnode_vtype(vp) != VREG)
  475                 return (EISDIR);
  476         np = VTONFS(vp);
  477         NFSLOCKCLSTATE();
  478         clp = nfscl_findcl(VFSTONFS(vnode_mount(vp)));
  479         if (clp == NULL) {
  480                 NFSUNLOCKCLSTATE();
  481                 return (EACCES);
  482         }
  483 
  484         /*
  485          * Wait for recovery to complete.
  486          */
  487         while ((clp->nfsc_flags & NFSCLFLAGS_RECVRINPROG))
  488                 (void) nfsmsleep(&clp->nfsc_flags, NFSCLSTATEMUTEXPTR,
  489                     PZERO, "nfsrecvr", NULL);
  490 
  491         /*
  492          * First, look for a delegation.
  493          */
  494         LIST_FOREACH(dp, NFSCLDELEGHASH(clp, nfhp, fhlen), nfsdl_hash) {
  495                 if (dp->nfsdl_fhlen == fhlen &&
  496                     !NFSBCMP(nfhp, dp->nfsdl_fh, fhlen)) {
  497                         if (!(mode & NFSV4OPEN_ACCESSWRITE) ||
  498                             (dp->nfsdl_flags & NFSCLDL_WRITE)) {
  499                                 stateidp->seqid = dp->nfsdl_stateid.seqid;
  500                                 stateidp->other[0] = dp->nfsdl_stateid.other[0];
  501                                 stateidp->other[1] = dp->nfsdl_stateid.other[1];
  502                                 stateidp->other[2] = dp->nfsdl_stateid.other[2];
  503                                 if (!(np->n_flag & NDELEGRECALL)) {
  504                                         TAILQ_REMOVE(&clp->nfsc_deleg, dp,
  505                                             nfsdl_list);
  506                                         TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp,
  507                                             nfsdl_list);
  508                                         dp->nfsdl_timestamp = NFSD_MONOSEC +
  509                                             120;
  510                                         dp->nfsdl_rwlock.nfslock_usecnt++;
  511                                         *lckpp = (void *)&dp->nfsdl_rwlock;
  512                                 }
  513                                 NFSUNLOCKCLSTATE();
  514                                 return (0);
  515                         }
  516                         break;
  517                 }
  518         }
  519 
  520         if (p != NULL) {
  521                 /*
  522                  * If p != NULL, we want to search the parentage tree
  523                  * for a matching OpenOwner and use that.
  524                  */
  525                 nfscl_filllockowner(p->td_proc, own, F_POSIX);
  526                 lp = NULL;
  527                 error = nfscl_getopen(&clp->nfsc_owner, nfhp, fhlen, own, own,
  528                     mode, &lp, &op);
  529                 if (error == 0 && lp != NULL) {
  530                         stateidp->seqid =
  531                             lp->nfsl_stateid.seqid;
  532                         stateidp->other[0] =
  533                             lp->nfsl_stateid.other[0];
  534                         stateidp->other[1] =
  535                             lp->nfsl_stateid.other[1];
  536                         stateidp->other[2] =
  537                             lp->nfsl_stateid.other[2];
  538                         NFSUNLOCKCLSTATE();
  539                         return (0);
  540                 }
  541         }
  542         if (op == NULL) {
  543                 /* If not found, just look for any OpenOwner that will work. */
  544                 done = 0;
  545                 owp = LIST_FIRST(&clp->nfsc_owner);
  546                 while (!done && owp != NULL) {
  547                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
  548                                 if (op->nfso_fhlen == fhlen &&
  549                                     !NFSBCMP(op->nfso_fh, nfhp, fhlen) &&
  550                                     (mode & op->nfso_mode) == mode) {
  551                                         done = 1;
  552                                         break;
  553                                 }
  554                         }
  555                         if (!done)
  556                                 owp = LIST_NEXT(owp, nfsow_list);
  557                 }
  558                 if (!done) {
  559                         NFSUNLOCKCLSTATE();
  560                         return (ENOENT);
  561                 }
  562                 /*
  563                  * For read aheads or write behinds, use the open cred.
  564                  * A read ahead or write behind is indicated by p == NULL.
  565                  */
  566                 if (p == NULL)
  567                         newnfs_copycred(&op->nfso_cred, cred);
  568         }
  569 
  570         /*
  571          * No lock stateid, so return the open stateid.
  572          */
  573         stateidp->seqid = op->nfso_stateid.seqid;
  574         stateidp->other[0] = op->nfso_stateid.other[0];
  575         stateidp->other[1] = op->nfso_stateid.other[1];
  576         stateidp->other[2] = op->nfso_stateid.other[2];
  577         NFSUNLOCKCLSTATE();
  578         return (0);
  579 }
  580 
  581 /*
  582  * Search for a matching file, mode and, optionally, lockowner.
  583  */
  584 static int
  585 nfscl_getopen(struct nfsclownerhead *ohp, u_int8_t *nfhp, int fhlen,
  586     u_int8_t *openown, u_int8_t *lockown, u_int32_t mode,
  587     struct nfscllockowner **lpp, struct nfsclopen **opp)
  588 {
  589         struct nfsclowner *owp;
  590         struct nfsclopen *op, *rop, *rop2;
  591         struct nfscllockowner *lp;
  592         int keep_looping;
  593 
  594         if (lpp != NULL)
  595                 *lpp = NULL;
  596         /*
  597          * rop will be set to the open to be returned. There are three
  598          * variants of this, all for an open of the correct file:
  599          * 1 - A match of lockown.
  600          * 2 - A match of the openown, when no lockown match exists.
  601          * 3 - A match for any open, if no openown or lockown match exists.
  602          * Looking for #2 over #3 probably isn't necessary, but since
  603          * RFC3530 is vague w.r.t. the relationship between openowners and
  604          * lockowners, I think this is the safer way to go.
  605          */
  606         rop = NULL;
  607         rop2 = NULL;
  608         keep_looping = 1;
  609         /* Search the client list */
  610         owp = LIST_FIRST(ohp);
  611         while (owp != NULL && keep_looping != 0) {
  612                 /* and look for the correct open */
  613                 op = LIST_FIRST(&owp->nfsow_open);
  614                 while (op != NULL && keep_looping != 0) {
  615                         if (op->nfso_fhlen == fhlen &&
  616                             !NFSBCMP(op->nfso_fh, nfhp, fhlen)
  617                             && (op->nfso_mode & mode) == mode) {
  618                                 if (lpp != NULL) {
  619                                         /* Now look for a matching lockowner. */
  620                                         LIST_FOREACH(lp, &op->nfso_lock,
  621                                             nfsl_list) {
  622                                                 if (!NFSBCMP(lp->nfsl_owner,
  623                                                     lockown,
  624                                                     NFSV4CL_LOCKNAMELEN)) {
  625                                                         *lpp = lp;
  626                                                         rop = op;
  627                                                         keep_looping = 0;
  628                                                         break;
  629                                                 }
  630                                         }
  631                                 }
  632                                 if (rop == NULL && !NFSBCMP(owp->nfsow_owner,
  633                                     openown, NFSV4CL_LOCKNAMELEN)) {
  634                                         rop = op;
  635                                         if (lpp == NULL)
  636                                                 keep_looping = 0;
  637                                 }
  638                                 if (rop2 == NULL)
  639                                         rop2 = op;
  640                         }
  641                         op = LIST_NEXT(op, nfso_list);
  642                 }
  643                 owp = LIST_NEXT(owp, nfsow_list);
  644         }
  645         if (rop == NULL)
  646                 rop = rop2;
  647         if (rop == NULL)
  648                 return (EBADF);
  649         *opp = rop;
  650         return (0);
  651 }
  652 
  653 /*
  654  * Release use of an open owner. Called when open operations are done
  655  * with the open owner.
  656  */
  657 APPLESTATIC void
  658 nfscl_ownerrelease(struct nfsclowner *owp, __unused int error,
  659     __unused int candelete, int unlocked)
  660 {
  661 
  662         if (owp == NULL)
  663                 return;
  664         NFSLOCKCLSTATE();
  665         if (!unlocked)
  666                 nfscl_lockunlock(&owp->nfsow_rwlock);
  667         nfscl_clrelease(owp->nfsow_clp);
  668         NFSUNLOCKCLSTATE();
  669 }
  670 
  671 /*
  672  * Release use of an open structure under an open owner.
  673  */
  674 APPLESTATIC void
  675 nfscl_openrelease(struct nfsclopen *op, int error, int candelete)
  676 {
  677         struct nfsclclient *clp;
  678         struct nfsclowner *owp;
  679 
  680         if (op == NULL)
  681                 return;
  682         NFSLOCKCLSTATE();
  683         owp = op->nfso_own;
  684         nfscl_lockunlock(&owp->nfsow_rwlock);
  685         clp = owp->nfsow_clp;
  686         if (error && candelete && op->nfso_opencnt == 0)
  687                 nfscl_freeopen(op, 0);
  688         nfscl_clrelease(clp);
  689         NFSUNLOCKCLSTATE();
  690 }
  691 
  692 /*
  693  * Called to get a clientid structure. It will optionally lock the
  694  * client data structures to do the SetClientId/SetClientId_confirm,
  695  * but will release that lock and return the clientid with a refernce
  696  * count on it.
  697  * If the "cred" argument is NULL, a new clientid should not be created.
  698  * If the "p" argument is NULL, a SetClientID/SetClientIDConfirm cannot
  699  * be done.
  700  * It always clpp with a reference count on it, unless returning an error.
  701  */
  702 APPLESTATIC int
  703 nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
  704     struct nfsclclient **clpp)
  705 {
  706         struct nfsclclient *clp;
  707         struct nfsclclient *newclp = NULL;
  708         struct mount *mp;
  709         struct nfsmount *nmp;
  710         char uuid[HOSTUUIDLEN];
  711         int igotlock = 0, error, trystalecnt, clidinusedelay, i;
  712         u_int16_t idlen = 0;
  713 
  714         mp = vnode_mount(vp);
  715         nmp = VFSTONFS(mp);
  716         if (cred != NULL) {
  717                 getcredhostuuid(cred, uuid, sizeof uuid);
  718                 idlen = strlen(uuid);
  719                 if (idlen > 0)
  720                         idlen += sizeof (u_int64_t);
  721                 else
  722                         idlen += sizeof (u_int64_t) + 16; /* 16 random bytes */
  723                 MALLOC(newclp, struct nfsclclient *,
  724                     sizeof (struct nfsclclient) + idlen - 1, M_NFSCLCLIENT,
  725                     M_WAITOK);
  726         }
  727         NFSLOCKCLSTATE();
  728         /*
  729          * If a forced dismount is already in progress, don't
  730          * allocate a new clientid and get out now. For the case where
  731          * clp != NULL, this is a harmless optimization.
  732          */
  733         if ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0) {
  734                 NFSUNLOCKCLSTATE();
  735                 if (newclp != NULL)
  736                         free(newclp, M_NFSCLCLIENT);
  737                 return (EBADF);
  738         }
  739         clp = nmp->nm_clp;
  740         if (clp == NULL) {
  741                 if (newclp == NULL) {
  742                         NFSUNLOCKCLSTATE();
  743                         return (EACCES);
  744                 }
  745                 clp = newclp;
  746                 NFSBZERO((caddr_t)clp, sizeof(struct nfsclclient) + idlen - 1);
  747                 clp->nfsc_idlen = idlen;
  748                 LIST_INIT(&clp->nfsc_owner);
  749                 TAILQ_INIT(&clp->nfsc_deleg);
  750                 for (i = 0; i < NFSCLDELEGHASHSIZE; i++)
  751                         LIST_INIT(&clp->nfsc_deleghash[i]);
  752                 clp->nfsc_flags = NFSCLFLAGS_INITED;
  753                 clp->nfsc_clientidrev = 1;
  754                 clp->nfsc_cbident = nfscl_nextcbident();
  755                 nfscl_fillclid(nmp->nm_clval, uuid, clp->nfsc_id,
  756                     clp->nfsc_idlen);
  757                 LIST_INSERT_HEAD(&nfsclhead, clp, nfsc_list);
  758                 nmp->nm_clp = clp;
  759                 clp->nfsc_nmp = nmp;
  760                 NFSUNLOCKCLSTATE();
  761                 nfscl_start_renewthread(clp);
  762         } else {
  763                 NFSUNLOCKCLSTATE();
  764                 if (newclp != NULL)
  765                         FREE((caddr_t)newclp, M_NFSCLCLIENT);
  766         }
  767         NFSLOCKCLSTATE();
  768         while ((clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID) == 0 && !igotlock &&
  769             (mp->mnt_kern_flag & MNTK_UNMOUNTF) == 0)
  770                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
  771                     NFSCLSTATEMUTEXPTR, mp);
  772         if (!igotlock)
  773                 nfsv4_getref(&clp->nfsc_lock, NULL, NFSCLSTATEMUTEXPTR, mp);
  774         if (igotlock == 0 && (mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0) {
  775                 /*
  776                  * Both nfsv4_lock() and nfsv4_getref() know to check
  777                  * for MNTK_UNMOUNTF and return without sleeping to
  778                  * wait for the exclusive lock to be released, since it
  779                  * might be held by nfscl_umount() and we need to get out
  780                  * now for that case and not wait until nfscl_umount()
  781                  * releases it.
  782                  */
  783                 NFSUNLOCKCLSTATE();
  784                 return (EBADF);
  785         }
  786         NFSUNLOCKCLSTATE();
  787 
  788         /*
  789          * If it needs a clientid, do the setclientid now.
  790          */
  791         if ((clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID) == 0) {
  792                 if (!igotlock)
  793                         panic("nfscl_clget");
  794                 if (p == NULL || cred == NULL) {
  795                         NFSLOCKCLSTATE();
  796                         nfsv4_unlock(&clp->nfsc_lock, 0);
  797                         NFSUNLOCKCLSTATE();
  798                         return (EACCES);
  799                 }
  800                 /*
  801                  * If RFC3530 Sec. 14.2.33 is taken literally,
  802                  * NFSERR_CLIDINUSE will be returned persistently for the
  803                  * case where a new mount of the same file system is using
  804                  * a different principal. In practice, NFSERR_CLIDINUSE is
  805                  * only returned when there is outstanding unexpired state
  806                  * on the clientid. As such, try for twice the lease
  807                  * interval, if we know what that is. Otherwise, make a
  808                  * wild ass guess.
  809                  * The case of returning NFSERR_STALECLIENTID is far less
  810                  * likely, but might occur if there is a significant delay
  811                  * between doing the SetClientID and SetClientIDConfirm Ops,
  812                  * such that the server throws away the clientid before
  813                  * receiving the SetClientIDConfirm.
  814                  */
  815                 if (clp->nfsc_renew > 0)
  816                         clidinusedelay = NFSCL_LEASE(clp->nfsc_renew) * 2;
  817                 else
  818                         clidinusedelay = 120;
  819                 trystalecnt = 3;
  820                 do {
  821                         error = nfsrpc_setclient(VFSTONFS(vnode_mount(vp)),
  822                             clp, cred, p);
  823                         if (error == NFSERR_STALECLIENTID ||
  824                             error == NFSERR_STALEDONTRECOVER ||
  825                             error == NFSERR_CLIDINUSE) {
  826                                 (void) nfs_catnap(PZERO, error, "nfs_setcl");
  827                         }
  828                 } while (((error == NFSERR_STALECLIENTID ||
  829                      error == NFSERR_STALEDONTRECOVER) && --trystalecnt > 0) ||
  830                     (error == NFSERR_CLIDINUSE && --clidinusedelay > 0));
  831                 if (error) {
  832                         NFSLOCKCLSTATE();
  833                         nfsv4_unlock(&clp->nfsc_lock, 0);
  834                         NFSUNLOCKCLSTATE();
  835                         return (error);
  836                 }
  837                 clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
  838         }
  839         if (igotlock) {
  840                 NFSLOCKCLSTATE();
  841                 nfsv4_unlock(&clp->nfsc_lock, 1);
  842                 NFSUNLOCKCLSTATE();
  843         }
  844 
  845         *clpp = clp;
  846         return (0);
  847 }
  848 
  849 /*
  850  * Get a reference to a clientid and return it, if valid.
  851  */
  852 APPLESTATIC struct nfsclclient *
  853 nfscl_findcl(struct nfsmount *nmp)
  854 {
  855         struct nfsclclient *clp;
  856 
  857         clp = nmp->nm_clp;
  858         if (clp == NULL || !(clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID))
  859                 return (NULL);
  860         return (clp);
  861 }
  862 
  863 /*
  864  * Release the clientid structure. It may be locked or reference counted.
  865  */
  866 static void
  867 nfscl_clrelease(struct nfsclclient *clp)
  868 {
  869 
  870         if (clp->nfsc_lock.nfslock_lock & NFSV4LOCK_LOCK)
  871                 nfsv4_unlock(&clp->nfsc_lock, 0);
  872         else
  873                 nfsv4_relref(&clp->nfsc_lock);
  874 }
  875 
  876 /*
  877  * External call for nfscl_clrelease.
  878  */
  879 APPLESTATIC void
  880 nfscl_clientrelease(struct nfsclclient *clp)
  881 {
  882 
  883         NFSLOCKCLSTATE();
  884         if (clp->nfsc_lock.nfslock_lock & NFSV4LOCK_LOCK)
  885                 nfsv4_unlock(&clp->nfsc_lock, 0);
  886         else
  887                 nfsv4_relref(&clp->nfsc_lock);
  888         NFSUNLOCKCLSTATE();
  889 }
  890 
  891 /*
  892  * Called when wanting to lock a byte region.
  893  */
  894 APPLESTATIC int
  895 nfscl_getbytelock(vnode_t vp, u_int64_t off, u_int64_t len,
  896     short type, struct ucred *cred, NFSPROC_T *p, struct nfsclclient *rclp,
  897     int recovery, void *id, int flags, u_int8_t *rownp, u_int8_t *ropenownp,
  898     struct nfscllockowner **lpp, int *newonep, int *donelocallyp)
  899 {
  900         struct nfscllockowner *lp;
  901         struct nfsclopen *op;
  902         struct nfsclclient *clp;
  903         struct nfscllockowner *nlp;
  904         struct nfscllock *nlop, *otherlop;
  905         struct nfscldeleg *dp = NULL, *ldp = NULL;
  906         struct nfscllockownerhead *lhp = NULL;
  907         struct nfsnode *np;
  908         u_int8_t own[NFSV4CL_LOCKNAMELEN], *ownp, openown[NFSV4CL_LOCKNAMELEN];
  909         u_int8_t *openownp;
  910         int error = 0, ret, donelocally = 0;
  911         u_int32_t mode;
  912 
  913         /* For Lock Ops, the open mode doesn't matter, so use 0 to match any. */
  914         mode = 0;
  915         np = VTONFS(vp);
  916         *lpp = NULL;
  917         lp = NULL;
  918         *newonep = 0;
  919         *donelocallyp = 0;
  920 
  921         /*
  922          * Might need these, so MALLOC them now, to
  923          * avoid a tsleep() in MALLOC later.
  924          */
  925         MALLOC(nlp, struct nfscllockowner *,
  926             sizeof (struct nfscllockowner), M_NFSCLLOCKOWNER, M_WAITOK);
  927         MALLOC(otherlop, struct nfscllock *,
  928             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
  929         MALLOC(nlop, struct nfscllock *,
  930             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
  931         nlop->nfslo_type = type;
  932         nlop->nfslo_first = off;
  933         if (len == NFS64BITSSET) {
  934                 nlop->nfslo_end = NFS64BITSSET;
  935         } else {
  936                 nlop->nfslo_end = off + len;
  937                 if (nlop->nfslo_end <= nlop->nfslo_first)
  938                         error = NFSERR_INVAL;
  939         }
  940 
  941         if (!error) {
  942                 if (recovery)
  943                         clp = rclp;
  944                 else
  945                         error = nfscl_getcl(vp, cred, p, &clp);
  946         }
  947         if (error) {
  948                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
  949                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
  950                 FREE((caddr_t)nlop, M_NFSCLLOCK);
  951                 return (error);
  952         }
  953 
  954         op = NULL;
  955         if (recovery) {
  956                 ownp = rownp;
  957                 openownp = ropenownp;
  958         } else {
  959                 nfscl_filllockowner(id, own, flags);
  960                 ownp = own;
  961                 nfscl_filllockowner(p->td_proc, openown, F_POSIX);
  962                 openownp = openown;
  963         }
  964         if (!recovery) {
  965                 NFSLOCKCLSTATE();
  966                 /*
  967                  * First, search for a delegation. If one exists for this file,
  968                  * the lock can be done locally against it, so long as there
  969                  * isn't a local lock conflict.
  970                  */
  971                 ldp = dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
  972                     np->n_fhp->nfh_len);
  973                 /* Just sanity check for correct type of delegation */
  974                 if (dp != NULL && ((dp->nfsdl_flags &
  975                     (NFSCLDL_RECALL | NFSCLDL_DELEGRET)) != 0 ||
  976                      (type == F_WRLCK &&
  977                       (dp->nfsdl_flags & NFSCLDL_WRITE) == 0)))
  978                         dp = NULL;
  979         }
  980         if (dp != NULL) {
  981                 /* Now, find an open and maybe a lockowner. */
  982                 ret = nfscl_getopen(&dp->nfsdl_owner, np->n_fhp->nfh_fh,
  983                     np->n_fhp->nfh_len, openownp, ownp, mode, NULL, &op);
  984                 if (ret)
  985                         ret = nfscl_getopen(&clp->nfsc_owner,
  986                             np->n_fhp->nfh_fh, np->n_fhp->nfh_len, openownp,
  987                             ownp, mode, NULL, &op);
  988                 if (!ret) {
  989                         lhp = &dp->nfsdl_lock;
  990                         TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
  991                         TAILQ_INSERT_HEAD(&clp->nfsc_deleg, dp, nfsdl_list);
  992                         dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
  993                         donelocally = 1;
  994                 } else {
  995                         dp = NULL;
  996                 }
  997         }
  998         if (!donelocally) {
  999                 /*
 1000                  * Get the related Open and maybe lockowner.
 1001                  */
 1002                 error = nfscl_getopen(&clp->nfsc_owner,
 1003                     np->n_fhp->nfh_fh, np->n_fhp->nfh_len, openownp,
 1004                     ownp, mode, &lp, &op);
 1005                 if (!error)
 1006                         lhp = &op->nfso_lock;
 1007         }
 1008         if (!error && !recovery)
 1009                 error = nfscl_localconflict(clp, np->n_fhp->nfh_fh,
 1010                     np->n_fhp->nfh_len, nlop, ownp, ldp, NULL);
 1011         if (error) {
 1012                 if (!recovery) {
 1013                         nfscl_clrelease(clp);
 1014                         NFSUNLOCKCLSTATE();
 1015                 }
 1016                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
 1017                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
 1018                 FREE((caddr_t)nlop, M_NFSCLLOCK);
 1019                 return (error);
 1020         }
 1021 
 1022         /*
 1023          * Ok, see if a lockowner exists and create one, as required.
 1024          */
 1025         if (lp == NULL)
 1026                 LIST_FOREACH(lp, lhp, nfsl_list) {
 1027                         if (!NFSBCMP(lp->nfsl_owner, ownp, NFSV4CL_LOCKNAMELEN))
 1028                                 break;
 1029                 }
 1030         if (lp == NULL) {
 1031                 NFSBCOPY(ownp, nlp->nfsl_owner, NFSV4CL_LOCKNAMELEN);
 1032                 if (recovery)
 1033                         NFSBCOPY(ropenownp, nlp->nfsl_openowner,
 1034                             NFSV4CL_LOCKNAMELEN);
 1035                 else
 1036                         NFSBCOPY(op->nfso_own->nfsow_owner, nlp->nfsl_openowner,
 1037                             NFSV4CL_LOCKNAMELEN);
 1038                 nlp->nfsl_seqid = 0;
 1039                 nlp->nfsl_lockflags = flags;
 1040                 nlp->nfsl_inprog = NULL;
 1041                 nfscl_lockinit(&nlp->nfsl_rwlock);
 1042                 LIST_INIT(&nlp->nfsl_lock);
 1043                 if (donelocally) {
 1044                         nlp->nfsl_open = NULL;
 1045                         newnfsstats.cllocallockowners++;
 1046                 } else {
 1047                         nlp->nfsl_open = op;
 1048                         newnfsstats.cllockowners++;
 1049                 }
 1050                 LIST_INSERT_HEAD(lhp, nlp, nfsl_list);
 1051                 lp = nlp;
 1052                 nlp = NULL;
 1053                 *newonep = 1;
 1054         }
 1055 
 1056         /*
 1057          * Now, update the byte ranges for locks.
 1058          */
 1059         ret = nfscl_updatelock(lp, &nlop, &otherlop, donelocally);
 1060         if (!ret)
 1061                 donelocally = 1;
 1062         if (donelocally) {
 1063                 *donelocallyp = 1;
 1064                 if (!recovery)
 1065                         nfscl_clrelease(clp);
 1066         } else {
 1067                 /*
 1068                  * Serial modifications on the lock owner for multiple threads
 1069                  * for the same process using a read/write lock.
 1070                  */
 1071                 if (!recovery)
 1072                         nfscl_lockexcl(&lp->nfsl_rwlock, NFSCLSTATEMUTEXPTR);
 1073         }
 1074         if (!recovery)
 1075                 NFSUNLOCKCLSTATE();
 1076 
 1077         if (nlp)
 1078                 FREE((caddr_t)nlp, M_NFSCLLOCKOWNER);
 1079         if (nlop)
 1080                 FREE((caddr_t)nlop, M_NFSCLLOCK);
 1081         if (otherlop)
 1082                 FREE((caddr_t)otherlop, M_NFSCLLOCK);
 1083 
 1084         *lpp = lp;
 1085         return (0);
 1086 }
 1087 
 1088 /*
 1089  * Called to unlock a byte range, for LockU.
 1090  */
 1091 APPLESTATIC int
 1092 nfscl_relbytelock(vnode_t vp, u_int64_t off, u_int64_t len,
 1093     __unused struct ucred *cred, NFSPROC_T *p, int callcnt,
 1094     struct nfsclclient *clp, void *id, int flags,
 1095     struct nfscllockowner **lpp, int *dorpcp)
 1096 {
 1097         struct nfscllockowner *lp;
 1098         struct nfsclowner *owp;
 1099         struct nfsclopen *op;
 1100         struct nfscllock *nlop, *other_lop = NULL;
 1101         struct nfscldeleg *dp;
 1102         struct nfsnode *np;
 1103         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 1104         int ret = 0, fnd;
 1105 
 1106         np = VTONFS(vp);
 1107         *lpp = NULL;
 1108         *dorpcp = 0;
 1109 
 1110         /*
 1111          * Might need these, so MALLOC them now, to
 1112          * avoid a tsleep() in MALLOC later.
 1113          */
 1114         MALLOC(nlop, struct nfscllock *,
 1115             sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
 1116         nlop->nfslo_type = F_UNLCK;
 1117         nlop->nfslo_first = off;
 1118         if (len == NFS64BITSSET) {
 1119                 nlop->nfslo_end = NFS64BITSSET;
 1120         } else {
 1121                 nlop->nfslo_end = off + len;
 1122                 if (nlop->nfslo_end <= nlop->nfslo_first) {
 1123                         FREE((caddr_t)nlop, M_NFSCLLOCK);
 1124                         return (NFSERR_INVAL);
 1125                 }
 1126         }
 1127         if (callcnt == 0) {
 1128                 MALLOC(other_lop, struct nfscllock *,
 1129                     sizeof (struct nfscllock), M_NFSCLLOCK, M_WAITOK);
 1130                 *other_lop = *nlop;
 1131         }
 1132         nfscl_filllockowner(id, own, flags);
 1133         dp = NULL;
 1134         NFSLOCKCLSTATE();
 1135         if (callcnt == 0)
 1136                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 1137                     np->n_fhp->nfh_len);
 1138 
 1139         /*
 1140          * First, unlock any local regions on a delegation.
 1141          */
 1142         if (dp != NULL) {
 1143                 /* Look for this lockowner. */
 1144                 LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 1145                         if (!NFSBCMP(lp->nfsl_owner, own,
 1146                             NFSV4CL_LOCKNAMELEN))
 1147                                 break;
 1148                 }
 1149                 if (lp != NULL)
 1150                         /* Use other_lop, so nlop is still available */
 1151                         (void)nfscl_updatelock(lp, &other_lop, NULL, 1);
 1152         }
 1153 
 1154         /*
 1155          * Now, find a matching open/lockowner that hasn't already been done,
 1156          * as marked by nfsl_inprog.
 1157          */
 1158         lp = NULL;
 1159         fnd = 0;
 1160         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 1161             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 1162                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
 1163                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
 1164                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 1165                         if (lp->nfsl_inprog == NULL &&
 1166                             !NFSBCMP(lp->nfsl_owner, own,
 1167                              NFSV4CL_LOCKNAMELEN)) {
 1168                                 fnd = 1;
 1169                                 break;
 1170                         }
 1171                     }
 1172                     if (fnd)
 1173                         break;
 1174                 }
 1175             }
 1176             if (fnd)
 1177                 break;
 1178         }
 1179 
 1180         if (lp != NULL) {
 1181                 ret = nfscl_updatelock(lp, &nlop, NULL, 0);
 1182                 if (ret)
 1183                         *dorpcp = 1;
 1184                 /*
 1185                  * Serial modifications on the lock owner for multiple
 1186                  * threads for the same process using a read/write lock.
 1187                  */
 1188                 lp->nfsl_inprog = p;
 1189                 nfscl_lockexcl(&lp->nfsl_rwlock, NFSCLSTATEMUTEXPTR);
 1190                 *lpp = lp;
 1191         }
 1192         NFSUNLOCKCLSTATE();
 1193         if (nlop)
 1194                 FREE((caddr_t)nlop, M_NFSCLLOCK);
 1195         if (other_lop)
 1196                 FREE((caddr_t)other_lop, M_NFSCLLOCK);
 1197         return (0);
 1198 }
 1199 
 1200 /*
 1201  * Release all lockowners marked in progess for this process and file.
 1202  */
 1203 APPLESTATIC void
 1204 nfscl_releasealllocks(struct nfsclclient *clp, vnode_t vp, NFSPROC_T *p,
 1205     void *id, int flags)
 1206 {
 1207         struct nfsclowner *owp;
 1208         struct nfsclopen *op;
 1209         struct nfscllockowner *lp;
 1210         struct nfsnode *np;
 1211         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 1212 
 1213         np = VTONFS(vp);
 1214         nfscl_filllockowner(id, own, flags);
 1215         NFSLOCKCLSTATE();
 1216         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 1217             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 1218                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
 1219                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
 1220                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 1221                         if (lp->nfsl_inprog == p &&
 1222                             !NFSBCMP(lp->nfsl_owner, own,
 1223                             NFSV4CL_LOCKNAMELEN)) {
 1224                             lp->nfsl_inprog = NULL;
 1225                             nfscl_lockunlock(&lp->nfsl_rwlock);
 1226                         }
 1227                     }
 1228                 }
 1229             }
 1230         }
 1231         nfscl_clrelease(clp);
 1232         NFSUNLOCKCLSTATE();
 1233 }
 1234 
 1235 /*
 1236  * Called to find out if any bytes within the byte range specified are
 1237  * write locked by the calling process. Used to determine if flushing
 1238  * is required before a LockU.
 1239  * If in doubt, return 1, so the flush will occur.
 1240  */
 1241 APPLESTATIC int
 1242 nfscl_checkwritelocked(vnode_t vp, struct flock *fl,
 1243     struct ucred *cred, NFSPROC_T *p, void *id, int flags)
 1244 {
 1245         struct nfsclowner *owp;
 1246         struct nfscllockowner *lp;
 1247         struct nfsclopen *op;
 1248         struct nfsclclient *clp;
 1249         struct nfscllock *lop;
 1250         struct nfscldeleg *dp;
 1251         struct nfsnode *np;
 1252         u_int64_t off, end;
 1253         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 1254         int error = 0;
 1255 
 1256         np = VTONFS(vp);
 1257         switch (fl->l_whence) {
 1258         case SEEK_SET:
 1259         case SEEK_CUR:
 1260                 /*
 1261                  * Caller is responsible for adding any necessary offset
 1262                  * when SEEK_CUR is used.
 1263                  */
 1264                 off = fl->l_start;
 1265                 break;
 1266         case SEEK_END:
 1267                 off = np->n_size + fl->l_start;
 1268                 break;
 1269         default:
 1270                 return (1);
 1271         };
 1272         if (fl->l_len != 0) {
 1273                 end = off + fl->l_len;
 1274                 if (end < off)
 1275                         return (1);
 1276         } else {
 1277                 end = NFS64BITSSET;
 1278         }
 1279 
 1280         error = nfscl_getcl(vp, cred, p, &clp);
 1281         if (error)
 1282                 return (1);
 1283         nfscl_filllockowner(id, own, flags);
 1284         NFSLOCKCLSTATE();
 1285 
 1286         /*
 1287          * First check the delegation locks.
 1288          */
 1289         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 1290         if (dp != NULL) {
 1291                 LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 1292                         if (!NFSBCMP(lp->nfsl_owner, own,
 1293                             NFSV4CL_LOCKNAMELEN))
 1294                                 break;
 1295                 }
 1296                 if (lp != NULL) {
 1297                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
 1298                                 if (lop->nfslo_first >= end)
 1299                                         break;
 1300                                 if (lop->nfslo_end <= off)
 1301                                         continue;
 1302                                 if (lop->nfslo_type == F_WRLCK) {
 1303                                         nfscl_clrelease(clp);
 1304                                         NFSUNLOCKCLSTATE();
 1305                                         return (1);
 1306                                 }
 1307                         }
 1308                 }
 1309         }
 1310 
 1311         /*
 1312          * Now, check state against the server.
 1313          */
 1314         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 1315             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 1316                 if (op->nfso_fhlen == np->n_fhp->nfh_len &&
 1317                     !NFSBCMP(op->nfso_fh, np->n_fhp->nfh_fh, op->nfso_fhlen)) {
 1318                     LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 1319                         if (!NFSBCMP(lp->nfsl_owner, own,
 1320                             NFSV4CL_LOCKNAMELEN))
 1321                             break;
 1322                     }
 1323                     if (lp != NULL) {
 1324                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
 1325                             if (lop->nfslo_first >= end)
 1326                                 break;
 1327                             if (lop->nfslo_end <= off)
 1328                                 continue;
 1329                             if (lop->nfslo_type == F_WRLCK) {
 1330                                 nfscl_clrelease(clp);
 1331                                 NFSUNLOCKCLSTATE();
 1332                                 return (1);
 1333                             }
 1334                         }
 1335                     }
 1336                 }
 1337             }
 1338         }
 1339         nfscl_clrelease(clp);
 1340         NFSUNLOCKCLSTATE();
 1341         return (0);
 1342 }
 1343 
 1344 /*
 1345  * Release a byte range lock owner structure.
 1346  */
 1347 APPLESTATIC void
 1348 nfscl_lockrelease(struct nfscllockowner *lp, int error, int candelete)
 1349 {
 1350         struct nfsclclient *clp;
 1351 
 1352         if (lp == NULL)
 1353                 return;
 1354         NFSLOCKCLSTATE();
 1355         clp = lp->nfsl_open->nfso_own->nfsow_clp;
 1356         if (error != 0 && candelete &&
 1357             (lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED) == 0)
 1358                 nfscl_freelockowner(lp, 0);
 1359         else
 1360                 nfscl_lockunlock(&lp->nfsl_rwlock);
 1361         nfscl_clrelease(clp);
 1362         NFSUNLOCKCLSTATE();
 1363 }
 1364 
 1365 /*
 1366  * Free up an open structure and any associated byte range lock structures.
 1367  */
 1368 APPLESTATIC void
 1369 nfscl_freeopen(struct nfsclopen *op, int local)
 1370 {
 1371 
 1372         LIST_REMOVE(op, nfso_list);
 1373         nfscl_freealllocks(&op->nfso_lock, local);
 1374         FREE((caddr_t)op, M_NFSCLOPEN);
 1375         if (local)
 1376                 newnfsstats.cllocalopens--;
 1377         else
 1378                 newnfsstats.clopens--;
 1379 }
 1380 
 1381 /*
 1382  * Free up all lock owners and associated locks.
 1383  */
 1384 static void
 1385 nfscl_freealllocks(struct nfscllockownerhead *lhp, int local)
 1386 {
 1387         struct nfscllockowner *lp, *nlp;
 1388 
 1389         LIST_FOREACH_SAFE(lp, lhp, nfsl_list, nlp) {
 1390                 if ((lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED))
 1391                         panic("nfscllckw");
 1392                 nfscl_freelockowner(lp, local);
 1393         }
 1394 }
 1395 
 1396 /*
 1397  * Called for an Open when NFSERR_EXPIRED is received from the server.
 1398  * If there are no byte range locks nor a Share Deny lost, try to do a
 1399  * fresh Open. Otherwise, free the open.
 1400  */
 1401 static int
 1402 nfscl_expireopen(struct nfsclclient *clp, struct nfsclopen *op,
 1403     struct nfsmount *nmp, struct ucred *cred, NFSPROC_T *p)
 1404 {
 1405         struct nfscllockowner *lp;
 1406         struct nfscldeleg *dp;
 1407         int mustdelete = 0, error;
 1408 
 1409         /*
 1410          * Look for any byte range lock(s).
 1411          */
 1412         LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 1413                 if (!LIST_EMPTY(&lp->nfsl_lock)) {
 1414                         mustdelete = 1;
 1415                         break;
 1416                 }
 1417         }
 1418 
 1419         /*
 1420          * If no byte range lock(s) nor a Share deny, try to re-open.
 1421          */
 1422         if (!mustdelete && (op->nfso_mode & NFSLCK_DENYBITS) == 0) {
 1423                 newnfs_copycred(&op->nfso_cred, cred);
 1424                 dp = NULL;
 1425                 error = nfsrpc_reopen(nmp, op->nfso_fh,
 1426                     op->nfso_fhlen, op->nfso_mode, op, &dp, cred, p);
 1427                 if (error) {
 1428                         mustdelete = 1;
 1429                         if (dp != NULL) {
 1430                                 FREE((caddr_t)dp, M_NFSCLDELEG);
 1431                                 dp = NULL;
 1432                         }
 1433                 }
 1434                 if (dp != NULL)
 1435                         nfscl_deleg(nmp->nm_mountp, clp, op->nfso_fh,
 1436                             op->nfso_fhlen, cred, p, &dp);
 1437         }
 1438 
 1439         /*
 1440          * If a byte range lock or Share deny or couldn't re-open, free it.
 1441          */
 1442         if (mustdelete)
 1443                 nfscl_freeopen(op, 0);
 1444         return (mustdelete);
 1445 }
 1446 
 1447 /*
 1448  * Free up an open owner structure.
 1449  */
 1450 static void
 1451 nfscl_freeopenowner(struct nfsclowner *owp, int local)
 1452 {
 1453 
 1454         LIST_REMOVE(owp, nfsow_list);
 1455         FREE((caddr_t)owp, M_NFSCLOWNER);
 1456         if (local)
 1457                 newnfsstats.cllocalopenowners--;
 1458         else
 1459                 newnfsstats.clopenowners--;
 1460 }
 1461 
 1462 /*
 1463  * Free up a byte range lock owner structure.
 1464  */
 1465 APPLESTATIC void
 1466 nfscl_freelockowner(struct nfscllockowner *lp, int local)
 1467 {
 1468         struct nfscllock *lop, *nlop;
 1469 
 1470         LIST_REMOVE(lp, nfsl_list);
 1471         LIST_FOREACH_SAFE(lop, &lp->nfsl_lock, nfslo_list, nlop) {
 1472                 nfscl_freelock(lop, local);
 1473         }
 1474         FREE((caddr_t)lp, M_NFSCLLOCKOWNER);
 1475         if (local)
 1476                 newnfsstats.cllocallockowners--;
 1477         else
 1478                 newnfsstats.cllockowners--;
 1479 }
 1480 
 1481 /*
 1482  * Free up a byte range lock structure.
 1483  */
 1484 APPLESTATIC void
 1485 nfscl_freelock(struct nfscllock *lop, int local)
 1486 {
 1487 
 1488         LIST_REMOVE(lop, nfslo_list);
 1489         FREE((caddr_t)lop, M_NFSCLLOCK);
 1490         if (local)
 1491                 newnfsstats.cllocallocks--;
 1492         else
 1493                 newnfsstats.cllocks--;
 1494 }
 1495 
 1496 /*
 1497  * Clean out the state related to a delegation.
 1498  */
 1499 static void
 1500 nfscl_cleandeleg(struct nfscldeleg *dp)
 1501 {
 1502         struct nfsclowner *owp, *nowp;
 1503         struct nfsclopen *op;
 1504 
 1505         LIST_FOREACH_SAFE(owp, &dp->nfsdl_owner, nfsow_list, nowp) {
 1506                 op = LIST_FIRST(&owp->nfsow_open);
 1507                 if (op != NULL) {
 1508                         if (LIST_NEXT(op, nfso_list) != NULL)
 1509                                 panic("nfscleandel");
 1510                         nfscl_freeopen(op, 1);
 1511                 }
 1512                 nfscl_freeopenowner(owp, 1);
 1513         }
 1514         nfscl_freealllocks(&dp->nfsdl_lock, 1);
 1515 }
 1516 
 1517 /*
 1518  * Free a delegation.
 1519  */
 1520 static void
 1521 nfscl_freedeleg(struct nfscldeleghead *hdp, struct nfscldeleg *dp)
 1522 {
 1523 
 1524         TAILQ_REMOVE(hdp, dp, nfsdl_list);
 1525         LIST_REMOVE(dp, nfsdl_hash);
 1526         FREE((caddr_t)dp, M_NFSCLDELEG);
 1527         newnfsstats.cldelegates--;
 1528         nfscl_delegcnt--;
 1529 }
 1530 
 1531 /*
 1532  * Free up all state related to this client structure.
 1533  */
 1534 static void
 1535 nfscl_cleanclient(struct nfsclclient *clp)
 1536 {
 1537         struct nfsclowner *owp, *nowp;
 1538         struct nfsclopen *op, *nop;
 1539 
 1540         /* Now, all the OpenOwners, etc. */
 1541         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
 1542                 LIST_FOREACH_SAFE(op, &owp->nfsow_open, nfso_list, nop) {
 1543                         nfscl_freeopen(op, 0);
 1544                 }
 1545                 nfscl_freeopenowner(owp, 0);
 1546         }
 1547 }
 1548 
 1549 /*
 1550  * Called when an NFSERR_EXPIRED is received from the server.
 1551  */
 1552 static void
 1553 nfscl_expireclient(struct nfsclclient *clp, struct nfsmount *nmp,
 1554     struct ucred *cred, NFSPROC_T *p)
 1555 {
 1556         struct nfsclowner *owp, *nowp, *towp;
 1557         struct nfsclopen *op, *nop, *top;
 1558         struct nfscldeleg *dp, *ndp;
 1559         int ret, printed = 0;
 1560 
 1561         /*
 1562          * First, merge locally issued Opens into the list for the server.
 1563          */
 1564         dp = TAILQ_FIRST(&clp->nfsc_deleg);
 1565         while (dp != NULL) {
 1566             ndp = TAILQ_NEXT(dp, nfsdl_list);
 1567             owp = LIST_FIRST(&dp->nfsdl_owner);
 1568             while (owp != NULL) {
 1569                 nowp = LIST_NEXT(owp, nfsow_list);
 1570                 op = LIST_FIRST(&owp->nfsow_open);
 1571                 if (op != NULL) {
 1572                     if (LIST_NEXT(op, nfso_list) != NULL)
 1573                         panic("nfsclexp");
 1574                     LIST_FOREACH(towp, &clp->nfsc_owner, nfsow_list) {
 1575                         if (!NFSBCMP(towp->nfsow_owner, owp->nfsow_owner,
 1576                             NFSV4CL_LOCKNAMELEN))
 1577                             break;
 1578                     }
 1579                     if (towp != NULL) {
 1580                         /* Merge opens in */
 1581                         LIST_FOREACH(top, &towp->nfsow_open, nfso_list) {
 1582                             if (top->nfso_fhlen == op->nfso_fhlen &&
 1583                                 !NFSBCMP(top->nfso_fh, op->nfso_fh,
 1584                                  op->nfso_fhlen)) {
 1585                                 top->nfso_mode |= op->nfso_mode;
 1586                                 top->nfso_opencnt += op->nfso_opencnt;
 1587                                 break;
 1588                             }
 1589                         }
 1590                         if (top == NULL) {
 1591                             /* Just add the open to the owner list */
 1592                             LIST_REMOVE(op, nfso_list);
 1593                             op->nfso_own = towp;
 1594                             LIST_INSERT_HEAD(&towp->nfsow_open, op, nfso_list);
 1595                             newnfsstats.cllocalopens--;
 1596                             newnfsstats.clopens++;
 1597                         }
 1598                     } else {
 1599                         /* Just add the openowner to the client list */
 1600                         LIST_REMOVE(owp, nfsow_list);
 1601                         owp->nfsow_clp = clp;
 1602                         LIST_INSERT_HEAD(&clp->nfsc_owner, owp, nfsow_list);
 1603                         newnfsstats.cllocalopenowners--;
 1604                         newnfsstats.clopenowners++;
 1605                         newnfsstats.cllocalopens--;
 1606                         newnfsstats.clopens++;
 1607                     }
 1608                 }
 1609                 owp = nowp;
 1610             }
 1611             if (!printed && !LIST_EMPTY(&dp->nfsdl_lock)) {
 1612                 printed = 1;
 1613                 printf("nfsv4 expired locks lost\n");
 1614             }
 1615             nfscl_cleandeleg(dp);
 1616             nfscl_freedeleg(&clp->nfsc_deleg, dp);
 1617             dp = ndp;
 1618         }
 1619         if (!TAILQ_EMPTY(&clp->nfsc_deleg))
 1620             panic("nfsclexp");
 1621 
 1622         /*
 1623          * Now, try and reopen against the server.
 1624          */
 1625         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
 1626                 owp->nfsow_seqid = 0;
 1627                 LIST_FOREACH_SAFE(op, &owp->nfsow_open, nfso_list, nop) {
 1628                         ret = nfscl_expireopen(clp, op, nmp, cred, p);
 1629                         if (ret && !printed) {
 1630                                 printed = 1;
 1631                                 printf("nfsv4 expired locks lost\n");
 1632                         }
 1633                 }
 1634                 if (LIST_EMPTY(&owp->nfsow_open))
 1635                         nfscl_freeopenowner(owp, 0);
 1636         }
 1637 }
 1638 
 1639 /*
 1640  * This function must be called after the process represented by "own" has
 1641  * exited. Must be called with CLSTATE lock held.
 1642  */
 1643 static void
 1644 nfscl_cleanup_common(struct nfsclclient *clp, u_int8_t *own)
 1645 {
 1646         struct nfsclowner *owp, *nowp;
 1647         struct nfscllockowner *lp, *nlp;
 1648         struct nfscldeleg *dp;
 1649 
 1650         /* First, get rid of local locks on delegations. */
 1651         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
 1652                 LIST_FOREACH_SAFE(lp, &dp->nfsdl_lock, nfsl_list, nlp) {
 1653                     if (!NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN)) {
 1654                         if ((lp->nfsl_rwlock.nfslock_lock & NFSV4LOCK_WANTED))
 1655                             panic("nfscllckw");
 1656                         nfscl_freelockowner(lp, 1);
 1657                     }
 1658                 }
 1659         }
 1660         owp = LIST_FIRST(&clp->nfsc_owner);
 1661         while (owp != NULL) {
 1662                 nowp = LIST_NEXT(owp, nfsow_list);
 1663                 if (!NFSBCMP(owp->nfsow_owner, own,
 1664                     NFSV4CL_LOCKNAMELEN)) {
 1665                         /*
 1666                          * If there are children that haven't closed the
 1667                          * file descriptors yet, the opens will still be
 1668                          * here. For that case, let the renew thread clear
 1669                          * out the OpenOwner later.
 1670                          */
 1671                         if (LIST_EMPTY(&owp->nfsow_open))
 1672                                 nfscl_freeopenowner(owp, 0);
 1673                         else
 1674                                 owp->nfsow_defunct = 1;
 1675                 }
 1676                 owp = nowp;
 1677         }
 1678 }
 1679 
 1680 /*
 1681  * Find open/lock owners for processes that have exited.
 1682  */
 1683 static void
 1684 nfscl_cleanupkext(struct nfsclclient *clp, struct nfscllockownerfhhead *lhp)
 1685 {
 1686         struct nfsclowner *owp, *nowp;
 1687         struct nfsclopen *op;
 1688         struct nfscllockowner *lp, *nlp;
 1689 
 1690         NFSPROCLISTLOCK();
 1691         NFSLOCKCLSTATE();
 1692         LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
 1693                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 1694                         LIST_FOREACH_SAFE(lp, &op->nfso_lock, nfsl_list, nlp) {
 1695                                 if (LIST_EMPTY(&lp->nfsl_lock))
 1696                                         nfscl_emptylockowner(lp, lhp);
 1697                         }
 1698                 }
 1699                 if (nfscl_procdoesntexist(owp->nfsow_owner))
 1700                         nfscl_cleanup_common(clp, owp->nfsow_owner);
 1701         }
 1702         NFSUNLOCKCLSTATE();
 1703         NFSPROCLISTUNLOCK();
 1704 }
 1705 
 1706 /*
 1707  * Take the empty lock owner and move it to the local lhp list if the
 1708  * associated process no longer exists.
 1709  */
 1710 static void
 1711 nfscl_emptylockowner(struct nfscllockowner *lp,
 1712     struct nfscllockownerfhhead *lhp)
 1713 {
 1714         struct nfscllockownerfh *lfhp, *mylfhp;
 1715         struct nfscllockowner *nlp;
 1716         int fnd_it;
 1717 
 1718         /* If not a Posix lock owner, just return. */
 1719         if ((lp->nfsl_lockflags & F_POSIX) == 0)
 1720                 return;
 1721 
 1722         fnd_it = 0;
 1723         mylfhp = NULL;
 1724         /*
 1725          * First, search to see if this lock owner is already in the list.
 1726          * If it is, then the associated process no longer exists.
 1727          */
 1728         SLIST_FOREACH(lfhp, lhp, nfslfh_list) {
 1729                 if (lfhp->nfslfh_len == lp->nfsl_open->nfso_fhlen &&
 1730                     !NFSBCMP(lfhp->nfslfh_fh, lp->nfsl_open->nfso_fh,
 1731                     lfhp->nfslfh_len))
 1732                         mylfhp = lfhp;
 1733                 LIST_FOREACH(nlp, &lfhp->nfslfh_lock, nfsl_list)
 1734                         if (!NFSBCMP(nlp->nfsl_owner, lp->nfsl_owner,
 1735                             NFSV4CL_LOCKNAMELEN))
 1736                                 fnd_it = 1;
 1737         }
 1738         /* If not found, check if process still exists. */
 1739         if (fnd_it == 0 && nfscl_procdoesntexist(lp->nfsl_owner) == 0)
 1740                 return;
 1741 
 1742         /* Move the lock owner over to the local list. */
 1743         if (mylfhp == NULL) {
 1744                 mylfhp = malloc(sizeof(struct nfscllockownerfh), M_TEMP,
 1745                     M_NOWAIT);
 1746                 if (mylfhp == NULL)
 1747                         return;
 1748                 mylfhp->nfslfh_len = lp->nfsl_open->nfso_fhlen;
 1749                 NFSBCOPY(lp->nfsl_open->nfso_fh, mylfhp->nfslfh_fh,
 1750                     mylfhp->nfslfh_len);
 1751                 LIST_INIT(&mylfhp->nfslfh_lock);
 1752                 SLIST_INSERT_HEAD(lhp, mylfhp, nfslfh_list);
 1753         }
 1754         LIST_REMOVE(lp, nfsl_list);
 1755         LIST_INSERT_HEAD(&mylfhp->nfslfh_lock, lp, nfsl_list);
 1756 }
 1757 
 1758 static int      fake_global;    /* Used to force visibility of MNTK_UNMOUNTF */
 1759 /*
 1760  * Called from nfs umount to free up the clientid.
 1761  */
 1762 APPLESTATIC void
 1763 nfscl_umount(struct nfsmount *nmp, NFSPROC_T *p)
 1764 {
 1765         struct nfsclclient *clp;
 1766         struct ucred *cred;
 1767         int igotlock;
 1768 
 1769         /*
 1770          * For the case that matters, this is the thread that set
 1771          * MNTK_UNMOUNTF, so it will see it set. The code that follows is
 1772          * done to ensure that any thread executing nfscl_getcl() after
 1773          * this time, will see MNTK_UNMOUNTF set. nfscl_getcl() uses the
 1774          * mutex for NFSLOCKCLSTATE(), so it is "m" for the following
 1775          * explanation, courtesy of Alan Cox.
 1776          * What follows is a snippet from Alan Cox's email at:
 1777          * http://docs.FreeBSD.org/cgi/
 1778          *     mid.cgi?BANLkTikR3d65zPHo9==08ZfJ2vmqZucEvw
 1779          * 
 1780          * 1. Set MNTK_UNMOUNTF
 1781          * 2. Acquire a standard FreeBSD mutex "m".
 1782          * 3. Update some data structures.
 1783          * 4. Release mutex "m".
 1784          * 
 1785          * Then, other threads that acquire "m" after step 4 has occurred will
 1786          * see MNTK_UNMOUNTF as set.  But, other threads that beat thread X to
 1787          * step 2 may or may not see MNTK_UNMOUNTF as set.
 1788          */
 1789         NFSLOCKCLSTATE();
 1790         if ((nmp->nm_mountp->mnt_kern_flag & MNTK_UNMOUNTF) != 0) {
 1791                 fake_global++;
 1792                 NFSUNLOCKCLSTATE();
 1793                 NFSLOCKCLSTATE();
 1794         }
 1795 
 1796         clp = nmp->nm_clp;
 1797         if (clp != NULL) {
 1798                 if ((clp->nfsc_flags & NFSCLFLAGS_INITED) == 0)
 1799                         panic("nfscl umount");
 1800         
 1801                 /*
 1802                  * First, handshake with the nfscl renew thread, to terminate
 1803                  * it.
 1804                  */
 1805                 clp->nfsc_flags |= NFSCLFLAGS_UMOUNT;
 1806                 while (clp->nfsc_flags & NFSCLFLAGS_HASTHREAD)
 1807                         (void)mtx_sleep(clp, NFSCLSTATEMUTEXPTR, PWAIT,
 1808                             "nfsclumnt", hz);
 1809         
 1810                 /*
 1811                  * Now, get the exclusive lock on the client state, so
 1812                  * that no uses of the state are still in progress.
 1813                  */
 1814                 do {
 1815                         igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
 1816                             NFSCLSTATEMUTEXPTR, NULL);
 1817                 } while (!igotlock);
 1818                 NFSUNLOCKCLSTATE();
 1819         
 1820                 /*
 1821                  * Free up all the state. It will expire on the server, but
 1822                  * maybe we should do a SetClientId/SetClientIdConfirm so
 1823                  * the server throws it away?
 1824                  */
 1825                 LIST_REMOVE(clp, nfsc_list);
 1826                 nfscl_delegreturnall(clp, p);
 1827                 cred = newnfs_getcred();
 1828                 (void) nfsrpc_setclient(nmp, clp, cred, p);
 1829                 nfscl_cleanclient(clp);
 1830                 nmp->nm_clp = NULL;
 1831                 NFSFREECRED(cred);
 1832                 FREE((caddr_t)clp, M_NFSCLCLIENT);
 1833         } else
 1834                 NFSUNLOCKCLSTATE();
 1835 }
 1836 
 1837 /*
 1838  * This function is called when a server replies with NFSERR_STALECLIENTID
 1839  * or NFSERR_STALESTATEID. It traverses the clientid lists, doing Opens
 1840  * and Locks with reclaim. If these fail, it deletes the corresponding state.
 1841  */
 1842 static void
 1843 nfscl_recover(struct nfsclclient *clp, struct ucred *cred, NFSPROC_T *p)
 1844 {
 1845         struct nfsclowner *owp, *nowp;
 1846         struct nfsclopen *op, *nop;
 1847         struct nfscllockowner *lp, *nlp;
 1848         struct nfscllock *lop, *nlop;
 1849         struct nfscldeleg *dp, *ndp, *tdp;
 1850         struct nfsmount *nmp;
 1851         struct ucred *tcred;
 1852         struct nfsclopenhead extra_open;
 1853         struct nfscldeleghead extra_deleg;
 1854         struct nfsreq *rep;
 1855         u_int64_t len;
 1856         u_int32_t delegtype = NFSV4OPEN_DELEGATEWRITE, mode;
 1857         int igotlock = 0, error, trycnt, firstlock, s;
 1858 
 1859         /*
 1860          * First, lock the client structure, so everyone else will
 1861          * block when trying to use state.
 1862          */
 1863         NFSLOCKCLSTATE();
 1864         clp->nfsc_flags |= NFSCLFLAGS_RECVRINPROG;
 1865         do {
 1866                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
 1867                     NFSCLSTATEMUTEXPTR, NULL);
 1868         } while (!igotlock);
 1869         NFSUNLOCKCLSTATE();
 1870 
 1871         nmp = clp->nfsc_nmp;
 1872         if (nmp == NULL)
 1873                 panic("nfscl recover");
 1874         trycnt = 5;
 1875         do {
 1876                 error = nfsrpc_setclient(nmp, clp, cred, p);
 1877         } while ((error == NFSERR_STALECLIENTID ||
 1878              error == NFSERR_STALEDONTRECOVER) && --trycnt > 0);
 1879         if (error) {
 1880                 nfscl_cleanclient(clp);
 1881                 NFSLOCKCLSTATE();
 1882                 clp->nfsc_flags &= ~(NFSCLFLAGS_HASCLIENTID |
 1883                     NFSCLFLAGS_RECOVER | NFSCLFLAGS_RECVRINPROG);
 1884                 wakeup(&clp->nfsc_flags);
 1885                 nfsv4_unlock(&clp->nfsc_lock, 0);
 1886                 NFSUNLOCKCLSTATE();
 1887                 return;
 1888         }
 1889         clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
 1890         clp->nfsc_flags &= ~NFSCLFLAGS_RECOVER;
 1891 
 1892         /*
 1893          * Mark requests already queued on the server, so that they don't
 1894          * initiate another recovery cycle. Any requests already in the
 1895          * queue that handle state information will have the old stale
 1896          * clientid/stateid and will get a NFSERR_STALESTATEID or
 1897          * NFSERR_STALECLIENTID reply from the server. This will be
 1898          * translated to NFSERR_STALEDONTRECOVER when R_DONTRECOVER is set.
 1899          */
 1900         s = splsoftclock();
 1901         NFSLOCKREQ();
 1902         TAILQ_FOREACH(rep, &nfsd_reqq, r_chain) {
 1903                 if (rep->r_nmp == nmp)
 1904                         rep->r_flags |= R_DONTRECOVER;
 1905         }
 1906         NFSUNLOCKREQ();
 1907         splx(s);
 1908 
 1909         /*
 1910          * Now, mark all delegations "need reclaim".
 1911          */
 1912         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list)
 1913                 dp->nfsdl_flags |= NFSCLDL_NEEDRECLAIM;
 1914 
 1915         TAILQ_INIT(&extra_deleg);
 1916         LIST_INIT(&extra_open);
 1917         /*
 1918          * Now traverse the state lists, doing Open and Lock Reclaims.
 1919          */
 1920         tcred = newnfs_getcred();
 1921         owp = LIST_FIRST(&clp->nfsc_owner);
 1922         while (owp != NULL) {
 1923             nowp = LIST_NEXT(owp, nfsow_list);
 1924             owp->nfsow_seqid = 0;
 1925             op = LIST_FIRST(&owp->nfsow_open);
 1926             while (op != NULL) {
 1927                 nop = LIST_NEXT(op, nfso_list);
 1928                 if (error != NFSERR_NOGRACE) {
 1929                     /* Search for a delegation to reclaim with the open */
 1930                     TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
 1931                         if (!(dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM))
 1932                             continue;
 1933                         if ((dp->nfsdl_flags & NFSCLDL_WRITE)) {
 1934                             mode = NFSV4OPEN_ACCESSWRITE;
 1935                             delegtype = NFSV4OPEN_DELEGATEWRITE;
 1936                         } else {
 1937                             mode = NFSV4OPEN_ACCESSREAD;
 1938                             delegtype = NFSV4OPEN_DELEGATEREAD;
 1939                         }
 1940                         if ((op->nfso_mode & mode) == mode &&
 1941                             op->nfso_fhlen == dp->nfsdl_fhlen &&
 1942                             !NFSBCMP(op->nfso_fh, dp->nfsdl_fh, op->nfso_fhlen))
 1943                             break;
 1944                     }
 1945                     ndp = dp;
 1946                     if (dp == NULL)
 1947                         delegtype = NFSV4OPEN_DELEGATENONE;
 1948                     newnfs_copycred(&op->nfso_cred, tcred);
 1949                     error = nfscl_tryopen(nmp, NULL, op->nfso_fh,
 1950                         op->nfso_fhlen, op->nfso_fh, op->nfso_fhlen,
 1951                         op->nfso_mode, op, NULL, 0, &ndp, 1, delegtype,
 1952                         tcred, p);
 1953                     if (!error) {
 1954                         /* Handle any replied delegation */
 1955                         if (ndp != NULL && ((ndp->nfsdl_flags & NFSCLDL_WRITE)
 1956                             || NFSMNT_RDONLY(nmp->nm_mountp))) {
 1957                             if ((ndp->nfsdl_flags & NFSCLDL_WRITE))
 1958                                 mode = NFSV4OPEN_ACCESSWRITE;
 1959                             else
 1960                                 mode = NFSV4OPEN_ACCESSREAD;
 1961                             TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
 1962                                 if (!(dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM))
 1963                                     continue;
 1964                                 if ((op->nfso_mode & mode) == mode &&
 1965                                     op->nfso_fhlen == dp->nfsdl_fhlen &&
 1966                                     !NFSBCMP(op->nfso_fh, dp->nfsdl_fh,
 1967                                     op->nfso_fhlen)) {
 1968                                     dp->nfsdl_stateid = ndp->nfsdl_stateid;
 1969                                     dp->nfsdl_sizelimit = ndp->nfsdl_sizelimit;
 1970                                     dp->nfsdl_ace = ndp->nfsdl_ace;
 1971                                     dp->nfsdl_change = ndp->nfsdl_change;
 1972                                     dp->nfsdl_flags &= ~NFSCLDL_NEEDRECLAIM;
 1973                                     if ((ndp->nfsdl_flags & NFSCLDL_RECALL))
 1974                                         dp->nfsdl_flags |= NFSCLDL_RECALL;
 1975                                     FREE((caddr_t)ndp, M_NFSCLDELEG);
 1976                                     ndp = NULL;
 1977                                     break;
 1978                                 }
 1979                             }
 1980                         }
 1981                         if (ndp != NULL)
 1982                             TAILQ_INSERT_HEAD(&extra_deleg, ndp, nfsdl_list);
 1983 
 1984                         /* and reclaim all byte range locks */
 1985                         lp = LIST_FIRST(&op->nfso_lock);
 1986                         while (lp != NULL) {
 1987                             nlp = LIST_NEXT(lp, nfsl_list);
 1988                             lp->nfsl_seqid = 0;
 1989                             firstlock = 1;
 1990                             lop = LIST_FIRST(&lp->nfsl_lock);
 1991                             while (lop != NULL) {
 1992                                 nlop = LIST_NEXT(lop, nfslo_list);
 1993                                 if (lop->nfslo_end == NFS64BITSSET)
 1994                                     len = NFS64BITSSET;
 1995                                 else
 1996                                     len = lop->nfslo_end - lop->nfslo_first;
 1997                                 if (error != NFSERR_NOGRACE)
 1998                                     error = nfscl_trylock(nmp, NULL,
 1999                                         op->nfso_fh, op->nfso_fhlen, lp,
 2000                                         firstlock, 1, lop->nfslo_first, len,
 2001                                         lop->nfslo_type, tcred, p);
 2002                                 if (error != 0)
 2003                                     nfscl_freelock(lop, 0);
 2004                                 else
 2005                                     firstlock = 0;
 2006                                 lop = nlop;
 2007                             }
 2008                             /* If no locks, but a lockowner, just delete it. */
 2009                             if (LIST_EMPTY(&lp->nfsl_lock))
 2010                                 nfscl_freelockowner(lp, 0);
 2011                             lp = nlp;
 2012                         }
 2013                     } else {
 2014                         nfscl_freeopen(op, 0);
 2015                     }
 2016                 }
 2017                 op = nop;
 2018             }
 2019             owp = nowp;
 2020         }
 2021 
 2022         /*
 2023          * Now, try and get any delegations not yet reclaimed by cobbling
 2024          * to-gether an appropriate open.
 2025          */
 2026         nowp = NULL;
 2027         dp = TAILQ_FIRST(&clp->nfsc_deleg);
 2028         while (dp != NULL) {
 2029             ndp = TAILQ_NEXT(dp, nfsdl_list);
 2030             if ((dp->nfsdl_flags & NFSCLDL_NEEDRECLAIM)) {
 2031                 if (nowp == NULL) {
 2032                     MALLOC(nowp, struct nfsclowner *,
 2033                         sizeof (struct nfsclowner), M_NFSCLOWNER, M_WAITOK);
 2034                     /*
 2035                      * Name must be as long an largest possible
 2036                      * NFSV4CL_LOCKNAMELEN. 12 for now.
 2037                      */
 2038                     NFSBCOPY("RECLAIMDELEG", nowp->nfsow_owner,
 2039                         NFSV4CL_LOCKNAMELEN);
 2040                     LIST_INIT(&nowp->nfsow_open);
 2041                     nowp->nfsow_clp = clp;
 2042                     nowp->nfsow_seqid = 0;
 2043                     nowp->nfsow_defunct = 0;
 2044                     nfscl_lockinit(&nowp->nfsow_rwlock);
 2045                 }
 2046                 nop = NULL;
 2047                 if (error != NFSERR_NOGRACE) {
 2048                     MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
 2049                         dp->nfsdl_fhlen - 1, M_NFSCLOPEN, M_WAITOK);
 2050                     nop->nfso_own = nowp;
 2051                     if ((dp->nfsdl_flags & NFSCLDL_WRITE)) {
 2052                         nop->nfso_mode = NFSV4OPEN_ACCESSWRITE;
 2053                         delegtype = NFSV4OPEN_DELEGATEWRITE;
 2054                     } else {
 2055                         nop->nfso_mode = NFSV4OPEN_ACCESSREAD;
 2056                         delegtype = NFSV4OPEN_DELEGATEREAD;
 2057                     }
 2058                     nop->nfso_opencnt = 0;
 2059                     nop->nfso_posixlock = 1;
 2060                     nop->nfso_fhlen = dp->nfsdl_fhlen;
 2061                     NFSBCOPY(dp->nfsdl_fh, nop->nfso_fh, dp->nfsdl_fhlen);
 2062                     LIST_INIT(&nop->nfso_lock);
 2063                     nop->nfso_stateid.seqid = 0;
 2064                     nop->nfso_stateid.other[0] = 0;
 2065                     nop->nfso_stateid.other[1] = 0;
 2066                     nop->nfso_stateid.other[2] = 0;
 2067                     newnfs_copycred(&dp->nfsdl_cred, tcred);
 2068                     newnfs_copyincred(tcred, &nop->nfso_cred);
 2069                     tdp = NULL;
 2070                     error = nfscl_tryopen(nmp, NULL, nop->nfso_fh,
 2071                         nop->nfso_fhlen, nop->nfso_fh, nop->nfso_fhlen,
 2072                         nop->nfso_mode, nop, NULL, 0, &tdp, 1,
 2073                         delegtype, tcred, p);
 2074                     if (tdp != NULL) {
 2075                         if ((tdp->nfsdl_flags & NFSCLDL_WRITE))
 2076                             mode = NFSV4OPEN_ACCESSWRITE;
 2077                         else
 2078                             mode = NFSV4OPEN_ACCESSREAD;
 2079                         if ((nop->nfso_mode & mode) == mode &&
 2080                             nop->nfso_fhlen == tdp->nfsdl_fhlen &&
 2081                             !NFSBCMP(nop->nfso_fh, tdp->nfsdl_fh,
 2082                             nop->nfso_fhlen)) {
 2083                             dp->nfsdl_stateid = tdp->nfsdl_stateid;
 2084                             dp->nfsdl_sizelimit = tdp->nfsdl_sizelimit;
 2085                             dp->nfsdl_ace = tdp->nfsdl_ace;
 2086                             dp->nfsdl_change = tdp->nfsdl_change;
 2087                             dp->nfsdl_flags &= ~NFSCLDL_NEEDRECLAIM;
 2088                             if ((tdp->nfsdl_flags & NFSCLDL_RECALL))
 2089                                 dp->nfsdl_flags |= NFSCLDL_RECALL;
 2090                             FREE((caddr_t)tdp, M_NFSCLDELEG);
 2091                         } else {
 2092                             TAILQ_INSERT_HEAD(&extra_deleg, tdp, nfsdl_list);
 2093                         }
 2094                     }
 2095                 }
 2096                 if (error) {
 2097                     if (nop != NULL)
 2098                         FREE((caddr_t)nop, M_NFSCLOPEN);
 2099                     /*
 2100                      * Couldn't reclaim it, so throw the state
 2101                      * away. Ouch!!
 2102                      */
 2103                     nfscl_cleandeleg(dp);
 2104                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
 2105                 } else {
 2106                     LIST_INSERT_HEAD(&extra_open, nop, nfso_list);
 2107                 }
 2108             }
 2109             dp = ndp;
 2110         }
 2111 
 2112         /*
 2113          * Now, get rid of extra Opens and Delegations.
 2114          */
 2115         LIST_FOREACH_SAFE(op, &extra_open, nfso_list, nop) {
 2116                 do {
 2117                         newnfs_copycred(&op->nfso_cred, tcred);
 2118                         error = nfscl_tryclose(op, tcred, nmp, p);
 2119                         if (error == NFSERR_GRACE)
 2120                                 (void) nfs_catnap(PZERO, error, "nfsexcls");
 2121                 } while (error == NFSERR_GRACE);
 2122                 LIST_REMOVE(op, nfso_list);
 2123                 FREE((caddr_t)op, M_NFSCLOPEN);
 2124         }
 2125         if (nowp != NULL)
 2126                 FREE((caddr_t)nowp, M_NFSCLOWNER);
 2127 
 2128         TAILQ_FOREACH_SAFE(dp, &extra_deleg, nfsdl_list, ndp) {
 2129                 do {
 2130                         newnfs_copycred(&dp->nfsdl_cred, tcred);
 2131                         error = nfscl_trydelegreturn(dp, tcred, nmp, p);
 2132                         if (error == NFSERR_GRACE)
 2133                                 (void) nfs_catnap(PZERO, error, "nfsexdlg");
 2134                 } while (error == NFSERR_GRACE);
 2135                 TAILQ_REMOVE(&extra_deleg, dp, nfsdl_list);
 2136                 FREE((caddr_t)dp, M_NFSCLDELEG);
 2137         }
 2138 
 2139         NFSLOCKCLSTATE();
 2140         clp->nfsc_flags &= ~NFSCLFLAGS_RECVRINPROG;
 2141         wakeup(&clp->nfsc_flags);
 2142         nfsv4_unlock(&clp->nfsc_lock, 0);
 2143         NFSUNLOCKCLSTATE();
 2144         NFSFREECRED(tcred);
 2145 }
 2146 
 2147 /*
 2148  * This function is called when a server replies with NFSERR_EXPIRED.
 2149  * It deletes all state for the client and does a fresh SetClientId/confirm.
 2150  * XXX Someday it should post a signal to the process(es) that hold the
 2151  * state, so they know that lock state has been lost.
 2152  */
 2153 APPLESTATIC int
 2154 nfscl_hasexpired(struct nfsclclient *clp, u_int32_t clidrev, NFSPROC_T *p)
 2155 {
 2156         struct nfsmount *nmp;
 2157         struct ucred *cred;
 2158         int igotlock = 0, error, trycnt;
 2159 
 2160         /*
 2161          * If the clientid has gone away or a new SetClientid has already
 2162          * been done, just return ok.
 2163          */
 2164         if (clp == NULL || clidrev != clp->nfsc_clientidrev)
 2165                 return (0);
 2166 
 2167         /*
 2168          * First, lock the client structure, so everyone else will
 2169          * block when trying to use state. Also, use NFSCLFLAGS_EXPIREIT so
 2170          * that only one thread does the work.
 2171          */
 2172         NFSLOCKCLSTATE();
 2173         clp->nfsc_flags |= NFSCLFLAGS_EXPIREIT;
 2174         do {
 2175                 igotlock = nfsv4_lock(&clp->nfsc_lock, 1, NULL,
 2176                     NFSCLSTATEMUTEXPTR, NULL);
 2177         } while (!igotlock && (clp->nfsc_flags & NFSCLFLAGS_EXPIREIT));
 2178         if ((clp->nfsc_flags & NFSCLFLAGS_EXPIREIT) == 0) {
 2179                 if (igotlock)
 2180                         nfsv4_unlock(&clp->nfsc_lock, 0);
 2181                 NFSUNLOCKCLSTATE();
 2182                 return (0);
 2183         }
 2184         clp->nfsc_flags |= NFSCLFLAGS_RECVRINPROG;
 2185         NFSUNLOCKCLSTATE();
 2186 
 2187         nmp = clp->nfsc_nmp;
 2188         if (nmp == NULL)
 2189                 panic("nfscl expired");
 2190         cred = newnfs_getcred();
 2191         trycnt = 5;
 2192         do {
 2193                 error = nfsrpc_setclient(nmp, clp, cred, p);
 2194         } while ((error == NFSERR_STALECLIENTID ||
 2195              error == NFSERR_STALEDONTRECOVER) && --trycnt > 0);
 2196         if (error) {
 2197                 /*
 2198                  * Clear out any state.
 2199                  */
 2200                 nfscl_cleanclient(clp);
 2201                 NFSLOCKCLSTATE();
 2202                 clp->nfsc_flags &= ~(NFSCLFLAGS_HASCLIENTID |
 2203                     NFSCLFLAGS_RECOVER);
 2204         } else {
 2205                 /*
 2206                  * Expire the state for the client.
 2207                  */
 2208                 nfscl_expireclient(clp, nmp, cred, p);
 2209                 NFSLOCKCLSTATE();
 2210                 clp->nfsc_flags |= NFSCLFLAGS_HASCLIENTID;
 2211                 clp->nfsc_flags &= ~NFSCLFLAGS_RECOVER;
 2212         }
 2213         clp->nfsc_flags &= ~(NFSCLFLAGS_EXPIREIT | NFSCLFLAGS_RECVRINPROG);
 2214         wakeup(&clp->nfsc_flags);
 2215         nfsv4_unlock(&clp->nfsc_lock, 0);
 2216         NFSUNLOCKCLSTATE();
 2217         NFSFREECRED(cred);
 2218         return (error);
 2219 }
 2220 
 2221 /*
 2222  * This function inserts a lock in the list after insert_lop.
 2223  */
 2224 static void
 2225 nfscl_insertlock(struct nfscllockowner *lp, struct nfscllock *new_lop,
 2226     struct nfscllock *insert_lop, int local)
 2227 {
 2228 
 2229         if ((struct nfscllockowner *)insert_lop == lp)
 2230                 LIST_INSERT_HEAD(&lp->nfsl_lock, new_lop, nfslo_list);
 2231         else
 2232                 LIST_INSERT_AFTER(insert_lop, new_lop, nfslo_list);
 2233         if (local)
 2234                 newnfsstats.cllocallocks++;
 2235         else
 2236                 newnfsstats.cllocks++;
 2237 }
 2238 
 2239 /*
 2240  * This function updates the locking for a lock owner and given file. It
 2241  * maintains a list of lock ranges ordered on increasing file offset that
 2242  * are NFSCLLOCK_READ or NFSCLLOCK_WRITE and non-overlapping (aka POSIX style).
 2243  * It always adds new_lop to the list and sometimes uses the one pointed
 2244  * at by other_lopp.
 2245  * Returns 1 if the locks were modified, 0 otherwise.
 2246  */
 2247 static int
 2248 nfscl_updatelock(struct nfscllockowner *lp, struct nfscllock **new_lopp,
 2249     struct nfscllock **other_lopp, int local)
 2250 {
 2251         struct nfscllock *new_lop = *new_lopp;
 2252         struct nfscllock *lop, *tlop, *ilop;
 2253         struct nfscllock *other_lop;
 2254         int unlock = 0, modified = 0;
 2255         u_int64_t tmp;
 2256 
 2257         /*
 2258          * Work down the list until the lock is merged.
 2259          */
 2260         if (new_lop->nfslo_type == F_UNLCK)
 2261                 unlock = 1;
 2262         ilop = (struct nfscllock *)lp;
 2263         lop = LIST_FIRST(&lp->nfsl_lock);
 2264         while (lop != NULL) {
 2265             /*
 2266              * Only check locks for this file that aren't before the start of
 2267              * new lock's range.
 2268              */
 2269             if (lop->nfslo_end >= new_lop->nfslo_first) {
 2270                 if (new_lop->nfslo_end < lop->nfslo_first) {
 2271                     /*
 2272                      * If the new lock ends before the start of the
 2273                      * current lock's range, no merge, just insert
 2274                      * the new lock.
 2275                      */
 2276                     break;
 2277                 }
 2278                 if (new_lop->nfslo_type == lop->nfslo_type ||
 2279                     (new_lop->nfslo_first <= lop->nfslo_first &&
 2280                      new_lop->nfslo_end >= lop->nfslo_end)) {
 2281                     /*
 2282                      * This lock can be absorbed by the new lock/unlock.
 2283                      * This happens when it covers the entire range
 2284                      * of the old lock or is contiguous
 2285                      * with the old lock and is of the same type or an
 2286                      * unlock.
 2287                      */
 2288                     if (new_lop->nfslo_type != lop->nfslo_type ||
 2289                         new_lop->nfslo_first != lop->nfslo_first ||
 2290                         new_lop->nfslo_end != lop->nfslo_end)
 2291                         modified = 1;
 2292                     if (lop->nfslo_first < new_lop->nfslo_first)
 2293                         new_lop->nfslo_first = lop->nfslo_first;
 2294                     if (lop->nfslo_end > new_lop->nfslo_end)
 2295                         new_lop->nfslo_end = lop->nfslo_end;
 2296                     tlop = lop;
 2297                     lop = LIST_NEXT(lop, nfslo_list);
 2298                     nfscl_freelock(tlop, local);
 2299                     continue;
 2300                 }
 2301 
 2302                 /*
 2303                  * All these cases are for contiguous locks that are not the
 2304                  * same type, so they can't be merged.
 2305                  */
 2306                 if (new_lop->nfslo_first <= lop->nfslo_first) {
 2307                     /*
 2308                      * This case is where the new lock overlaps with the
 2309                      * first part of the old lock. Move the start of the
 2310                      * old lock to just past the end of the new lock. The
 2311                      * new lock will be inserted in front of the old, since
 2312                      * ilop hasn't been updated. (We are done now.)
 2313                      */
 2314                     if (lop->nfslo_first != new_lop->nfslo_end) {
 2315                         lop->nfslo_first = new_lop->nfslo_end;
 2316                         modified = 1;
 2317                     }
 2318                     break;
 2319                 }
 2320                 if (new_lop->nfslo_end >= lop->nfslo_end) {
 2321                     /*
 2322                      * This case is where the new lock overlaps with the
 2323                      * end of the old lock's range. Move the old lock's
 2324                      * end to just before the new lock's first and insert
 2325                      * the new lock after the old lock.
 2326                      * Might not be done yet, since the new lock could
 2327                      * overlap further locks with higher ranges.
 2328                      */
 2329                     if (lop->nfslo_end != new_lop->nfslo_first) {
 2330                         lop->nfslo_end = new_lop->nfslo_first;
 2331                         modified = 1;
 2332                     }
 2333                     ilop = lop;
 2334                     lop = LIST_NEXT(lop, nfslo_list);
 2335                     continue;
 2336                 }
 2337                 /*
 2338                  * The final case is where the new lock's range is in the
 2339                  * middle of the current lock's and splits the current lock
 2340                  * up. Use *other_lopp to handle the second part of the
 2341                  * split old lock range. (We are done now.)
 2342                  * For unlock, we use new_lop as other_lop and tmp, since
 2343                  * other_lop and new_lop are the same for this case.
 2344                  * We noted the unlock case above, so we don't need
 2345                  * new_lop->nfslo_type any longer.
 2346                  */
 2347                 tmp = new_lop->nfslo_first;
 2348                 if (unlock) {
 2349                     other_lop = new_lop;
 2350                     *new_lopp = NULL;
 2351                 } else {
 2352                     other_lop = *other_lopp;
 2353                     *other_lopp = NULL;
 2354                 }
 2355                 other_lop->nfslo_first = new_lop->nfslo_end;
 2356                 other_lop->nfslo_end = lop->nfslo_end;
 2357                 other_lop->nfslo_type = lop->nfslo_type;
 2358                 lop->nfslo_end = tmp;
 2359                 nfscl_insertlock(lp, other_lop, lop, local);
 2360                 ilop = lop;
 2361                 modified = 1;
 2362                 break;
 2363             }
 2364             ilop = lop;
 2365             lop = LIST_NEXT(lop, nfslo_list);
 2366             if (lop == NULL)
 2367                 break;
 2368         }
 2369 
 2370         /*
 2371          * Insert the new lock in the list at the appropriate place.
 2372          */
 2373         if (!unlock) {
 2374                 nfscl_insertlock(lp, new_lop, ilop, local);
 2375                 *new_lopp = NULL;
 2376                 modified = 1;
 2377         }
 2378         return (modified);
 2379 }
 2380 
 2381 /*
 2382  * This function must be run as a kernel thread.
 2383  * It does Renew Ops and recovery, when required.
 2384  */
 2385 APPLESTATIC void
 2386 nfscl_renewthread(struct nfsclclient *clp, NFSPROC_T *p)
 2387 {
 2388         struct nfsclowner *owp, *nowp;
 2389         struct nfsclopen *op;
 2390         struct nfscllockowner *lp, *nlp;
 2391         struct nfscldeleghead dh;
 2392         struct nfscldeleg *dp, *ndp;
 2393         struct ucred *cred;
 2394         u_int32_t clidrev;
 2395         int error, cbpathdown, islept, igotlock, ret, clearok;
 2396         uint32_t recover_done_time = 0;
 2397         time_t mytime;
 2398         static time_t prevsec = 0;
 2399         struct nfscllockownerfh *lfhp, *nlfhp;
 2400         struct nfscllockownerfhhead lfh;
 2401 
 2402         cred = newnfs_getcred();
 2403         NFSLOCKCLSTATE();
 2404         clp->nfsc_flags |= NFSCLFLAGS_HASTHREAD;
 2405         NFSUNLOCKCLSTATE();
 2406         for(;;) {
 2407                 newnfs_setroot(cred);
 2408                 cbpathdown = 0;
 2409                 if (clp->nfsc_flags & NFSCLFLAGS_RECOVER) {
 2410                         /*
 2411                          * Only allow one recover within 1/2 of the lease
 2412                          * duration (nfsc_renew).
 2413                          */
 2414                         if (recover_done_time < NFSD_MONOSEC) {
 2415                                 recover_done_time = NFSD_MONOSEC +
 2416                                     clp->nfsc_renew;
 2417                                 nfscl_recover(clp, cred, p);
 2418                         } else {
 2419                                 NFSLOCKCLSTATE();
 2420                                 clp->nfsc_flags &= ~NFSCLFLAGS_RECOVER;
 2421                                 NFSUNLOCKCLSTATE();
 2422                         }
 2423                 }
 2424                 if (clp->nfsc_expire <= NFSD_MONOSEC &&
 2425                     (clp->nfsc_flags & NFSCLFLAGS_HASCLIENTID)) {
 2426                         clp->nfsc_expire = NFSD_MONOSEC + clp->nfsc_renew;
 2427                         clidrev = clp->nfsc_clientidrev;
 2428                         error = nfsrpc_renew(clp, cred, p);
 2429                         if (error == NFSERR_CBPATHDOWN)
 2430                             cbpathdown = 1;
 2431                         else if (error == NFSERR_STALECLIENTID) {
 2432                             NFSLOCKCLSTATE();
 2433                             clp->nfsc_flags |= NFSCLFLAGS_RECOVER;
 2434                             NFSUNLOCKCLSTATE();
 2435                         } else if (error == NFSERR_EXPIRED)
 2436                             (void) nfscl_hasexpired(clp, clidrev, p);
 2437                 }
 2438 
 2439                 TAILQ_INIT(&dh);
 2440                 NFSLOCKCLSTATE();
 2441                 if (cbpathdown)
 2442                         /* It's a Total Recall! */
 2443                         nfscl_totalrecall(clp);
 2444 
 2445                 /*
 2446                  * Now, handle defunct owners.
 2447                  */
 2448                 LIST_FOREACH_SAFE(owp, &clp->nfsc_owner, nfsow_list, nowp) {
 2449                         if (LIST_EMPTY(&owp->nfsow_open)) {
 2450                                 if (owp->nfsow_defunct != 0)
 2451                                         nfscl_freeopenowner(owp, 0);
 2452                         }
 2453                 }
 2454 
 2455                 /*
 2456                  * Do the recall on any delegations. To avoid trouble, always
 2457                  * come back up here after having slept.
 2458                  */
 2459                 igotlock = 0;
 2460 tryagain:
 2461                 dp = TAILQ_FIRST(&clp->nfsc_deleg);
 2462                 while (dp != NULL) {
 2463                         ndp = TAILQ_NEXT(dp, nfsdl_list);
 2464                         if ((dp->nfsdl_flags & NFSCLDL_RECALL)) {
 2465                                 /*
 2466                                  * Wait for outstanding I/O ops to be done.
 2467                                  */
 2468                                 if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
 2469                                     if (igotlock) {
 2470                                         nfsv4_unlock(&clp->nfsc_lock, 0);
 2471                                         igotlock = 0;
 2472                                     }
 2473                                     dp->nfsdl_rwlock.nfslock_lock |=
 2474                                         NFSV4LOCK_WANTED;
 2475                                     (void) nfsmsleep(&dp->nfsdl_rwlock,
 2476                                         NFSCLSTATEMUTEXPTR, PZERO, "nfscld",
 2477                                         NULL);
 2478                                     goto tryagain;
 2479                                 }
 2480                                 while (!igotlock) {
 2481                                     igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
 2482                                         &islept, NFSCLSTATEMUTEXPTR, NULL);
 2483                                     if (islept)
 2484                                         goto tryagain;
 2485                                 }
 2486                                 NFSUNLOCKCLSTATE();
 2487                                 newnfs_copycred(&dp->nfsdl_cred, cred);
 2488                                 ret = nfscl_recalldeleg(clp, clp->nfsc_nmp, dp,
 2489                                     NULL, cred, p, 1);
 2490                                 if (!ret) {
 2491                                     nfscl_cleandeleg(dp);
 2492                                     TAILQ_REMOVE(&clp->nfsc_deleg, dp,
 2493                                         nfsdl_list);
 2494                                     LIST_REMOVE(dp, nfsdl_hash);
 2495                                     TAILQ_INSERT_HEAD(&dh, dp, nfsdl_list);
 2496                                     nfscl_delegcnt--;
 2497                                     newnfsstats.cldelegates--;
 2498                                 }
 2499                                 NFSLOCKCLSTATE();
 2500                         }
 2501                         dp = ndp;
 2502                 }
 2503 
 2504                 /*
 2505                  * Clear out old delegations, if we are above the high water
 2506                  * mark. Only clear out ones with no state related to them.
 2507                  * The tailq list is in LRU order.
 2508                  */
 2509                 dp = TAILQ_LAST(&clp->nfsc_deleg, nfscldeleghead);
 2510                 while (nfscl_delegcnt > nfscl_deleghighwater && dp != NULL) {
 2511                     ndp = TAILQ_PREV(dp, nfscldeleghead, nfsdl_list);
 2512                     if (dp->nfsdl_rwlock.nfslock_usecnt == 0 &&
 2513                         dp->nfsdl_rwlock.nfslock_lock == 0 &&
 2514                         dp->nfsdl_timestamp < NFSD_MONOSEC &&
 2515                         (dp->nfsdl_flags & (NFSCLDL_RECALL | NFSCLDL_ZAPPED |
 2516                           NFSCLDL_NEEDRECLAIM | NFSCLDL_DELEGRET)) == 0) {
 2517                         clearok = 1;
 2518                         LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 2519                             op = LIST_FIRST(&owp->nfsow_open);
 2520                             if (op != NULL) {
 2521                                 clearok = 0;
 2522                                 break;
 2523                             }
 2524                         }
 2525                         if (clearok) {
 2526                             LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 2527                                 if (!LIST_EMPTY(&lp->nfsl_lock)) {
 2528                                     clearok = 0;
 2529                                     break;
 2530                                 }
 2531                             }
 2532                         }
 2533                         if (clearok) {
 2534                             TAILQ_REMOVE(&clp->nfsc_deleg, dp, nfsdl_list);
 2535                             LIST_REMOVE(dp, nfsdl_hash);
 2536                             TAILQ_INSERT_HEAD(&dh, dp, nfsdl_list);
 2537                             nfscl_delegcnt--;
 2538                             newnfsstats.cldelegates--;
 2539                         }
 2540                     }
 2541                     dp = ndp;
 2542                 }
 2543                 if (igotlock)
 2544                         nfsv4_unlock(&clp->nfsc_lock, 0);
 2545                 NFSUNLOCKCLSTATE();
 2546 
 2547                 /*
 2548                  * Delegreturn any delegations cleaned out or recalled.
 2549                  */
 2550                 TAILQ_FOREACH_SAFE(dp, &dh, nfsdl_list, ndp) {
 2551                         newnfs_copycred(&dp->nfsdl_cred, cred);
 2552                         (void) nfscl_trydelegreturn(dp, cred, clp->nfsc_nmp, p);
 2553                         TAILQ_REMOVE(&dh, dp, nfsdl_list);
 2554                         FREE((caddr_t)dp, M_NFSCLDELEG);
 2555                 }
 2556 
 2557                 SLIST_INIT(&lfh);
 2558                 /*
 2559                  * Call nfscl_cleanupkext() once per second to check for
 2560                  * open/lock owners where the process has exited.
 2561                  */
 2562                 mytime = NFSD_MONOSEC;
 2563                 if (prevsec != mytime) {
 2564                         prevsec = mytime;
 2565                         nfscl_cleanupkext(clp, &lfh);
 2566                 }
 2567 
 2568                 /*
 2569                  * Do a ReleaseLockOwner for all lock owners where the
 2570                  * associated process no longer exists, as found by
 2571                  * nfscl_cleanupkext().
 2572                  */
 2573                 newnfs_setroot(cred);
 2574                 SLIST_FOREACH_SAFE(lfhp, &lfh, nfslfh_list, nlfhp) {
 2575                         LIST_FOREACH_SAFE(lp, &lfhp->nfslfh_lock, nfsl_list,
 2576                             nlp) {
 2577                                 (void)nfsrpc_rellockown(clp->nfsc_nmp, lp,
 2578                                     lfhp->nfslfh_fh, lfhp->nfslfh_len, cred,
 2579                                     p);
 2580                                 nfscl_freelockowner(lp, 0);
 2581                         }
 2582                         free(lfhp, M_TEMP);
 2583                 }
 2584                 SLIST_INIT(&lfh);
 2585 
 2586                 NFSLOCKCLSTATE();
 2587                 if ((clp->nfsc_flags & NFSCLFLAGS_RECOVER) == 0)
 2588                         (void)mtx_sleep(clp, NFSCLSTATEMUTEXPTR, PWAIT, "nfscl",
 2589                             hz);
 2590                 if (clp->nfsc_flags & NFSCLFLAGS_UMOUNT) {
 2591                         clp->nfsc_flags &= ~NFSCLFLAGS_HASTHREAD;
 2592                         NFSUNLOCKCLSTATE();
 2593                         NFSFREECRED(cred);
 2594                         wakeup((caddr_t)clp);
 2595                         return;
 2596                 }
 2597                 NFSUNLOCKCLSTATE();
 2598         }
 2599 }
 2600 
 2601 /*
 2602  * Initiate state recovery. Called when NFSERR_STALECLIENTID or
 2603  * NFSERR_STALESTATEID is received.
 2604  */
 2605 APPLESTATIC void
 2606 nfscl_initiate_recovery(struct nfsclclient *clp)
 2607 {
 2608 
 2609         if (clp == NULL)
 2610                 return;
 2611         NFSLOCKCLSTATE();
 2612         clp->nfsc_flags |= NFSCLFLAGS_RECOVER;
 2613         NFSUNLOCKCLSTATE();
 2614         wakeup((caddr_t)clp);
 2615 }
 2616 
 2617 /*
 2618  * Dump out the state stuff for debugging.
 2619  */
 2620 APPLESTATIC void
 2621 nfscl_dumpstate(struct nfsmount *nmp, int openowner, int opens,
 2622     int lockowner, int locks)
 2623 {
 2624         struct nfsclclient *clp;
 2625         struct nfsclowner *owp;
 2626         struct nfsclopen *op;
 2627         struct nfscllockowner *lp;
 2628         struct nfscllock *lop;
 2629         struct nfscldeleg *dp;
 2630 
 2631         clp = nmp->nm_clp;
 2632         if (clp == NULL) {
 2633                 printf("nfscl dumpstate NULL clp\n");
 2634                 return;
 2635         }
 2636         NFSLOCKCLSTATE();
 2637         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
 2638           LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 2639             if (openowner && !LIST_EMPTY(&owp->nfsow_open))
 2640                 printf("owner=0x%x 0x%x 0x%x 0x%x seqid=%d\n",
 2641                     owp->nfsow_owner[0], owp->nfsow_owner[1],
 2642                     owp->nfsow_owner[2], owp->nfsow_owner[3],
 2643                     owp->nfsow_seqid);
 2644             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 2645                 if (opens)
 2646                     printf("open st=0x%x 0x%x 0x%x cnt=%d fh12=0x%x\n",
 2647                         op->nfso_stateid.other[0], op->nfso_stateid.other[1],
 2648                         op->nfso_stateid.other[2], op->nfso_opencnt,
 2649                         op->nfso_fh[12]);
 2650                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 2651                     if (lockowner)
 2652                         printf("lckown=0x%x 0x%x 0x%x 0x%x seqid=%d st=0x%x 0x%x 0x%x\n",
 2653                             lp->nfsl_owner[0], lp->nfsl_owner[1],
 2654                             lp->nfsl_owner[2], lp->nfsl_owner[3],
 2655                             lp->nfsl_seqid,
 2656                             lp->nfsl_stateid.other[0], lp->nfsl_stateid.other[1],
 2657                             lp->nfsl_stateid.other[2]);
 2658                     LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
 2659                         if (locks)
 2660 #ifdef __FreeBSD__
 2661                             printf("lck typ=%d fst=%ju end=%ju\n",
 2662                                 lop->nfslo_type, (intmax_t)lop->nfslo_first,
 2663                                 (intmax_t)lop->nfslo_end);
 2664 #else
 2665                             printf("lck typ=%d fst=%qd end=%qd\n",
 2666                                 lop->nfslo_type, lop->nfslo_first,
 2667                                 lop->nfslo_end);
 2668 #endif
 2669                     }
 2670                 }
 2671             }
 2672           }
 2673         }
 2674         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2675             if (openowner && !LIST_EMPTY(&owp->nfsow_open))
 2676                 printf("owner=0x%x 0x%x 0x%x 0x%x seqid=%d\n",
 2677                     owp->nfsow_owner[0], owp->nfsow_owner[1],
 2678                     owp->nfsow_owner[2], owp->nfsow_owner[3],
 2679                     owp->nfsow_seqid);
 2680             LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 2681                 if (opens)
 2682                     printf("open st=0x%x 0x%x 0x%x cnt=%d fh12=0x%x\n",
 2683                         op->nfso_stateid.other[0], op->nfso_stateid.other[1],
 2684                         op->nfso_stateid.other[2], op->nfso_opencnt,
 2685                         op->nfso_fh[12]);
 2686                 LIST_FOREACH(lp, &op->nfso_lock, nfsl_list) {
 2687                     if (lockowner)
 2688                         printf("lckown=0x%x 0x%x 0x%x 0x%x seqid=%d st=0x%x 0x%x 0x%x\n",
 2689                             lp->nfsl_owner[0], lp->nfsl_owner[1],
 2690                             lp->nfsl_owner[2], lp->nfsl_owner[3],
 2691                             lp->nfsl_seqid,
 2692                             lp->nfsl_stateid.other[0], lp->nfsl_stateid.other[1],
 2693                             lp->nfsl_stateid.other[2]);
 2694                     LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
 2695                         if (locks)
 2696 #ifdef __FreeBSD__
 2697                             printf("lck typ=%d fst=%ju end=%ju\n",
 2698                                 lop->nfslo_type, (intmax_t)lop->nfslo_first,
 2699                                 (intmax_t)lop->nfslo_end);
 2700 #else
 2701                             printf("lck typ=%d fst=%qd end=%qd\n",
 2702                                 lop->nfslo_type, lop->nfslo_first,
 2703                                 lop->nfslo_end);
 2704 #endif
 2705                     }
 2706                 }
 2707             }
 2708         }
 2709         NFSUNLOCKCLSTATE();
 2710 }
 2711 
 2712 /*
 2713  * Check for duplicate open owners and opens.
 2714  * (Only used as a diagnostic aid.)
 2715  */
 2716 APPLESTATIC void
 2717 nfscl_dupopen(vnode_t vp, int dupopens)
 2718 {
 2719         struct nfsclclient *clp;
 2720         struct nfsclowner *owp, *owp2;
 2721         struct nfsclopen *op, *op2;
 2722         struct nfsfh *nfhp;
 2723 
 2724         clp = VFSTONFS(vnode_mount(vp))->nm_clp;
 2725         if (clp == NULL) {
 2726                 printf("nfscl dupopen NULL clp\n");
 2727                 return;
 2728         }
 2729         nfhp = VTONFS(vp)->n_fhp;
 2730         NFSLOCKCLSTATE();
 2731 
 2732         /*
 2733          * First, search for duplicate owners.
 2734          * These should never happen!
 2735          */
 2736         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
 2737             LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2738                 if (owp != owp2 &&
 2739                     !NFSBCMP(owp->nfsow_owner, owp2->nfsow_owner,
 2740                     NFSV4CL_LOCKNAMELEN)) {
 2741                         NFSUNLOCKCLSTATE();
 2742                         printf("DUP OWNER\n");
 2743                         nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1, 0, 0);
 2744                         return;
 2745                 }
 2746             }
 2747         }
 2748 
 2749         /*
 2750          * Now, search for duplicate stateids.
 2751          * These shouldn't happen, either.
 2752          */
 2753         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
 2754             LIST_FOREACH(op2, &owp2->nfsow_open, nfso_list) {
 2755                 LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2756                     LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 2757                         if (op != op2 &&
 2758                             (op->nfso_stateid.other[0] != 0 ||
 2759                              op->nfso_stateid.other[1] != 0 ||
 2760                              op->nfso_stateid.other[2] != 0) &&
 2761                             op->nfso_stateid.other[0] == op2->nfso_stateid.other[0] &&
 2762                             op->nfso_stateid.other[1] == op2->nfso_stateid.other[1] &&
 2763                             op->nfso_stateid.other[2] == op2->nfso_stateid.other[2]) {
 2764                             NFSUNLOCKCLSTATE();
 2765                             printf("DUP STATEID\n");
 2766                             nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1, 0,
 2767                                 0);
 2768                             return;
 2769                         }
 2770                     }
 2771                 }
 2772             }
 2773         }
 2774 
 2775         /*
 2776          * Now search for duplicate opens.
 2777          * Duplicate opens for the same owner
 2778          * should never occur. Other duplicates are
 2779          * possible and are checked for if "dupopens"
 2780          * is true.
 2781          */
 2782         LIST_FOREACH(owp2, &clp->nfsc_owner, nfsow_list) {
 2783             LIST_FOREACH(op2, &owp2->nfsow_open, nfso_list) {
 2784                 if (nfhp->nfh_len == op2->nfso_fhlen &&
 2785                     !NFSBCMP(nfhp->nfh_fh, op2->nfso_fh, nfhp->nfh_len)) {
 2786                     LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2787                         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 2788                             if (op != op2 && nfhp->nfh_len == op->nfso_fhlen &&
 2789                                 !NFSBCMP(nfhp->nfh_fh, op->nfso_fh, nfhp->nfh_len) &&
 2790                                 (!NFSBCMP(op->nfso_own->nfsow_owner,
 2791                                  op2->nfso_own->nfsow_owner, NFSV4CL_LOCKNAMELEN) ||
 2792                                  dupopens)) {
 2793                                 if (!NFSBCMP(op->nfso_own->nfsow_owner,
 2794                                     op2->nfso_own->nfsow_owner, NFSV4CL_LOCKNAMELEN)) {
 2795                                     NFSUNLOCKCLSTATE();
 2796                                     printf("BADDUP OPEN\n");
 2797                                 } else {
 2798                                     NFSUNLOCKCLSTATE();
 2799                                     printf("DUP OPEN\n");
 2800                                 }
 2801                                 nfscl_dumpstate(VFSTONFS(vnode_mount(vp)), 1, 1,
 2802                                     0, 0);
 2803                                 return;
 2804                             }
 2805                         }
 2806                     }
 2807                 }
 2808             }
 2809         }
 2810         NFSUNLOCKCLSTATE();
 2811 }
 2812 
 2813 /*
 2814  * During close, find an open that needs to be dereferenced and
 2815  * dereference it. If there are no more opens for this file,
 2816  * log a message to that effect.
 2817  * Opens aren't actually Close'd until VOP_INACTIVE() is performed
 2818  * on the file's vnode.
 2819  * This is the safe way, since it is difficult to identify
 2820  * which open the close is for and I/O can be performed after the
 2821  * close(2) system call when a file is mmap'd.
 2822  * If it returns 0 for success, there will be a referenced
 2823  * clp returned via clpp.
 2824  */
 2825 APPLESTATIC int
 2826 nfscl_getclose(vnode_t vp, struct nfsclclient **clpp)
 2827 {
 2828         struct nfsclclient *clp;
 2829         struct nfsclowner *owp;
 2830         struct nfsclopen *op;
 2831         struct nfscldeleg *dp;
 2832         struct nfsfh *nfhp;
 2833         int error, notdecr;
 2834 
 2835         error = nfscl_getcl(vp, NULL, NULL, &clp);
 2836         if (error)
 2837                 return (error);
 2838         *clpp = clp;
 2839 
 2840         nfhp = VTONFS(vp)->n_fhp;
 2841         notdecr = 1;
 2842         NFSLOCKCLSTATE();
 2843         /*
 2844          * First, look for one under a delegation that was locally issued
 2845          * and just decrement the opencnt for it. Since all my Opens against
 2846          * the server are DENY_NONE, I don't see a problem with hanging
 2847          * onto them. (It is much easier to use one of the extant Opens
 2848          * that I already have on the server when a Delegation is recalled
 2849          * than to do fresh Opens.) Someday, I might need to rethink this, but.
 2850          */
 2851         dp = nfscl_finddeleg(clp, nfhp->nfh_fh, nfhp->nfh_len);
 2852         if (dp != NULL) {
 2853                 LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 2854                         op = LIST_FIRST(&owp->nfsow_open);
 2855                         if (op != NULL) {
 2856                                 /*
 2857                                  * Since a delegation is for a file, there
 2858                                  * should never be more than one open for
 2859                                  * each openowner.
 2860                                  */
 2861                                 if (LIST_NEXT(op, nfso_list) != NULL)
 2862                                         panic("nfscdeleg opens");
 2863                                 if (notdecr && op->nfso_opencnt > 0) {
 2864                                         notdecr = 0;
 2865                                         op->nfso_opencnt--;
 2866                                         break;
 2867                                 }
 2868                         }
 2869                 }
 2870         }
 2871 
 2872         /* Now process the opens against the server. */
 2873         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2874                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 2875                         if (op->nfso_fhlen == nfhp->nfh_len &&
 2876                             !NFSBCMP(op->nfso_fh, nfhp->nfh_fh,
 2877                             nfhp->nfh_len)) {
 2878                                 /* Found an open, decrement cnt if possible */
 2879                                 if (notdecr && op->nfso_opencnt > 0) {
 2880                                         notdecr = 0;
 2881                                         op->nfso_opencnt--;
 2882                                 }
 2883                                 /*
 2884                                  * There are more opens, so just return.
 2885                                  */
 2886                                 if (op->nfso_opencnt > 0) {
 2887                                         NFSUNLOCKCLSTATE();
 2888                                         return (0);
 2889                                 }
 2890                         }
 2891                 }
 2892         }
 2893         NFSUNLOCKCLSTATE();
 2894         if (notdecr)
 2895                 printf("nfscl: never fnd open\n");
 2896         return (0);
 2897 }
 2898 
 2899 APPLESTATIC int
 2900 nfscl_doclose(vnode_t vp, struct nfsclclient **clpp, NFSPROC_T *p)
 2901 {
 2902         struct nfsclclient *clp;
 2903         struct nfsclowner *owp, *nowp;
 2904         struct nfsclopen *op;
 2905         struct nfscldeleg *dp;
 2906         struct nfsfh *nfhp;
 2907         int error;
 2908 
 2909         error = nfscl_getcl(vp, NULL, NULL, &clp);
 2910         if (error)
 2911                 return (error);
 2912         *clpp = clp;
 2913 
 2914         nfhp = VTONFS(vp)->n_fhp;
 2915         NFSLOCKCLSTATE();
 2916         /*
 2917          * First get rid of the local Open structures, which should be no
 2918          * longer in use.
 2919          */
 2920         dp = nfscl_finddeleg(clp, nfhp->nfh_fh, nfhp->nfh_len);
 2921         if (dp != NULL) {
 2922                 LIST_FOREACH_SAFE(owp, &dp->nfsdl_owner, nfsow_list, nowp) {
 2923                         op = LIST_FIRST(&owp->nfsow_open);
 2924                         if (op != NULL) {
 2925                                 KASSERT((op->nfso_opencnt == 0),
 2926                                     ("nfscl: bad open cnt on deleg"));
 2927                                 nfscl_freeopen(op, 1);
 2928                         }
 2929                         nfscl_freeopenowner(owp, 1);
 2930                 }
 2931         }
 2932 
 2933         /* Now process the opens against the server. */
 2934 lookformore:
 2935         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 2936                 op = LIST_FIRST(&owp->nfsow_open);
 2937                 while (op != NULL) {
 2938                         if (op->nfso_fhlen == nfhp->nfh_len &&
 2939                             !NFSBCMP(op->nfso_fh, nfhp->nfh_fh,
 2940                             nfhp->nfh_len)) {
 2941                                 /* Found an open, close it. */
 2942                                 KASSERT((op->nfso_opencnt == 0),
 2943                                     ("nfscl: bad open cnt on server"));
 2944                                 NFSUNLOCKCLSTATE();
 2945                                 nfsrpc_doclose(VFSTONFS(vnode_mount(vp)), op,
 2946                                     p);
 2947                                 NFSLOCKCLSTATE();
 2948                                 goto lookformore;
 2949                         }
 2950                         op = LIST_NEXT(op, nfso_list);
 2951                 }
 2952         }
 2953         NFSUNLOCKCLSTATE();
 2954         return (0);
 2955 }
 2956 
 2957 /*
 2958  * Return all delegations on this client.
 2959  * (Must be called with client sleep lock.)
 2960  */
 2961 static void
 2962 nfscl_delegreturnall(struct nfsclclient *clp, NFSPROC_T *p)
 2963 {
 2964         struct nfscldeleg *dp, *ndp;
 2965         struct ucred *cred;
 2966 
 2967         cred = newnfs_getcred();
 2968         TAILQ_FOREACH_SAFE(dp, &clp->nfsc_deleg, nfsdl_list, ndp) {
 2969                 nfscl_cleandeleg(dp);
 2970                 (void) nfscl_trydelegreturn(dp, cred, clp->nfsc_nmp, p);
 2971                 nfscl_freedeleg(&clp->nfsc_deleg, dp);
 2972         }
 2973         NFSFREECRED(cred);
 2974 }
 2975 
 2976 /*
 2977  * Do a callback RPC.
 2978  */
 2979 APPLESTATIC void
 2980 nfscl_docb(struct nfsrv_descript *nd, NFSPROC_T *p)
 2981 {
 2982         int i, op;
 2983         u_int32_t *tl;
 2984         struct nfsclclient *clp;
 2985         struct nfscldeleg *dp = NULL;
 2986         int numops, taglen = -1, error = 0, trunc, ret = 0;
 2987         u_int32_t minorvers, retops = 0, *retopsp = NULL, *repp, cbident;
 2988         u_char tag[NFSV4_SMALLSTR + 1], *tagstr;
 2989         vnode_t vp = NULL;
 2990         struct nfsnode *np;
 2991         struct vattr va;
 2992         struct nfsfh *nfhp;
 2993         mount_t mp;
 2994         nfsattrbit_t attrbits, rattrbits;
 2995         nfsv4stateid_t stateid;
 2996 
 2997         nfsrvd_rephead(nd);
 2998         NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 2999         taglen = fxdr_unsigned(int, *tl);
 3000         if (taglen < 0) {
 3001                 error = EBADRPC;
 3002                 goto nfsmout;
 3003         }
 3004         if (taglen <= NFSV4_SMALLSTR)
 3005                 tagstr = tag;
 3006         else
 3007                 tagstr = malloc(taglen + 1, M_TEMP, M_WAITOK);
 3008         error = nfsrv_mtostr(nd, tagstr, taglen);
 3009         if (error) {
 3010                 if (taglen > NFSV4_SMALLSTR)
 3011                         free(tagstr, M_TEMP);
 3012                 taglen = -1;
 3013                 goto nfsmout;
 3014         }
 3015         (void) nfsm_strtom(nd, tag, taglen);
 3016         if (taglen > NFSV4_SMALLSTR) {
 3017                 free(tagstr, M_TEMP);
 3018         }
 3019         NFSM_BUILD(retopsp, u_int32_t *, NFSX_UNSIGNED);
 3020         NFSM_DISSECT(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
 3021         minorvers = fxdr_unsigned(u_int32_t, *tl++);
 3022         if (minorvers != NFSV4_MINORVERSION)
 3023                 nd->nd_repstat = NFSERR_MINORVERMISMATCH;
 3024         cbident = fxdr_unsigned(u_int32_t, *tl++);
 3025         if (nd->nd_repstat)
 3026                 numops = 0;
 3027         else
 3028                 numops = fxdr_unsigned(int, *tl);
 3029         /*
 3030          * Loop around doing the sub ops.
 3031          */
 3032         for (i = 0; i < numops; i++) {
 3033                 NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 3034                 NFSM_BUILD(repp, u_int32_t *, 2 * NFSX_UNSIGNED);
 3035                 *repp++ = *tl;
 3036                 op = fxdr_unsigned(int, *tl);
 3037                 if (op < NFSV4OP_CBGETATTR || op > NFSV4OP_CBRECALL) {
 3038                     nd->nd_repstat = NFSERR_OPILLEGAL;
 3039                     *repp = nfscl_errmap(nd);
 3040                     retops++;
 3041                     break;
 3042                 }
 3043                 nd->nd_procnum = op;
 3044                 newnfsstats.cbrpccnt[nd->nd_procnum]++;
 3045                 switch (op) {
 3046                 case NFSV4OP_CBGETATTR:
 3047                         clp = NULL;
 3048                         error = nfsm_getfh(nd, &nfhp);
 3049                         if (!error)
 3050                                 error = nfsrv_getattrbits(nd, &attrbits,
 3051                                     NULL, NULL);
 3052                         if (!error) {
 3053                                 mp = nfscl_getmnt(cbident);
 3054                                 if (mp == NULL)
 3055                                         error = NFSERR_SERVERFAULT;
 3056                         }
 3057                         if (!error) {
 3058                                 dp = NULL;
 3059                                 NFSLOCKCLSTATE();
 3060                                 clp = nfscl_findcl(VFSTONFS(mp));
 3061                                 if (clp != NULL)
 3062                                         dp = nfscl_finddeleg(clp, nfhp->nfh_fh,
 3063                                             nfhp->nfh_len);
 3064                                 NFSUNLOCKCLSTATE();
 3065                                 if (dp == NULL)
 3066                                         error = NFSERR_SERVERFAULT;
 3067                         }
 3068                         if (!error) {
 3069                                 ret = nfscl_ngetreopen(mp, nfhp->nfh_fh,
 3070                                     nfhp->nfh_len, p, &np);
 3071                                 if (!ret)
 3072                                         vp = NFSTOV(np);
 3073                         }
 3074                         if (nfhp != NULL)
 3075                                 FREE((caddr_t)nfhp, M_NFSFH);
 3076                         if (!error) {
 3077                                 NFSZERO_ATTRBIT(&rattrbits);
 3078                                 if (NFSISSET_ATTRBIT(&attrbits,
 3079                                     NFSATTRBIT_SIZE)) {
 3080                                         if (!ret)
 3081                                                 va.va_size = np->n_size;
 3082                                         else
 3083                                                 va.va_size = dp->nfsdl_size;
 3084                                         NFSSETBIT_ATTRBIT(&rattrbits,
 3085                                             NFSATTRBIT_SIZE);
 3086                                 }
 3087                                 if (NFSISSET_ATTRBIT(&attrbits,
 3088                                     NFSATTRBIT_CHANGE)) {
 3089                                         va.va_filerev = dp->nfsdl_change;
 3090                                         if (ret || (np->n_flag & NDELEGMOD))
 3091                                                 va.va_filerev++;
 3092                                         NFSSETBIT_ATTRBIT(&rattrbits,
 3093                                             NFSATTRBIT_CHANGE);
 3094                                 }
 3095                                 (void) nfsv4_fillattr(nd, NULL, NULL, NULL, &va,
 3096                                     NULL, 0, &rattrbits, NULL, NULL, 0, 0, 0, 0,
 3097                                     (uint64_t)0);
 3098                                 if (!ret)
 3099                                         vrele(vp);
 3100                         }
 3101                         break;
 3102                 case NFSV4OP_CBRECALL:
 3103                         clp = NULL;
 3104                         NFSM_DISSECT(tl, u_int32_t *, NFSX_STATEID +
 3105                             NFSX_UNSIGNED);
 3106                         stateid.seqid = *tl++;
 3107                         NFSBCOPY((caddr_t)tl, (caddr_t)stateid.other,
 3108                             NFSX_STATEIDOTHER);
 3109                         tl += (NFSX_STATEIDOTHER / NFSX_UNSIGNED);
 3110                         trunc = fxdr_unsigned(int, *tl);
 3111                         error = nfsm_getfh(nd, &nfhp);
 3112                         if (!error) {
 3113                                 mp = nfscl_getmnt(cbident);
 3114                                 if (mp == NULL)
 3115                                         error = NFSERR_SERVERFAULT;
 3116                         }
 3117                         if (!error) {
 3118                                 NFSLOCKCLSTATE();
 3119                                 clp = nfscl_findcl(VFSTONFS(mp));
 3120                                 if (clp != NULL) {
 3121                                         dp = nfscl_finddeleg(clp, nfhp->nfh_fh,
 3122                                             nfhp->nfh_len);
 3123                                         if (dp != NULL && (dp->nfsdl_flags &
 3124                                             NFSCLDL_DELEGRET) == 0) {
 3125                                                 dp->nfsdl_flags |=
 3126                                                     NFSCLDL_RECALL;
 3127                                                 wakeup((caddr_t)clp);
 3128                                         }
 3129                                 } else {
 3130                                         error = NFSERR_SERVERFAULT;
 3131                                 }
 3132                                 NFSUNLOCKCLSTATE();
 3133                         }
 3134                         if (nfhp != NULL)
 3135                                 FREE((caddr_t)nfhp, M_NFSFH);
 3136                         break;
 3137                 };
 3138                 if (error) {
 3139                         if (error == EBADRPC || error == NFSERR_BADXDR) {
 3140                                 nd->nd_repstat = NFSERR_BADXDR;
 3141                         } else {
 3142                                 nd->nd_repstat = error;
 3143                         }
 3144                         error = 0;
 3145                 }
 3146                 retops++;
 3147                 if (nd->nd_repstat) {
 3148                         *repp = nfscl_errmap(nd);
 3149                         break;
 3150                 } else
 3151                         *repp = 0;      /* NFS4_OK */
 3152         }
 3153 nfsmout:
 3154         if (error) {
 3155                 if (error == EBADRPC || error == NFSERR_BADXDR)
 3156                         nd->nd_repstat = NFSERR_BADXDR;
 3157                 else
 3158                         printf("nfsv4 comperr1=%d\n", error);
 3159         }
 3160         if (taglen == -1) {
 3161                 NFSM_BUILD(tl, u_int32_t *, 2 * NFSX_UNSIGNED);
 3162                 *tl++ = 0;
 3163                 *tl = 0;
 3164         } else {
 3165                 *retopsp = txdr_unsigned(retops);
 3166         }
 3167         *nd->nd_errp = nfscl_errmap(nd);
 3168 }
 3169 
 3170 /*
 3171  * Generate the next cbident value. Basically just increment a static value
 3172  * and then check that it isn't already in the list, if it has wrapped around.
 3173  */
 3174 static u_int32_t
 3175 nfscl_nextcbident(void)
 3176 {
 3177         struct nfsclclient *clp;
 3178         int matched;
 3179         static u_int32_t nextcbident = 0;
 3180         static int haswrapped = 0;
 3181 
 3182         nextcbident++;
 3183         if (nextcbident == 0)
 3184                 haswrapped = 1;
 3185         if (haswrapped) {
 3186                 /*
 3187                  * Search the clientid list for one already using this cbident.
 3188                  */
 3189                 do {
 3190                         matched = 0;
 3191                         NFSLOCKCLSTATE();
 3192                         LIST_FOREACH(clp, &nfsclhead, nfsc_list) {
 3193                                 if (clp->nfsc_cbident == nextcbident) {
 3194                                         matched = 1;
 3195                                         break;
 3196                                 }
 3197                         }
 3198                         NFSUNLOCKCLSTATE();
 3199                         if (matched == 1)
 3200                                 nextcbident++;
 3201                 } while (matched);
 3202         }
 3203         return (nextcbident);
 3204 }
 3205 
 3206 /*
 3207  * Get the mount point related to a given cbident.
 3208  */
 3209 static mount_t
 3210 nfscl_getmnt(u_int32_t cbident)
 3211 {
 3212         struct nfsclclient *clp;
 3213         struct nfsmount *nmp;
 3214 
 3215         NFSLOCKCLSTATE();
 3216         LIST_FOREACH(clp, &nfsclhead, nfsc_list) {
 3217                 if (clp->nfsc_cbident == cbident)
 3218                         break;
 3219         }
 3220         if (clp == NULL) {
 3221                 NFSUNLOCKCLSTATE();
 3222                 return (NULL);
 3223         }
 3224         nmp = clp->nfsc_nmp;
 3225         NFSUNLOCKCLSTATE();
 3226         return (nmp->nm_mountp);
 3227 }
 3228 
 3229 /*
 3230  * Search for a lock conflict locally on the client. A conflict occurs if
 3231  * - not same owner and overlapping byte range and at least one of them is
 3232  *   a write lock or this is an unlock.
 3233  */
 3234 static int
 3235 nfscl_localconflict(struct nfsclclient *clp, u_int8_t *fhp, int fhlen,
 3236     struct nfscllock *nlop, u_int8_t *own, struct nfscldeleg *dp,
 3237     struct nfscllock **lopp)
 3238 {
 3239         struct nfsclowner *owp;
 3240         struct nfsclopen *op;
 3241         int ret;
 3242 
 3243         if (dp != NULL) {
 3244                 ret = nfscl_checkconflict(&dp->nfsdl_lock, nlop, own, lopp);
 3245                 if (ret)
 3246                         return (ret);
 3247         }
 3248         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 3249                 LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 3250                         if (op->nfso_fhlen == fhlen &&
 3251                             !NFSBCMP(op->nfso_fh, fhp, fhlen)) {
 3252                                 ret = nfscl_checkconflict(&op->nfso_lock, nlop,
 3253                                     own, lopp);
 3254                                 if (ret)
 3255                                         return (ret);
 3256                         }
 3257                 }
 3258         }
 3259         return (0);
 3260 }
 3261 
 3262 static int
 3263 nfscl_checkconflict(struct nfscllockownerhead *lhp, struct nfscllock *nlop,
 3264     u_int8_t *own, struct nfscllock **lopp)
 3265 {
 3266         struct nfscllockowner *lp;
 3267         struct nfscllock *lop;
 3268 
 3269         LIST_FOREACH(lp, lhp, nfsl_list) {
 3270                 if (NFSBCMP(lp->nfsl_owner, own, NFSV4CL_LOCKNAMELEN)) {
 3271                         LIST_FOREACH(lop, &lp->nfsl_lock, nfslo_list) {
 3272                                 if (lop->nfslo_first >= nlop->nfslo_end)
 3273                                         break;
 3274                                 if (lop->nfslo_end <= nlop->nfslo_first)
 3275                                         continue;
 3276                                 if (lop->nfslo_type == F_WRLCK ||
 3277                                     nlop->nfslo_type == F_WRLCK ||
 3278                                     nlop->nfslo_type == F_UNLCK) {
 3279                                         if (lopp != NULL)
 3280                                                 *lopp = lop;
 3281                                         return (NFSERR_DENIED);
 3282                                 }
 3283                         }
 3284                 }
 3285         }
 3286         return (0);
 3287 }
 3288 
 3289 /*
 3290  * Check for a local conflicting lock.
 3291  */
 3292 APPLESTATIC int
 3293 nfscl_lockt(vnode_t vp, struct nfsclclient *clp, u_int64_t off,
 3294     u_int64_t len, struct flock *fl, NFSPROC_T *p, void *id, int flags)
 3295 {
 3296         struct nfscllock *lop, nlck;
 3297         struct nfscldeleg *dp;
 3298         struct nfsnode *np;
 3299         u_int8_t own[NFSV4CL_LOCKNAMELEN];
 3300         int error;
 3301 
 3302         nlck.nfslo_type = fl->l_type;
 3303         nlck.nfslo_first = off;
 3304         if (len == NFS64BITSSET) {
 3305                 nlck.nfslo_end = NFS64BITSSET;
 3306         } else {
 3307                 nlck.nfslo_end = off + len;
 3308                 if (nlck.nfslo_end <= nlck.nfslo_first)
 3309                         return (NFSERR_INVAL);
 3310         }
 3311         np = VTONFS(vp);
 3312         nfscl_filllockowner(id, own, flags);
 3313         NFSLOCKCLSTATE();
 3314         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 3315         error = nfscl_localconflict(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len,
 3316             &nlck, own, dp, &lop);
 3317         if (error != 0) {
 3318                 fl->l_whence = SEEK_SET;
 3319                 fl->l_start = lop->nfslo_first;
 3320                 if (lop->nfslo_end == NFS64BITSSET)
 3321                         fl->l_len = 0;
 3322                 else
 3323                         fl->l_len = lop->nfslo_end - lop->nfslo_first;
 3324                 fl->l_pid = (pid_t)0;
 3325                 fl->l_type = lop->nfslo_type;
 3326                 error = -1;                     /* no RPC required */
 3327         } else if (dp != NULL && ((dp->nfsdl_flags & NFSCLDL_WRITE) ||
 3328             fl->l_type == F_RDLCK)) {
 3329                 /*
 3330                  * The delegation ensures that there isn't a conflicting
 3331                  * lock on the server, so return -1 to indicate an RPC
 3332                  * isn't required.
 3333                  */
 3334                 fl->l_type = F_UNLCK;
 3335                 error = -1;
 3336         }
 3337         NFSUNLOCKCLSTATE();
 3338         return (error);
 3339 }
 3340 
 3341 /*
 3342  * Handle Recall of a delegation.
 3343  * The clp must be exclusive locked when this is called.
 3344  */
 3345 static int
 3346 nfscl_recalldeleg(struct nfsclclient *clp, struct nfsmount *nmp,
 3347     struct nfscldeleg *dp, vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 3348     int called_from_renewthread)
 3349 {
 3350         struct nfsclowner *owp, *lowp, *nowp;
 3351         struct nfsclopen *op, *lop;
 3352         struct nfscllockowner *lp;
 3353         struct nfscllock *lckp;
 3354         struct nfsnode *np;
 3355         int error = 0, ret, gotvp = 0;
 3356 
 3357         if (vp == NULL) {
 3358                 /*
 3359                  * First, get a vnode for the file. This is needed to do RPCs.
 3360                  */
 3361                 ret = nfscl_ngetreopen(nmp->nm_mountp, dp->nfsdl_fh,
 3362                     dp->nfsdl_fhlen, p, &np);
 3363                 if (ret) {
 3364                         /*
 3365                          * File isn't open, so nothing to move over to the
 3366                          * server.
 3367                          */
 3368                         return (0);
 3369                 }
 3370                 vp = NFSTOV(np);
 3371                 gotvp = 1;
 3372         } else {
 3373                 np = VTONFS(vp);
 3374         }
 3375         dp->nfsdl_flags &= ~NFSCLDL_MODTIMESET;
 3376 
 3377         /*
 3378          * Ok, if it's a write delegation, flush data to the server, so
 3379          * that close/open consistency is retained.
 3380          */
 3381         ret = 0;
 3382         NFSLOCKNODE(np);
 3383         if ((dp->nfsdl_flags & NFSCLDL_WRITE) && (np->n_flag & NMODIFIED)) {
 3384                 np->n_flag |= NDELEGRECALL;
 3385                 NFSUNLOCKNODE(np);
 3386                 ret = ncl_flush(vp, MNT_WAIT, cred, p, 1,
 3387                     called_from_renewthread);
 3388                 NFSLOCKNODE(np);
 3389                 np->n_flag &= ~NDELEGRECALL;
 3390         }
 3391         NFSINVALATTRCACHE(np);
 3392         NFSUNLOCKNODE(np);
 3393         if (ret == EIO && called_from_renewthread != 0) {
 3394                 /*
 3395                  * If the flush failed with EIO for the renew thread,
 3396                  * return now, so that the dirty buffer will be flushed
 3397                  * later.
 3398                  */
 3399                 if (gotvp != 0)
 3400                         vrele(vp);
 3401                 return (ret);
 3402         }
 3403 
 3404         /*
 3405          * Now, for each openowner with opens issued locally, move them
 3406          * over to state against the server.
 3407          */
 3408         LIST_FOREACH(lowp, &dp->nfsdl_owner, nfsow_list) {
 3409                 lop = LIST_FIRST(&lowp->nfsow_open);
 3410                 if (lop != NULL) {
 3411                         if (LIST_NEXT(lop, nfso_list) != NULL)
 3412                                 panic("nfsdlg mult opens");
 3413                         /*
 3414                          * Look for the same openowner against the server.
 3415                          */
 3416                         LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
 3417                                 if (!NFSBCMP(lowp->nfsow_owner,
 3418                                     owp->nfsow_owner, NFSV4CL_LOCKNAMELEN)) {
 3419                                         newnfs_copycred(&dp->nfsdl_cred, cred);
 3420                                         ret = nfscl_moveopen(vp, clp, nmp, lop,
 3421                                             owp, dp, cred, p);
 3422                                         if (ret == NFSERR_STALECLIENTID ||
 3423                                             ret == NFSERR_STALEDONTRECOVER) {
 3424                                                 if (gotvp)
 3425                                                         vrele(vp);
 3426                                                 return (ret);
 3427                                         }
 3428                                         if (ret) {
 3429                                                 nfscl_freeopen(lop, 1);
 3430                                                 if (!error)
 3431                                                         error = ret;
 3432                                         }
 3433                                         break;
 3434                                 }
 3435                         }
 3436 
 3437                         /*
 3438                          * If no openowner found, create one and get an open
 3439                          * for it.
 3440                          */
 3441                         if (owp == NULL) {
 3442                                 MALLOC(nowp, struct nfsclowner *,
 3443                                     sizeof (struct nfsclowner), M_NFSCLOWNER,
 3444                                     M_WAITOK);
 3445                                 nfscl_newopen(clp, NULL, &owp, &nowp, &op, 
 3446                                     NULL, lowp->nfsow_owner, dp->nfsdl_fh,
 3447                                     dp->nfsdl_fhlen, NULL);
 3448                                 newnfs_copycred(&dp->nfsdl_cred, cred);
 3449                                 ret = nfscl_moveopen(vp, clp, nmp, lop,
 3450                                     owp, dp, cred, p);
 3451                                 if (ret) {
 3452                                         nfscl_freeopenowner(owp, 0);
 3453                                         if (ret == NFSERR_STALECLIENTID ||
 3454                                             ret == NFSERR_STALEDONTRECOVER) {
 3455                                                 if (gotvp)
 3456                                                         vrele(vp);
 3457                                                 return (ret);
 3458                                         }
 3459                                         if (ret) {
 3460                                                 nfscl_freeopen(lop, 1);
 3461                                                 if (!error)
 3462                                                         error = ret;
 3463                                         }
 3464                                 }
 3465                         }
 3466                 }
 3467         }
 3468 
 3469         /*
 3470          * Now, get byte range locks for any locks done locally.
 3471          */
 3472         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 3473                 LIST_FOREACH(lckp, &lp->nfsl_lock, nfslo_list) {
 3474                         newnfs_copycred(&dp->nfsdl_cred, cred);
 3475                         ret = nfscl_relock(vp, clp, nmp, lp, lckp, cred, p);
 3476                         if (ret == NFSERR_STALESTATEID ||
 3477                             ret == NFSERR_STALEDONTRECOVER ||
 3478                             ret == NFSERR_STALECLIENTID) {
 3479                                 if (gotvp)
 3480                                         vrele(vp);
 3481                                 return (ret);
 3482                         }
 3483                         if (ret && !error)
 3484                                 error = ret;
 3485                 }
 3486         }
 3487         if (gotvp)
 3488                 vrele(vp);
 3489         return (error);
 3490 }
 3491 
 3492 /*
 3493  * Move a locally issued open over to an owner on the state list.
 3494  * SIDE EFFECT: If it needs to sleep (do an rpc), it unlocks clstate and
 3495  * returns with it unlocked.
 3496  */
 3497 static int
 3498 nfscl_moveopen(vnode_t vp, struct nfsclclient *clp, struct nfsmount *nmp,
 3499     struct nfsclopen *lop, struct nfsclowner *owp, struct nfscldeleg *dp,
 3500     struct ucred *cred, NFSPROC_T *p)
 3501 {
 3502         struct nfsclopen *op, *nop;
 3503         struct nfscldeleg *ndp;
 3504         struct nfsnode *np;
 3505         int error = 0, newone;
 3506 
 3507         /*
 3508          * First, look for an appropriate open, If found, just increment the
 3509          * opencnt in it.
 3510          */
 3511         LIST_FOREACH(op, &owp->nfsow_open, nfso_list) {
 3512                 if ((op->nfso_mode & lop->nfso_mode) == lop->nfso_mode &&
 3513                     op->nfso_fhlen == lop->nfso_fhlen &&
 3514                     !NFSBCMP(op->nfso_fh, lop->nfso_fh, op->nfso_fhlen)) {
 3515                         op->nfso_opencnt += lop->nfso_opencnt;
 3516                         nfscl_freeopen(lop, 1);
 3517                         return (0);
 3518                 }
 3519         }
 3520 
 3521         /* No appropriate open, so we have to do one against the server. */
 3522         np = VTONFS(vp);
 3523         MALLOC(nop, struct nfsclopen *, sizeof (struct nfsclopen) +
 3524             lop->nfso_fhlen - 1, M_NFSCLOPEN, M_WAITOK);
 3525         newone = 0;
 3526         nfscl_newopen(clp, NULL, &owp, NULL, &op, &nop, owp->nfsow_owner,
 3527             lop->nfso_fh, lop->nfso_fhlen, &newone);
 3528         ndp = dp;
 3529         error = nfscl_tryopen(nmp, vp, np->n_v4->n4_data, np->n_v4->n4_fhlen,
 3530             lop->nfso_fh, lop->nfso_fhlen, lop->nfso_mode, op,
 3531             NFS4NODENAME(np->n_v4), np->n_v4->n4_namelen, &ndp, 0, 0, cred, p);
 3532         if (error) {
 3533                 if (newone)
 3534                         nfscl_freeopen(op, 0);
 3535         } else {
 3536                 if (newone)
 3537                         newnfs_copyincred(cred, &op->nfso_cred);
 3538                 op->nfso_mode |= lop->nfso_mode;
 3539                 op->nfso_opencnt += lop->nfso_opencnt;
 3540                 nfscl_freeopen(lop, 1);
 3541         }
 3542         if (nop != NULL)
 3543                 FREE((caddr_t)nop, M_NFSCLOPEN);
 3544         if (ndp != NULL) {
 3545                 /*
 3546                  * What should I do with the returned delegation, since the
 3547                  * delegation is being recalled? For now, just printf and
 3548                  * through it away.
 3549                  */
 3550                 printf("Moveopen returned deleg\n");
 3551                 FREE((caddr_t)ndp, M_NFSCLDELEG);
 3552         }
 3553         return (error);
 3554 }
 3555 
 3556 /*
 3557  * Recall all delegations on this client.
 3558  */
 3559 static void
 3560 nfscl_totalrecall(struct nfsclclient *clp)
 3561 {
 3562         struct nfscldeleg *dp;
 3563 
 3564         TAILQ_FOREACH(dp, &clp->nfsc_deleg, nfsdl_list) {
 3565                 if ((dp->nfsdl_flags & NFSCLDL_DELEGRET) == 0)
 3566                         dp->nfsdl_flags |= NFSCLDL_RECALL;
 3567         }
 3568 }
 3569 
 3570 /*
 3571  * Relock byte ranges. Called for delegation recall and state expiry.
 3572  */
 3573 static int
 3574 nfscl_relock(vnode_t vp, struct nfsclclient *clp, struct nfsmount *nmp,
 3575     struct nfscllockowner *lp, struct nfscllock *lop, struct ucred *cred,
 3576     NFSPROC_T *p)
 3577 {
 3578         struct nfscllockowner *nlp;
 3579         struct nfsfh *nfhp;
 3580         u_int64_t off, len;
 3581         u_int32_t clidrev = 0;
 3582         int error, newone, donelocally;
 3583 
 3584         off = lop->nfslo_first;
 3585         len = lop->nfslo_end - lop->nfslo_first;
 3586         error = nfscl_getbytelock(vp, off, len, lop->nfslo_type, cred, p,
 3587             clp, 1, NULL, lp->nfsl_lockflags, lp->nfsl_owner,
 3588             lp->nfsl_openowner, &nlp, &newone, &donelocally);
 3589         if (error || donelocally)
 3590                 return (error);
 3591         if (nmp->nm_clp != NULL)
 3592                 clidrev = nmp->nm_clp->nfsc_clientidrev;
 3593         else
 3594                 clidrev = 0;
 3595         nfhp = VTONFS(vp)->n_fhp;
 3596         error = nfscl_trylock(nmp, vp, nfhp->nfh_fh,
 3597             nfhp->nfh_len, nlp, newone, 0, off,
 3598             len, lop->nfslo_type, cred, p);
 3599         if (error)
 3600                 nfscl_freelockowner(nlp, 0);
 3601         return (error);
 3602 }
 3603 
 3604 /*
 3605  * Called to re-open a file. Basically get a vnode for the file handle
 3606  * and then call nfsrpc_openrpc() to do the rest.
 3607  */
 3608 static int
 3609 nfsrpc_reopen(struct nfsmount *nmp, u_int8_t *fhp, int fhlen,
 3610     u_int32_t mode, struct nfsclopen *op, struct nfscldeleg **dpp,
 3611     struct ucred *cred, NFSPROC_T *p)
 3612 {
 3613         struct nfsnode *np;
 3614         vnode_t vp;
 3615         int error;
 3616 
 3617         error = nfscl_ngetreopen(nmp->nm_mountp, fhp, fhlen, p, &np);
 3618         if (error)
 3619                 return (error);
 3620         vp = NFSTOV(np);
 3621         if (np->n_v4 != NULL) {
 3622                 error = nfscl_tryopen(nmp, vp, np->n_v4->n4_data,
 3623                     np->n_v4->n4_fhlen, fhp, fhlen, mode, op,
 3624                     NFS4NODENAME(np->n_v4), np->n_v4->n4_namelen, dpp, 0, 0,
 3625                     cred, p);
 3626         } else {
 3627                 error = EINVAL;
 3628         }
 3629         vrele(vp);
 3630         return (error);
 3631 }
 3632 
 3633 /*
 3634  * Try an open against the server. Just call nfsrpc_openrpc(), retrying while
 3635  * NFSERR_DELAY. Also, try system credentials, if the passed in credentials
 3636  * fail.
 3637  */
 3638 static int
 3639 nfscl_tryopen(struct nfsmount *nmp, vnode_t vp, u_int8_t *fhp, int fhlen,
 3640     u_int8_t *newfhp, int newfhlen, u_int32_t mode, struct nfsclopen *op,
 3641     u_int8_t *name, int namelen, struct nfscldeleg **ndpp,
 3642     int reclaim, u_int32_t delegtype, struct ucred *cred, NFSPROC_T *p)
 3643 {
 3644         int error;
 3645 
 3646         do {
 3647                 error = nfsrpc_openrpc(nmp, vp, fhp, fhlen, newfhp, newfhlen,
 3648                     mode, op, name, namelen, ndpp, reclaim, delegtype, cred, p,
 3649                     0, 0);
 3650                 if (error == NFSERR_DELAY)
 3651                         (void) nfs_catnap(PZERO, error, "nfstryop");
 3652         } while (error == NFSERR_DELAY);
 3653         if (error == EAUTH || error == EACCES) {
 3654                 /* Try again using system credentials */
 3655                 newnfs_setroot(cred);
 3656                 do {
 3657                     error = nfsrpc_openrpc(nmp, vp, fhp, fhlen, newfhp,
 3658                         newfhlen, mode, op, name, namelen, ndpp, reclaim,
 3659                         delegtype, cred, p, 1, 0);
 3660                     if (error == NFSERR_DELAY)
 3661                         (void) nfs_catnap(PZERO, error, "nfstryop");
 3662                 } while (error == NFSERR_DELAY);
 3663         }
 3664         return (error);
 3665 }
 3666 
 3667 /*
 3668  * Try a byte range lock. Just loop on nfsrpc_lock() while it returns
 3669  * NFSERR_DELAY. Also, retry with system credentials, if the provided
 3670  * cred don't work.
 3671  */
 3672 static int
 3673 nfscl_trylock(struct nfsmount *nmp, vnode_t vp, u_int8_t *fhp,
 3674     int fhlen, struct nfscllockowner *nlp, int newone, int reclaim,
 3675     u_int64_t off, u_int64_t len, short type, struct ucred *cred, NFSPROC_T *p)
 3676 {
 3677         struct nfsrv_descript nfsd, *nd = &nfsd;
 3678         int error;
 3679 
 3680         do {
 3681                 error = nfsrpc_lock(nd, nmp, vp, fhp, fhlen, nlp, newone,
 3682                     reclaim, off, len, type, cred, p, 0);
 3683                 if (!error && nd->nd_repstat == NFSERR_DELAY)
 3684                         (void) nfs_catnap(PZERO, (int)nd->nd_repstat,
 3685                             "nfstrylck");
 3686         } while (!error && nd->nd_repstat == NFSERR_DELAY);
 3687         if (!error)
 3688                 error = nd->nd_repstat;
 3689         if (error == EAUTH || error == EACCES) {
 3690                 /* Try again using root credentials */
 3691                 newnfs_setroot(cred);
 3692                 do {
 3693                         error = nfsrpc_lock(nd, nmp, vp, fhp, fhlen, nlp,
 3694                             newone, reclaim, off, len, type, cred, p, 1);
 3695                         if (!error && nd->nd_repstat == NFSERR_DELAY)
 3696                                 (void) nfs_catnap(PZERO, (int)nd->nd_repstat,
 3697                                     "nfstrylck");
 3698                 } while (!error && nd->nd_repstat == NFSERR_DELAY);
 3699                 if (!error)
 3700                         error = nd->nd_repstat;
 3701         }
 3702         return (error);
 3703 }
 3704 
 3705 /*
 3706  * Try a delegreturn against the server. Just call nfsrpc_delegreturn(),
 3707  * retrying while NFSERR_DELAY. Also, try system credentials, if the passed in
 3708  * credentials fail.
 3709  */
 3710 static int
 3711 nfscl_trydelegreturn(struct nfscldeleg *dp, struct ucred *cred,
 3712     struct nfsmount *nmp, NFSPROC_T *p)
 3713 {
 3714         int error;
 3715 
 3716         do {
 3717                 error = nfsrpc_delegreturn(dp, cred, nmp, p, 0);
 3718                 if (error == NFSERR_DELAY)
 3719                         (void) nfs_catnap(PZERO, error, "nfstrydp");
 3720         } while (error == NFSERR_DELAY);
 3721         if (error == EAUTH || error == EACCES) {
 3722                 /* Try again using system credentials */
 3723                 newnfs_setroot(cred);
 3724                 do {
 3725                         error = nfsrpc_delegreturn(dp, cred, nmp, p, 1);
 3726                         if (error == NFSERR_DELAY)
 3727                                 (void) nfs_catnap(PZERO, error, "nfstrydp");
 3728                 } while (error == NFSERR_DELAY);
 3729         }
 3730         return (error);
 3731 }
 3732 
 3733 /*
 3734  * Try a close against the server. Just call nfsrpc_closerpc(),
 3735  * retrying while NFSERR_DELAY. Also, try system credentials, if the passed in
 3736  * credentials fail.
 3737  */
 3738 APPLESTATIC int
 3739 nfscl_tryclose(struct nfsclopen *op, struct ucred *cred,
 3740     struct nfsmount *nmp, NFSPROC_T *p)
 3741 {
 3742         struct nfsrv_descript nfsd, *nd = &nfsd;
 3743         int error;
 3744 
 3745         do {
 3746                 error = nfsrpc_closerpc(nd, nmp, op, cred, p, 0);
 3747                 if (error == NFSERR_DELAY)
 3748                         (void) nfs_catnap(PZERO, error, "nfstrycl");
 3749         } while (error == NFSERR_DELAY);
 3750         if (error == EAUTH || error == EACCES) {
 3751                 /* Try again using system credentials */
 3752                 newnfs_setroot(cred);
 3753                 do {
 3754                         error = nfsrpc_closerpc(nd, nmp, op, cred, p, 1);
 3755                         if (error == NFSERR_DELAY)
 3756                                 (void) nfs_catnap(PZERO, error, "nfstrycl");
 3757                 } while (error == NFSERR_DELAY);
 3758         }
 3759         return (error);
 3760 }
 3761 
 3762 /*
 3763  * Decide if a delegation on a file permits close without flushing writes
 3764  * to the server. This might be a big performance win in some environments.
 3765  * (Not useful until the client does caching on local stable storage.)
 3766  */
 3767 APPLESTATIC int
 3768 nfscl_mustflush(vnode_t vp)
 3769 {
 3770         struct nfsclclient *clp;
 3771         struct nfscldeleg *dp;
 3772         struct nfsnode *np;
 3773         struct nfsmount *nmp;
 3774 
 3775         np = VTONFS(vp);
 3776         nmp = VFSTONFS(vnode_mount(vp));
 3777         if (!NFSHASNFSV4(nmp))
 3778                 return (1);
 3779         NFSLOCKCLSTATE();
 3780         clp = nfscl_findcl(nmp);
 3781         if (clp == NULL) {
 3782                 NFSUNLOCKCLSTATE();
 3783                 return (1);
 3784         }
 3785         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 3786         if (dp != NULL && (dp->nfsdl_flags &
 3787             (NFSCLDL_WRITE | NFSCLDL_RECALL | NFSCLDL_DELEGRET)) ==
 3788              NFSCLDL_WRITE &&
 3789             (dp->nfsdl_sizelimit >= np->n_size ||
 3790              !NFSHASSTRICT3530(nmp))) {
 3791                 NFSUNLOCKCLSTATE();
 3792                 return (0);
 3793         }
 3794         NFSUNLOCKCLSTATE();
 3795         return (1);
 3796 }
 3797 
 3798 /*
 3799  * See if a (write) delegation exists for this file.
 3800  */
 3801 APPLESTATIC int
 3802 nfscl_nodeleg(vnode_t vp, int writedeleg)
 3803 {
 3804         struct nfsclclient *clp;
 3805         struct nfscldeleg *dp;
 3806         struct nfsnode *np;
 3807         struct nfsmount *nmp;
 3808 
 3809         np = VTONFS(vp);
 3810         nmp = VFSTONFS(vnode_mount(vp));
 3811         if (!NFSHASNFSV4(nmp))
 3812                 return (1);
 3813         NFSLOCKCLSTATE();
 3814         clp = nfscl_findcl(nmp);
 3815         if (clp == NULL) {
 3816                 NFSUNLOCKCLSTATE();
 3817                 return (1);
 3818         }
 3819         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 3820         if (dp != NULL &&
 3821             (dp->nfsdl_flags & (NFSCLDL_RECALL | NFSCLDL_DELEGRET)) == 0 &&
 3822             (writedeleg == 0 || (dp->nfsdl_flags & NFSCLDL_WRITE) ==
 3823              NFSCLDL_WRITE)) {
 3824                 NFSUNLOCKCLSTATE();
 3825                 return (0);
 3826         }
 3827         NFSUNLOCKCLSTATE();
 3828         return (1);
 3829 }
 3830 
 3831 /*
 3832  * Look for an associated delegation that should be DelegReturned.
 3833  */
 3834 APPLESTATIC int
 3835 nfscl_removedeleg(vnode_t vp, NFSPROC_T *p, nfsv4stateid_t *stp)
 3836 {
 3837         struct nfsclclient *clp;
 3838         struct nfscldeleg *dp;
 3839         struct nfsclowner *owp;
 3840         struct nfscllockowner *lp;
 3841         struct nfsmount *nmp;
 3842         struct ucred *cred;
 3843         struct nfsnode *np;
 3844         int igotlock = 0, triedrecall = 0, needsrecall, retcnt = 0, islept;
 3845 
 3846         nmp = VFSTONFS(vnode_mount(vp));
 3847         np = VTONFS(vp);
 3848         NFSLOCKCLSTATE();
 3849         /*
 3850          * Loop around waiting for:
 3851          * - outstanding I/O operations on delegations to complete
 3852          * - for a delegation on vp that has state, lock the client and
 3853          *   do a recall
 3854          * - return delegation with no state
 3855          */
 3856         while (1) {
 3857                 clp = nfscl_findcl(nmp);
 3858                 if (clp == NULL) {
 3859                         NFSUNLOCKCLSTATE();
 3860                         return (retcnt);
 3861                 }
 3862                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 3863                     np->n_fhp->nfh_len);
 3864                 if (dp != NULL) {
 3865                     /*
 3866                      * Wait for outstanding I/O ops to be done.
 3867                      */
 3868                     if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
 3869                         if (igotlock) {
 3870                             nfsv4_unlock(&clp->nfsc_lock, 0);
 3871                             igotlock = 0;
 3872                         }
 3873                         dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
 3874                         (void) nfsmsleep(&dp->nfsdl_rwlock,
 3875                             NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
 3876                         continue;
 3877                     }
 3878                     needsrecall = 0;
 3879                     LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 3880                         if (!LIST_EMPTY(&owp->nfsow_open)) {
 3881                             needsrecall = 1;
 3882                             break;
 3883                         }
 3884                     }
 3885                     if (!needsrecall) {
 3886                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 3887                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
 3888                                 needsrecall = 1;
 3889                                 break;
 3890                             }
 3891                         }
 3892                     }
 3893                     if (needsrecall && !triedrecall) {
 3894                         dp->nfsdl_flags |= NFSCLDL_DELEGRET;
 3895                         islept = 0;
 3896                         while (!igotlock) {
 3897                             igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
 3898                                 &islept, NFSCLSTATEMUTEXPTR, NULL);
 3899                             if (islept)
 3900                                 break;
 3901                         }
 3902                         if (islept)
 3903                             continue;
 3904                         NFSUNLOCKCLSTATE();
 3905                         cred = newnfs_getcred();
 3906                         newnfs_copycred(&dp->nfsdl_cred, cred);
 3907                         (void) nfscl_recalldeleg(clp, nmp, dp, vp, cred, p, 0);
 3908                         NFSFREECRED(cred);
 3909                         triedrecall = 1;
 3910                         NFSLOCKCLSTATE();
 3911                         nfsv4_unlock(&clp->nfsc_lock, 0);
 3912                         igotlock = 0;
 3913                         continue;
 3914                     }
 3915                     *stp = dp->nfsdl_stateid;
 3916                     retcnt = 1;
 3917                     nfscl_cleandeleg(dp);
 3918                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
 3919                 }
 3920                 if (igotlock)
 3921                     nfsv4_unlock(&clp->nfsc_lock, 0);
 3922                 NFSUNLOCKCLSTATE();
 3923                 return (retcnt);
 3924         }
 3925 }
 3926 
 3927 /*
 3928  * Look for associated delegation(s) that should be DelegReturned.
 3929  */
 3930 APPLESTATIC int
 3931 nfscl_renamedeleg(vnode_t fvp, nfsv4stateid_t *fstp, int *gotfdp, vnode_t tvp,
 3932     nfsv4stateid_t *tstp, int *gottdp, NFSPROC_T *p)
 3933 {
 3934         struct nfsclclient *clp;
 3935         struct nfscldeleg *dp;
 3936         struct nfsclowner *owp;
 3937         struct nfscllockowner *lp;
 3938         struct nfsmount *nmp;
 3939         struct ucred *cred;
 3940         struct nfsnode *np;
 3941         int igotlock = 0, triedrecall = 0, needsrecall, retcnt = 0, islept;
 3942 
 3943         nmp = VFSTONFS(vnode_mount(fvp));
 3944         *gotfdp = 0;
 3945         *gottdp = 0;
 3946         NFSLOCKCLSTATE();
 3947         /*
 3948          * Loop around waiting for:
 3949          * - outstanding I/O operations on delegations to complete
 3950          * - for a delegation on fvp that has state, lock the client and
 3951          *   do a recall
 3952          * - return delegation(s) with no state.
 3953          */
 3954         while (1) {
 3955                 clp = nfscl_findcl(nmp);
 3956                 if (clp == NULL) {
 3957                         NFSUNLOCKCLSTATE();
 3958                         return (retcnt);
 3959                 }
 3960                 np = VTONFS(fvp);
 3961                 dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 3962                     np->n_fhp->nfh_len);
 3963                 if (dp != NULL && *gotfdp == 0) {
 3964                     /*
 3965                      * Wait for outstanding I/O ops to be done.
 3966                      */
 3967                     if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
 3968                         if (igotlock) {
 3969                             nfsv4_unlock(&clp->nfsc_lock, 0);
 3970                             igotlock = 0;
 3971                         }
 3972                         dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
 3973                         (void) nfsmsleep(&dp->nfsdl_rwlock,
 3974                             NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
 3975                         continue;
 3976                     }
 3977                     needsrecall = 0;
 3978                     LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 3979                         if (!LIST_EMPTY(&owp->nfsow_open)) {
 3980                             needsrecall = 1;
 3981                             break;
 3982                         }
 3983                     }
 3984                     if (!needsrecall) {
 3985                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 3986                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
 3987                                 needsrecall = 1;
 3988                                 break;
 3989                             }
 3990                         }
 3991                     }
 3992                     if (needsrecall && !triedrecall) {
 3993                         dp->nfsdl_flags |= NFSCLDL_DELEGRET;
 3994                         islept = 0;
 3995                         while (!igotlock) {
 3996                             igotlock = nfsv4_lock(&clp->nfsc_lock, 1,
 3997                                 &islept, NFSCLSTATEMUTEXPTR, NULL);
 3998                             if (islept)
 3999                                 break;
 4000                         }
 4001                         if (islept)
 4002                             continue;
 4003                         NFSUNLOCKCLSTATE();
 4004                         cred = newnfs_getcred();
 4005                         newnfs_copycred(&dp->nfsdl_cred, cred);
 4006                         (void) nfscl_recalldeleg(clp, nmp, dp, fvp, cred, p, 0);
 4007                         NFSFREECRED(cred);
 4008                         triedrecall = 1;
 4009                         NFSLOCKCLSTATE();
 4010                         nfsv4_unlock(&clp->nfsc_lock, 0);
 4011                         igotlock = 0;
 4012                         continue;
 4013                     }
 4014                     *fstp = dp->nfsdl_stateid;
 4015                     retcnt++;
 4016                     *gotfdp = 1;
 4017                     nfscl_cleandeleg(dp);
 4018                     nfscl_freedeleg(&clp->nfsc_deleg, dp);
 4019                 }
 4020                 if (igotlock) {
 4021                     nfsv4_unlock(&clp->nfsc_lock, 0);
 4022                     igotlock = 0;
 4023                 }
 4024                 if (tvp != NULL) {
 4025                     np = VTONFS(tvp);
 4026                     dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh,
 4027                         np->n_fhp->nfh_len);
 4028                     if (dp != NULL && *gottdp == 0) {
 4029                         /*
 4030                          * Wait for outstanding I/O ops to be done.
 4031                          */
 4032                         if (dp->nfsdl_rwlock.nfslock_usecnt > 0) {
 4033                             dp->nfsdl_rwlock.nfslock_lock |= NFSV4LOCK_WANTED;
 4034                             (void) nfsmsleep(&dp->nfsdl_rwlock,
 4035                                 NFSCLSTATEMUTEXPTR, PZERO, "nfscld", NULL);
 4036                             continue;
 4037                         }
 4038                         LIST_FOREACH(owp, &dp->nfsdl_owner, nfsow_list) {
 4039                             if (!LIST_EMPTY(&owp->nfsow_open)) {
 4040                                 NFSUNLOCKCLSTATE();
 4041                                 return (retcnt);
 4042                             }
 4043                         }
 4044                         LIST_FOREACH(lp, &dp->nfsdl_lock, nfsl_list) {
 4045                             if (!LIST_EMPTY(&lp->nfsl_lock)) {
 4046                                 NFSUNLOCKCLSTATE();
 4047                                 return (retcnt);
 4048                             }
 4049                         }
 4050                         *tstp = dp->nfsdl_stateid;
 4051                         retcnt++;
 4052                         *gottdp = 1;
 4053                         nfscl_cleandeleg(dp);
 4054                         nfscl_freedeleg(&clp->nfsc_deleg, dp);
 4055                     }
 4056                 }
 4057                 NFSUNLOCKCLSTATE();
 4058                 return (retcnt);
 4059         }
 4060 }
 4061 
 4062 /*
 4063  * Get a reference on the clientid associated with the mount point.
 4064  * Return 1 if success, 0 otherwise.
 4065  */
 4066 APPLESTATIC int
 4067 nfscl_getref(struct nfsmount *nmp)
 4068 {
 4069         struct nfsclclient *clp;
 4070 
 4071         NFSLOCKCLSTATE();
 4072         clp = nfscl_findcl(nmp);
 4073         if (clp == NULL) {
 4074                 NFSUNLOCKCLSTATE();
 4075                 return (0);
 4076         }
 4077         nfsv4_getref(&clp->nfsc_lock, NULL, NFSCLSTATEMUTEXPTR, NULL);
 4078         NFSUNLOCKCLSTATE();
 4079         return (1);
 4080 }
 4081 
 4082 /*
 4083  * Release a reference on a clientid acquired with the above call.
 4084  */
 4085 APPLESTATIC void
 4086 nfscl_relref(struct nfsmount *nmp)
 4087 {
 4088         struct nfsclclient *clp;
 4089 
 4090         NFSLOCKCLSTATE();
 4091         clp = nfscl_findcl(nmp);
 4092         if (clp == NULL) {
 4093                 NFSUNLOCKCLSTATE();
 4094                 return;
 4095         }
 4096         nfsv4_relref(&clp->nfsc_lock);
 4097         NFSUNLOCKCLSTATE();
 4098 }
 4099 
 4100 /*
 4101  * Save the size attribute in the delegation, since the nfsnode
 4102  * is going away.
 4103  */
 4104 APPLESTATIC void
 4105 nfscl_reclaimnode(vnode_t vp)
 4106 {
 4107         struct nfsclclient *clp;
 4108         struct nfscldeleg *dp;
 4109         struct nfsnode *np = VTONFS(vp);
 4110         struct nfsmount *nmp;
 4111 
 4112         nmp = VFSTONFS(vnode_mount(vp));
 4113         if (!NFSHASNFSV4(nmp))
 4114                 return;
 4115         NFSLOCKCLSTATE();
 4116         clp = nfscl_findcl(nmp);
 4117         if (clp == NULL) {
 4118                 NFSUNLOCKCLSTATE();
 4119                 return;
 4120         }
 4121         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 4122         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE))
 4123                 dp->nfsdl_size = np->n_size;
 4124         NFSUNLOCKCLSTATE();
 4125 }
 4126 
 4127 /*
 4128  * Get the saved size attribute in the delegation, since it is a
 4129  * newly allocated nfsnode.
 4130  */
 4131 APPLESTATIC void
 4132 nfscl_newnode(vnode_t vp)
 4133 {
 4134         struct nfsclclient *clp;
 4135         struct nfscldeleg *dp;
 4136         struct nfsnode *np = VTONFS(vp);
 4137         struct nfsmount *nmp;
 4138 
 4139         nmp = VFSTONFS(vnode_mount(vp));
 4140         if (!NFSHASNFSV4(nmp))
 4141                 return;
 4142         NFSLOCKCLSTATE();
 4143         clp = nfscl_findcl(nmp);
 4144         if (clp == NULL) {
 4145                 NFSUNLOCKCLSTATE();
 4146                 return;
 4147         }
 4148         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 4149         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE))
 4150                 np->n_size = dp->nfsdl_size;
 4151         NFSUNLOCKCLSTATE();
 4152 }
 4153 
 4154 /*
 4155  * If there is a valid write delegation for this file, set the modtime
 4156  * to the local clock time.
 4157  */
 4158 APPLESTATIC void
 4159 nfscl_delegmodtime(vnode_t vp)
 4160 {
 4161         struct nfsclclient *clp;
 4162         struct nfscldeleg *dp;
 4163         struct nfsnode *np = VTONFS(vp);
 4164         struct nfsmount *nmp;
 4165 
 4166         nmp = VFSTONFS(vnode_mount(vp));
 4167         if (!NFSHASNFSV4(nmp))
 4168                 return;
 4169         NFSLOCKCLSTATE();
 4170         clp = nfscl_findcl(nmp);
 4171         if (clp == NULL) {
 4172                 NFSUNLOCKCLSTATE();
 4173                 return;
 4174         }
 4175         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 4176         if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE)) {
 4177                 nanotime(&dp->nfsdl_modtime);
 4178                 dp->nfsdl_flags |= NFSCLDL_MODTIMESET;
 4179         }
 4180         NFSUNLOCKCLSTATE();
 4181 }
 4182 
 4183 /*
 4184  * If there is a valid write delegation for this file with a modtime set,
 4185  * put that modtime in mtime.
 4186  */
 4187 APPLESTATIC void
 4188 nfscl_deleggetmodtime(vnode_t vp, struct timespec *mtime)
 4189 {
 4190         struct nfsclclient *clp;
 4191         struct nfscldeleg *dp;
 4192         struct nfsnode *np = VTONFS(vp);
 4193         struct nfsmount *nmp;
 4194 
 4195         nmp = VFSTONFS(vnode_mount(vp));
 4196         if (!NFSHASNFSV4(nmp))
 4197                 return;
 4198         NFSLOCKCLSTATE();
 4199         clp = nfscl_findcl(nmp);
 4200         if (clp == NULL) {
 4201                 NFSUNLOCKCLSTATE();
 4202                 return;
 4203         }
 4204         dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 4205         if (dp != NULL &&
 4206             (dp->nfsdl_flags & (NFSCLDL_WRITE | NFSCLDL_MODTIMESET)) ==
 4207             (NFSCLDL_WRITE | NFSCLDL_MODTIMESET))
 4208                 *mtime = dp->nfsdl_modtime;
 4209         NFSUNLOCKCLSTATE();
 4210 }
 4211 
 4212 static int
 4213 nfscl_errmap(struct nfsrv_descript *nd)
 4214 {
 4215         short *defaulterrp, *errp;
 4216 
 4217         if (!nd->nd_repstat)
 4218                 return (0);
 4219         if (nd->nd_procnum == NFSPROC_NOOP)
 4220                 return (txdr_unsigned(nd->nd_repstat & 0xffff));
 4221         if (nd->nd_repstat == EBADRPC)
 4222                 return (txdr_unsigned(NFSERR_BADXDR));
 4223         if (nd->nd_repstat == NFSERR_MINORVERMISMATCH ||
 4224             nd->nd_repstat == NFSERR_OPILLEGAL)
 4225                 return (txdr_unsigned(nd->nd_repstat));
 4226         errp = defaulterrp = nfscl_cberrmap[nd->nd_procnum];
 4227         while (*++errp)
 4228                 if (*errp == (short)nd->nd_repstat)
 4229                         return (txdr_unsigned(nd->nd_repstat));
 4230         return (txdr_unsigned(*defaulterrp));
 4231 }
 4232 

Cache object: a14367d35109ce8c51c5c673a783d446


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.