The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_cache.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1989, 1993, 1995
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * Poul-Henning Kamp of the FreeBSD Project.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 3. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      @(#)vfs_cache.c 8.5 (Berkeley) 3/22/95
   33  */
   34 
   35 #include <sys/cdefs.h>
   36 __FBSDID("$FreeBSD$");
   37 
   38 #include "opt_ktrace.h"
   39 
   40 #include <sys/param.h>
   41 #include <sys/systm.h>
   42 #include <sys/counter.h>
   43 #include <sys/filedesc.h>
   44 #include <sys/fnv_hash.h>
   45 #include <sys/kernel.h>
   46 #include <sys/lock.h>
   47 #include <sys/malloc.h>
   48 #include <sys/fcntl.h>
   49 #include <sys/mount.h>
   50 #include <sys/namei.h>
   51 #include <sys/proc.h>
   52 #include <sys/rwlock.h>
   53 #include <sys/sdt.h>
   54 #include <sys/smp.h>
   55 #include <sys/syscallsubr.h>
   56 #include <sys/sysctl.h>
   57 #include <sys/sysproto.h>
   58 #include <sys/vnode.h>
   59 #ifdef KTRACE
   60 #include <sys/ktrace.h>
   61 #endif
   62 
   63 #include <vm/uma.h>
   64 
   65 SDT_PROVIDER_DECLARE(vfs);
   66 SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
   67     "struct vnode *");
   68 SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
   69     "char *");
   70 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
   71 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
   72     "char *", "struct vnode *");
   73 SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
   74 SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
   75     "struct vnode *", "char *");
   76 SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
   77     "struct vnode *");
   78 SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
   79     "struct vnode *", "char *");
   80 SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
   81     "char *");
   82 SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
   83 SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
   84 SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
   85 SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
   86     "struct vnode *");
   87 SDT_PROBE_DEFINE3(vfs, namecache, zap_negative, done, "struct vnode *",
   88     "char *", "int");
   89 SDT_PROBE_DEFINE3(vfs, namecache, shrink_negative, done, "struct vnode *",
   90     "char *", "int");
   91 
   92 /*
   93  * This structure describes the elements in the cache of recent
   94  * names looked up by namei.
   95  */
   96 
   97 struct  namecache {
   98         LIST_ENTRY(namecache) nc_hash;  /* hash chain */
   99         LIST_ENTRY(namecache) nc_src;   /* source vnode list */
  100         TAILQ_ENTRY(namecache) nc_dst;  /* destination vnode list */
  101         struct  vnode *nc_dvp;          /* vnode of parent of name */
  102         union {
  103                 struct  vnode *nu_vp;   /* vnode the name refers to */
  104                 u_int   nu_neghits;     /* negative entry hits */
  105         } n_un;
  106         u_char  nc_flag;                /* flag bits */
  107         u_char  nc_nlen;                /* length of name */
  108         char    nc_name[0];             /* segment name + nul */
  109 };
  110 
  111 /*
  112  * struct namecache_ts repeats struct namecache layout up to the
  113  * nc_nlen member.
  114  * struct namecache_ts is used in place of struct namecache when time(s) need
  115  * to be stored.  The nc_dotdottime field is used when a cache entry is mapping
  116  * both a non-dotdot directory name plus dotdot for the directory's
  117  * parent.
  118  */
  119 struct  namecache_ts {
  120         struct  timespec nc_time;       /* timespec provided by fs */
  121         struct  timespec nc_dotdottime; /* dotdot timespec provided by fs */
  122         int     nc_ticks;               /* ticks value when entry was added */
  123         struct namecache nc_nc;
  124 };
  125 
  126 #define nc_vp           n_un.nu_vp
  127 #define nc_neghits      n_un.nu_neghits
  128 
  129 /*
  130  * Flags in namecache.nc_flag
  131  */
  132 #define NCF_WHITE       0x01
  133 #define NCF_ISDOTDOT    0x02
  134 #define NCF_TS          0x04
  135 #define NCF_DTS         0x08
  136 #define NCF_DVDROP      0x10
  137 #define NCF_NEGATIVE    0x20
  138 #define NCF_HOTNEGATIVE 0x40
  139 
  140 /*
  141  * Name caching works as follows:
  142  *
  143  * Names found by directory scans are retained in a cache
  144  * for future reference.  It is managed LRU, so frequently
  145  * used names will hang around.  Cache is indexed by hash value
  146  * obtained from (dvp, name) where dvp refers to the directory
  147  * containing name.
  148  *
  149  * If it is a "negative" entry, (i.e. for a name that is known NOT to
  150  * exist) the vnode pointer will be NULL.
  151  *
  152  * Upon reaching the last segment of a path, if the reference
  153  * is for DELETE, or NOCACHE is set (rewrite), and the
  154  * name is located in the cache, it will be dropped.
  155  *
  156  * These locks are used (in the order in which they can be taken):
  157  * NAME         TYPE    ROLE
  158  * vnodelock    mtx     vnode lists and v_cache_dd field protection
  159  * bucketlock   rwlock  for access to given set of hash buckets
  160  * neglist      mtx     negative entry LRU management
  161  *
  162  * Additionally, ncneg_shrink_lock mtx is used to have at most one thread
  163  * shrinking the LRU list.
  164  *
  165  * It is legal to take multiple vnodelock and bucketlock locks. The locking
  166  * order is lower address first. Both are recursive.
  167  *
  168  * "." lookups are lockless.
  169  *
  170  * ".." and vnode -> name lookups require vnodelock.
  171  *
  172  * name -> vnode lookup requires the relevant bucketlock to be held for reading.
  173  *
  174  * Insertions and removals of entries require involved vnodes and bucketlocks
  175  * to be write-locked to prevent other threads from seeing the entry.
  176  *
  177  * Some lookups result in removal of the found entry (e.g. getting rid of a
  178  * negative entry with the intent to create a positive one), which poses a
  179  * problem when multiple threads reach the state. Similarly, two different
  180  * threads can purge two different vnodes and try to remove the same name.
  181  *
  182  * If the already held vnode lock is lower than the second required lock, we
  183  * can just take the other lock. However, in the opposite case, this could
  184  * deadlock. As such, this is resolved by trylocking and if that fails unlocking
  185  * the first node, locking everything in order and revalidating the state.
  186  */
  187 
  188 /*
  189  * Structures associated with name caching.
  190  */
  191 #define NCHHASH(hash) \
  192         (&nchashtbl[(hash) & nchash])
  193 static __read_mostly LIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */
  194 static u_long __read_mostly     nchash;                 /* size of hash table */
  195 SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
  196     "Size of namecache hash table");
  197 static u_long __read_mostly     ncnegfactor = 12; /* ratio of negative entries */
  198 SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0,
  199     "Ratio of negative namecache entries");
  200 static u_long __exclusive_cache_line    numneg; /* number of negative entries allocated */
  201 SYSCTL_ULONG(_debug, OID_AUTO, numneg, CTLFLAG_RD, &numneg, 0,
  202     "Number of negative entries in namecache");
  203 static u_long __exclusive_cache_line    numcache;/* number of cache entries allocated */
  204 SYSCTL_ULONG(_debug, OID_AUTO, numcache, CTLFLAG_RD, &numcache, 0,
  205     "Number of namecache entries");
  206 static u_long __exclusive_cache_line    numcachehv;/* number of cache entries with vnodes held */
  207 SYSCTL_ULONG(_debug, OID_AUTO, numcachehv, CTLFLAG_RD, &numcachehv, 0,
  208     "Number of namecache entries with vnodes held");
  209 u_int __read_mostly     ncsizefactor = 2;
  210 SYSCTL_UINT(_vfs, OID_AUTO, ncsizefactor, CTLFLAG_RW, &ncsizefactor, 0,
  211     "Size factor for namecache");
  212 static u_int __read_mostly      ncpurgeminvnodes;
  213 SYSCTL_UINT(_vfs, OID_AUTO, ncpurgeminvnodes, CTLFLAG_RW, &ncpurgeminvnodes, 0,
  214     "Number of vnodes below which purgevfs ignores the request");
  215 static u_int __read_mostly      ncneghitsrequeue = 8;
  216 SYSCTL_UINT(_vfs, OID_AUTO, ncneghitsrequeue, CTLFLAG_RW, &ncneghitsrequeue, 0,
  217     "Number of hits to requeue a negative entry in the LRU list");
  218 
  219 struct nchstats nchstats;               /* cache effectiveness statistics */
  220 
  221 static struct mtx       ncneg_shrink_lock;
  222 static int      shrink_list_turn;
  223 
  224 struct neglist {
  225         struct mtx              nl_lock;
  226         TAILQ_HEAD(, namecache) nl_list;
  227 } __aligned(CACHE_LINE_SIZE);
  228 
  229 static struct neglist __read_mostly     *neglists;
  230 static struct neglist ncneg_hot;
  231 
  232 #define numneglists (ncneghash + 1)
  233 static u_int __read_mostly      ncneghash;
  234 static inline struct neglist *
  235 NCP2NEGLIST(struct namecache *ncp)
  236 {
  237 
  238         return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
  239 }
  240 
  241 #define numbucketlocks (ncbuckethash + 1)
  242 static u_int __read_mostly  ncbuckethash;
  243 static struct rwlock_padalign __read_mostly  *bucketlocks;
  244 #define HASH2BUCKETLOCK(hash) \
  245         ((struct rwlock *)(&bucketlocks[((hash) & ncbuckethash)]))
  246 
  247 #define numvnodelocks (ncvnodehash + 1)
  248 static u_int __read_mostly  ncvnodehash;
  249 static struct mtx __read_mostly *vnodelocks;
  250 static inline struct mtx *
  251 VP2VNODELOCK(struct vnode *vp)
  252 {
  253 
  254         return (&vnodelocks[(((uintptr_t)(vp) >> 8) & ncvnodehash)]);
  255 }
  256 
  257 /*
  258  * UMA zones for the VFS cache.
  259  *
  260  * The small cache is used for entries with short names, which are the
  261  * most common.  The large cache is used for entries which are too big to
  262  * fit in the small cache.
  263  */
  264 static uma_zone_t __read_mostly cache_zone_small;
  265 static uma_zone_t __read_mostly cache_zone_small_ts;
  266 static uma_zone_t __read_mostly cache_zone_large;
  267 static uma_zone_t __read_mostly cache_zone_large_ts;
  268 
  269 #define CACHE_PATH_CUTOFF       35
  270 
  271 static struct namecache *
  272 cache_alloc(int len, int ts)
  273 {
  274         struct namecache_ts *ncp_ts;
  275         struct namecache *ncp;
  276 
  277         if (__predict_false(ts)) {
  278                 if (len <= CACHE_PATH_CUTOFF)
  279                         ncp_ts = uma_zalloc(cache_zone_small_ts, M_WAITOK);
  280                 else
  281                         ncp_ts = uma_zalloc(cache_zone_large_ts, M_WAITOK);
  282                 ncp = &ncp_ts->nc_nc;
  283         } else {
  284                 if (len <= CACHE_PATH_CUTOFF)
  285                         ncp = uma_zalloc(cache_zone_small, M_WAITOK);
  286                 else
  287                         ncp = uma_zalloc(cache_zone_large, M_WAITOK);
  288         }
  289         return (ncp);
  290 }
  291 
  292 static void
  293 cache_free(struct namecache *ncp)
  294 {
  295         struct namecache_ts *ncp_ts;
  296 
  297         if (ncp == NULL)
  298                 return;
  299         if ((ncp->nc_flag & NCF_DVDROP) != 0)
  300                 vdrop(ncp->nc_dvp);
  301         if (__predict_false(ncp->nc_flag & NCF_TS)) {
  302                 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
  303                 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
  304                         uma_zfree(cache_zone_small_ts, ncp_ts);
  305                 else
  306                         uma_zfree(cache_zone_large_ts, ncp_ts);
  307         } else {
  308                 if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
  309                         uma_zfree(cache_zone_small, ncp);
  310                 else
  311                         uma_zfree(cache_zone_large, ncp);
  312         }
  313 }
  314 
  315 static void
  316 cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
  317 {
  318         struct namecache_ts *ncp_ts;
  319 
  320         KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
  321             (tsp == NULL && ticksp == NULL),
  322             ("No NCF_TS"));
  323 
  324         if (tsp == NULL && ticksp == NULL)
  325                 return;
  326 
  327         ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
  328         if (tsp != NULL)
  329                 *tsp = ncp_ts->nc_time;
  330         if (ticksp != NULL)
  331                 *ticksp = ncp_ts->nc_ticks;
  332 }
  333 
  334 static int __read_mostly        doingcache = 1; /* 1 => enable the cache */
  335 SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
  336     "VFS namecache enabled");
  337 
  338 /* Export size information to userland */
  339 SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
  340     sizeof(struct namecache), "sizeof(struct namecache)");
  341 
  342 /*
  343  * The new name cache statistics
  344  */
  345 static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW, 0,
  346     "Name cache statistics");
  347 #define STATNODE_ULONG(name, descr)     \
  348         SYSCTL_ULONG(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, 0, descr);
  349 #define STATNODE_COUNTER(name, descr)   \
  350         static counter_u64_t __read_mostly name; \
  351         SYSCTL_COUNTER_U64(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, descr);
  352 STATNODE_ULONG(numneg, "Number of negative cache entries");
  353 STATNODE_ULONG(numcache, "Number of cache entries");
  354 STATNODE_COUNTER(numcalls, "Number of cache lookups");
  355 STATNODE_COUNTER(dothits, "Number of '.' hits");
  356 STATNODE_COUNTER(dotdothits, "Number of '..' hits");
  357 STATNODE_COUNTER(numchecks, "Number of checks in lookup");
  358 STATNODE_COUNTER(nummiss, "Number of cache misses");
  359 STATNODE_COUNTER(nummisszap, "Number of cache misses we do not want to cache");
  360 STATNODE_COUNTER(numposzaps,
  361     "Number of cache hits (positive) we do not want to cache");
  362 STATNODE_COUNTER(numposhits, "Number of cache hits (positive)");
  363 STATNODE_COUNTER(numnegzaps,
  364     "Number of cache hits (negative) we do not want to cache");
  365 STATNODE_COUNTER(numneghits, "Number of cache hits (negative)");
  366 /* These count for kern___getcwd(), too. */
  367 STATNODE_COUNTER(numfullpathcalls, "Number of fullpath search calls");
  368 STATNODE_COUNTER(numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
  369 STATNODE_COUNTER(numfullpathfail2,
  370     "Number of fullpath search errors (VOP_VPTOCNP failures)");
  371 STATNODE_COUNTER(numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
  372 STATNODE_COUNTER(numfullpathfound, "Number of successful fullpath calls");
  373 static long zap_and_exit_bucket_fail; STATNODE_ULONG(zap_and_exit_bucket_fail,
  374     "Number of times zap_and_exit failed to lock");
  375 static long cache_lock_vnodes_cel_3_failures;
  376 STATNODE_ULONG(cache_lock_vnodes_cel_3_failures,
  377     "Number of times 3-way vnode locking failed");
  378 
  379 static void cache_zap_locked(struct namecache *ncp, bool neg_locked);
  380 static int vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
  381     char *buf, char **retbuf, u_int buflen);
  382 
  383 static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
  384 
  385 static int cache_yield;
  386 SYSCTL_INT(_vfs_cache, OID_AUTO, yield, CTLFLAG_RD, &cache_yield, 0,
  387     "Number of times cache called yield");
  388 
  389 static void
  390 cache_maybe_yield(void)
  391 {
  392 
  393         if (should_yield()) {
  394                 cache_yield++;
  395                 kern_yield(PRI_USER);
  396         }
  397 }
  398 
  399 static inline void
  400 cache_assert_vlp_locked(struct mtx *vlp)
  401 {
  402 
  403         if (vlp != NULL)
  404                 mtx_assert(vlp, MA_OWNED);
  405 }
  406 
  407 static inline void
  408 cache_assert_vnode_locked(struct vnode *vp)
  409 {
  410         struct mtx *vlp;
  411 
  412         vlp = VP2VNODELOCK(vp);
  413         cache_assert_vlp_locked(vlp);
  414 }
  415 
  416 static uint32_t
  417 cache_get_hash(char *name, u_char len, struct vnode *dvp)
  418 {
  419         uint32_t hash;
  420 
  421         hash = fnv_32_buf(name, len, FNV1_32_INIT);
  422         hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
  423         return (hash);
  424 }
  425 
  426 static inline struct rwlock *
  427 NCP2BUCKETLOCK(struct namecache *ncp)
  428 {
  429         uint32_t hash;
  430 
  431         hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
  432         return (HASH2BUCKETLOCK(hash));
  433 }
  434 
  435 #ifdef INVARIANTS
  436 static void
  437 cache_assert_bucket_locked(struct namecache *ncp, int mode)
  438 {
  439         struct rwlock *blp;
  440 
  441         blp = NCP2BUCKETLOCK(ncp);
  442         rw_assert(blp, mode);
  443 }
  444 #else
  445 #define cache_assert_bucket_locked(x, y) do { } while (0)
  446 #endif
  447 
  448 #define cache_sort(x, y)        _cache_sort((void **)(x), (void **)(y))
  449 static void
  450 _cache_sort(void **p1, void **p2)
  451 {
  452         void *tmp;
  453 
  454         if (*p1 > *p2) {
  455                 tmp = *p2;
  456                 *p2 = *p1;
  457                 *p1 = tmp;
  458         }
  459 }
  460 
  461 static void
  462 cache_lock_all_buckets(void)
  463 {
  464         u_int i;
  465 
  466         for (i = 0; i < numbucketlocks; i++)
  467                 rw_wlock(&bucketlocks[i]);
  468 }
  469 
  470 static void
  471 cache_unlock_all_buckets(void)
  472 {
  473         u_int i;
  474 
  475         for (i = 0; i < numbucketlocks; i++)
  476                 rw_wunlock(&bucketlocks[i]);
  477 }
  478 
  479 static void
  480 cache_lock_all_vnodes(void)
  481 {
  482         u_int i;
  483 
  484         for (i = 0; i < numvnodelocks; i++)
  485                 mtx_lock(&vnodelocks[i]);
  486 }
  487 
  488 static void
  489 cache_unlock_all_vnodes(void)
  490 {
  491         u_int i;
  492 
  493         for (i = 0; i < numvnodelocks; i++)
  494                 mtx_unlock(&vnodelocks[i]);
  495 }
  496 
  497 static int
  498 cache_trylock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
  499 {
  500 
  501         cache_sort(&vlp1, &vlp2);
  502         MPASS(vlp2 != NULL);
  503 
  504         if (vlp1 != NULL) {
  505                 if (!mtx_trylock(vlp1))
  506                         return (EAGAIN);
  507         }
  508         if (!mtx_trylock(vlp2)) {
  509                 if (vlp1 != NULL)
  510                         mtx_unlock(vlp1);
  511                 return (EAGAIN);
  512         }
  513 
  514         return (0);
  515 }
  516 
  517 static void
  518 cache_unlock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
  519 {
  520 
  521         MPASS(vlp1 != NULL || vlp2 != NULL);
  522 
  523         if (vlp1 != NULL)
  524                 mtx_unlock(vlp1);
  525         if (vlp2 != NULL)
  526                 mtx_unlock(vlp2);
  527 }
  528 
  529 static int
  530 sysctl_nchstats(SYSCTL_HANDLER_ARGS)
  531 {
  532         struct nchstats snap;
  533 
  534         if (req->oldptr == NULL)
  535                 return (SYSCTL_OUT(req, 0, sizeof(snap)));
  536 
  537         snap = nchstats;
  538         snap.ncs_goodhits = counter_u64_fetch(numposhits);
  539         snap.ncs_neghits = counter_u64_fetch(numneghits);
  540         snap.ncs_badhits = counter_u64_fetch(numposzaps) +
  541             counter_u64_fetch(numnegzaps);
  542         snap.ncs_miss = counter_u64_fetch(nummisszap) +
  543             counter_u64_fetch(nummiss);
  544 
  545         return (SYSCTL_OUT(req, &snap, sizeof(snap)));
  546 }
  547 SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE | CTLFLAG_RD |
  548     CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU",
  549     "VFS cache effectiveness statistics");
  550 
  551 #ifdef DIAGNOSTIC
  552 /*
  553  * Grab an atomic snapshot of the name cache hash chain lengths
  554  */
  555 static SYSCTL_NODE(_debug, OID_AUTO, hashstat, CTLFLAG_RW, NULL,
  556     "hash table stats");
  557 
  558 static int
  559 sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
  560 {
  561         struct nchashhead *ncpp;
  562         struct namecache *ncp;
  563         int i, error, n_nchash, *cntbuf;
  564 
  565 retry:
  566         n_nchash = nchash + 1;  /* nchash is max index, not count */
  567         if (req->oldptr == NULL)
  568                 return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
  569         cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK);
  570         cache_lock_all_buckets();
  571         if (n_nchash != nchash + 1) {
  572                 cache_unlock_all_buckets();
  573                 free(cntbuf, M_TEMP);
  574                 goto retry;
  575         }
  576         /* Scan hash tables counting entries */
  577         for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
  578                 LIST_FOREACH(ncp, ncpp, nc_hash)
  579                         cntbuf[i]++;
  580         cache_unlock_all_buckets();
  581         for (error = 0, i = 0; i < n_nchash; i++)
  582                 if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
  583                         break;
  584         free(cntbuf, M_TEMP);
  585         return (error);
  586 }
  587 SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD|
  588     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
  589     "nchash chain lengths");
  590 
  591 static int
  592 sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
  593 {
  594         int error;
  595         struct nchashhead *ncpp;
  596         struct namecache *ncp;
  597         int n_nchash;
  598         int count, maxlength, used, pct;
  599 
  600         if (!req->oldptr)
  601                 return SYSCTL_OUT(req, 0, 4 * sizeof(int));
  602 
  603         cache_lock_all_buckets();
  604         n_nchash = nchash + 1;  /* nchash is max index, not count */
  605         used = 0;
  606         maxlength = 0;
  607 
  608         /* Scan hash tables for applicable entries */
  609         for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
  610                 count = 0;
  611                 LIST_FOREACH(ncp, ncpp, nc_hash) {
  612                         count++;
  613                 }
  614                 if (count)
  615                         used++;
  616                 if (maxlength < count)
  617                         maxlength = count;
  618         }
  619         n_nchash = nchash + 1;
  620         cache_unlock_all_buckets();
  621         pct = (used * 100) / (n_nchash / 100);
  622         error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
  623         if (error)
  624                 return (error);
  625         error = SYSCTL_OUT(req, &used, sizeof(used));
  626         if (error)
  627                 return (error);
  628         error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
  629         if (error)
  630                 return (error);
  631         error = SYSCTL_OUT(req, &pct, sizeof(pct));
  632         if (error)
  633                 return (error);
  634         return (0);
  635 }
  636 SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD|
  637     CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
  638     "nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
  639 #endif
  640 
  641 /*
  642  * Negative entries management
  643  *
  644  * A variation of LRU scheme is used. New entries are hashed into one of
  645  * numneglists cold lists. Entries get promoted to the hot list on first hit.
  646  * Partial LRU for the hot list is maintained by requeueing them every
  647  * ncneghitsrequeue hits.
  648  *
  649  * The shrinker will demote hot list head and evict from the cold list in a
  650  * round-robin manner.
  651  */
  652 static void
  653 cache_negative_hit(struct namecache *ncp)
  654 {
  655         struct neglist *neglist;
  656         u_int hits;
  657 
  658         MPASS(ncp->nc_flag & NCF_NEGATIVE);
  659         hits = atomic_fetchadd_int(&ncp->nc_neghits, 1);
  660         if (ncp->nc_flag & NCF_HOTNEGATIVE) {
  661                 if ((hits % ncneghitsrequeue) != 0)
  662                         return;
  663                 mtx_lock(&ncneg_hot.nl_lock);
  664                 if (ncp->nc_flag & NCF_HOTNEGATIVE) {
  665                         TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
  666                         TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
  667                         mtx_unlock(&ncneg_hot.nl_lock);
  668                         return;
  669                 }
  670                 /*
  671                  * The shrinker cleared the flag and removed the entry from
  672                  * the hot list. Put it back.
  673                  */
  674         } else {
  675                 mtx_lock(&ncneg_hot.nl_lock);
  676         }
  677         neglist = NCP2NEGLIST(ncp);
  678         mtx_lock(&neglist->nl_lock);
  679         if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
  680                 TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
  681                 TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
  682                 ncp->nc_flag |= NCF_HOTNEGATIVE;
  683         }
  684         mtx_unlock(&neglist->nl_lock);
  685         mtx_unlock(&ncneg_hot.nl_lock);
  686 }
  687 
  688 static void
  689 cache_negative_insert(struct namecache *ncp, bool neg_locked)
  690 {
  691         struct neglist *neglist;
  692 
  693         MPASS(ncp->nc_flag & NCF_NEGATIVE);
  694         cache_assert_bucket_locked(ncp, RA_WLOCKED);
  695         neglist = NCP2NEGLIST(ncp);
  696         if (!neg_locked) {
  697                 mtx_lock(&neglist->nl_lock);
  698         } else {
  699                 mtx_assert(&neglist->nl_lock, MA_OWNED);
  700         }
  701         TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
  702         if (!neg_locked)
  703                 mtx_unlock(&neglist->nl_lock);
  704         atomic_add_rel_long(&numneg, 1);
  705 }
  706 
  707 static void
  708 cache_negative_remove(struct namecache *ncp, bool neg_locked)
  709 {
  710         struct neglist *neglist;
  711         bool hot_locked = false;
  712         bool list_locked = false;
  713 
  714         MPASS(ncp->nc_flag & NCF_NEGATIVE);
  715         cache_assert_bucket_locked(ncp, RA_WLOCKED);
  716         neglist = NCP2NEGLIST(ncp);
  717         if (!neg_locked) {
  718                 if (ncp->nc_flag & NCF_HOTNEGATIVE) {
  719                         hot_locked = true;
  720                         mtx_lock(&ncneg_hot.nl_lock);
  721                         if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
  722                                 list_locked = true;
  723                                 mtx_lock(&neglist->nl_lock);
  724                         }
  725                 } else {
  726                         list_locked = true;
  727                         mtx_lock(&neglist->nl_lock);
  728                 }
  729         }
  730         if (ncp->nc_flag & NCF_HOTNEGATIVE) {
  731                 mtx_assert(&ncneg_hot.nl_lock, MA_OWNED);
  732                 TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
  733         } else {
  734                 mtx_assert(&neglist->nl_lock, MA_OWNED);
  735                 TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
  736         }
  737         if (list_locked)
  738                 mtx_unlock(&neglist->nl_lock);
  739         if (hot_locked)
  740                 mtx_unlock(&ncneg_hot.nl_lock);
  741         atomic_subtract_rel_long(&numneg, 1);
  742 }
  743 
  744 static void
  745 cache_negative_shrink_select(int start, struct namecache **ncpp,
  746     struct neglist **neglistpp)
  747 {
  748         struct neglist *neglist;
  749         struct namecache *ncp;
  750         int i;
  751 
  752         *ncpp = ncp = NULL;
  753 
  754         for (i = start; i < numneglists; i++) {
  755                 neglist = &neglists[i];
  756                 if (TAILQ_FIRST(&neglist->nl_list) == NULL)
  757                         continue;
  758                 mtx_lock(&neglist->nl_lock);
  759                 ncp = TAILQ_FIRST(&neglist->nl_list);
  760                 if (ncp != NULL)
  761                         break;
  762                 mtx_unlock(&neglist->nl_lock);
  763         }
  764 
  765         *neglistpp = neglist;
  766         *ncpp = ncp;
  767 }
  768 
  769 static void
  770 cache_negative_zap_one(void)
  771 {
  772         struct namecache *ncp, *ncp2;
  773         struct neglist *neglist;
  774         struct mtx *dvlp;
  775         struct rwlock *blp;
  776 
  777         if (!mtx_trylock(&ncneg_shrink_lock))
  778                 return;
  779 
  780         mtx_lock(&ncneg_hot.nl_lock);
  781         ncp = TAILQ_FIRST(&ncneg_hot.nl_list);
  782         if (ncp != NULL) {
  783                 neglist = NCP2NEGLIST(ncp);
  784                 mtx_lock(&neglist->nl_lock);
  785                 TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
  786                 TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
  787                 ncp->nc_flag &= ~NCF_HOTNEGATIVE;
  788                 mtx_unlock(&neglist->nl_lock);
  789         }
  790 
  791         cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
  792         shrink_list_turn++;
  793         if (shrink_list_turn == numneglists)
  794                 shrink_list_turn = 0;
  795         if (ncp == NULL && shrink_list_turn == 0)
  796                 cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
  797         if (ncp == NULL) {
  798                 mtx_unlock(&ncneg_hot.nl_lock);
  799                 goto out;
  800         }
  801 
  802         MPASS(ncp->nc_flag & NCF_NEGATIVE);
  803         dvlp = VP2VNODELOCK(ncp->nc_dvp);
  804         blp = NCP2BUCKETLOCK(ncp);
  805         mtx_unlock(&neglist->nl_lock);
  806         mtx_unlock(&ncneg_hot.nl_lock);
  807         mtx_lock(dvlp);
  808         rw_wlock(blp);
  809         mtx_lock(&neglist->nl_lock);
  810         ncp2 = TAILQ_FIRST(&neglist->nl_list);
  811         if (ncp != ncp2 || dvlp != VP2VNODELOCK(ncp2->nc_dvp) ||
  812             blp != NCP2BUCKETLOCK(ncp2) || !(ncp2->nc_flag & NCF_NEGATIVE)) {
  813                 ncp = NULL;
  814                 goto out_unlock_all;
  815         }
  816         SDT_PROBE3(vfs, namecache, shrink_negative, done, ncp->nc_dvp,
  817             ncp->nc_name, ncp->nc_neghits);
  818 
  819         cache_zap_locked(ncp, true);
  820 out_unlock_all:
  821         mtx_unlock(&neglist->nl_lock);
  822         rw_wunlock(blp);
  823         mtx_unlock(dvlp);
  824 out:
  825         mtx_unlock(&ncneg_shrink_lock);
  826         cache_free(ncp);
  827 }
  828 
  829 /*
  830  * cache_zap_locked():
  831  *
  832  *   Removes a namecache entry from cache, whether it contains an actual
  833  *   pointer to a vnode or if it is just a negative cache entry.
  834  */
  835 static void
  836 cache_zap_locked(struct namecache *ncp, bool neg_locked)
  837 {
  838 
  839         if (!(ncp->nc_flag & NCF_NEGATIVE))
  840                 cache_assert_vnode_locked(ncp->nc_vp);
  841         cache_assert_vnode_locked(ncp->nc_dvp);
  842         cache_assert_bucket_locked(ncp, RA_WLOCKED);
  843 
  844         CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp,
  845             (ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp);
  846         if (!(ncp->nc_flag & NCF_NEGATIVE)) {
  847                 SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
  848                     ncp->nc_name, ncp->nc_vp);
  849         } else {
  850                 SDT_PROBE3(vfs, namecache, zap_negative, done, ncp->nc_dvp,
  851                     ncp->nc_name, ncp->nc_neghits);
  852         }
  853         LIST_REMOVE(ncp, nc_hash);
  854         if (!(ncp->nc_flag & NCF_NEGATIVE)) {
  855                 TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
  856                 if (ncp == ncp->nc_vp->v_cache_dd)
  857                         ncp->nc_vp->v_cache_dd = NULL;
  858         } else {
  859                 cache_negative_remove(ncp, neg_locked);
  860         }
  861         if (ncp->nc_flag & NCF_ISDOTDOT) {
  862                 if (ncp == ncp->nc_dvp->v_cache_dd)
  863                         ncp->nc_dvp->v_cache_dd = NULL;
  864         } else {
  865                 LIST_REMOVE(ncp, nc_src);
  866                 if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
  867                         ncp->nc_flag |= NCF_DVDROP;
  868                         atomic_subtract_rel_long(&numcachehv, 1);
  869                 }
  870         }
  871         atomic_subtract_rel_long(&numcache, 1);
  872 }
  873 
  874 static void
  875 cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp)
  876 {
  877         struct rwlock *blp;
  878 
  879         MPASS(ncp->nc_dvp == vp);
  880         MPASS(ncp->nc_flag & NCF_NEGATIVE);
  881         cache_assert_vnode_locked(vp);
  882 
  883         blp = NCP2BUCKETLOCK(ncp);
  884         rw_wlock(blp);
  885         cache_zap_locked(ncp, false);
  886         rw_wunlock(blp);
  887 }
  888 
  889 static bool
  890 cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp,
  891     struct mtx **vlpp)
  892 {
  893         struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
  894         struct rwlock *blp;
  895 
  896         MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
  897         cache_assert_vnode_locked(vp);
  898 
  899         if (ncp->nc_flag & NCF_NEGATIVE) {
  900                 if (*vlpp != NULL) {
  901                         mtx_unlock(*vlpp);
  902                         *vlpp = NULL;
  903                 }
  904                 cache_zap_negative_locked_vnode_kl(ncp, vp);
  905                 return (true);
  906         }
  907 
  908         pvlp = VP2VNODELOCK(vp);
  909         blp = NCP2BUCKETLOCK(ncp);
  910         vlp1 = VP2VNODELOCK(ncp->nc_dvp);
  911         vlp2 = VP2VNODELOCK(ncp->nc_vp);
  912 
  913         if (*vlpp == vlp1 || *vlpp == vlp2) {
  914                 to_unlock = *vlpp;
  915                 *vlpp = NULL;
  916         } else {
  917                 if (*vlpp != NULL) {
  918                         mtx_unlock(*vlpp);
  919                         *vlpp = NULL;
  920                 }
  921                 cache_sort(&vlp1, &vlp2);
  922                 if (vlp1 == pvlp) {
  923                         mtx_lock(vlp2);
  924                         to_unlock = vlp2;
  925                 } else {
  926                         if (!mtx_trylock(vlp1))
  927                                 goto out_relock;
  928                         to_unlock = vlp1;
  929                 }
  930         }
  931         rw_wlock(blp);
  932         cache_zap_locked(ncp, false);
  933         rw_wunlock(blp);
  934         if (to_unlock != NULL)
  935                 mtx_unlock(to_unlock);
  936         return (true);
  937 
  938 out_relock:
  939         mtx_unlock(vlp2);
  940         mtx_lock(vlp1);
  941         mtx_lock(vlp2);
  942         MPASS(*vlpp == NULL);
  943         *vlpp = vlp1;
  944         return (false);
  945 }
  946 
  947 static int
  948 cache_zap_locked_vnode(struct namecache *ncp, struct vnode *vp)
  949 {
  950         struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
  951         struct rwlock *blp;
  952         int error = 0;
  953 
  954         MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
  955         cache_assert_vnode_locked(vp);
  956 
  957         pvlp = VP2VNODELOCK(vp);
  958         if (ncp->nc_flag & NCF_NEGATIVE) {
  959                 cache_zap_negative_locked_vnode_kl(ncp, vp);
  960                 goto out;
  961         }
  962 
  963         blp = NCP2BUCKETLOCK(ncp);
  964         vlp1 = VP2VNODELOCK(ncp->nc_dvp);
  965         vlp2 = VP2VNODELOCK(ncp->nc_vp);
  966         cache_sort(&vlp1, &vlp2);
  967         if (vlp1 == pvlp) {
  968                 mtx_lock(vlp2);
  969                 to_unlock = vlp2;
  970         } else {
  971                 if (!mtx_trylock(vlp1)) {
  972                         error = EAGAIN;
  973                         goto out;
  974                 }
  975                 to_unlock = vlp1;
  976         }
  977         rw_wlock(blp);
  978         cache_zap_locked(ncp, false);
  979         rw_wunlock(blp);
  980         mtx_unlock(to_unlock);
  981 out:
  982         mtx_unlock(pvlp);
  983         return (error);
  984 }
  985 
  986 static int
  987 cache_zap_wlocked_bucket(struct namecache *ncp, struct rwlock *blp)
  988 {
  989         struct mtx *dvlp, *vlp;
  990 
  991         cache_assert_bucket_locked(ncp, RA_WLOCKED);
  992 
  993         dvlp = VP2VNODELOCK(ncp->nc_dvp);
  994         vlp = NULL;
  995         if (!(ncp->nc_flag & NCF_NEGATIVE))
  996                 vlp = VP2VNODELOCK(ncp->nc_vp);
  997         if (cache_trylock_vnodes(dvlp, vlp) == 0) {
  998                 cache_zap_locked(ncp, false);
  999                 rw_wunlock(blp);
 1000                 cache_unlock_vnodes(dvlp, vlp);
 1001                 return (0);
 1002         }
 1003 
 1004         rw_wunlock(blp);
 1005         return (EAGAIN);
 1006 }
 1007 
 1008 static int
 1009 cache_zap_rlocked_bucket(struct namecache *ncp, struct rwlock *blp)
 1010 {
 1011         struct mtx *dvlp, *vlp;
 1012 
 1013         cache_assert_bucket_locked(ncp, RA_RLOCKED);
 1014 
 1015         dvlp = VP2VNODELOCK(ncp->nc_dvp);
 1016         vlp = NULL;
 1017         if (!(ncp->nc_flag & NCF_NEGATIVE))
 1018                 vlp = VP2VNODELOCK(ncp->nc_vp);
 1019         if (cache_trylock_vnodes(dvlp, vlp) == 0) {
 1020                 rw_runlock(blp);
 1021                 rw_wlock(blp);
 1022                 cache_zap_locked(ncp, false);
 1023                 rw_wunlock(blp);
 1024                 cache_unlock_vnodes(dvlp, vlp);
 1025                 return (0);
 1026         }
 1027 
 1028         rw_runlock(blp);
 1029         return (EAGAIN);
 1030 }
 1031 
 1032 static int
 1033 cache_zap_wlocked_bucket_kl(struct namecache *ncp, struct rwlock *blp,
 1034     struct mtx **vlpp1, struct mtx **vlpp2)
 1035 {
 1036         struct mtx *dvlp, *vlp;
 1037 
 1038         cache_assert_bucket_locked(ncp, RA_WLOCKED);
 1039 
 1040         dvlp = VP2VNODELOCK(ncp->nc_dvp);
 1041         vlp = NULL;
 1042         if (!(ncp->nc_flag & NCF_NEGATIVE))
 1043                 vlp = VP2VNODELOCK(ncp->nc_vp);
 1044         cache_sort(&dvlp, &vlp);
 1045 
 1046         if (*vlpp1 == dvlp && *vlpp2 == vlp) {
 1047                 cache_zap_locked(ncp, false);
 1048                 cache_unlock_vnodes(dvlp, vlp);
 1049                 *vlpp1 = NULL;
 1050                 *vlpp2 = NULL;
 1051                 return (0);
 1052         }
 1053 
 1054         if (*vlpp1 != NULL)
 1055                 mtx_unlock(*vlpp1);
 1056         if (*vlpp2 != NULL)
 1057                 mtx_unlock(*vlpp2);
 1058         *vlpp1 = NULL;
 1059         *vlpp2 = NULL;
 1060 
 1061         if (cache_trylock_vnodes(dvlp, vlp) == 0) {
 1062                 cache_zap_locked(ncp, false);
 1063                 cache_unlock_vnodes(dvlp, vlp);
 1064                 return (0);
 1065         }
 1066 
 1067         rw_wunlock(blp);
 1068         *vlpp1 = dvlp;
 1069         *vlpp2 = vlp;
 1070         if (*vlpp1 != NULL)
 1071                 mtx_lock(*vlpp1);
 1072         mtx_lock(*vlpp2);
 1073         rw_wlock(blp);
 1074         return (EAGAIN);
 1075 }
 1076 
 1077 static void
 1078 cache_lookup_unlock(struct rwlock *blp, struct mtx *vlp)
 1079 {
 1080 
 1081         if (blp != NULL) {
 1082                 rw_runlock(blp);
 1083         } else {
 1084                 mtx_unlock(vlp);
 1085         }
 1086 }
 1087 
 1088 static int __noinline
 1089 cache_lookup_dot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 1090     struct timespec *tsp, int *ticksp)
 1091 {
 1092         int ltype;
 1093 
 1094         *vpp = dvp;
 1095         CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
 1096                         dvp, cnp->cn_nameptr);
 1097         counter_u64_add(dothits, 1);
 1098         SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
 1099         if (tsp != NULL)
 1100                 timespecclear(tsp);
 1101         if (ticksp != NULL)
 1102                 *ticksp = ticks;
 1103         vrefact(*vpp);
 1104         /*
 1105          * When we lookup "." we still can be asked to lock it
 1106          * differently...
 1107          */
 1108         ltype = cnp->cn_lkflags & LK_TYPE_MASK;
 1109         if (ltype != VOP_ISLOCKED(*vpp)) {
 1110                 if (ltype == LK_EXCLUSIVE) {
 1111                         vn_lock(*vpp, LK_UPGRADE | LK_RETRY);
 1112                         if ((*vpp)->v_iflag & VI_DOOMED) {
 1113                                 /* forced unmount */
 1114                                 vrele(*vpp);
 1115                                 *vpp = NULL;
 1116                                 return (ENOENT);
 1117                         }
 1118                 } else
 1119                         vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
 1120         }
 1121         return (-1);
 1122 }
 1123 
 1124 static __noinline int
 1125 cache_lookup_nomakeentry(struct vnode *dvp, struct vnode **vpp,
 1126     struct componentname *cnp, struct timespec *tsp, int *ticksp)
 1127 {
 1128         struct namecache *ncp;
 1129         struct rwlock *blp;
 1130         struct mtx *dvlp, *dvlp2;
 1131         uint32_t hash;
 1132         int error;
 1133 
 1134         if (cnp->cn_namelen == 2 &&
 1135             cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
 1136                 counter_u64_add(dotdothits, 1);
 1137                 dvlp = VP2VNODELOCK(dvp);
 1138                 dvlp2 = NULL;
 1139                 mtx_lock(dvlp);
 1140 retry_dotdot:
 1141                 ncp = dvp->v_cache_dd;
 1142                 if (ncp == NULL) {
 1143                         SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
 1144                             "..", NULL);
 1145                         mtx_unlock(dvlp);
 1146                         if (dvlp2 != NULL)
 1147                                 mtx_unlock(dvlp2);
 1148                         return (0);
 1149                 }
 1150                 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
 1151                         if (ncp->nc_dvp != dvp)
 1152                                 panic("dvp %p v_cache_dd %p\n", dvp, ncp);
 1153                         if (!cache_zap_locked_vnode_kl2(ncp,
 1154                             dvp, &dvlp2))
 1155                                 goto retry_dotdot;
 1156                         MPASS(dvp->v_cache_dd == NULL);
 1157                         mtx_unlock(dvlp);
 1158                         if (dvlp2 != NULL)
 1159                                 mtx_unlock(dvlp2);
 1160                         cache_free(ncp);
 1161                 } else {
 1162                         dvp->v_cache_dd = NULL;
 1163                         mtx_unlock(dvlp);
 1164                         if (dvlp2 != NULL)
 1165                                 mtx_unlock(dvlp2);
 1166                 }
 1167                 return (0);
 1168         }
 1169 
 1170         hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
 1171         blp = HASH2BUCKETLOCK(hash);
 1172 retry:
 1173         if (LIST_EMPTY(NCHHASH(hash)))
 1174                 goto out_no_entry;
 1175 
 1176         rw_wlock(blp);
 1177 
 1178         LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
 1179                 counter_u64_add(numchecks, 1);
 1180                 if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
 1181                     !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
 1182                         break;
 1183         }
 1184 
 1185         /* We failed to find an entry */
 1186         if (ncp == NULL) {
 1187                 rw_wunlock(blp);
 1188                 goto out_no_entry;
 1189         }
 1190 
 1191         counter_u64_add(numposzaps, 1);
 1192 
 1193         error = cache_zap_wlocked_bucket(ncp, blp);
 1194         if (error != 0) {
 1195                 zap_and_exit_bucket_fail++;
 1196                 cache_maybe_yield();
 1197                 goto retry;
 1198         }
 1199         cache_free(ncp);
 1200         return (0);
 1201 out_no_entry:
 1202         SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, NULL);
 1203         counter_u64_add(nummisszap, 1);
 1204         return (0);
 1205 }
 1206 
 1207 /**
 1208  * Lookup a name in the name cache
 1209  *
 1210  * # Arguments
 1211  *
 1212  * - dvp:       Parent directory in which to search.
 1213  * - vpp:       Return argument.  Will contain desired vnode on cache hit.
 1214  * - cnp:       Parameters of the name search.  The most interesting bits of
 1215  *              the cn_flags field have the following meanings:
 1216  *      - MAKEENTRY:    If clear, free an entry from the cache rather than look
 1217  *                      it up.
 1218  *      - ISDOTDOT:     Must be set if and only if cn_nameptr == ".."
 1219  * - tsp:       Return storage for cache timestamp.  On a successful (positive
 1220  *              or negative) lookup, tsp will be filled with any timespec that
 1221  *              was stored when this cache entry was created.  However, it will
 1222  *              be clear for "." entries.
 1223  * - ticks:     Return storage for alternate cache timestamp.  On a successful
 1224  *              (positive or negative) lookup, it will contain the ticks value
 1225  *              that was current when the cache entry was created, unless cnp
 1226  *              was ".".
 1227  *
 1228  * # Returns
 1229  *
 1230  * - -1:        A positive cache hit.  vpp will contain the desired vnode.
 1231  * - ENOENT:    A negative cache hit, or dvp was recycled out from under us due
 1232  *              to a forced unmount.  vpp will not be modified.  If the entry
 1233  *              is a whiteout, then the ISWHITEOUT flag will be set in
 1234  *              cnp->cn_flags.
 1235  * - 0:         A cache miss.  vpp will not be modified.
 1236  *
 1237  * # Locking
 1238  *
 1239  * On a cache hit, vpp will be returned locked and ref'd.  If we're looking up
 1240  * .., dvp is unlocked.  If we're looking up . an extra ref is taken, but the
 1241  * lock is not recursively acquired.
 1242  */
 1243 int
 1244 cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 1245     struct timespec *tsp, int *ticksp)
 1246 {
 1247         struct namecache_ts *ncp_ts;
 1248         struct namecache *ncp;
 1249         struct rwlock *blp;
 1250         struct mtx *dvlp, *dvlp2;
 1251         uint32_t hash;
 1252         int error, ltype;
 1253 
 1254         if (__predict_false(!doingcache)) {
 1255                 cnp->cn_flags &= ~MAKEENTRY;
 1256                 return (0);
 1257         }
 1258 
 1259         counter_u64_add(numcalls, 1);
 1260 
 1261         if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.'))
 1262                 return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp));
 1263 
 1264         if ((cnp->cn_flags & MAKEENTRY) == 0)
 1265                 return (cache_lookup_nomakeentry(dvp, vpp, cnp, tsp, ticksp));
 1266 
 1267 retry:
 1268         blp = NULL;
 1269         error = 0;
 1270         if (cnp->cn_namelen == 2 &&
 1271             cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
 1272                 counter_u64_add(dotdothits, 1);
 1273                 dvlp = VP2VNODELOCK(dvp);
 1274                 dvlp2 = NULL;
 1275                 mtx_lock(dvlp);
 1276                 ncp = dvp->v_cache_dd;
 1277                 if (ncp == NULL) {
 1278                         SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
 1279                             "..", NULL);
 1280                         mtx_unlock(dvlp);
 1281                         return (0);
 1282                 }
 1283                 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
 1284                         if (ncp->nc_flag & NCF_NEGATIVE)
 1285                                 *vpp = NULL;
 1286                         else
 1287                                 *vpp = ncp->nc_vp;
 1288                 } else
 1289                         *vpp = ncp->nc_dvp;
 1290                 /* Return failure if negative entry was found. */
 1291                 if (*vpp == NULL)
 1292                         goto negative_success;
 1293                 CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
 1294                     dvp, cnp->cn_nameptr, *vpp);
 1295                 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..",
 1296                     *vpp);
 1297                 cache_out_ts(ncp, tsp, ticksp);
 1298                 if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
 1299                     NCF_DTS && tsp != NULL) {
 1300                         ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
 1301                         *tsp = ncp_ts->nc_dotdottime;
 1302                 }
 1303                 goto success;
 1304         }
 1305 
 1306         hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
 1307         blp = HASH2BUCKETLOCK(hash);
 1308         rw_rlock(blp);
 1309 
 1310         LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
 1311                 counter_u64_add(numchecks, 1);
 1312                 if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
 1313                     !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
 1314                         break;
 1315         }
 1316 
 1317         /* We failed to find an entry */
 1318         if (ncp == NULL) {
 1319                 rw_runlock(blp);
 1320                 SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
 1321                     NULL);
 1322                 counter_u64_add(nummiss, 1);
 1323                 return (0);
 1324         }
 1325 
 1326         /* We found a "positive" match, return the vnode */
 1327         if (!(ncp->nc_flag & NCF_NEGATIVE)) {
 1328                 counter_u64_add(numposhits, 1);
 1329                 *vpp = ncp->nc_vp;
 1330                 CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
 1331                     dvp, cnp->cn_nameptr, *vpp, ncp);
 1332                 SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
 1333                     *vpp);
 1334                 cache_out_ts(ncp, tsp, ticksp);
 1335                 goto success;
 1336         }
 1337 
 1338 negative_success:
 1339         /* We found a negative match, and want to create it, so purge */
 1340         if (cnp->cn_nameiop == CREATE) {
 1341                 counter_u64_add(numnegzaps, 1);
 1342                 goto zap_and_exit;
 1343         }
 1344 
 1345         counter_u64_add(numneghits, 1);
 1346         cache_negative_hit(ncp);
 1347         if (ncp->nc_flag & NCF_WHITE)
 1348                 cnp->cn_flags |= ISWHITEOUT;
 1349         SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp,
 1350             ncp->nc_name);
 1351         cache_out_ts(ncp, tsp, ticksp);
 1352         cache_lookup_unlock(blp, dvlp);
 1353         return (ENOENT);
 1354 
 1355 success:
 1356         /*
 1357          * On success we return a locked and ref'd vnode as per the lookup
 1358          * protocol.
 1359          */
 1360         MPASS(dvp != *vpp);
 1361         ltype = 0;      /* silence gcc warning */
 1362         if (cnp->cn_flags & ISDOTDOT) {
 1363                 ltype = VOP_ISLOCKED(dvp);
 1364                 VOP_UNLOCK(dvp, 0);
 1365         }
 1366         vhold(*vpp);
 1367         cache_lookup_unlock(blp, dvlp);
 1368         error = vget(*vpp, cnp->cn_lkflags | LK_VNHELD, cnp->cn_thread);
 1369         if (cnp->cn_flags & ISDOTDOT) {
 1370                 vn_lock(dvp, ltype | LK_RETRY);
 1371                 if (dvp->v_iflag & VI_DOOMED) {
 1372                         if (error == 0)
 1373                                 vput(*vpp);
 1374                         *vpp = NULL;
 1375                         return (ENOENT);
 1376                 }
 1377         }
 1378         if (error) {
 1379                 *vpp = NULL;
 1380                 goto retry;
 1381         }
 1382         if ((cnp->cn_flags & ISLASTCN) &&
 1383             (cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
 1384                 ASSERT_VOP_ELOCKED(*vpp, "cache_lookup");
 1385         }
 1386         return (-1);
 1387 
 1388 zap_and_exit:
 1389         if (blp != NULL)
 1390                 error = cache_zap_rlocked_bucket(ncp, blp);
 1391         else
 1392                 error = cache_zap_locked_vnode(ncp, dvp);
 1393         if (error != 0) {
 1394                 zap_and_exit_bucket_fail++;
 1395                 cache_maybe_yield();
 1396                 goto retry;
 1397         }
 1398         cache_free(ncp);
 1399         return (0);
 1400 }
 1401 
 1402 struct celockstate {
 1403         struct mtx *vlp[3];
 1404         struct rwlock *blp[2];
 1405 };
 1406 CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
 1407 CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));
 1408 
 1409 static inline void
 1410 cache_celockstate_init(struct celockstate *cel)
 1411 {
 1412 
 1413         bzero(cel, sizeof(*cel));
 1414 }
 1415 
 1416 static void
 1417 cache_lock_vnodes_cel(struct celockstate *cel, struct vnode *vp,
 1418     struct vnode *dvp)
 1419 {
 1420         struct mtx *vlp1, *vlp2;
 1421 
 1422         MPASS(cel->vlp[0] == NULL);
 1423         MPASS(cel->vlp[1] == NULL);
 1424         MPASS(cel->vlp[2] == NULL);
 1425 
 1426         MPASS(vp != NULL || dvp != NULL);
 1427 
 1428         vlp1 = VP2VNODELOCK(vp);
 1429         vlp2 = VP2VNODELOCK(dvp);
 1430         cache_sort(&vlp1, &vlp2);
 1431 
 1432         if (vlp1 != NULL) {
 1433                 mtx_lock(vlp1);
 1434                 cel->vlp[0] = vlp1;
 1435         }
 1436         mtx_lock(vlp2);
 1437         cel->vlp[1] = vlp2;
 1438 }
 1439 
 1440 static void
 1441 cache_unlock_vnodes_cel(struct celockstate *cel)
 1442 {
 1443 
 1444         MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL);
 1445 
 1446         if (cel->vlp[0] != NULL)
 1447                 mtx_unlock(cel->vlp[0]);
 1448         if (cel->vlp[1] != NULL)
 1449                 mtx_unlock(cel->vlp[1]);
 1450         if (cel->vlp[2] != NULL)
 1451                 mtx_unlock(cel->vlp[2]);
 1452 }
 1453 
 1454 static bool
 1455 cache_lock_vnodes_cel_3(struct celockstate *cel, struct vnode *vp)
 1456 {
 1457         struct mtx *vlp;
 1458         bool ret;
 1459 
 1460         cache_assert_vlp_locked(cel->vlp[0]);
 1461         cache_assert_vlp_locked(cel->vlp[1]);
 1462         MPASS(cel->vlp[2] == NULL);
 1463 
 1464         MPASS(vp != NULL);
 1465         vlp = VP2VNODELOCK(vp);
 1466 
 1467         ret = true;
 1468         if (vlp >= cel->vlp[1]) {
 1469                 mtx_lock(vlp);
 1470         } else {
 1471                 if (mtx_trylock(vlp))
 1472                         goto out;
 1473                 cache_lock_vnodes_cel_3_failures++;
 1474                 cache_unlock_vnodes_cel(cel);
 1475                 if (vlp < cel->vlp[0]) {
 1476                         mtx_lock(vlp);
 1477                         mtx_lock(cel->vlp[0]);
 1478                         mtx_lock(cel->vlp[1]);
 1479                 } else {
 1480                         if (cel->vlp[0] != NULL)
 1481                                 mtx_lock(cel->vlp[0]);
 1482                         mtx_lock(vlp);
 1483                         mtx_lock(cel->vlp[1]);
 1484                 }
 1485                 ret = false;
 1486         }
 1487 out:
 1488         cel->vlp[2] = vlp;
 1489         return (ret);
 1490 }
 1491 
 1492 static void
 1493 cache_lock_buckets_cel(struct celockstate *cel, struct rwlock *blp1,
 1494     struct rwlock *blp2)
 1495 {
 1496 
 1497         MPASS(cel->blp[0] == NULL);
 1498         MPASS(cel->blp[1] == NULL);
 1499 
 1500         cache_sort(&blp1, &blp2);
 1501 
 1502         if (blp1 != NULL) {
 1503                 rw_wlock(blp1);
 1504                 cel->blp[0] = blp1;
 1505         }
 1506         rw_wlock(blp2);
 1507         cel->blp[1] = blp2;
 1508 }
 1509 
 1510 static void
 1511 cache_unlock_buckets_cel(struct celockstate *cel)
 1512 {
 1513 
 1514         if (cel->blp[0] != NULL)
 1515                 rw_wunlock(cel->blp[0]);
 1516         rw_wunlock(cel->blp[1]);
 1517 }
 1518 
 1519 /*
 1520  * Lock part of the cache affected by the insertion.
 1521  *
 1522  * This means vnodelocks for dvp, vp and the relevant bucketlock.
 1523  * However, insertion can result in removal of an old entry. In this
 1524  * case we have an additional vnode and bucketlock pair to lock. If the
 1525  * entry is negative, ncelock is locked instead of the vnode.
 1526  *
 1527  * That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while
 1528  * preserving the locking order (smaller address first).
 1529  */
 1530 static void
 1531 cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
 1532     uint32_t hash)
 1533 {
 1534         struct namecache *ncp;
 1535         struct rwlock *blps[2];
 1536 
 1537         blps[0] = HASH2BUCKETLOCK(hash);
 1538         for (;;) {
 1539                 blps[1] = NULL;
 1540                 cache_lock_vnodes_cel(cel, dvp, vp);
 1541                 if (vp == NULL || vp->v_type != VDIR)
 1542                         break;
 1543                 ncp = vp->v_cache_dd;
 1544                 if (ncp == NULL)
 1545                         break;
 1546                 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
 1547                         break;
 1548                 MPASS(ncp->nc_dvp == vp);
 1549                 blps[1] = NCP2BUCKETLOCK(ncp);
 1550                 if (ncp->nc_flag & NCF_NEGATIVE)
 1551                         break;
 1552                 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
 1553                         break;
 1554                 /*
 1555                  * All vnodes got re-locked. Re-validate the state and if
 1556                  * nothing changed we are done. Otherwise restart.
 1557                  */
 1558                 if (ncp == vp->v_cache_dd &&
 1559                     (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
 1560                     blps[1] == NCP2BUCKETLOCK(ncp) &&
 1561                     VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
 1562                         break;
 1563                 cache_unlock_vnodes_cel(cel);
 1564                 cel->vlp[0] = NULL;
 1565                 cel->vlp[1] = NULL;
 1566                 cel->vlp[2] = NULL;
 1567         }
 1568         cache_lock_buckets_cel(cel, blps[0], blps[1]);
 1569 }
 1570 
 1571 static void
 1572 cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
 1573     uint32_t hash)
 1574 {
 1575         struct namecache *ncp;
 1576         struct rwlock *blps[2];
 1577 
 1578         blps[0] = HASH2BUCKETLOCK(hash);
 1579         for (;;) {
 1580                 blps[1] = NULL;
 1581                 cache_lock_vnodes_cel(cel, dvp, vp);
 1582                 ncp = dvp->v_cache_dd;
 1583                 if (ncp == NULL)
 1584                         break;
 1585                 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
 1586                         break;
 1587                 MPASS(ncp->nc_dvp == dvp);
 1588                 blps[1] = NCP2BUCKETLOCK(ncp);
 1589                 if (ncp->nc_flag & NCF_NEGATIVE)
 1590                         break;
 1591                 if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
 1592                         break;
 1593                 if (ncp == dvp->v_cache_dd &&
 1594                     (ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
 1595                     blps[1] == NCP2BUCKETLOCK(ncp) &&
 1596                     VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
 1597                         break;
 1598                 cache_unlock_vnodes_cel(cel);
 1599                 cel->vlp[0] = NULL;
 1600                 cel->vlp[1] = NULL;
 1601                 cel->vlp[2] = NULL;
 1602         }
 1603         cache_lock_buckets_cel(cel, blps[0], blps[1]);
 1604 }
 1605 
 1606 static void
 1607 cache_enter_unlock(struct celockstate *cel)
 1608 {
 1609 
 1610         cache_unlock_buckets_cel(cel);
 1611         cache_unlock_vnodes_cel(cel);
 1612 }
 1613 
 1614 /*
 1615  * Add an entry to the cache.
 1616  */
 1617 void
 1618 cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 1619     struct timespec *tsp, struct timespec *dtsp)
 1620 {
 1621         struct celockstate cel;
 1622         struct namecache *ncp, *n2, *ndd;
 1623         struct namecache_ts *ncp_ts, *n2_ts;
 1624         struct nchashhead *ncpp;
 1625         struct neglist *neglist;
 1626         uint32_t hash;
 1627         int flag;
 1628         int len;
 1629         bool neg_locked;
 1630         int lnumcache;
 1631 
 1632         CTR3(KTR_VFS, "cache_enter(%p, %p, %s)", dvp, vp, cnp->cn_nameptr);
 1633         VNASSERT(vp == NULL || (vp->v_iflag & VI_DOOMED) == 0, vp,
 1634             ("cache_enter: Adding a doomed vnode"));
 1635         VNASSERT(dvp == NULL || (dvp->v_iflag & VI_DOOMED) == 0, dvp,
 1636             ("cache_enter: Doomed vnode used as src"));
 1637 
 1638         if (__predict_false(!doingcache))
 1639                 return;
 1640 
 1641         /*
 1642          * Avoid blowout in namecache entries.
 1643          */
 1644         if (__predict_false(numcache >= desiredvnodes * ncsizefactor))
 1645                 return;
 1646 
 1647         cache_celockstate_init(&cel);
 1648         ndd = NULL;
 1649         flag = 0;
 1650         if (cnp->cn_nameptr[0] == '.') {
 1651                 if (cnp->cn_namelen == 1)
 1652                         return;
 1653                 if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
 1654                         len = cnp->cn_namelen;
 1655                         hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
 1656                         cache_enter_lock_dd(&cel, dvp, vp, hash);
 1657                         /*
 1658                          * If dotdot entry already exists, just retarget it
 1659                          * to new parent vnode, otherwise continue with new
 1660                          * namecache entry allocation.
 1661                          */
 1662                         if ((ncp = dvp->v_cache_dd) != NULL &&
 1663                             ncp->nc_flag & NCF_ISDOTDOT) {
 1664                                 KASSERT(ncp->nc_dvp == dvp,
 1665                                     ("wrong isdotdot parent"));
 1666                                 neg_locked = false;
 1667                                 if (ncp->nc_flag & NCF_NEGATIVE || vp == NULL) {
 1668                                         neglist = NCP2NEGLIST(ncp);
 1669                                         mtx_lock(&ncneg_hot.nl_lock);
 1670                                         mtx_lock(&neglist->nl_lock);
 1671                                         neg_locked = true;
 1672                                 }
 1673                                 if (!(ncp->nc_flag & NCF_NEGATIVE)) {
 1674                                         TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst,
 1675                                             ncp, nc_dst);
 1676                                 } else {
 1677                                         cache_negative_remove(ncp, true);
 1678                                 }
 1679                                 if (vp != NULL) {
 1680                                         TAILQ_INSERT_HEAD(&vp->v_cache_dst,
 1681                                             ncp, nc_dst);
 1682                                         ncp->nc_flag &= ~(NCF_NEGATIVE|NCF_HOTNEGATIVE);
 1683                                 } else {
 1684                                         ncp->nc_flag &= ~(NCF_HOTNEGATIVE);
 1685                                         ncp->nc_flag |= NCF_NEGATIVE;
 1686                                         cache_negative_insert(ncp, true);
 1687                                 }
 1688                                 if (neg_locked) {
 1689                                         mtx_unlock(&neglist->nl_lock);
 1690                                         mtx_unlock(&ncneg_hot.nl_lock);
 1691                                 }
 1692                                 ncp->nc_vp = vp;
 1693                                 cache_enter_unlock(&cel);
 1694                                 return;
 1695                         }
 1696                         dvp->v_cache_dd = NULL;
 1697                         cache_enter_unlock(&cel);
 1698                         cache_celockstate_init(&cel);
 1699                         SDT_PROBE3(vfs, namecache, enter, done, dvp, "..", vp);
 1700                         flag = NCF_ISDOTDOT;
 1701                 }
 1702         }
 1703 
 1704         /*
 1705          * Calculate the hash key and setup as much of the new
 1706          * namecache entry as possible before acquiring the lock.
 1707          */
 1708         ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
 1709         ncp->nc_flag = flag;
 1710         ncp->nc_vp = vp;
 1711         if (vp == NULL)
 1712                 ncp->nc_flag |= NCF_NEGATIVE;
 1713         ncp->nc_dvp = dvp;
 1714         if (tsp != NULL) {
 1715                 ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
 1716                 ncp_ts->nc_time = *tsp;
 1717                 ncp_ts->nc_ticks = ticks;
 1718                 ncp_ts->nc_nc.nc_flag |= NCF_TS;
 1719                 if (dtsp != NULL) {
 1720                         ncp_ts->nc_dotdottime = *dtsp;
 1721                         ncp_ts->nc_nc.nc_flag |= NCF_DTS;
 1722                 }
 1723         }
 1724         len = ncp->nc_nlen = cnp->cn_namelen;
 1725         hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
 1726         strlcpy(ncp->nc_name, cnp->cn_nameptr, len + 1);
 1727         cache_enter_lock(&cel, dvp, vp, hash);
 1728 
 1729         /*
 1730          * See if this vnode or negative entry is already in the cache
 1731          * with this name.  This can happen with concurrent lookups of
 1732          * the same path name.
 1733          */
 1734         ncpp = NCHHASH(hash);
 1735         LIST_FOREACH(n2, ncpp, nc_hash) {
 1736                 if (n2->nc_dvp == dvp &&
 1737                     n2->nc_nlen == cnp->cn_namelen &&
 1738                     !bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) {
 1739                         if (tsp != NULL) {
 1740                                 KASSERT((n2->nc_flag & NCF_TS) != 0,
 1741                                     ("no NCF_TS"));
 1742                                 n2_ts = __containerof(n2, struct namecache_ts, nc_nc);
 1743                                 n2_ts->nc_time = ncp_ts->nc_time;
 1744                                 n2_ts->nc_ticks = ncp_ts->nc_ticks;
 1745                                 if (dtsp != NULL) {
 1746                                         n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
 1747                                         if (ncp->nc_flag & NCF_NEGATIVE)
 1748                                                 mtx_lock(&ncneg_hot.nl_lock);
 1749                                         n2_ts->nc_nc.nc_flag |= NCF_DTS;
 1750                                         if (ncp->nc_flag & NCF_NEGATIVE)
 1751                                                 mtx_unlock(&ncneg_hot.nl_lock);
 1752                                 }
 1753                         }
 1754                         goto out_unlock_free;
 1755                 }
 1756         }
 1757 
 1758         if (flag == NCF_ISDOTDOT) {
 1759                 /*
 1760                  * See if we are trying to add .. entry, but some other lookup
 1761                  * has populated v_cache_dd pointer already.
 1762                  */
 1763                 if (dvp->v_cache_dd != NULL)
 1764                         goto out_unlock_free;
 1765                 KASSERT(vp == NULL || vp->v_type == VDIR,
 1766                     ("wrong vnode type %p", vp));
 1767                 dvp->v_cache_dd = ncp;
 1768         }
 1769 
 1770         if (vp != NULL) {
 1771                 if (vp->v_type == VDIR) {
 1772                         if (flag != NCF_ISDOTDOT) {
 1773                                 /*
 1774                                  * For this case, the cache entry maps both the
 1775                                  * directory name in it and the name ".." for the
 1776                                  * directory's parent.
 1777                                  */
 1778                                 if ((ndd = vp->v_cache_dd) != NULL) {
 1779                                         if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
 1780                                                 cache_zap_locked(ndd, false);
 1781                                         else
 1782                                                 ndd = NULL;
 1783                                 }
 1784                                 vp->v_cache_dd = ncp;
 1785                         }
 1786                 } else {
 1787                         vp->v_cache_dd = NULL;
 1788                 }
 1789         }
 1790 
 1791         if (flag != NCF_ISDOTDOT) {
 1792                 if (LIST_EMPTY(&dvp->v_cache_src)) {
 1793                         vhold(dvp);
 1794                         atomic_add_rel_long(&numcachehv, 1);
 1795                 }
 1796                 LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
 1797         }
 1798 
 1799         /*
 1800          * Insert the new namecache entry into the appropriate chain
 1801          * within the cache entries table.
 1802          */
 1803         LIST_INSERT_HEAD(ncpp, ncp, nc_hash);
 1804 
 1805         /*
 1806          * If the entry is "negative", we place it into the
 1807          * "negative" cache queue, otherwise, we place it into the
 1808          * destination vnode's cache entries queue.
 1809          */
 1810         if (vp != NULL) {
 1811                 TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
 1812                 SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
 1813                     vp);
 1814         } else {
 1815                 if (cnp->cn_flags & ISWHITEOUT)
 1816                         ncp->nc_flag |= NCF_WHITE;
 1817                 cache_negative_insert(ncp, false);
 1818                 SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
 1819                     ncp->nc_name);
 1820         }
 1821         cache_enter_unlock(&cel);
 1822         lnumcache = atomic_fetchadd_long(&numcache, 1) + 1;
 1823         if (numneg * ncnegfactor > lnumcache)
 1824                 cache_negative_zap_one();
 1825         cache_free(ndd);
 1826         return;
 1827 out_unlock_free:
 1828         cache_enter_unlock(&cel);
 1829         cache_free(ncp);
 1830         return;
 1831 }
 1832 
 1833 static u_int
 1834 cache_roundup_2(u_int val)
 1835 {
 1836         u_int res;
 1837 
 1838         for (res = 1; res <= val; res <<= 1)
 1839                 continue;
 1840 
 1841         return (res);
 1842 }
 1843 
 1844 /*
 1845  * Name cache initialization, from vfs_init() when we are booting
 1846  */
 1847 static void
 1848 nchinit(void *dummy __unused)
 1849 {
 1850         u_int i;
 1851 
 1852         cache_zone_small = uma_zcreate("S VFS Cache",
 1853             sizeof(struct namecache) + CACHE_PATH_CUTOFF + 1,
 1854             NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
 1855             UMA_ZONE_ZINIT);
 1856         cache_zone_small_ts = uma_zcreate("STS VFS Cache",
 1857             sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1,
 1858             NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
 1859             UMA_ZONE_ZINIT);
 1860         cache_zone_large = uma_zcreate("L VFS Cache",
 1861             sizeof(struct namecache) + NAME_MAX + 1,
 1862             NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
 1863             UMA_ZONE_ZINIT);
 1864         cache_zone_large_ts = uma_zcreate("LTS VFS Cache",
 1865             sizeof(struct namecache_ts) + NAME_MAX + 1,
 1866             NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
 1867             UMA_ZONE_ZINIT);
 1868 
 1869         nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash);
 1870         ncbuckethash = cache_roundup_2(mp_ncpus * 64) - 1;
 1871         if (ncbuckethash > nchash)
 1872                 ncbuckethash = nchash;
 1873         bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE,
 1874             M_WAITOK | M_ZERO);
 1875         for (i = 0; i < numbucketlocks; i++)
 1876                 rw_init_flags(&bucketlocks[i], "ncbuc", RW_DUPOK | RW_RECURSE);
 1877         ncvnodehash = cache_roundup_2(mp_ncpus * 64) - 1;
 1878         vnodelocks = malloc(sizeof(*vnodelocks) * numvnodelocks, M_VFSCACHE,
 1879             M_WAITOK | M_ZERO);
 1880         for (i = 0; i < numvnodelocks; i++)
 1881                 mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK | MTX_RECURSE);
 1882         ncpurgeminvnodes = numbucketlocks;
 1883 
 1884         ncneghash = 3;
 1885         neglists = malloc(sizeof(*neglists) * numneglists, M_VFSCACHE,
 1886             M_WAITOK | M_ZERO);
 1887         for (i = 0; i < numneglists; i++) {
 1888                 mtx_init(&neglists[i].nl_lock, "ncnegl", NULL, MTX_DEF);
 1889                 TAILQ_INIT(&neglists[i].nl_list);
 1890         }
 1891         mtx_init(&ncneg_hot.nl_lock, "ncneglh", NULL, MTX_DEF);
 1892         TAILQ_INIT(&ncneg_hot.nl_list);
 1893 
 1894         mtx_init(&ncneg_shrink_lock, "ncnegs", NULL, MTX_DEF);
 1895 
 1896         numcalls = counter_u64_alloc(M_WAITOK);
 1897         dothits = counter_u64_alloc(M_WAITOK);
 1898         dotdothits = counter_u64_alloc(M_WAITOK);
 1899         numchecks = counter_u64_alloc(M_WAITOK);
 1900         nummiss = counter_u64_alloc(M_WAITOK);
 1901         nummisszap = counter_u64_alloc(M_WAITOK);
 1902         numposzaps = counter_u64_alloc(M_WAITOK);
 1903         numposhits = counter_u64_alloc(M_WAITOK);
 1904         numnegzaps = counter_u64_alloc(M_WAITOK);
 1905         numneghits = counter_u64_alloc(M_WAITOK);
 1906         numfullpathcalls = counter_u64_alloc(M_WAITOK);
 1907         numfullpathfail1 = counter_u64_alloc(M_WAITOK);
 1908         numfullpathfail2 = counter_u64_alloc(M_WAITOK);
 1909         numfullpathfail4 = counter_u64_alloc(M_WAITOK);
 1910         numfullpathfound = counter_u64_alloc(M_WAITOK);
 1911 }
 1912 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);
 1913 
 1914 void
 1915 cache_changesize(int newmaxvnodes)
 1916 {
 1917         struct nchashhead *new_nchashtbl, *old_nchashtbl;
 1918         u_long new_nchash, old_nchash;
 1919         struct namecache *ncp;
 1920         uint32_t hash;
 1921         int i;
 1922 
 1923         newmaxvnodes = cache_roundup_2(newmaxvnodes * 2);
 1924         if (newmaxvnodes < numbucketlocks)
 1925                 newmaxvnodes = numbucketlocks;
 1926 
 1927         new_nchashtbl = hashinit(newmaxvnodes, M_VFSCACHE, &new_nchash);
 1928         /* If same hash table size, nothing to do */
 1929         if (nchash == new_nchash) {
 1930                 free(new_nchashtbl, M_VFSCACHE);
 1931                 return;
 1932         }
 1933         /*
 1934          * Move everything from the old hash table to the new table.
 1935          * None of the namecache entries in the table can be removed
 1936          * because to do so, they have to be removed from the hash table.
 1937          */
 1938         cache_lock_all_vnodes();
 1939         cache_lock_all_buckets();
 1940         old_nchashtbl = nchashtbl;
 1941         old_nchash = nchash;
 1942         nchashtbl = new_nchashtbl;
 1943         nchash = new_nchash;
 1944         for (i = 0; i <= old_nchash; i++) {
 1945                 while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) {
 1946                         hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
 1947                             ncp->nc_dvp);
 1948                         LIST_REMOVE(ncp, nc_hash);
 1949                         LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash);
 1950                 }
 1951         }
 1952         cache_unlock_all_buckets();
 1953         cache_unlock_all_vnodes();
 1954         free(old_nchashtbl, M_VFSCACHE);
 1955 }
 1956 
 1957 /*
 1958  * Invalidate all entries to a particular vnode.
 1959  */
 1960 void
 1961 cache_purge(struct vnode *vp)
 1962 {
 1963         TAILQ_HEAD(, namecache) ncps;
 1964         struct namecache *ncp, *nnp;
 1965         struct mtx *vlp, *vlp2;
 1966 
 1967         CTR1(KTR_VFS, "cache_purge(%p)", vp);
 1968         SDT_PROBE1(vfs, namecache, purge, done, vp);
 1969         if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
 1970             vp->v_cache_dd == NULL)
 1971                 return;
 1972         TAILQ_INIT(&ncps);
 1973         vlp = VP2VNODELOCK(vp);
 1974         vlp2 = NULL;
 1975         mtx_lock(vlp);
 1976 retry:
 1977         while (!LIST_EMPTY(&vp->v_cache_src)) {
 1978                 ncp = LIST_FIRST(&vp->v_cache_src);
 1979                 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
 1980                         goto retry;
 1981                 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
 1982         }
 1983         while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
 1984                 ncp = TAILQ_FIRST(&vp->v_cache_dst);
 1985                 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
 1986                         goto retry;
 1987                 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
 1988         }
 1989         ncp = vp->v_cache_dd;
 1990         if (ncp != NULL) {
 1991                 KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
 1992                    ("lost dotdot link"));
 1993                 if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
 1994                         goto retry;
 1995                 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
 1996         }
 1997         KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
 1998         mtx_unlock(vlp);
 1999         if (vlp2 != NULL)
 2000                 mtx_unlock(vlp2);
 2001         TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
 2002                 cache_free(ncp);
 2003         }
 2004 }
 2005 
 2006 /*
 2007  * Invalidate all negative entries for a particular directory vnode.
 2008  */
 2009 void
 2010 cache_purge_negative(struct vnode *vp)
 2011 {
 2012         TAILQ_HEAD(, namecache) ncps;
 2013         struct namecache *ncp, *nnp;
 2014         struct mtx *vlp;
 2015 
 2016         CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
 2017         SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
 2018         if (LIST_EMPTY(&vp->v_cache_src))
 2019                 return;
 2020         TAILQ_INIT(&ncps);
 2021         vlp = VP2VNODELOCK(vp);
 2022         mtx_lock(vlp);
 2023         LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
 2024                 if (!(ncp->nc_flag & NCF_NEGATIVE))
 2025                         continue;
 2026                 cache_zap_negative_locked_vnode_kl(ncp, vp);
 2027                 TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
 2028         }
 2029         mtx_unlock(vlp);
 2030         TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
 2031                 cache_free(ncp);
 2032         }
 2033 }
 2034 
 2035 /*
 2036  * Flush all entries referencing a particular filesystem.
 2037  */
 2038 void
 2039 cache_purgevfs(struct mount *mp, bool force)
 2040 {
 2041         TAILQ_HEAD(, namecache) ncps;
 2042         struct mtx *vlp1, *vlp2;
 2043         struct rwlock *blp;
 2044         struct nchashhead *bucket;
 2045         struct namecache *ncp, *nnp;
 2046         u_long i, j, n_nchash;
 2047         int error;
 2048 
 2049         /* Scan hash tables for applicable entries */
 2050         SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
 2051         if (!force && mp->mnt_nvnodelistsize <= ncpurgeminvnodes)
 2052                 return;
 2053         TAILQ_INIT(&ncps);
 2054         n_nchash = nchash + 1;
 2055         vlp1 = vlp2 = NULL;
 2056         for (i = 0; i < numbucketlocks; i++) {
 2057                 blp = (struct rwlock *)&bucketlocks[i];
 2058                 rw_wlock(blp);
 2059                 for (j = i; j < n_nchash; j += numbucketlocks) {
 2060 retry:
 2061                         bucket = &nchashtbl[j];
 2062                         LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) {
 2063                                 cache_assert_bucket_locked(ncp, RA_WLOCKED);
 2064                                 if (ncp->nc_dvp->v_mount != mp)
 2065                                         continue;
 2066                                 error = cache_zap_wlocked_bucket_kl(ncp, blp,
 2067                                     &vlp1, &vlp2);
 2068                                 if (error != 0)
 2069                                         goto retry;
 2070                                 TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst);
 2071                         }
 2072                 }
 2073                 rw_wunlock(blp);
 2074                 if (vlp1 == NULL && vlp2 == NULL)
 2075                         cache_maybe_yield();
 2076         }
 2077         if (vlp1 != NULL)
 2078                 mtx_unlock(vlp1);
 2079         if (vlp2 != NULL)
 2080                 mtx_unlock(vlp2);
 2081 
 2082         TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
 2083                 cache_free(ncp);
 2084         }
 2085 }
 2086 
 2087 /*
 2088  * Perform canonical checks and cache lookup and pass on to filesystem
 2089  * through the vop_cachedlookup only if needed.
 2090  */
 2091 
 2092 int
 2093 vfs_cache_lookup(struct vop_lookup_args *ap)
 2094 {
 2095         struct vnode *dvp;
 2096         int error;
 2097         struct vnode **vpp = ap->a_vpp;
 2098         struct componentname *cnp = ap->a_cnp;
 2099         int flags = cnp->cn_flags;
 2100 
 2101         *vpp = NULL;
 2102         dvp = ap->a_dvp;
 2103 
 2104         if (dvp->v_type != VDIR)
 2105                 return (ENOTDIR);
 2106 
 2107         if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
 2108             (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
 2109                 return (EROFS);
 2110 
 2111         error = vn_dir_check_exec(dvp, cnp);
 2112         if (error != 0)
 2113                 return (error);
 2114 
 2115         error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
 2116         if (error == 0)
 2117                 return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
 2118         if (error == -1)
 2119                 return (0);
 2120         return (error);
 2121 }
 2122 
 2123 /*
 2124  * XXX All of these sysctls would probably be more productive dead.
 2125  */
 2126 static int __read_mostly disablecwd;
 2127 SYSCTL_INT(_debug, OID_AUTO, disablecwd, CTLFLAG_RW, &disablecwd, 0,
 2128    "Disable the getcwd syscall");
 2129 
 2130 /* Implementation of the getcwd syscall. */
 2131 int
 2132 sys___getcwd(struct thread *td, struct __getcwd_args *uap)
 2133 {
 2134 
 2135         return (kern___getcwd(td, uap->buf, UIO_USERSPACE, uap->buflen,
 2136             MAXPATHLEN));
 2137 }
 2138 
 2139 int
 2140 kern___getcwd(struct thread *td, char *buf, enum uio_seg bufseg, u_int buflen,
 2141     u_int path_max)
 2142 {
 2143         char *bp, *tmpbuf;
 2144         struct filedesc *fdp;
 2145         struct vnode *cdir, *rdir;
 2146         int error;
 2147 
 2148         if (__predict_false(disablecwd))
 2149                 return (ENODEV);
 2150         if (__predict_false(buflen < 2))
 2151                 return (EINVAL);
 2152         if (buflen > path_max)
 2153                 buflen = path_max;
 2154 
 2155         tmpbuf = malloc(buflen, M_TEMP, M_WAITOK);
 2156         fdp = td->td_proc->p_fd;
 2157         FILEDESC_SLOCK(fdp);
 2158         cdir = fdp->fd_cdir;
 2159         vrefact(cdir);
 2160         rdir = fdp->fd_rdir;
 2161         vrefact(rdir);
 2162         FILEDESC_SUNLOCK(fdp);
 2163         error = vn_fullpath1(td, cdir, rdir, tmpbuf, &bp, buflen);
 2164         vrele(rdir);
 2165         vrele(cdir);
 2166 
 2167         if (!error) {
 2168                 if (bufseg == UIO_SYSSPACE)
 2169                         bcopy(bp, buf, strlen(bp) + 1);
 2170                 else
 2171                         error = copyout(bp, buf, strlen(bp) + 1);
 2172 #ifdef KTRACE
 2173         if (KTRPOINT(curthread, KTR_NAMEI))
 2174                 ktrnamei(bp);
 2175 #endif
 2176         }
 2177         free(tmpbuf, M_TEMP);
 2178         return (error);
 2179 }
 2180 
 2181 /*
 2182  * Thus begins the fullpath magic.
 2183  */
 2184 
 2185 static int __read_mostly disablefullpath;
 2186 SYSCTL_INT(_debug, OID_AUTO, disablefullpath, CTLFLAG_RW, &disablefullpath, 0,
 2187     "Disable the vn_fullpath function");
 2188 
 2189 /*
 2190  * Retrieve the full filesystem path that correspond to a vnode from the name
 2191  * cache (if available)
 2192  */
 2193 int
 2194 vn_fullpath(struct thread *td, struct vnode *vn, char **retbuf, char **freebuf)
 2195 {
 2196         char *buf;
 2197         struct filedesc *fdp;
 2198         struct vnode *rdir;
 2199         int error;
 2200 
 2201         if (__predict_false(disablefullpath))
 2202                 return (ENODEV);
 2203         if (__predict_false(vn == NULL))
 2204                 return (EINVAL);
 2205 
 2206         buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
 2207         fdp = td->td_proc->p_fd;
 2208         FILEDESC_SLOCK(fdp);
 2209         rdir = fdp->fd_rdir;
 2210         vrefact(rdir);
 2211         FILEDESC_SUNLOCK(fdp);
 2212         error = vn_fullpath1(td, vn, rdir, buf, retbuf, MAXPATHLEN);
 2213         vrele(rdir);
 2214 
 2215         if (!error)
 2216                 *freebuf = buf;
 2217         else
 2218                 free(buf, M_TEMP);
 2219         return (error);
 2220 }
 2221 
 2222 /*
 2223  * This function is similar to vn_fullpath, but it attempts to lookup the
 2224  * pathname relative to the global root mount point.  This is required for the
 2225  * auditing sub-system, as audited pathnames must be absolute, relative to the
 2226  * global root mount point.
 2227  */
 2228 int
 2229 vn_fullpath_global(struct thread *td, struct vnode *vn,
 2230     char **retbuf, char **freebuf)
 2231 {
 2232         char *buf;
 2233         int error;
 2234 
 2235         if (__predict_false(disablefullpath))
 2236                 return (ENODEV);
 2237         if (__predict_false(vn == NULL))
 2238                 return (EINVAL);
 2239         buf = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
 2240         error = vn_fullpath1(td, vn, rootvnode, buf, retbuf, MAXPATHLEN);
 2241         if (!error)
 2242                 *freebuf = buf;
 2243         else
 2244                 free(buf, M_TEMP);
 2245         return (error);
 2246 }
 2247 
 2248 int
 2249 vn_vptocnp(struct vnode **vp, struct ucred *cred, char *buf, u_int *buflen)
 2250 {
 2251         struct vnode *dvp;
 2252         struct namecache *ncp;
 2253         struct mtx *vlp;
 2254         int error;
 2255 
 2256         vlp = VP2VNODELOCK(*vp);
 2257         mtx_lock(vlp);
 2258         TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) {
 2259                 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
 2260                         break;
 2261         }
 2262         if (ncp != NULL) {
 2263                 if (*buflen < ncp->nc_nlen) {
 2264                         mtx_unlock(vlp);
 2265                         vrele(*vp);
 2266                         counter_u64_add(numfullpathfail4, 1);
 2267                         error = ENOMEM;
 2268                         SDT_PROBE3(vfs, namecache, fullpath, return, error,
 2269                             vp, NULL);
 2270                         return (error);
 2271                 }
 2272                 *buflen -= ncp->nc_nlen;
 2273                 memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
 2274                 SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
 2275                     ncp->nc_name, vp);
 2276                 dvp = *vp;
 2277                 *vp = ncp->nc_dvp;
 2278                 vref(*vp);
 2279                 mtx_unlock(vlp);
 2280                 vrele(dvp);
 2281                 return (0);
 2282         }
 2283         SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
 2284 
 2285         mtx_unlock(vlp);
 2286         vn_lock(*vp, LK_SHARED | LK_RETRY);
 2287         error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen);
 2288         vput(*vp);
 2289         if (error) {
 2290                 counter_u64_add(numfullpathfail2, 1);
 2291                 SDT_PROBE3(vfs, namecache, fullpath, return,  error, vp, NULL);
 2292                 return (error);
 2293         }
 2294 
 2295         *vp = dvp;
 2296         if (dvp->v_iflag & VI_DOOMED) {
 2297                 /* forced unmount */
 2298                 vrele(dvp);
 2299                 error = ENOENT;
 2300                 SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
 2301                 return (error);
 2302         }
 2303         /*
 2304          * *vp has its use count incremented still.
 2305          */
 2306 
 2307         return (0);
 2308 }
 2309 
 2310 /*
 2311  * The magic behind kern___getcwd() and vn_fullpath().
 2312  */
 2313 static int
 2314 vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 2315     char *buf, char **retbuf, u_int buflen)
 2316 {
 2317         int error, slash_prefixed;
 2318 #ifdef KDTRACE_HOOKS
 2319         struct vnode *startvp = vp;
 2320 #endif
 2321         struct vnode *vp1;
 2322 
 2323         buflen--;
 2324         buf[buflen] = '\0';
 2325         error = 0;
 2326         slash_prefixed = 0;
 2327 
 2328         SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
 2329         counter_u64_add(numfullpathcalls, 1);
 2330         vref(vp);
 2331         if (vp->v_type != VDIR) {
 2332                 error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen);
 2333                 if (error)
 2334                         return (error);
 2335                 if (buflen == 0) {
 2336                         vrele(vp);
 2337                         return (ENOMEM);
 2338                 }
 2339                 buf[--buflen] = '/';
 2340                 slash_prefixed = 1;
 2341         }
 2342         while (vp != rdir && vp != rootvnode) {
 2343                 /*
 2344                  * The vp vnode must be already fully constructed,
 2345                  * since it is either found in namecache or obtained
 2346                  * from VOP_VPTOCNP().  We may test for VV_ROOT safely
 2347                  * without obtaining the vnode lock.
 2348                  */
 2349                 if ((vp->v_vflag & VV_ROOT) != 0) {
 2350                         vn_lock(vp, LK_RETRY | LK_SHARED);
 2351 
 2352                         /*
 2353                          * With the vnode locked, check for races with
 2354                          * unmount, forced or not.  Note that we
 2355                          * already verified that vp is not equal to
 2356                          * the root vnode, which means that
 2357                          * mnt_vnodecovered can be NULL only for the
 2358                          * case of unmount.
 2359                          */
 2360                         if ((vp->v_iflag & VI_DOOMED) != 0 ||
 2361                             (vp1 = vp->v_mount->mnt_vnodecovered) == NULL ||
 2362                             vp1->v_mountedhere != vp->v_mount) {
 2363                                 vput(vp);
 2364                                 error = ENOENT;
 2365                                 SDT_PROBE3(vfs, namecache, fullpath, return,
 2366                                     error, vp, NULL);
 2367                                 break;
 2368                         }
 2369 
 2370                         vref(vp1);
 2371                         vput(vp);
 2372                         vp = vp1;
 2373                         continue;
 2374                 }
 2375                 if (vp->v_type != VDIR) {
 2376                         vrele(vp);
 2377                         counter_u64_add(numfullpathfail1, 1);
 2378                         error = ENOTDIR;
 2379                         SDT_PROBE3(vfs, namecache, fullpath, return,
 2380                             error, vp, NULL);
 2381                         break;
 2382                 }
 2383                 error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen);
 2384                 if (error)
 2385                         break;
 2386                 if (buflen == 0) {
 2387                         vrele(vp);
 2388                         error = ENOMEM;
 2389                         SDT_PROBE3(vfs, namecache, fullpath, return, error,
 2390                             startvp, NULL);
 2391                         break;
 2392                 }
 2393                 buf[--buflen] = '/';
 2394                 slash_prefixed = 1;
 2395         }
 2396         if (error)
 2397                 return (error);
 2398         if (!slash_prefixed) {
 2399                 if (buflen == 0) {
 2400                         vrele(vp);
 2401                         counter_u64_add(numfullpathfail4, 1);
 2402                         SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
 2403                             startvp, NULL);
 2404                         return (ENOMEM);
 2405                 }
 2406                 buf[--buflen] = '/';
 2407         }
 2408         counter_u64_add(numfullpathfound, 1);
 2409         vrele(vp);
 2410 
 2411         SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, buf + buflen);
 2412         *retbuf = buf + buflen;
 2413         return (0);
 2414 }
 2415 
 2416 struct vnode *
 2417 vn_dir_dd_ino(struct vnode *vp)
 2418 {
 2419         struct namecache *ncp;
 2420         struct vnode *ddvp;
 2421         struct mtx *vlp;
 2422 
 2423         ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
 2424         vlp = VP2VNODELOCK(vp);
 2425         mtx_lock(vlp);
 2426         TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
 2427                 if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
 2428                         continue;
 2429                 ddvp = ncp->nc_dvp;
 2430                 vhold(ddvp);
 2431                 mtx_unlock(vlp);
 2432                 if (vget(ddvp, LK_SHARED | LK_NOWAIT | LK_VNHELD, curthread))
 2433                         return (NULL);
 2434                 return (ddvp);
 2435         }
 2436         mtx_unlock(vlp);
 2437         return (NULL);
 2438 }
 2439 
 2440 int
 2441 vn_commname(struct vnode *vp, char *buf, u_int buflen)
 2442 {
 2443         struct namecache *ncp;
 2444         struct mtx *vlp;
 2445         int l;
 2446 
 2447         vlp = VP2VNODELOCK(vp);
 2448         mtx_lock(vlp);
 2449         TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
 2450                 if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
 2451                         break;
 2452         if (ncp == NULL) {
 2453                 mtx_unlock(vlp);
 2454                 return (ENOENT);
 2455         }
 2456         l = min(ncp->nc_nlen, buflen - 1);
 2457         memcpy(buf, ncp->nc_name, l);
 2458         mtx_unlock(vlp);
 2459         buf[l] = '\0';
 2460         return (0);
 2461 }
 2462 
 2463 /* ABI compat shims for old kernel modules. */
 2464 #undef cache_enter
 2465 
 2466 void    cache_enter(struct vnode *dvp, struct vnode *vp,
 2467             struct componentname *cnp);
 2468 
 2469 void
 2470 cache_enter(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
 2471 {
 2472 
 2473         cache_enter_time(dvp, vp, cnp, NULL, NULL);
 2474 }
 2475 
 2476 /*
 2477  * This function updates path string to vnode's full global path
 2478  * and checks the size of the new path string against the pathlen argument.
 2479  *
 2480  * Requires a locked, referenced vnode.
 2481  * Vnode is re-locked on success or ENODEV, otherwise unlocked.
 2482  *
 2483  * If sysctl debug.disablefullpath is set, ENODEV is returned,
 2484  * vnode is left locked and path remain untouched.
 2485  *
 2486  * If vp is a directory, the call to vn_fullpath_global() always succeeds
 2487  * because it falls back to the ".." lookup if the namecache lookup fails.
 2488  */
 2489 int
 2490 vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path,
 2491     u_int pathlen)
 2492 {
 2493         struct nameidata nd;
 2494         struct vnode *vp1;
 2495         char *rpath, *fbuf;
 2496         int error;
 2497 
 2498         ASSERT_VOP_ELOCKED(vp, __func__);
 2499 
 2500         /* Return ENODEV if sysctl debug.disablefullpath==1 */
 2501         if (__predict_false(disablefullpath))
 2502                 return (ENODEV);
 2503 
 2504         /* Construct global filesystem path from vp. */
 2505         VOP_UNLOCK(vp, 0);
 2506         error = vn_fullpath_global(td, vp, &rpath, &fbuf);
 2507 
 2508         if (error != 0) {
 2509                 vrele(vp);
 2510                 return (error);
 2511         }
 2512 
 2513         if (strlen(rpath) >= pathlen) {
 2514                 vrele(vp);
 2515                 error = ENAMETOOLONG;
 2516                 goto out;
 2517         }
 2518 
 2519         /*
 2520          * Re-lookup the vnode by path to detect a possible rename.
 2521          * As a side effect, the vnode is relocked.
 2522          * If vnode was renamed, return ENOENT.
 2523          */
 2524         NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
 2525             UIO_SYSSPACE, path, td);
 2526         error = namei(&nd);
 2527         if (error != 0) {
 2528                 vrele(vp);
 2529                 goto out;
 2530         }
 2531         NDFREE(&nd, NDF_ONLY_PNBUF);
 2532         vp1 = nd.ni_vp;
 2533         vrele(vp);
 2534         if (vp1 == vp)
 2535                 strcpy(path, rpath);
 2536         else {
 2537                 vput(vp1);
 2538                 error = ENOENT;
 2539         }
 2540 
 2541 out:
 2542         free(fbuf, M_TEMP);
 2543         return (error);
 2544 }

Cache object: ae5a9b3b69fb6ceeeddb863eb57bc831


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.