[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_pdpolicy_clockpro.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_pdpolicy_clockpro.c,v 1.16 2011/02/05 13:33:47 yamt Exp $  */
    2 
    3 /*-
    4  * Copyright (c)2005, 2006 YAMAMOTO Takashi,
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  */
   28 
   29 /*
   30  * CLOCK-Pro replacement policy:
   31  *      http://www.cs.wm.edu/hpcs/WWW/HTML/publications/abs05-3.html
   32  *
   33  * approximation of the list of non-resident pages using hash:
   34  *      http://linux-mm.org/ClockProApproximation
   35  */
   36 
   37 /* #define      CLOCKPRO_DEBUG */
   38 
   39 #if defined(PDSIM)
   40 
   41 #include "pdsim.h"
   42 
   43 #else /* defined(PDSIM) */
   44 
   45 #include <sys/cdefs.h>
   46 __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.16 2011/02/05 13:33:47 yamt Exp $");
   47 
   48 #include "opt_ddb.h"
   49 
   50 #include <sys/param.h>
   51 #include <sys/proc.h>
   52 #include <sys/systm.h>
   53 #include <sys/kernel.h>
   54 #include <sys/hash.h>
   55 
   56 #include <uvm/uvm.h>
   57 #include <uvm/uvm_pdpolicy.h>
   58 #include <uvm/uvm_pdpolicy_impl.h>
   59 
   60 #if ((__STDC_VERSION__ - 0) >= 199901L)
   61 #define DPRINTF(...)    /* nothing */
   62 #define WARN(...)       printf(__VA_ARGS__)
   63 #else /* ((__STDC_VERSION__ - 0) >= 199901L) */
   64 #define DPRINTF(a...)   /* nothing */   /* GCC */
   65 #define WARN(a...)      printf(a)
   66 #endif /* ((__STDC_VERSION__ - 0) >= 199901L) */
   67 
   68 #define dump(a)         /* nothing */
   69 
   70 #undef  USEONCE2
   71 #define LISTQ
   72 #undef  ADAPTIVE
   73 
   74 #endif /* defined(PDSIM) */
   75 
   76 #if !defined(CLOCKPRO_COLDPCT)
   77 #define CLOCKPRO_COLDPCT        10
   78 #endif /* !defined(CLOCKPRO_COLDPCT) */
   79 
   80 #define CLOCKPRO_COLDPCTMAX     90
   81 
   82 #if !defined(CLOCKPRO_HASHFACTOR)
   83 #define CLOCKPRO_HASHFACTOR     2
   84 #endif /* !defined(CLOCKPRO_HASHFACTOR) */
   85 
   86 #define CLOCKPRO_NEWQMIN        ((1024 * 1024) >> PAGE_SHIFT)   /* XXX */
   87 
   88 int clockpro_hashfactor = CLOCKPRO_HASHFACTOR;
   89 
   90 PDPOL_EVCNT_DEFINE(nresrecordobj)
   91 PDPOL_EVCNT_DEFINE(nresrecordanon)
   92 PDPOL_EVCNT_DEFINE(nreslookupobj)
   93 PDPOL_EVCNT_DEFINE(nreslookupanon)
   94 PDPOL_EVCNT_DEFINE(nresfoundobj)
   95 PDPOL_EVCNT_DEFINE(nresfoundanon)
   96 PDPOL_EVCNT_DEFINE(nresanonfree)
   97 PDPOL_EVCNT_DEFINE(nresconflict)
   98 PDPOL_EVCNT_DEFINE(nresoverwritten)
   99 PDPOL_EVCNT_DEFINE(nreshandhot)
  100 
  101 PDPOL_EVCNT_DEFINE(hhottakeover)
  102 PDPOL_EVCNT_DEFINE(hhotref)
  103 PDPOL_EVCNT_DEFINE(hhotunref)
  104 PDPOL_EVCNT_DEFINE(hhotcold)
  105 PDPOL_EVCNT_DEFINE(hhotcoldtest)
  106 
  107 PDPOL_EVCNT_DEFINE(hcoldtakeover)
  108 PDPOL_EVCNT_DEFINE(hcoldref)
  109 PDPOL_EVCNT_DEFINE(hcoldunref)
  110 PDPOL_EVCNT_DEFINE(hcoldreftest)
  111 PDPOL_EVCNT_DEFINE(hcoldunreftest)
  112 PDPOL_EVCNT_DEFINE(hcoldunreftestspeculative)
  113 PDPOL_EVCNT_DEFINE(hcoldhot)
  114 
  115 PDPOL_EVCNT_DEFINE(speculativeenqueue)
  116 PDPOL_EVCNT_DEFINE(speculativehit1)
  117 PDPOL_EVCNT_DEFINE(speculativehit2)
  118 PDPOL_EVCNT_DEFINE(speculativemiss)
  119 
  120 #define PQ_REFERENCED   PQ_PRIVATE1
  121 #define PQ_HOT          PQ_PRIVATE2
  122 #define PQ_TEST         PQ_PRIVATE3
  123 #define PQ_INITIALREF   PQ_PRIVATE4
  124 #if PQ_PRIVATE6 != PQ_PRIVATE5 * 2 || PQ_PRIVATE7 != PQ_PRIVATE6 * 2
  125 #error PQ_PRIVATE
  126 #endif
  127 #define PQ_QMASK        (PQ_PRIVATE5|PQ_PRIVATE6|PQ_PRIVATE7)
  128 #define PQ_QFACTOR      PQ_PRIVATE5
  129 #define PQ_SPECULATIVE  PQ_PRIVATE8
  130 
  131 #define CLOCKPRO_NOQUEUE        0
  132 #define CLOCKPRO_NEWQ           1       /* small queue to clear initial ref. */
  133 #if defined(LISTQ)
  134 #define CLOCKPRO_COLDQ          2
  135 #define CLOCKPRO_HOTQ           3
  136 #else /* defined(LISTQ) */
  137 #define CLOCKPRO_COLDQ          (2 + coldqidx)  /* XXX */
  138 #define CLOCKPRO_HOTQ           (3 - coldqidx)  /* XXX */
  139 #endif /* defined(LISTQ) */
  140 #define CLOCKPRO_LISTQ          4
  141 #define CLOCKPRO_NQUEUE         4
  142 
  143 static inline void
  144 clockpro_setq(struct vm_page *pg, int qidx)
  145 {
  146         KASSERT(qidx >= CLOCKPRO_NOQUEUE);
  147         KASSERT(qidx <= CLOCKPRO_NQUEUE);
  148 
  149         pg->pqflags = (pg->pqflags & ~PQ_QMASK) | (qidx * PQ_QFACTOR);
  150 }
  151 
  152 static inline int
  153 clockpro_getq(struct vm_page *pg)
  154 {
  155         int qidx;
  156 
  157         qidx = (pg->pqflags & PQ_QMASK) / PQ_QFACTOR;
  158         KASSERT(qidx >= CLOCKPRO_NOQUEUE);
  159         KASSERT(qidx <= CLOCKPRO_NQUEUE);
  160         return qidx;
  161 }
  162 
  163 typedef struct {
  164         struct pglist q_q;
  165         int q_len;
  166 } pageq_t;
  167 
  168 struct clockpro_state {
  169         int s_npages;
  170         int s_coldtarget;
  171         int s_ncold;
  172 
  173         int s_newqlenmax;
  174         pageq_t s_q[CLOCKPRO_NQUEUE];
  175 
  176         struct uvm_pctparam s_coldtargetpct;
  177 };
  178 
  179 static pageq_t *
  180 clockpro_queue(struct clockpro_state *s, int qidx)
  181 {
  182 
  183         KASSERT(CLOCKPRO_NOQUEUE < qidx);
  184         KASSERT(qidx <= CLOCKPRO_NQUEUE);
  185 
  186         return &s->s_q[qidx - 1];
  187 }
  188 
  189 #if !defined(LISTQ)
  190 
  191 static int coldqidx;
  192 
  193 static void
  194 clockpro_switchqueue(void)
  195 {
  196 
  197         coldqidx = 1 - coldqidx;
  198 }
  199 
  200 #endif /* !defined(LISTQ) */
  201 
  202 static struct clockpro_state clockpro;
  203 static struct clockpro_scanstate {
  204         int ss_nscanned;
  205 } scanstate;
  206 
  207 /* ---------------------------------------- */
  208 
  209 static void
  210 pageq_init(pageq_t *q)
  211 {
  212 
  213         TAILQ_INIT(&q->q_q);
  214         q->q_len = 0;
  215 }
  216 
  217 static int
  218 pageq_len(const pageq_t *q)
  219 {
  220 
  221         return q->q_len;
  222 }
  223 
  224 static struct vm_page *
  225 pageq_first(const pageq_t *q)
  226 {
  227 
  228         return TAILQ_FIRST(&q->q_q);
  229 }
  230 
  231 static void
  232 pageq_insert_tail(pageq_t *q, struct vm_page *pg)
  233 {
  234 
  235         TAILQ_INSERT_TAIL(&q->q_q, pg, pageq.queue);
  236         q->q_len++;
  237 }
  238 
  239 #if defined(LISTQ)
  240 static void
  241 pageq_insert_head(pageq_t *q, struct vm_page *pg)
  242 {
  243 
  244         TAILQ_INSERT_HEAD(&q->q_q, pg, pageq.queue);
  245         q->q_len++;
  246 }
  247 #endif
  248 
  249 static void
  250 pageq_remove(pageq_t *q, struct vm_page *pg)
  251 {
  252 
  253 #if 1
  254         KASSERT(clockpro_queue(&clockpro, clockpro_getq(pg)) == q);
  255 #endif
  256         KASSERT(q->q_len > 0);
  257         TAILQ_REMOVE(&q->q_q, pg, pageq.queue);
  258         q->q_len--;
  259 }
  260 
  261 static struct vm_page *
  262 pageq_remove_head(pageq_t *q)
  263 {
  264         struct vm_page *pg;
  265 
  266         pg = TAILQ_FIRST(&q->q_q);
  267         if (pg == NULL) {
  268                 KASSERT(q->q_len == 0);
  269                 return NULL;
  270         }
  271         pageq_remove(q, pg);
  272         return pg;
  273 }
  274 
  275 /* ---------------------------------------- */
  276 
  277 static void
  278 clockpro_insert_tail(struct clockpro_state *s, int qidx, struct vm_page *pg)
  279 {
  280         pageq_t *q = clockpro_queue(s, qidx);
  281         
  282         clockpro_setq(pg, qidx);
  283         pageq_insert_tail(q, pg);
  284 }
  285 
  286 #if defined(LISTQ)
  287 static void
  288 clockpro_insert_head(struct clockpro_state *s, int qidx, struct vm_page *pg)
  289 {
  290         pageq_t *q = clockpro_queue(s, qidx);
  291         
  292         clockpro_setq(pg, qidx);
  293         pageq_insert_head(q, pg);
  294 }
  295 
  296 #endif
  297 /* ---------------------------------------- */
  298 
  299 typedef uint32_t nonres_cookie_t;
  300 #define NONRES_COOKIE_INVAL     0
  301 
  302 typedef uintptr_t objid_t;
  303 
  304 /*
  305  * XXX maybe these hash functions need reconsideration,
  306  * given that hash distribution is critical here.
  307  */
  308 
  309 static uint32_t
  310 pageidentityhash1(objid_t obj, off_t idx)
  311 {
  312         uint32_t hash = HASH32_BUF_INIT;
  313 
  314 #if 1
  315         hash = hash32_buf(&idx, sizeof(idx), hash);
  316         hash = hash32_buf(&obj, sizeof(obj), hash);
  317 #else
  318         hash = hash32_buf(&obj, sizeof(obj), hash);
  319         hash = hash32_buf(&idx, sizeof(idx), hash);
  320 #endif
  321         return hash;
  322 }
  323 
  324 static uint32_t
  325 pageidentityhash2(objid_t obj, off_t idx)
  326 {
  327         uint32_t hash = HASH32_BUF_INIT;
  328 
  329         hash = hash32_buf(&obj, sizeof(obj), hash);
  330         hash = hash32_buf(&idx, sizeof(idx), hash);
  331         return hash;
  332 }
  333 
  334 static nonres_cookie_t
  335 calccookie(objid_t obj, off_t idx)
  336 {
  337         uint32_t hash = pageidentityhash2(obj, idx);
  338         nonres_cookie_t cookie = hash;
  339 
  340         if (__predict_false(cookie == NONRES_COOKIE_INVAL)) {
  341                 cookie++; /* XXX */
  342         }
  343         return cookie;
  344 }
  345 
  346 #define BUCKETSIZE      14
  347 struct bucket {
  348         int cycle;
  349         int cur;
  350         nonres_cookie_t pages[BUCKETSIZE];
  351 };
  352 static int cycle_target;
  353 static int cycle_target_frac;
  354 
  355 static struct bucket static_bucket;
  356 static struct bucket *buckets = &static_bucket;
  357 static size_t hashsize = 1;
  358 
  359 static int coldadj;
  360 #define COLDTARGET_ADJ(d)       coldadj += (d)
  361 
  362 #if defined(PDSIM)
  363 
  364 static void *
  365 clockpro_hashalloc(int n)
  366 {
  367         size_t allocsz = sizeof(*buckets) * n;
  368 
  369         return malloc(allocsz);
  370 }
  371 
  372 static void
  373 clockpro_hashfree(void *p, int n)
  374 {
  375 
  376         free(p);
  377 }
  378 
  379 #else /* defined(PDSIM) */
  380 
  381 static void *
  382 clockpro_hashalloc(int n)
  383 {
  384         size_t allocsz = round_page(sizeof(*buckets) * n);
  385 
  386         return (void *)uvm_km_alloc(kernel_map, allocsz, 0, UVM_KMF_WIRED);
  387 }
  388 
  389 static void
  390 clockpro_hashfree(void *p, int n)
  391 {
  392         size_t allocsz = round_page(sizeof(*buckets) * n);
  393 
  394         uvm_km_free(kernel_map, (vaddr_t)p, allocsz, UVM_KMF_WIRED);
  395 }
  396 
  397 #endif /* defined(PDSIM) */
  398 
  399 static void
  400 clockpro_hashinit(uint64_t n)
  401 {
  402         struct bucket *newbuckets;
  403         struct bucket *oldbuckets;
  404         size_t sz;
  405         size_t oldsz;
  406         int i;
  407 
  408         sz = howmany(n, BUCKETSIZE);
  409         sz *= clockpro_hashfactor;
  410         newbuckets = clockpro_hashalloc(sz);
  411         if (newbuckets == NULL) {
  412                 panic("%s: allocation failure", __func__);
  413         }
  414         for (i = 0; i < sz; i++) {
  415                 struct bucket *b = &newbuckets[i];
  416                 int j;
  417 
  418                 b->cycle = cycle_target;
  419                 b->cur = 0;
  420                 for (j = 0; j < BUCKETSIZE; j++) {
  421                         b->pages[j] = NONRES_COOKIE_INVAL;
  422                 }
  423         }
  424         /* XXX lock */
  425         oldbuckets = buckets;
  426         oldsz = hashsize;
  427         buckets = newbuckets;
  428         hashsize = sz;
  429         /* XXX unlock */
  430         if (oldbuckets != &static_bucket) {
  431                 clockpro_hashfree(oldbuckets, oldsz);
  432         }
  433 }
  434 
  435 static struct bucket *
  436 nonresident_getbucket(objid_t obj, off_t idx)
  437 {
  438         uint32_t hash;
  439 
  440         hash = pageidentityhash1(obj, idx);
  441         return &buckets[hash % hashsize];
  442 }
  443 
  444 static void
  445 nonresident_rotate(struct bucket *b)
  446 {
  447         const int target = cycle_target;
  448         const int cycle = b->cycle;
  449         int cur;
  450         int todo;
  451 
  452         todo = target - cycle;
  453         if (todo >= BUCKETSIZE * 2) {
  454                 todo = (todo % BUCKETSIZE) + BUCKETSIZE;
  455         }
  456         cur = b->cur;
  457         while (todo > 0) {
  458                 if (b->pages[cur] != NONRES_COOKIE_INVAL) {
  459                         PDPOL_EVCNT_INCR(nreshandhot);
  460                         COLDTARGET_ADJ(-1);
  461                 }
  462                 b->pages[cur] = NONRES_COOKIE_INVAL;
  463                 cur++;
  464                 if (cur == BUCKETSIZE) {
  465                         cur = 0;
  466                 }
  467                 todo--;
  468         }
  469         b->cycle = target;
  470         b->cur = cur;
  471 }
  472 
  473 static bool
  474 nonresident_lookupremove(objid_t obj, off_t idx)
  475 {
  476         struct bucket *b = nonresident_getbucket(obj, idx);
  477         nonres_cookie_t cookie = calccookie(obj, idx);
  478         int i;
  479 
  480         nonresident_rotate(b);
  481         for (i = 0; i < BUCKETSIZE; i++) {
  482                 if (b->pages[i] == cookie) {
  483                         b->pages[i] = NONRES_COOKIE_INVAL;
  484                         return true;
  485                 }
  486         }
  487         return false;
  488 }
  489 
  490 static objid_t
  491 pageobj(struct vm_page *pg)
  492 {
  493         const void *obj;
  494 
  495         /*
  496          * XXX object pointer is often freed and reused for unrelated object.
  497          * for vnodes, it would be better to use something like
  498          * a hash of fsid/fileid/generation.
  499          */
  500 
  501         obj = pg->uobject;
  502         if (obj == NULL) {
  503                 obj = pg->uanon;
  504                 KASSERT(obj != NULL);
  505         }
  506         return (objid_t)obj;
  507 }
  508 
  509 static off_t
  510 pageidx(struct vm_page *pg)
  511 {
  512 
  513         KASSERT((pg->offset & PAGE_MASK) == 0);
  514         return pg->offset >> PAGE_SHIFT;
  515 }
  516 
  517 static bool
  518 nonresident_pagelookupremove(struct vm_page *pg)
  519 {
  520         bool found = nonresident_lookupremove(pageobj(pg), pageidx(pg));
  521 
  522         if (pg->uobject) {
  523                 PDPOL_EVCNT_INCR(nreslookupobj);
  524         } else {
  525                 PDPOL_EVCNT_INCR(nreslookupanon);
  526         }
  527         if (found) {
  528                 if (pg->uobject) {
  529                         PDPOL_EVCNT_INCR(nresfoundobj);
  530                 } else {
  531                         PDPOL_EVCNT_INCR(nresfoundanon);
  532                 }
  533         }
  534         return found;
  535 }
  536 
  537 static void
  538 nonresident_pagerecord(struct vm_page *pg)
  539 {
  540         objid_t obj = pageobj(pg);
  541         off_t idx = pageidx(pg);
  542         struct bucket *b = nonresident_getbucket(obj, idx);
  543         nonres_cookie_t cookie = calccookie(obj, idx);
  544 
  545 #if defined(DEBUG)
  546         int i;
  547 
  548         for (i = 0; i < BUCKETSIZE; i++) {
  549                 if (b->pages[i] == cookie) {
  550                         PDPOL_EVCNT_INCR(nresconflict);
  551                 }
  552         }
  553 #endif /* defined(DEBUG) */
  554 
  555         if (pg->uobject) {
  556                 PDPOL_EVCNT_INCR(nresrecordobj);
  557         } else {
  558                 PDPOL_EVCNT_INCR(nresrecordanon);
  559         }
  560         nonresident_rotate(b);
  561         if (b->pages[b->cur] != NONRES_COOKIE_INVAL) {
  562                 PDPOL_EVCNT_INCR(nresoverwritten);
  563                 COLDTARGET_ADJ(-1);
  564         }
  565         b->pages[b->cur] = cookie;
  566         b->cur = (b->cur + 1) % BUCKETSIZE;
  567 }
  568 
  569 /* ---------------------------------------- */
  570 
  571 #if defined(CLOCKPRO_DEBUG)
  572 static void
  573 check_sanity(void)
  574 {
  575 }
  576 #else /* defined(CLOCKPRO_DEBUG) */
  577 #define check_sanity()  /* nothing */
  578 #endif /* defined(CLOCKPRO_DEBUG) */
  579 
  580 static void
  581 clockpro_reinit(void)
  582 {
  583 
  584         clockpro_hashinit(uvmexp.npages);
  585 }
  586 
  587 static void
  588 clockpro_init(void)
  589 {
  590         struct clockpro_state *s = &clockpro;
  591         int i;
  592 
  593         for (i = 0; i < CLOCKPRO_NQUEUE; i++) {
  594                 pageq_init(&s->s_q[i]);
  595         }
  596         s->s_newqlenmax = 1;
  597         s->s_coldtarget = 1;
  598         uvm_pctparam_init(&s->s_coldtargetpct, CLOCKPRO_COLDPCT, NULL);
  599 }
  600 
  601 static void
  602 clockpro_tune(void)
  603 {
  604         struct clockpro_state *s = &clockpro;
  605         int coldtarget;
  606 
  607 #if defined(ADAPTIVE)
  608         int coldmax = s->s_npages * CLOCKPRO_COLDPCTMAX / 100;
  609         int coldmin = 1;
  610 
  611         coldtarget = s->s_coldtarget;
  612         if (coldtarget + coldadj < coldmin) {
  613                 coldadj = coldmin - coldtarget;
  614         } else if (coldtarget + coldadj > coldmax) {
  615                 coldadj = coldmax - coldtarget;
  616         }
  617         coldtarget += coldadj;
  618 #else /* defined(ADAPTIVE) */
  619         coldtarget = UVM_PCTPARAM_APPLY(&s->s_coldtargetpct, s->s_npages);
  620         if (coldtarget < 1) {
  621                 coldtarget = 1;
  622         }
  623 #endif /* defined(ADAPTIVE) */
  624 
  625         s->s_coldtarget = coldtarget;
  626         s->s_newqlenmax = coldtarget / 4;
  627         if (s->s_newqlenmax < CLOCKPRO_NEWQMIN) {
  628                 s->s_newqlenmax = CLOCKPRO_NEWQMIN;
  629         }
  630 }
  631 
  632 static void
  633 clockpro_movereferencebit(struct vm_page *pg)
  634 {
  635         bool referenced;
  636 
  637         referenced = pmap_clear_reference(pg);
  638         if (referenced) {
  639                 pg->pqflags |= PQ_REFERENCED;
  640         }
  641 }
  642 
  643 static void
  644 clockpro_clearreferencebit(struct vm_page *pg)
  645 {
  646 
  647         clockpro_movereferencebit(pg);
  648         pg->pqflags &= ~PQ_REFERENCED;
  649 }
  650 
  651 static void
  652 clockpro___newqrotate(int len)
  653 {
  654         struct clockpro_state * const s = &clockpro;
  655         pageq_t * const newq = clockpro_queue(s, CLOCKPRO_NEWQ);
  656         struct vm_page *pg;
  657 
  658         while (pageq_len(newq) > len) {
  659                 pg = pageq_remove_head(newq);
  660                 KASSERT(pg != NULL);
  661                 KASSERT(clockpro_getq(pg) == CLOCKPRO_NEWQ);
  662                 if ((pg->pqflags & PQ_INITIALREF) != 0) {
  663                         clockpro_clearreferencebit(pg);
  664                         pg->pqflags &= ~PQ_INITIALREF;
  665                 }
  666                 /* place at the list head */
  667                 clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
  668         }
  669 }
  670 
  671 static void
  672 clockpro_newqrotate(void)
  673 {
  674         struct clockpro_state * const s = &clockpro;
  675 
  676         check_sanity();
  677         clockpro___newqrotate(s->s_newqlenmax);
  678         check_sanity();
  679 }
  680 
  681 static void
  682 clockpro_newqflush(int n)
  683 {
  684 
  685         check_sanity();
  686         clockpro___newqrotate(n);
  687         check_sanity();
  688 }
  689 
  690 static void
  691 clockpro_newqflushone(void)
  692 {
  693         struct clockpro_state * const s = &clockpro;
  694 
  695         clockpro_newqflush(
  696             MAX(pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) - 1, 0));
  697 }
  698 
  699 /*
  700  * our "tail" is called "list-head" in the paper.
  701  */
  702 
  703 static void
  704 clockpro___enqueuetail(struct vm_page *pg)
  705 {
  706         struct clockpro_state * const s = &clockpro;
  707 
  708         KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);
  709 
  710         check_sanity();
  711 #if !defined(USEONCE2)
  712         clockpro_insert_tail(s, CLOCKPRO_NEWQ, pg);
  713         clockpro_newqrotate();
  714 #else /* !defined(USEONCE2) */
  715 #if defined(LISTQ)
  716         KASSERT((pg->pqflags & PQ_REFERENCED) == 0);
  717 #endif /* defined(LISTQ) */
  718         clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
  719 #endif /* !defined(USEONCE2) */
  720         check_sanity();
  721 }
  722 
  723 static void
  724 clockpro_pageenqueue(struct vm_page *pg)
  725 {
  726         struct clockpro_state * const s = &clockpro;
  727         bool hot;
  728         bool speculative = (pg->pqflags & PQ_SPECULATIVE) != 0; /* XXX */
  729 
  730         KASSERT((~pg->pqflags & (PQ_INITIALREF|PQ_SPECULATIVE)) != 0);
  731         KASSERT(mutex_owned(&uvm_pageqlock));
  732         check_sanity();
  733         KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);
  734         s->s_npages++;
  735         pg->pqflags &= ~(PQ_HOT|PQ_TEST);
  736         if (speculative) {
  737                 hot = false;
  738                 PDPOL_EVCNT_INCR(speculativeenqueue);
  739         } else {
  740                 hot = nonresident_pagelookupremove(pg);
  741                 if (hot) {
  742                         COLDTARGET_ADJ(1);
  743                 }
  744         }
  745 
  746         /*
  747          * consider mmap'ed file:
  748          *
  749          * - read-ahead enqueues a page.
  750          *
  751          * - on the following read-ahead hit, the fault handler activates it.
  752          *
  753          * - finally, the userland code which caused the above fault
  754          *   actually accesses the page.  it makes its reference bit set.
  755          *
  756          * we want to count the above as a single access, rather than
  757          * three accesses with short reuse distances.
  758          */
  759 
  760 #if defined(USEONCE2)
  761         pg->pqflags &= ~PQ_INITIALREF;
  762         if (hot) {
  763                 pg->pqflags |= PQ_TEST;
  764         }
  765         s->s_ncold++;
  766         clockpro_clearreferencebit(pg);
  767         clockpro___enqueuetail(pg);
  768 #else /* defined(USEONCE2) */
  769         if (speculative) {
  770                 s->s_ncold++;
  771         } else if (hot) {
  772                 pg->pqflags |= PQ_HOT;
  773         } else {
  774                 pg->pqflags |= PQ_TEST;
  775                 s->s_ncold++;
  776         }
  777         clockpro___enqueuetail(pg);
  778 #endif /* defined(USEONCE2) */
  779         KASSERT(s->s_ncold <= s->s_npages);
  780 }
  781 
  782 static pageq_t *
  783 clockpro_pagequeue(struct vm_page *pg)
  784 {
  785         struct clockpro_state * const s = &clockpro;
  786         int qidx;
  787 
  788         qidx = clockpro_getq(pg);
  789         KASSERT(qidx != CLOCKPRO_NOQUEUE);
  790 
  791         return clockpro_queue(s, qidx);
  792 }
  793 
  794 static void
  795 clockpro_pagedequeue(struct vm_page *pg)
  796 {
  797         struct clockpro_state * const s = &clockpro;
  798         pageq_t *q;
  799 
  800         KASSERT(s->s_npages > 0);
  801         check_sanity();
  802         q = clockpro_pagequeue(pg);
  803         pageq_remove(q, pg);
  804         check_sanity();
  805         clockpro_setq(pg, CLOCKPRO_NOQUEUE);
  806         if ((pg->pqflags & PQ_HOT) == 0) {
  807                 KASSERT(s->s_ncold > 0);
  808                 s->s_ncold--;
  809         }
  810         KASSERT(s->s_npages > 0);
  811         s->s_npages--;
  812         check_sanity();
  813 }
  814 
  815 static void
  816 clockpro_pagerequeue(struct vm_page *pg)
  817 {
  818         struct clockpro_state * const s = &clockpro;
  819         int qidx;
  820 
  821         qidx = clockpro_getq(pg);
  822         KASSERT(qidx == CLOCKPRO_HOTQ || qidx == CLOCKPRO_COLDQ);
  823         pageq_remove(clockpro_queue(s, qidx), pg);
  824         check_sanity();
  825         clockpro_setq(pg, CLOCKPRO_NOQUEUE);
  826 
  827         clockpro___enqueuetail(pg);
  828 }
  829 
  830 static void
  831 handhot_endtest(struct vm_page *pg)
  832 {
  833 
  834         KASSERT((pg->pqflags & PQ_HOT) == 0);
  835         if ((pg->pqflags & PQ_TEST) != 0) {
  836                 PDPOL_EVCNT_INCR(hhotcoldtest);
  837                 COLDTARGET_ADJ(-1);
  838                 pg->pqflags &= ~PQ_TEST;
  839         } else {
  840                 PDPOL_EVCNT_INCR(hhotcold);
  841         }
  842 }
  843 
  844 static void
  845 handhot_advance(void)
  846 {
  847         struct clockpro_state * const s = &clockpro;
  848         struct vm_page *pg;
  849         pageq_t *hotq;
  850         int hotqlen;
  851 
  852         clockpro_tune();
  853 
  854         dump("hot called");
  855         if (s->s_ncold >= s->s_coldtarget) {
  856                 return;
  857         }
  858         hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
  859 again:
  860         pg = pageq_first(hotq);
  861         if (pg == NULL) {
  862                 DPRINTF("%s: HHOT TAKEOVER\n", __func__);
  863                 dump("hhottakeover");
  864                 PDPOL_EVCNT_INCR(hhottakeover);
  865 #if defined(LISTQ)
  866                 while (/* CONSTCOND */ 1) {
  867                         pageq_t *coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
  868 
  869                         pg = pageq_first(coldq);
  870                         if (pg == NULL) {
  871                                 clockpro_newqflushone();
  872                                 pg = pageq_first(coldq);
  873                                 if (pg == NULL) {
  874                                         WARN("hhot: no page?\n");
  875                                         return;
  876                                 }
  877                         }
  878                         KASSERT(clockpro_pagequeue(pg) == coldq);
  879                         pageq_remove(coldq, pg);
  880                         check_sanity();
  881                         if ((pg->pqflags & PQ_HOT) == 0) {
  882                                 handhot_endtest(pg);
  883                                 clockpro_insert_tail(s, CLOCKPRO_LISTQ, pg);
  884                         } else {
  885                                 clockpro_insert_head(s, CLOCKPRO_HOTQ, pg);
  886                                 break;
  887                         }
  888                 }
  889 #else /* defined(LISTQ) */
  890                 clockpro_newqflush(0); /* XXX XXX */
  891                 clockpro_switchqueue();
  892                 hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
  893                 goto again;
  894 #endif /* defined(LISTQ) */
  895         }
  896 
  897         KASSERT(clockpro_pagequeue(pg) == hotq);
  898 
  899         /*
  900          * terminate test period of nonresident pages by cycling them.
  901          */
  902 
  903         cycle_target_frac += BUCKETSIZE;
  904         hotqlen = pageq_len(hotq);
  905         while (cycle_target_frac >= hotqlen) {
  906                 cycle_target++;
  907                 cycle_target_frac -= hotqlen;
  908         }
  909 
  910         if ((pg->pqflags & PQ_HOT) == 0) {
  911 #if defined(LISTQ)
  912                 panic("cold page in hotq: %p", pg);
  913 #else /* defined(LISTQ) */
  914                 handhot_endtest(pg);
  915                 goto next;
  916 #endif /* defined(LISTQ) */
  917         }
  918         KASSERT((pg->pqflags & PQ_TEST) == 0);
  919         KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
  920         KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);
  921 
  922         /*
  923          * once we met our target,
  924          * stop at a hot page so that no cold pages in test period
  925          * have larger recency than any hot pages.
  926          */
  927 
  928         if (s->s_ncold >= s->s_coldtarget) {
  929                 dump("hot done");
  930                 return;
  931         }
  932         clockpro_movereferencebit(pg);
  933         if ((pg->pqflags & PQ_REFERENCED) == 0) {
  934                 PDPOL_EVCNT_INCR(hhotunref);
  935                 uvmexp.pddeact++;
  936                 pg->pqflags &= ~PQ_HOT;
  937                 clockpro.s_ncold++;
  938                 KASSERT(s->s_ncold <= s->s_npages);
  939         } else {
  940                 PDPOL_EVCNT_INCR(hhotref);
  941         }
  942         pg->pqflags &= ~PQ_REFERENCED;
  943 #if !defined(LISTQ)
  944 next:
  945 #endif /* !defined(LISTQ) */
  946         clockpro_pagerequeue(pg);
  947         dump("hot");
  948         goto again;
  949 }
  950 
  951 static struct vm_page *
  952 handcold_advance(void)
  953 {
  954         struct clockpro_state * const s = &clockpro;
  955         struct vm_page *pg;
  956 
  957         for (;;) {
  958 #if defined(LISTQ)
  959                 pageq_t *listq = clockpro_queue(s, CLOCKPRO_LISTQ);
  960 #endif /* defined(LISTQ) */
  961                 pageq_t *coldq;
  962 
  963                 clockpro_newqrotate();
  964                 handhot_advance();
  965 #if defined(LISTQ)
  966                 pg = pageq_first(listq);
  967                 if (pg != NULL) {
  968                         KASSERT(clockpro_getq(pg) == CLOCKPRO_LISTQ);
  969                         KASSERT((pg->pqflags & PQ_TEST) == 0);
  970                         KASSERT((pg->pqflags & PQ_HOT) == 0);
  971                         KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
  972                         pageq_remove(listq, pg);
  973                         check_sanity();
  974                         clockpro_insert_head(s, CLOCKPRO_COLDQ, pg); /* XXX */
  975                         goto gotcold;
  976                 }
  977 #endif /* defined(LISTQ) */
  978                 check_sanity();
  979                 coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
  980                 pg = pageq_first(coldq);
  981                 if (pg == NULL) {
  982                         clockpro_newqflushone();
  983                         pg = pageq_first(coldq);
  984                 }
  985                 if (pg == NULL) {
  986                         DPRINTF("%s: HCOLD TAKEOVER\n", __func__);
  987                         dump("hcoldtakeover");
  988                         PDPOL_EVCNT_INCR(hcoldtakeover);
  989                         KASSERT(
  990                             pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) == 0);
  991 #if defined(LISTQ)
  992                         KASSERT(
  993                             pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)) == 0);
  994 #else /* defined(LISTQ) */
  995                         clockpro_switchqueue();
  996                         coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
  997                         pg = pageq_first(coldq);
  998 #endif /* defined(LISTQ) */
  999                 }
 1000                 if (pg == NULL) {
 1001                         WARN("hcold: no page?\n");
 1002                         return NULL;
 1003                 }
 1004                 KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
 1005                 if ((pg->pqflags & PQ_HOT) != 0) {
 1006                         PDPOL_EVCNT_INCR(hcoldhot);
 1007                         pageq_remove(coldq, pg);
 1008                         clockpro_insert_tail(s, CLOCKPRO_HOTQ, pg);
 1009                         check_sanity();
 1010                         KASSERT((pg->pqflags & PQ_TEST) == 0);
 1011                         uvmexp.pdscans++;
 1012                         continue;
 1013                 }
 1014 #if defined(LISTQ)
 1015 gotcold:
 1016 #endif /* defined(LISTQ) */
 1017                 KASSERT((pg->pqflags & PQ_HOT) == 0);
 1018                 uvmexp.pdscans++;
 1019                 clockpro_movereferencebit(pg);
 1020                 if ((pg->pqflags & PQ_SPECULATIVE) != 0) {
 1021                         KASSERT((pg->pqflags & PQ_TEST) == 0);
 1022                         if ((pg->pqflags & PQ_REFERENCED) != 0) {
 1023                                 PDPOL_EVCNT_INCR(speculativehit2);
 1024                                 pg->pqflags &= ~(PQ_SPECULATIVE|PQ_REFERENCED);
 1025                                 clockpro_pagedequeue(pg);
 1026                                 clockpro_pageenqueue(pg);
 1027                                 continue;
 1028                         }
 1029                         PDPOL_EVCNT_INCR(speculativemiss);
 1030                 }
 1031                 switch (pg->pqflags & (PQ_REFERENCED|PQ_TEST)) {
 1032                 case PQ_TEST:
 1033                         PDPOL_EVCNT_INCR(hcoldunreftest);
 1034                         nonresident_pagerecord(pg);
 1035                         goto gotit;
 1036                 case 0:
 1037                         PDPOL_EVCNT_INCR(hcoldunref);
 1038 gotit:
 1039                         KASSERT(s->s_ncold > 0);
 1040                         clockpro_pagerequeue(pg); /* XXX */
 1041                         dump("cold done");
 1042                         /* XXX "pg" is still in queue */
 1043                         handhot_advance();
 1044                         goto done;
 1045 
 1046                 case PQ_REFERENCED|PQ_TEST:
 1047                         PDPOL_EVCNT_INCR(hcoldreftest);
 1048                         s->s_ncold--;
 1049                         COLDTARGET_ADJ(1);
 1050                         pg->pqflags |= PQ_HOT;
 1051                         pg->pqflags &= ~PQ_TEST;
 1052                         break;
 1053 
 1054                 case PQ_REFERENCED:
 1055                         PDPOL_EVCNT_INCR(hcoldref);
 1056                         pg->pqflags |= PQ_TEST;
 1057                         break;
 1058                 }
 1059                 pg->pqflags &= ~PQ_REFERENCED;
 1060                 uvmexp.pdreact++;
 1061                 /* move to the list head */
 1062                 clockpro_pagerequeue(pg);
 1063                 dump("cold");
 1064         }
 1065 done:;
 1066         return pg;
 1067 }
 1068 
 1069 void
 1070 uvmpdpol_pageactivate(struct vm_page *pg)
 1071 {
 1072 
 1073         if (!uvmpdpol_pageisqueued_p(pg)) {
 1074                 KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);
 1075                 pg->pqflags |= PQ_INITIALREF;
 1076                 clockpro_pageenqueue(pg);
 1077         } else if ((pg->pqflags & PQ_SPECULATIVE)) {
 1078                 PDPOL_EVCNT_INCR(speculativehit1);
 1079                 pg->pqflags &= ~PQ_SPECULATIVE;
 1080                 pg->pqflags |= PQ_INITIALREF;
 1081                 clockpro_pagedequeue(pg);
 1082                 clockpro_pageenqueue(pg);
 1083         }
 1084         pg->pqflags |= PQ_REFERENCED;
 1085 }
 1086 
 1087 void
 1088 uvmpdpol_pagedeactivate(struct vm_page *pg)
 1089 {
 1090 
 1091         clockpro_clearreferencebit(pg);
 1092 }
 1093 
 1094 void
 1095 uvmpdpol_pagedequeue(struct vm_page *pg)
 1096 {
 1097 
 1098         if (!uvmpdpol_pageisqueued_p(pg)) {
 1099                 return;
 1100         }
 1101         clockpro_pagedequeue(pg);
 1102         pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE);
 1103 }
 1104 
 1105 void
 1106 uvmpdpol_pageenqueue(struct vm_page *pg)
 1107 {
 1108 
 1109 #if 1
 1110         if (uvmpdpol_pageisqueued_p(pg)) {
 1111                 return;
 1112         }
 1113         clockpro_clearreferencebit(pg);
 1114         pg->pqflags |= PQ_SPECULATIVE;
 1115         clockpro_pageenqueue(pg);
 1116 #else
 1117         uvmpdpol_pageactivate(pg);
 1118 #endif
 1119 }
 1120 
 1121 void
 1122 uvmpdpol_anfree(struct vm_anon *an)
 1123 {
 1124 
 1125         KASSERT(an->an_page == NULL);
 1126         if (nonresident_lookupremove((objid_t)an, 0)) {
 1127                 PDPOL_EVCNT_INCR(nresanonfree);
 1128         }
 1129 }
 1130 
 1131 void
 1132 uvmpdpol_init(void)
 1133 {
 1134 
 1135         clockpro_init();
 1136 }
 1137 
 1138 void
 1139 uvmpdpol_reinit(void)
 1140 {
 1141 
 1142         clockpro_reinit();
 1143 }
 1144 
 1145 void
 1146 uvmpdpol_estimatepageable(int *active, int *inactive)
 1147 {
 1148         struct clockpro_state * const s = &clockpro;
 1149 
 1150         if (active) {
 1151                 *active = s->s_npages - s->s_ncold;
 1152         }
 1153         if (inactive) {
 1154                 *inactive = s->s_ncold;
 1155         }
 1156 }
 1157 
 1158 bool
 1159 uvmpdpol_pageisqueued_p(struct vm_page *pg)
 1160 {
 1161 
 1162         return clockpro_getq(pg) != CLOCKPRO_NOQUEUE;
 1163 }
 1164 
 1165 void
 1166 uvmpdpol_scaninit(void)
 1167 {
 1168         struct clockpro_scanstate * const ss = &scanstate;
 1169 
 1170         ss->ss_nscanned = 0;
 1171 }
 1172 
 1173 struct vm_page *
 1174 uvmpdpol_selectvictim(void)
 1175 {
 1176         struct clockpro_state * const s = &clockpro;
 1177         struct clockpro_scanstate * const ss = &scanstate;
 1178         struct vm_page *pg;
 1179 
 1180         if (ss->ss_nscanned > s->s_npages) {
 1181                 DPRINTF("scan too much\n");
 1182                 return NULL;
 1183         }
 1184         pg = handcold_advance();
 1185         ss->ss_nscanned++;
 1186         return pg;
 1187 }
 1188 
 1189 static void
 1190 clockpro_dropswap(pageq_t *q, int *todo)
 1191 {
 1192         struct vm_page *pg;
 1193 
 1194         TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pageq.queue) {
 1195                 if (*todo <= 0) {
 1196                         break;
 1197                 }
 1198                 if ((pg->pqflags & PQ_HOT) == 0) {
 1199                         continue;
 1200                 }
 1201                 if ((pg->pqflags & PQ_SWAPBACKED) == 0) {
 1202                         continue;
 1203                 }
 1204                 if (uvmpd_trydropswap(pg)) {
 1205                         (*todo)--;
 1206                 }
 1207         }
 1208 }
 1209 
 1210 void
 1211 uvmpdpol_balancequeue(int swap_shortage)
 1212 {
 1213         struct clockpro_state * const s = &clockpro;
 1214         int todo = swap_shortage;
 1215 
 1216         if (todo == 0) {
 1217                 return;
 1218         }
 1219 
 1220         /*
 1221          * reclaim swap slots from hot pages
 1222          */
 1223 
 1224         DPRINTF("%s: swap_shortage=%d\n", __func__, swap_shortage);
 1225 
 1226         clockpro_dropswap(clockpro_queue(s, CLOCKPRO_NEWQ), &todo);
 1227         clockpro_dropswap(clockpro_queue(s, CLOCKPRO_COLDQ), &todo);
 1228         clockpro_dropswap(clockpro_queue(s, CLOCKPRO_HOTQ), &todo);
 1229 
 1230         DPRINTF("%s: done=%d\n", __func__, swap_shortage - todo);
 1231 }
 1232 
 1233 bool
 1234 uvmpdpol_needsscan_p(void)
 1235 {
 1236         struct clockpro_state * const s = &clockpro;
 1237 
 1238         if (s->s_ncold < s->s_coldtarget) {
 1239                 return true;
 1240         }
 1241         return false;
 1242 }
 1243 
 1244 void
 1245 uvmpdpol_tune(void)
 1246 {
 1247 
 1248         clockpro_tune();
 1249 }
 1250 
 1251 #if !defined(PDSIM)
 1252 
 1253 #include <sys/sysctl.h> /* XXX SYSCTL_DESCR */
 1254 
 1255 void
 1256 uvmpdpol_sysctlsetup(void)
 1257 {
 1258 #if !defined(ADAPTIVE)
 1259         struct clockpro_state * const s = &clockpro;
 1260 
 1261         uvm_pctparam_createsysctlnode(&s->s_coldtargetpct, "coldtargetpct",
 1262             SYSCTL_DESCR("Percentage cold target queue of the entire queue"));
 1263 #endif /* !defined(ADAPTIVE) */
 1264 }
 1265 
 1266 #endif /* !defined(PDSIM) */
 1267 
 1268 #if defined(DDB)
 1269 
 1270 void clockpro_dump(void);
 1271 
 1272 void
 1273 clockpro_dump(void)
 1274 {
 1275         struct clockpro_state * const s = &clockpro;
 1276 
 1277         struct vm_page *pg;
 1278         int ncold, nhot, ntest, nspeculative, ninitialref, nref;
 1279         int newqlen, coldqlen, hotqlen, listqlen;
 1280 
 1281         newqlen = coldqlen = hotqlen = listqlen = 0;
 1282         printf("npages=%d, ncold=%d, coldtarget=%d, newqlenmax=%d\n",
 1283             s->s_npages, s->s_ncold, s->s_coldtarget, s->s_newqlenmax);
 1284 
 1285 #define INITCOUNT()     \
 1286         ncold = nhot = ntest = nspeculative = ninitialref = nref = 0
 1287 
 1288 #define COUNT(pg)       \
 1289         if ((pg->pqflags & PQ_HOT) != 0) { \
 1290                 nhot++; \
 1291         } else { \
 1292                 ncold++; \
 1293                 if ((pg->pqflags & PQ_TEST) != 0) { \
 1294                         ntest++; \
 1295                 } \
 1296                 if ((pg->pqflags & PQ_SPECULATIVE) != 0) { \
 1297                         nspeculative++; \
 1298                 } \
 1299                 if ((pg->pqflags & PQ_INITIALREF) != 0) { \
 1300                         ninitialref++; \
 1301                 } else if ((pg->pqflags & PQ_REFERENCED) != 0 || \
 1302                     pmap_is_referenced(pg)) { \
 1303                         nref++; \
 1304                 } \
 1305         }
 1306 
 1307 #define PRINTCOUNT(name)        \
 1308         printf("%s hot=%d, cold=%d, test=%d, speculative=%d, initialref=%d, " \
 1309             "nref=%d\n", \
 1310             (name), nhot, ncold, ntest, nspeculative, ninitialref, nref)
 1311 
 1312         INITCOUNT();
 1313         TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_NEWQ)->q_q, pageq.queue) {
 1314                 if (clockpro_getq(pg) != CLOCKPRO_NEWQ) {
 1315                         printf("newq corrupt %p\n", pg);
 1316                 }
 1317                 COUNT(pg)
 1318                 newqlen++;
 1319         }
 1320         PRINTCOUNT("newq");
 1321 
 1322         INITCOUNT();
 1323         TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_COLDQ)->q_q, pageq.queue) {
 1324                 if (clockpro_getq(pg) != CLOCKPRO_COLDQ) {
 1325                         printf("coldq corrupt %p\n", pg);
 1326                 }
 1327                 COUNT(pg)
 1328                 coldqlen++;
 1329         }
 1330         PRINTCOUNT("coldq");
 1331 
 1332         INITCOUNT();
 1333         TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_HOTQ)->q_q, pageq.queue) {
 1334                 if (clockpro_getq(pg) != CLOCKPRO_HOTQ) {
 1335                         printf("hotq corrupt %p\n", pg);
 1336                 }
 1337 #if defined(LISTQ)
 1338                 if ((pg->pqflags & PQ_HOT) == 0) {
 1339                         printf("cold page in hotq: %p\n", pg);
 1340                 }
 1341 #endif /* defined(LISTQ) */
 1342                 COUNT(pg)
 1343                 hotqlen++;
 1344         }
 1345         PRINTCOUNT("hotq");
 1346 
 1347         INITCOUNT();
 1348         TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_LISTQ)->q_q, pageq.queue) {
 1349 #if !defined(LISTQ)
 1350                 printf("listq %p\n", pg);
 1351 #endif /* !defined(LISTQ) */
 1352                 if (clockpro_getq(pg) != CLOCKPRO_LISTQ) {
 1353                         printf("listq corrupt %p\n", pg);
 1354                 }
 1355                 COUNT(pg)
 1356                 listqlen++;
 1357         }
 1358         PRINTCOUNT("listq");
 1359 
 1360         printf("newqlen=%d/%d, coldqlen=%d/%d, hotqlen=%d/%d, listqlen=%d/%d\n",
 1361             newqlen, pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)),
 1362             coldqlen, pageq_len(clockpro_queue(s, CLOCKPRO_COLDQ)),
 1363             hotqlen, pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)),
 1364             listqlen, pageq_len(clockpro_queue(s, CLOCKPRO_LISTQ)));
 1365 }
 1366 
 1367 #endif /* defined(DDB) */
 1368 
 1369 #if defined(PDSIM)
 1370 #if defined(DEBUG)
 1371 static void
 1372 pdsim_dumpq(int qidx)
 1373 {
 1374         struct clockpro_state * const s = &clockpro;
 1375         pageq_t *q = clockpro_queue(s, qidx);
 1376         struct vm_page *pg;
 1377 
 1378         TAILQ_FOREACH(pg, &q->q_q, pageq.queue) {
 1379                 DPRINTF(" %" PRIu64 "%s%s%s%s%s%s",
 1380                     pg->offset >> PAGE_SHIFT,
 1381                     (pg->pqflags & PQ_HOT) ? "H" : "",
 1382                     (pg->pqflags & PQ_TEST) ? "T" : "",
 1383                     (pg->pqflags & PQ_REFERENCED) ? "R" : "",
 1384                     pmap_is_referenced(pg) ? "r" : "",
 1385                     (pg->pqflags & PQ_INITIALREF) ? "I" : "",
 1386                     (pg->pqflags & PQ_SPECULATIVE) ? "S" : ""
 1387                     );
 1388         }
 1389 }
 1390 #endif /* defined(DEBUG) */
 1391 
 1392 void
 1393 pdsim_dump(const char *id)
 1394 {
 1395 #if defined(DEBUG)
 1396         struct clockpro_state * const s = &clockpro;
 1397 
 1398         DPRINTF("  %s L(", id);
 1399         pdsim_dumpq(CLOCKPRO_LISTQ);
 1400         DPRINTF(" ) H(");
 1401         pdsim_dumpq(CLOCKPRO_HOTQ);
 1402         DPRINTF(" ) C(");
 1403         pdsim_dumpq(CLOCKPRO_COLDQ);
 1404         DPRINTF(" ) N(");
 1405         pdsim_dumpq(CLOCKPRO_NEWQ);
 1406         DPRINTF(" ) ncold=%d/%d, coldadj=%d\n",
 1407             s->s_ncold, s->s_coldtarget, coldadj);
 1408 #endif /* defined(DEBUG) */
 1409 }
 1410 #endif /* defined(PDSIM) */

Cache object: 2f26ed1b13c7179cda45ff9dea039e62


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.