1 /* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.15 2008/06/04 12:41:40 ad Exp $ */
2
3 /*-
4 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * CLOCK-Pro replacement policy:
31 * http://www.cs.wm.edu/hpcs/WWW/HTML/publications/abs05-3.html
32 *
33 * approximation of the list of non-resident pages using hash:
34 * http://linux-mm.org/ClockProApproximation
35 */
36
37 /* #define CLOCKPRO_DEBUG */
38
39 #if defined(PDSIM)
40
41 #include "pdsim.h"
42
43 #else /* defined(PDSIM) */
44
45 #include <sys/cdefs.h>
46 __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.15 2008/06/04 12:41:40 ad Exp $");
47
48 #include "opt_ddb.h"
49
50 #include <sys/param.h>
51 #include <sys/proc.h>
52 #include <sys/systm.h>
53 #include <sys/kernel.h>
54 #include <sys/hash.h>
55
56 #include <uvm/uvm.h>
57 #include <uvm/uvm_pdpolicy.h>
58 #include <uvm/uvm_pdpolicy_impl.h>
59
60 #if ((__STDC_VERSION__ - 0) >= 199901L)
61 #define DPRINTF(...) /* nothing */
62 #define WARN(...) printf(__VA_ARGS__)
63 #else /* ((__STDC_VERSION__ - 0) >= 199901L) */
64 #define DPRINTF(a...) /* nothing */ /* GCC */
65 #define WARN(a...) printf(a)
66 #endif /* ((__STDC_VERSION__ - 0) >= 199901L) */
67
68 #define dump(a) /* nothing */
69
70 #undef USEONCE2
71 #define LISTQ
72 #undef ADAPTIVE
73
74 #endif /* defined(PDSIM) */
75
76 #if !defined(CLOCKPRO_COLDPCT)
77 #define CLOCKPRO_COLDPCT 10
78 #endif /* !defined(CLOCKPRO_COLDPCT) */
79
80 #define CLOCKPRO_COLDPCTMAX 90
81
82 #if !defined(CLOCKPRO_HASHFACTOR)
83 #define CLOCKPRO_HASHFACTOR 2
84 #endif /* !defined(CLOCKPRO_HASHFACTOR) */
85
86 #define CLOCKPRO_NEWQMIN ((1024 * 1024) >> PAGE_SHIFT) /* XXX */
87
88 int clockpro_hashfactor = CLOCKPRO_HASHFACTOR;
89
90 PDPOL_EVCNT_DEFINE(nresrecordobj)
91 PDPOL_EVCNT_DEFINE(nresrecordanon)
92 PDPOL_EVCNT_DEFINE(nreslookupobj)
93 PDPOL_EVCNT_DEFINE(nreslookupanon)
94 PDPOL_EVCNT_DEFINE(nresfoundobj)
95 PDPOL_EVCNT_DEFINE(nresfoundanon)
96 PDPOL_EVCNT_DEFINE(nresanonfree)
97 PDPOL_EVCNT_DEFINE(nresconflict)
98 PDPOL_EVCNT_DEFINE(nresoverwritten)
99 PDPOL_EVCNT_DEFINE(nreshandhot)
100
101 PDPOL_EVCNT_DEFINE(hhottakeover)
102 PDPOL_EVCNT_DEFINE(hhotref)
103 PDPOL_EVCNT_DEFINE(hhotunref)
104 PDPOL_EVCNT_DEFINE(hhotcold)
105 PDPOL_EVCNT_DEFINE(hhotcoldtest)
106
107 PDPOL_EVCNT_DEFINE(hcoldtakeover)
108 PDPOL_EVCNT_DEFINE(hcoldref)
109 PDPOL_EVCNT_DEFINE(hcoldunref)
110 PDPOL_EVCNT_DEFINE(hcoldreftest)
111 PDPOL_EVCNT_DEFINE(hcoldunreftest)
112 PDPOL_EVCNT_DEFINE(hcoldunreftestspeculative)
113 PDPOL_EVCNT_DEFINE(hcoldhot)
114
115 PDPOL_EVCNT_DEFINE(speculativeenqueue)
116 PDPOL_EVCNT_DEFINE(speculativehit1)
117 PDPOL_EVCNT_DEFINE(speculativehit2)
118 PDPOL_EVCNT_DEFINE(speculativemiss)
119
120 #define PQ_REFERENCED PQ_PRIVATE1
121 #define PQ_HOT PQ_PRIVATE2
122 #define PQ_TEST PQ_PRIVATE3
123 #define PQ_INITIALREF PQ_PRIVATE4
124 #if PQ_PRIVATE6 != PQ_PRIVATE5 * 2 || PQ_PRIVATE7 != PQ_PRIVATE6 * 2
125 #error PQ_PRIVATE
126 #endif
127 #define PQ_QMASK (PQ_PRIVATE5|PQ_PRIVATE6|PQ_PRIVATE7)
128 #define PQ_QFACTOR PQ_PRIVATE5
129 #define PQ_SPECULATIVE PQ_PRIVATE8
130
131 #define CLOCKPRO_NOQUEUE 0
132 #define CLOCKPRO_NEWQ 1 /* small queue to clear initial ref. */
133 #if defined(LISTQ)
134 #define CLOCKPRO_COLDQ 2
135 #define CLOCKPRO_HOTQ 3
136 #else /* defined(LISTQ) */
137 #define CLOCKPRO_COLDQ (2 + coldqidx) /* XXX */
138 #define CLOCKPRO_HOTQ (3 - coldqidx) /* XXX */
139 #endif /* defined(LISTQ) */
140 #define CLOCKPRO_LISTQ 4
141 #define CLOCKPRO_NQUEUE 4
142
143 static inline void
144 clockpro_setq(struct vm_page *pg, int qidx)
145 {
146 KASSERT(qidx >= CLOCKPRO_NOQUEUE);
147 KASSERT(qidx <= CLOCKPRO_NQUEUE);
148
149 pg->pqflags = (pg->pqflags & ~PQ_QMASK) | (qidx * PQ_QFACTOR);
150 }
151
152 static inline int
153 clockpro_getq(struct vm_page *pg)
154 {
155 int qidx;
156
157 qidx = (pg->pqflags & PQ_QMASK) / PQ_QFACTOR;
158 KASSERT(qidx >= CLOCKPRO_NOQUEUE);
159 KASSERT(qidx <= CLOCKPRO_NQUEUE);
160 return qidx;
161 }
162
163 typedef struct {
164 struct pglist q_q;
165 int q_len;
166 } pageq_t;
167
168 struct clockpro_state {
169 int s_npages;
170 int s_coldtarget;
171 int s_ncold;
172
173 int s_newqlenmax;
174 pageq_t s_q[CLOCKPRO_NQUEUE];
175
176 struct uvm_pctparam s_coldtargetpct;
177 };
178
179 static pageq_t *
180 clockpro_queue(struct clockpro_state *s, int qidx)
181 {
182
183 KASSERT(CLOCKPRO_NOQUEUE < qidx);
184 KASSERT(qidx <= CLOCKPRO_NQUEUE);
185
186 return &s->s_q[qidx - 1];
187 }
188
189 #if !defined(LISTQ)
190
191 static int coldqidx;
192
193 static void
194 clockpro_switchqueue(void)
195 {
196
197 coldqidx = 1 - coldqidx;
198 }
199
200 #endif /* !defined(LISTQ) */
201
202 static struct clockpro_state clockpro;
203 static struct clockpro_scanstate {
204 int ss_nscanned;
205 } scanstate;
206
207 /* ---------------------------------------- */
208
209 static void
210 pageq_init(pageq_t *q)
211 {
212
213 TAILQ_INIT(&q->q_q);
214 q->q_len = 0;
215 }
216
217 static int
218 pageq_len(const pageq_t *q)
219 {
220
221 return q->q_len;
222 }
223
224 static struct vm_page *
225 pageq_first(const pageq_t *q)
226 {
227
228 return TAILQ_FIRST(&q->q_q);
229 }
230
231 static void
232 pageq_insert_tail(pageq_t *q, struct vm_page *pg)
233 {
234
235 TAILQ_INSERT_TAIL(&q->q_q, pg, pageq.queue);
236 q->q_len++;
237 }
238
239 #if defined(LISTQ)
240 static void
241 pageq_insert_head(pageq_t *q, struct vm_page *pg)
242 {
243
244 TAILQ_INSERT_HEAD(&q->q_q, pg, pageq.queue);
245 q->q_len++;
246 }
247 #endif
248
249 static void
250 pageq_remove(pageq_t *q, struct vm_page *pg)
251 {
252
253 #if 1
254 KASSERT(clockpro_queue(&clockpro, clockpro_getq(pg)) == q);
255 #endif
256 KASSERT(q->q_len > 0);
257 TAILQ_REMOVE(&q->q_q, pg, pageq.queue);
258 q->q_len--;
259 }
260
261 static struct vm_page *
262 pageq_remove_head(pageq_t *q)
263 {
264 struct vm_page *pg;
265
266 pg = TAILQ_FIRST(&q->q_q);
267 if (pg == NULL) {
268 KASSERT(q->q_len == 0);
269 return NULL;
270 }
271 pageq_remove(q, pg);
272 return pg;
273 }
274
275 /* ---------------------------------------- */
276
277 static void
278 clockpro_insert_tail(struct clockpro_state *s, int qidx, struct vm_page *pg)
279 {
280 pageq_t *q = clockpro_queue(s, qidx);
281
282 clockpro_setq(pg, qidx);
283 pageq_insert_tail(q, pg);
284 }
285
286 #if defined(LISTQ)
287 static void
288 clockpro_insert_head(struct clockpro_state *s, int qidx, struct vm_page *pg)
289 {
290 pageq_t *q = clockpro_queue(s, qidx);
291
292 clockpro_setq(pg, qidx);
293 pageq_insert_head(q, pg);
294 }
295
296 #endif
297 /* ---------------------------------------- */
298
299 typedef uint32_t nonres_cookie_t;
300 #define NONRES_COOKIE_INVAL 0
301
302 typedef uintptr_t objid_t;
303
304 /*
305 * XXX maybe these hash functions need reconsideration,
306 * given that hash distribution is critical here.
307 */
308
309 static uint32_t
310 pageidentityhash1(objid_t obj, off_t idx)
311 {
312 uint32_t hash = HASH32_BUF_INIT;
313
314 #if 1
315 hash = hash32_buf(&idx, sizeof(idx), hash);
316 hash = hash32_buf(&obj, sizeof(obj), hash);
317 #else
318 hash = hash32_buf(&obj, sizeof(obj), hash);
319 hash = hash32_buf(&idx, sizeof(idx), hash);
320 #endif
321 return hash;
322 }
323
324 static uint32_t
325 pageidentityhash2(objid_t obj, off_t idx)
326 {
327 uint32_t hash = HASH32_BUF_INIT;
328
329 hash = hash32_buf(&obj, sizeof(obj), hash);
330 hash = hash32_buf(&idx, sizeof(idx), hash);
331 return hash;
332 }
333
334 static nonres_cookie_t
335 calccookie(objid_t obj, off_t idx)
336 {
337 uint32_t hash = pageidentityhash2(obj, idx);
338 nonres_cookie_t cookie = hash;
339
340 if (__predict_false(cookie == NONRES_COOKIE_INVAL)) {
341 cookie++; /* XXX */
342 }
343 return cookie;
344 }
345
346 #define BUCKETSIZE 14
347 struct bucket {
348 int cycle;
349 int cur;
350 nonres_cookie_t pages[BUCKETSIZE];
351 };
352 static int cycle_target;
353 static int cycle_target_frac;
354
355 static struct bucket static_bucket;
356 static struct bucket *buckets = &static_bucket;
357 static size_t hashsize = 1;
358
359 static int coldadj;
360 #define COLDTARGET_ADJ(d) coldadj += (d)
361
362 #if defined(PDSIM)
363
364 static void *
365 clockpro_hashalloc(int n)
366 {
367 size_t allocsz = sizeof(*buckets) * n;
368
369 return malloc(allocsz);
370 }
371
372 static void
373 clockpro_hashfree(void *p, int n)
374 {
375
376 free(p);
377 }
378
379 #else /* defined(PDSIM) */
380
381 static void *
382 clockpro_hashalloc(int n)
383 {
384 size_t allocsz = round_page(sizeof(*buckets) * n);
385
386 return (void *)uvm_km_alloc(kernel_map, allocsz, 0, UVM_KMF_WIRED);
387 }
388
389 static void
390 clockpro_hashfree(void *p, int n)
391 {
392 size_t allocsz = round_page(sizeof(*buckets) * n);
393
394 uvm_km_free(kernel_map, (vaddr_t)p, allocsz, UVM_KMF_WIRED);
395 }
396
397 #endif /* defined(PDSIM) */
398
399 static void
400 clockpro_hashinit(uint64_t n)
401 {
402 struct bucket *newbuckets;
403 struct bucket *oldbuckets;
404 size_t sz;
405 size_t oldsz;
406 int i;
407
408 sz = howmany(n, BUCKETSIZE);
409 sz *= clockpro_hashfactor;
410 newbuckets = clockpro_hashalloc(sz);
411 if (newbuckets == NULL) {
412 panic("%s: allocation failure", __func__);
413 }
414 for (i = 0; i < sz; i++) {
415 struct bucket *b = &newbuckets[i];
416 int j;
417
418 b->cycle = cycle_target;
419 b->cur = 0;
420 for (j = 0; j < BUCKETSIZE; j++) {
421 b->pages[j] = NONRES_COOKIE_INVAL;
422 }
423 }
424 /* XXX lock */
425 oldbuckets = buckets;
426 oldsz = hashsize;
427 buckets = newbuckets;
428 hashsize = sz;
429 /* XXX unlock */
430 if (oldbuckets != &static_bucket) {
431 clockpro_hashfree(oldbuckets, oldsz);
432 }
433 }
434
435 static struct bucket *
436 nonresident_getbucket(objid_t obj, off_t idx)
437 {
438 uint32_t hash;
439
440 hash = pageidentityhash1(obj, idx);
441 return &buckets[hash % hashsize];
442 }
443
444 static void
445 nonresident_rotate(struct bucket *b)
446 {
447 const int target = cycle_target;
448 const int cycle = b->cycle;
449 int cur;
450 int todo;
451
452 todo = target - cycle;
453 if (todo >= BUCKETSIZE * 2) {
454 todo = (todo % BUCKETSIZE) + BUCKETSIZE;
455 }
456 cur = b->cur;
457 while (todo > 0) {
458 if (b->pages[cur] != NONRES_COOKIE_INVAL) {
459 PDPOL_EVCNT_INCR(nreshandhot);
460 COLDTARGET_ADJ(-1);
461 }
462 b->pages[cur] = NONRES_COOKIE_INVAL;
463 cur++;
464 if (cur == BUCKETSIZE) {
465 cur = 0;
466 }
467 todo--;
468 }
469 b->cycle = target;
470 b->cur = cur;
471 }
472
473 static bool
474 nonresident_lookupremove(objid_t obj, off_t idx)
475 {
476 struct bucket *b = nonresident_getbucket(obj, idx);
477 nonres_cookie_t cookie = calccookie(obj, idx);
478 int i;
479
480 nonresident_rotate(b);
481 for (i = 0; i < BUCKETSIZE; i++) {
482 if (b->pages[i] == cookie) {
483 b->pages[i] = NONRES_COOKIE_INVAL;
484 return true;
485 }
486 }
487 return false;
488 }
489
490 static objid_t
491 pageobj(struct vm_page *pg)
492 {
493 const void *obj;
494
495 /*
496 * XXX object pointer is often freed and reused for unrelated object.
497 * for vnodes, it would be better to use something like
498 * a hash of fsid/fileid/generation.
499 */
500
501 obj = pg->uobject;
502 if (obj == NULL) {
503 obj = pg->uanon;
504 KASSERT(obj != NULL);
505 KASSERT(pg->offset == 0);
506 }
507
508 return (objid_t)obj;
509 }
510
511 static off_t
512 pageidx(struct vm_page *pg)
513 {
514
515 KASSERT((pg->offset & PAGE_MASK) == 0);
516 return pg->offset >> PAGE_SHIFT;
517 }
518
519 static bool
520 nonresident_pagelookupremove(struct vm_page *pg)
521 {
522 bool found = nonresident_lookupremove(pageobj(pg), pageidx(pg));
523
524 if (pg->uobject) {
525 PDPOL_EVCNT_INCR(nreslookupobj);
526 } else {
527 PDPOL_EVCNT_INCR(nreslookupanon);
528 }
529 if (found) {
530 if (pg->uobject) {
531 PDPOL_EVCNT_INCR(nresfoundobj);
532 } else {
533 PDPOL_EVCNT_INCR(nresfoundanon);
534 }
535 }
536 return found;
537 }
538
539 static void
540 nonresident_pagerecord(struct vm_page *pg)
541 {
542 objid_t obj = pageobj(pg);
543 off_t idx = pageidx(pg);
544 struct bucket *b = nonresident_getbucket(obj, idx);
545 nonres_cookie_t cookie = calccookie(obj, idx);
546
547 #if defined(DEBUG)
548 int i;
549
550 for (i = 0; i < BUCKETSIZE; i++) {
551 if (b->pages[i] == cookie) {
552 PDPOL_EVCNT_INCR(nresconflict);
553 }
554 }
555 #endif /* defined(DEBUG) */
556
557 if (pg->uobject) {
558 PDPOL_EVCNT_INCR(nresrecordobj);
559 } else {
560 PDPOL_EVCNT_INCR(nresrecordanon);
561 }
562 nonresident_rotate(b);
563 if (b->pages[b->cur] != NONRES_COOKIE_INVAL) {
564 PDPOL_EVCNT_INCR(nresoverwritten);
565 COLDTARGET_ADJ(-1);
566 }
567 b->pages[b->cur] = cookie;
568 b->cur = (b->cur + 1) % BUCKETSIZE;
569 }
570
571 /* ---------------------------------------- */
572
573 #if defined(CLOCKPRO_DEBUG)
574 static void
575 check_sanity(void)
576 {
577 }
578 #else /* defined(CLOCKPRO_DEBUG) */
579 #define check_sanity() /* nothing */
580 #endif /* defined(CLOCKPRO_DEBUG) */
581
582 static void
583 clockpro_reinit(void)
584 {
585
586 clockpro_hashinit(uvmexp.npages);
587 }
588
589 static void
590 clockpro_init(void)
591 {
592 struct clockpro_state *s = &clockpro;
593 int i;
594
595 for (i = 0; i < CLOCKPRO_NQUEUE; i++) {
596 pageq_init(&s->s_q[i]);
597 }
598 s->s_newqlenmax = 1;
599 s->s_coldtarget = 1;
600 uvm_pctparam_init(&s->s_coldtargetpct, CLOCKPRO_COLDPCT, NULL);
601 }
602
603 static void
604 clockpro_tune(void)
605 {
606 struct clockpro_state *s = &clockpro;
607 int coldtarget;
608
609 #if defined(ADAPTIVE)
610 int coldmax = s->s_npages * CLOCKPRO_COLDPCTMAX / 100;
611 int coldmin = 1;
612
613 coldtarget = s->s_coldtarget;
614 if (coldtarget + coldadj < coldmin) {
615 coldadj = coldmin - coldtarget;
616 } else if (coldtarget + coldadj > coldmax) {
617 coldadj = coldmax - coldtarget;
618 }
619 coldtarget += coldadj;
620 #else /* defined(ADAPTIVE) */
621 coldtarget = UVM_PCTPARAM_APPLY(&s->s_coldtargetpct, s->s_npages);
622 if (coldtarget < 1) {
623 coldtarget = 1;
624 }
625 #endif /* defined(ADAPTIVE) */
626
627 s->s_coldtarget = coldtarget;
628 s->s_newqlenmax = coldtarget / 4;
629 if (s->s_newqlenmax < CLOCKPRO_NEWQMIN) {
630 s->s_newqlenmax = CLOCKPRO_NEWQMIN;
631 }
632 }
633
634 static void
635 clockpro_movereferencebit(struct vm_page *pg)
636 {
637 bool referenced;
638
639 referenced = pmap_clear_reference(pg);
640 if (referenced) {
641 pg->pqflags |= PQ_REFERENCED;
642 }
643 }
644
645 static void
646 clockpro_clearreferencebit(struct vm_page *pg)
647 {
648
649 clockpro_movereferencebit(pg);
650 pg->pqflags &= ~PQ_REFERENCED;
651 }
652
653 static void
654 clockpro___newqrotate(int len)
655 {
656 struct clockpro_state * const s = &clockpro;
657 pageq_t * const newq = clockpro_queue(s, CLOCKPRO_NEWQ);
658 struct vm_page *pg;
659
660 while (pageq_len(newq) > len) {
661 pg = pageq_remove_head(newq);
662 KASSERT(pg != NULL);
663 KASSERT(clockpro_getq(pg) == CLOCKPRO_NEWQ);
664 if ((pg->pqflags & PQ_INITIALREF) != 0) {
665 clockpro_clearreferencebit(pg);
666 pg->pqflags &= ~PQ_INITIALREF;
667 }
668 /* place at the list head */
669 clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
670 }
671 }
672
673 static void
674 clockpro_newqrotate(void)
675 {
676 struct clockpro_state * const s = &clockpro;
677
678 check_sanity();
679 clockpro___newqrotate(s->s_newqlenmax);
680 check_sanity();
681 }
682
683 static void
684 clockpro_newqflush(int n)
685 {
686
687 check_sanity();
688 clockpro___newqrotate(n);
689 check_sanity();
690 }
691
692 static void
693 clockpro_newqflushone(void)
694 {
695 struct clockpro_state * const s = &clockpro;
696
697 clockpro_newqflush(
698 MAX(pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) - 1, 0));
699 }
700
701 /*
702 * our "tail" is called "list-head" in the paper.
703 */
704
705 static void
706 clockpro___enqueuetail(struct vm_page *pg)
707 {
708 struct clockpro_state * const s = &clockpro;
709
710 KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);
711
712 check_sanity();
713 #if !defined(USEONCE2)
714 clockpro_insert_tail(s, CLOCKPRO_NEWQ, pg);
715 clockpro_newqrotate();
716 #else /* !defined(USEONCE2) */
717 #if defined(LISTQ)
718 KASSERT((pg->pqflags & PQ_REFERENCED) == 0);
719 #endif /* defined(LISTQ) */
720 clockpro_insert_tail(s, CLOCKPRO_COLDQ, pg);
721 #endif /* !defined(USEONCE2) */
722 check_sanity();
723 }
724
725 static void
726 clockpro_pageenqueue(struct vm_page *pg)
727 {
728 struct clockpro_state * const s = &clockpro;
729 bool hot;
730 bool speculative = (pg->pqflags & PQ_SPECULATIVE) != 0; /* XXX */
731
732 KASSERT((~pg->pqflags & (PQ_INITIALREF|PQ_SPECULATIVE)) != 0);
733 KASSERT(mutex_owned(&uvm_pageqlock));
734 check_sanity();
735 KASSERT(clockpro_getq(pg) == CLOCKPRO_NOQUEUE);
736 s->s_npages++;
737 pg->pqflags &= ~(PQ_HOT|PQ_TEST);
738 if (speculative) {
739 hot = false;
740 PDPOL_EVCNT_INCR(speculativeenqueue);
741 } else {
742 hot = nonresident_pagelookupremove(pg);
743 if (hot) {
744 COLDTARGET_ADJ(1);
745 }
746 }
747
748 /*
749 * consider mmap'ed file:
750 *
751 * - read-ahead enqueues a page.
752 *
753 * - on the following read-ahead hit, the fault handler activates it.
754 *
755 * - finally, the userland code which caused the above fault
756 * actually accesses the page. it makes its reference bit set.
757 *
758 * we want to count the above as a single access, rather than
759 * three accesses with short reuse distances.
760 */
761
762 #if defined(USEONCE2)
763 pg->pqflags &= ~PQ_INITIALREF;
764 if (hot) {
765 pg->pqflags |= PQ_TEST;
766 }
767 s->s_ncold++;
768 clockpro_clearreferencebit(pg);
769 clockpro___enqueuetail(pg);
770 #else /* defined(USEONCE2) */
771 if (speculative) {
772 s->s_ncold++;
773 } else if (hot) {
774 pg->pqflags |= PQ_HOT;
775 } else {
776 pg->pqflags |= PQ_TEST;
777 s->s_ncold++;
778 }
779 clockpro___enqueuetail(pg);
780 #endif /* defined(USEONCE2) */
781 KASSERT(s->s_ncold <= s->s_npages);
782 }
783
784 static pageq_t *
785 clockpro_pagequeue(struct vm_page *pg)
786 {
787 struct clockpro_state * const s = &clockpro;
788 int qidx;
789
790 qidx = clockpro_getq(pg);
791 KASSERT(qidx != CLOCKPRO_NOQUEUE);
792
793 return clockpro_queue(s, qidx);
794 }
795
796 static void
797 clockpro_pagedequeue(struct vm_page *pg)
798 {
799 struct clockpro_state * const s = &clockpro;
800 pageq_t *q;
801
802 KASSERT(s->s_npages > 0);
803 check_sanity();
804 q = clockpro_pagequeue(pg);
805 pageq_remove(q, pg);
806 check_sanity();
807 clockpro_setq(pg, CLOCKPRO_NOQUEUE);
808 if ((pg->pqflags & PQ_HOT) == 0) {
809 KASSERT(s->s_ncold > 0);
810 s->s_ncold--;
811 }
812 KASSERT(s->s_npages > 0);
813 s->s_npages--;
814 check_sanity();
815 }
816
817 static void
818 clockpro_pagerequeue(struct vm_page *pg)
819 {
820 struct clockpro_state * const s = &clockpro;
821 int qidx;
822
823 qidx = clockpro_getq(pg);
824 KASSERT(qidx == CLOCKPRO_HOTQ || qidx == CLOCKPRO_COLDQ);
825 pageq_remove(clockpro_queue(s, qidx), pg);
826 check_sanity();
827 clockpro_setq(pg, CLOCKPRO_NOQUEUE);
828
829 clockpro___enqueuetail(pg);
830 }
831
832 static void
833 handhot_endtest(struct vm_page *pg)
834 {
835
836 KASSERT((pg->pqflags & PQ_HOT) == 0);
837 if ((pg->pqflags & PQ_TEST) != 0) {
838 PDPOL_EVCNT_INCR(hhotcoldtest);
839 COLDTARGET_ADJ(-1);
840 pg->pqflags &= ~PQ_TEST;
841 } else {
842 PDPOL_EVCNT_INCR(hhotcold);
843 }
844 }
845
846 static void
847 handhot_advance(void)
848 {
849 struct clockpro_state * const s = &clockpro;
850 struct vm_page *pg;
851 pageq_t *hotq;
852 int hotqlen;
853
854 clockpro_tune();
855
856 dump("hot called");
857 if (s->s_ncold >= s->s_coldtarget) {
858 return;
859 }
860 hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
861 again:
862 pg = pageq_first(hotq);
863 if (pg == NULL) {
864 DPRINTF("%s: HHOT TAKEOVER\n", __func__);
865 dump("hhottakeover");
866 PDPOL_EVCNT_INCR(hhottakeover);
867 #if defined(LISTQ)
868 while (/* CONSTCOND */ 1) {
869 pageq_t *coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
870
871 pg = pageq_first(coldq);
872 if (pg == NULL) {
873 clockpro_newqflushone();
874 pg = pageq_first(coldq);
875 if (pg == NULL) {
876 WARN("hhot: no page?\n");
877 return;
878 }
879 }
880 KASSERT(clockpro_pagequeue(pg) == coldq);
881 pageq_remove(coldq, pg);
882 check_sanity();
883 if ((pg->pqflags & PQ_HOT) == 0) {
884 handhot_endtest(pg);
885 clockpro_insert_tail(s, CLOCKPRO_LISTQ, pg);
886 } else {
887 clockpro_insert_head(s, CLOCKPRO_HOTQ, pg);
888 break;
889 }
890 }
891 #else /* defined(LISTQ) */
892 clockpro_newqflush(0); /* XXX XXX */
893 clockpro_switchqueue();
894 hotq = clockpro_queue(s, CLOCKPRO_HOTQ);
895 goto again;
896 #endif /* defined(LISTQ) */
897 }
898
899 KASSERT(clockpro_pagequeue(pg) == hotq);
900
901 /*
902 * terminate test period of nonresident pages by cycling them.
903 */
904
905 cycle_target_frac += BUCKETSIZE;
906 hotqlen = pageq_len(hotq);
907 while (cycle_target_frac >= hotqlen) {
908 cycle_target++;
909 cycle_target_frac -= hotqlen;
910 }
911
912 if ((pg->pqflags & PQ_HOT) == 0) {
913 #if defined(LISTQ)
914 panic("cold page in hotq: %p", pg);
915 #else /* defined(LISTQ) */
916 handhot_endtest(pg);
917 goto next;
918 #endif /* defined(LISTQ) */
919 }
920 KASSERT((pg->pqflags & PQ_TEST) == 0);
921 KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
922 KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);
923
924 /*
925 * once we met our target,
926 * stop at a hot page so that no cold pages in test period
927 * have larger recency than any hot pages.
928 */
929
930 if (s->s_ncold >= s->s_coldtarget) {
931 dump("hot done");
932 return;
933 }
934 clockpro_movereferencebit(pg);
935 if ((pg->pqflags & PQ_REFERENCED) == 0) {
936 PDPOL_EVCNT_INCR(hhotunref);
937 uvmexp.pddeact++;
938 pg->pqflags &= ~PQ_HOT;
939 clockpro.s_ncold++;
940 KASSERT(s->s_ncold <= s->s_npages);
941 } else {
942 PDPOL_EVCNT_INCR(hhotref);
943 }
944 pg->pqflags &= ~PQ_REFERENCED;
945 #if !defined(LISTQ)
946 next:
947 #endif /* !defined(LISTQ) */
948 clockpro_pagerequeue(pg);
949 dump("hot");
950 goto again;
951 }
952
953 static struct vm_page *
954 handcold_advance(void)
955 {
956 struct clockpro_state * const s = &clockpro;
957 struct vm_page *pg;
958
959 for (;;) {
960 #if defined(LISTQ)
961 pageq_t *listq = clockpro_queue(s, CLOCKPRO_LISTQ);
962 #endif /* defined(LISTQ) */
963 pageq_t *coldq;
964
965 clockpro_newqrotate();
966 handhot_advance();
967 #if defined(LISTQ)
968 pg = pageq_first(listq);
969 if (pg != NULL) {
970 KASSERT(clockpro_getq(pg) == CLOCKPRO_LISTQ);
971 KASSERT((pg->pqflags & PQ_TEST) == 0);
972 KASSERT((pg->pqflags & PQ_HOT) == 0);
973 KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
974 pageq_remove(listq, pg);
975 check_sanity();
976 clockpro_insert_head(s, CLOCKPRO_COLDQ, pg); /* XXX */
977 goto gotcold;
978 }
979 #endif /* defined(LISTQ) */
980 check_sanity();
981 coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
982 pg = pageq_first(coldq);
983 if (pg == NULL) {
984 clockpro_newqflushone();
985 pg = pageq_first(coldq);
986 }
987 if (pg == NULL) {
988 DPRINTF("%s: HCOLD TAKEOVER\n", __func__);
989 dump("hcoldtakeover");
990 PDPOL_EVCNT_INCR(hcoldtakeover);
991 KASSERT(
992 pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)) == 0);
993 #if defined(LISTQ)
994 KASSERT(
995 pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)) == 0);
996 #else /* defined(LISTQ) */
997 clockpro_switchqueue();
998 coldq = clockpro_queue(s, CLOCKPRO_COLDQ);
999 pg = pageq_first(coldq);
1000 #endif /* defined(LISTQ) */
1001 }
1002 if (pg == NULL) {
1003 WARN("hcold: no page?\n");
1004 return NULL;
1005 }
1006 KASSERT((pg->pqflags & PQ_INITIALREF) == 0);
1007 if ((pg->pqflags & PQ_HOT) != 0) {
1008 PDPOL_EVCNT_INCR(hcoldhot);
1009 pageq_remove(coldq, pg);
1010 clockpro_insert_tail(s, CLOCKPRO_HOTQ, pg);
1011 check_sanity();
1012 KASSERT((pg->pqflags & PQ_TEST) == 0);
1013 uvmexp.pdscans++;
1014 continue;
1015 }
1016 #if defined(LISTQ)
1017 gotcold:
1018 #endif /* defined(LISTQ) */
1019 KASSERT((pg->pqflags & PQ_HOT) == 0);
1020 uvmexp.pdscans++;
1021 clockpro_movereferencebit(pg);
1022 if ((pg->pqflags & PQ_SPECULATIVE) != 0) {
1023 KASSERT((pg->pqflags & PQ_TEST) == 0);
1024 if ((pg->pqflags & PQ_REFERENCED) != 0) {
1025 PDPOL_EVCNT_INCR(speculativehit2);
1026 pg->pqflags &= ~(PQ_SPECULATIVE|PQ_REFERENCED);
1027 clockpro_pagedequeue(pg);
1028 clockpro_pageenqueue(pg);
1029 continue;
1030 }
1031 PDPOL_EVCNT_INCR(speculativemiss);
1032 }
1033 switch (pg->pqflags & (PQ_REFERENCED|PQ_TEST)) {
1034 case PQ_TEST:
1035 PDPOL_EVCNT_INCR(hcoldunreftest);
1036 nonresident_pagerecord(pg);
1037 goto gotit;
1038 case 0:
1039 PDPOL_EVCNT_INCR(hcoldunref);
1040 gotit:
1041 KASSERT(s->s_ncold > 0);
1042 clockpro_pagerequeue(pg); /* XXX */
1043 dump("cold done");
1044 /* XXX "pg" is still in queue */
1045 handhot_advance();
1046 goto done;
1047
1048 case PQ_REFERENCED|PQ_TEST:
1049 PDPOL_EVCNT_INCR(hcoldreftest);
1050 s->s_ncold--;
1051 COLDTARGET_ADJ(1);
1052 pg->pqflags |= PQ_HOT;
1053 pg->pqflags &= ~PQ_TEST;
1054 break;
1055
1056 case PQ_REFERENCED:
1057 PDPOL_EVCNT_INCR(hcoldref);
1058 pg->pqflags |= PQ_TEST;
1059 break;
1060 }
1061 pg->pqflags &= ~PQ_REFERENCED;
1062 uvmexp.pdreact++;
1063 /* move to the list head */
1064 clockpro_pagerequeue(pg);
1065 dump("cold");
1066 }
1067 done:;
1068 return pg;
1069 }
1070
1071 void
1072 uvmpdpol_pageactivate(struct vm_page *pg)
1073 {
1074
1075 if (!uvmpdpol_pageisqueued_p(pg)) {
1076 KASSERT((pg->pqflags & PQ_SPECULATIVE) == 0);
1077 pg->pqflags |= PQ_INITIALREF;
1078 clockpro_pageenqueue(pg);
1079 } else if ((pg->pqflags & PQ_SPECULATIVE)) {
1080 PDPOL_EVCNT_INCR(speculativehit1);
1081 pg->pqflags &= ~PQ_SPECULATIVE;
1082 pg->pqflags |= PQ_INITIALREF;
1083 clockpro_pagedequeue(pg);
1084 clockpro_pageenqueue(pg);
1085 }
1086 pg->pqflags |= PQ_REFERENCED;
1087 }
1088
1089 void
1090 uvmpdpol_pagedeactivate(struct vm_page *pg)
1091 {
1092
1093 clockpro_clearreferencebit(pg);
1094 }
1095
1096 void
1097 uvmpdpol_pagedequeue(struct vm_page *pg)
1098 {
1099
1100 if (!uvmpdpol_pageisqueued_p(pg)) {
1101 return;
1102 }
1103 clockpro_pagedequeue(pg);
1104 pg->pqflags &= ~(PQ_INITIALREF|PQ_SPECULATIVE);
1105 }
1106
1107 void
1108 uvmpdpol_pageenqueue(struct vm_page *pg)
1109 {
1110
1111 #if 1
1112 if (uvmpdpol_pageisqueued_p(pg)) {
1113 return;
1114 }
1115 clockpro_clearreferencebit(pg);
1116 pg->pqflags |= PQ_SPECULATIVE;
1117 clockpro_pageenqueue(pg);
1118 #else
1119 uvmpdpol_pageactivate(pg);
1120 #endif
1121 }
1122
1123 void
1124 uvmpdpol_anfree(struct vm_anon *an)
1125 {
1126
1127 KASSERT(an->an_page == NULL);
1128 if (nonresident_lookupremove((objid_t)an, 0)) {
1129 PDPOL_EVCNT_INCR(nresanonfree);
1130 }
1131 }
1132
1133 void
1134 uvmpdpol_init(void)
1135 {
1136
1137 clockpro_init();
1138 }
1139
1140 void
1141 uvmpdpol_reinit(void)
1142 {
1143
1144 clockpro_reinit();
1145 }
1146
1147 void
1148 uvmpdpol_estimatepageable(int *active, int *inactive)
1149 {
1150 struct clockpro_state * const s = &clockpro;
1151
1152 if (active) {
1153 *active = s->s_npages - s->s_ncold;
1154 }
1155 if (inactive) {
1156 *inactive = s->s_ncold;
1157 }
1158 }
1159
1160 bool
1161 uvmpdpol_pageisqueued_p(struct vm_page *pg)
1162 {
1163
1164 return clockpro_getq(pg) != CLOCKPRO_NOQUEUE;
1165 }
1166
1167 void
1168 uvmpdpol_scaninit(void)
1169 {
1170 struct clockpro_scanstate * const ss = &scanstate;
1171
1172 ss->ss_nscanned = 0;
1173 }
1174
1175 struct vm_page *
1176 uvmpdpol_selectvictim(void)
1177 {
1178 struct clockpro_state * const s = &clockpro;
1179 struct clockpro_scanstate * const ss = &scanstate;
1180 struct vm_page *pg;
1181
1182 if (ss->ss_nscanned > s->s_npages) {
1183 DPRINTF("scan too much\n");
1184 return NULL;
1185 }
1186 pg = handcold_advance();
1187 ss->ss_nscanned++;
1188 return pg;
1189 }
1190
1191 static void
1192 clockpro_dropswap(pageq_t *q, int *todo)
1193 {
1194 struct vm_page *pg;
1195
1196 TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pageq.queue) {
1197 if (*todo <= 0) {
1198 break;
1199 }
1200 if ((pg->pqflags & PQ_HOT) == 0) {
1201 continue;
1202 }
1203 if ((pg->pqflags & PQ_SWAPBACKED) == 0) {
1204 continue;
1205 }
1206 if (uvmpd_trydropswap(pg)) {
1207 (*todo)--;
1208 }
1209 }
1210 }
1211
1212 void
1213 uvmpdpol_balancequeue(int swap_shortage)
1214 {
1215 struct clockpro_state * const s = &clockpro;
1216 int todo = swap_shortage;
1217
1218 if (todo == 0) {
1219 return;
1220 }
1221
1222 /*
1223 * reclaim swap slots from hot pages
1224 */
1225
1226 DPRINTF("%s: swap_shortage=%d\n", __func__, swap_shortage);
1227
1228 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_NEWQ), &todo);
1229 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_COLDQ), &todo);
1230 clockpro_dropswap(clockpro_queue(s, CLOCKPRO_HOTQ), &todo);
1231
1232 DPRINTF("%s: done=%d\n", __func__, swap_shortage - todo);
1233 }
1234
1235 bool
1236 uvmpdpol_needsscan_p(void)
1237 {
1238 struct clockpro_state * const s = &clockpro;
1239
1240 if (s->s_ncold < s->s_coldtarget) {
1241 return true;
1242 }
1243 return false;
1244 }
1245
1246 void
1247 uvmpdpol_tune(void)
1248 {
1249
1250 clockpro_tune();
1251 }
1252
1253 #if !defined(PDSIM)
1254
1255 #include <sys/sysctl.h> /* XXX SYSCTL_DESCR */
1256
1257 void
1258 uvmpdpol_sysctlsetup(void)
1259 {
1260 #if !defined(ADAPTIVE)
1261 struct clockpro_state * const s = &clockpro;
1262
1263 uvm_pctparam_createsysctlnode(&s->s_coldtargetpct, "coldtargetpct",
1264 SYSCTL_DESCR("Percentage cold target queue of the entire queue"));
1265 #endif /* !defined(ADAPTIVE) */
1266 }
1267
1268 #endif /* !defined(PDSIM) */
1269
1270 #if defined(DDB)
1271
1272 void clockpro_dump(void);
1273
1274 void
1275 clockpro_dump(void)
1276 {
1277 struct clockpro_state * const s = &clockpro;
1278
1279 struct vm_page *pg;
1280 int ncold, nhot, ntest, nspeculative, ninitialref, nref;
1281 int newqlen, coldqlen, hotqlen, listqlen;
1282
1283 newqlen = coldqlen = hotqlen = listqlen = 0;
1284 printf("npages=%d, ncold=%d, coldtarget=%d, newqlenmax=%d\n",
1285 s->s_npages, s->s_ncold, s->s_coldtarget, s->s_newqlenmax);
1286
1287 #define INITCOUNT() \
1288 ncold = nhot = ntest = nspeculative = ninitialref = nref = 0
1289
1290 #define COUNT(pg) \
1291 if ((pg->pqflags & PQ_HOT) != 0) { \
1292 nhot++; \
1293 } else { \
1294 ncold++; \
1295 if ((pg->pqflags & PQ_TEST) != 0) { \
1296 ntest++; \
1297 } \
1298 if ((pg->pqflags & PQ_SPECULATIVE) != 0) { \
1299 nspeculative++; \
1300 } \
1301 if ((pg->pqflags & PQ_INITIALREF) != 0) { \
1302 ninitialref++; \
1303 } else if ((pg->pqflags & PQ_REFERENCED) != 0 || \
1304 pmap_is_referenced(pg)) { \
1305 nref++; \
1306 } \
1307 }
1308
1309 #define PRINTCOUNT(name) \
1310 printf("%s hot=%d, cold=%d, test=%d, speculative=%d, initialref=%d, " \
1311 "nref=%d\n", \
1312 (name), nhot, ncold, ntest, nspeculative, ninitialref, nref)
1313
1314 INITCOUNT();
1315 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_NEWQ)->q_q, pageq.queue) {
1316 if (clockpro_getq(pg) != CLOCKPRO_NEWQ) {
1317 printf("newq corrupt %p\n", pg);
1318 }
1319 COUNT(pg)
1320 newqlen++;
1321 }
1322 PRINTCOUNT("newq");
1323
1324 INITCOUNT();
1325 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_COLDQ)->q_q, pageq.queue) {
1326 if (clockpro_getq(pg) != CLOCKPRO_COLDQ) {
1327 printf("coldq corrupt %p\n", pg);
1328 }
1329 COUNT(pg)
1330 coldqlen++;
1331 }
1332 PRINTCOUNT("coldq");
1333
1334 INITCOUNT();
1335 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_HOTQ)->q_q, pageq.queue) {
1336 if (clockpro_getq(pg) != CLOCKPRO_HOTQ) {
1337 printf("hotq corrupt %p\n", pg);
1338 }
1339 #if defined(LISTQ)
1340 if ((pg->pqflags & PQ_HOT) == 0) {
1341 printf("cold page in hotq: %p\n", pg);
1342 }
1343 #endif /* defined(LISTQ) */
1344 COUNT(pg)
1345 hotqlen++;
1346 }
1347 PRINTCOUNT("hotq");
1348
1349 INITCOUNT();
1350 TAILQ_FOREACH(pg, &clockpro_queue(s, CLOCKPRO_LISTQ)->q_q, pageq.queue) {
1351 #if !defined(LISTQ)
1352 printf("listq %p\n", pg);
1353 #endif /* !defined(LISTQ) */
1354 if (clockpro_getq(pg) != CLOCKPRO_LISTQ) {
1355 printf("listq corrupt %p\n", pg);
1356 }
1357 COUNT(pg)
1358 listqlen++;
1359 }
1360 PRINTCOUNT("listq");
1361
1362 printf("newqlen=%d/%d, coldqlen=%d/%d, hotqlen=%d/%d, listqlen=%d/%d\n",
1363 newqlen, pageq_len(clockpro_queue(s, CLOCKPRO_NEWQ)),
1364 coldqlen, pageq_len(clockpro_queue(s, CLOCKPRO_COLDQ)),
1365 hotqlen, pageq_len(clockpro_queue(s, CLOCKPRO_HOTQ)),
1366 listqlen, pageq_len(clockpro_queue(s, CLOCKPRO_LISTQ)));
1367 }
1368
1369 #endif /* defined(DDB) */
1370
1371 #if defined(PDSIM)
1372 #if defined(DEBUG)
1373 static void
1374 pdsim_dumpq(int qidx)
1375 {
1376 struct clockpro_state * const s = &clockpro;
1377 pageq_t *q = clockpro_queue(s, qidx);
1378 struct vm_page *pg;
1379
1380 TAILQ_FOREACH(pg, &q->q_q, pageq.queue) {
1381 DPRINTF(" %" PRIu64 "%s%s%s%s%s%s",
1382 pg->offset >> PAGE_SHIFT,
1383 (pg->pqflags & PQ_HOT) ? "H" : "",
1384 (pg->pqflags & PQ_TEST) ? "T" : "",
1385 (pg->pqflags & PQ_REFERENCED) ? "R" : "",
1386 pmap_is_referenced(pg) ? "r" : "",
1387 (pg->pqflags & PQ_INITIALREF) ? "I" : "",
1388 (pg->pqflags & PQ_SPECULATIVE) ? "S" : ""
1389 );
1390 }
1391 }
1392 #endif /* defined(DEBUG) */
1393
1394 void
1395 pdsim_dump(const char *id)
1396 {
1397 #if defined(DEBUG)
1398 struct clockpro_state * const s = &clockpro;
1399
1400 DPRINTF(" %s L(", id);
1401 pdsim_dumpq(CLOCKPRO_LISTQ);
1402 DPRINTF(" ) H(");
1403 pdsim_dumpq(CLOCKPRO_HOTQ);
1404 DPRINTF(" ) C(");
1405 pdsim_dumpq(CLOCKPRO_COLDQ);
1406 DPRINTF(" ) N(");
1407 pdsim_dumpq(CLOCKPRO_NEWQ);
1408 DPRINTF(" ) ncold=%d/%d, coldadj=%d\n",
1409 s->s_ncold, s->s_coldtarget, coldadj);
1410 #endif /* defined(DEBUG) */
1411 }
1412 #endif /* defined(PDSIM) */
Cache object: 2f26ed1b13c7179cda45ff9dea039e62
|