FreeBSD/Linux Kernel Cross Reference
sys/geom/geom_subr.c
1 /*-
2 * Copyright (c) 2002 Poul-Henning Kamp
3 * Copyright (c) 2002 Networks Associates Technology, Inc.
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7 * and NAI Labs, the Security Research Division of Network Associates, Inc.
8 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9 * DARPA CHATS research program.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. The names of the authors may not be used to endorse or promote
20 * products derived from this software without specific prior written
21 * permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * SUCH DAMAGE.
34 */
35
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD: releng/8.3/sys/geom/geom_subr.c 222920 2011-06-10 09:12:09Z mav $");
38
39 #include "opt_ddb.h"
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/devicestat.h>
44 #include <sys/kernel.h>
45 #include <sys/malloc.h>
46 #include <sys/bio.h>
47 #include <sys/sysctl.h>
48 #include <sys/proc.h>
49 #include <sys/kthread.h>
50 #include <sys/lock.h>
51 #include <sys/mutex.h>
52 #include <sys/errno.h>
53 #include <sys/sbuf.h>
54 #include <geom/geom.h>
55 #include <geom/geom_int.h>
56 #include <machine/stdarg.h>
57
58 #ifdef DDB
59 #include <ddb/ddb.h>
60 #endif
61
62 #ifdef KDB
63 #include <sys/kdb.h>
64 #endif
65
66 struct class_list_head g_classes = LIST_HEAD_INITIALIZER(g_classes);
67 static struct g_tailq_head geoms = TAILQ_HEAD_INITIALIZER(geoms);
68 char *g_wait_event, *g_wait_up, *g_wait_down, *g_wait_sim;
69
70 struct g_hh00 {
71 struct g_class *mp;
72 int error;
73 int post;
74 };
75
76 /*
77 * This event offers a new class a chance to taste all preexisting providers.
78 */
79 static void
80 g_load_class(void *arg, int flag)
81 {
82 struct g_hh00 *hh;
83 struct g_class *mp2, *mp;
84 struct g_geom *gp;
85 struct g_provider *pp;
86
87 g_topology_assert();
88 if (flag == EV_CANCEL) /* XXX: can't happen ? */
89 return;
90 if (g_shutdown)
91 return;
92
93 hh = arg;
94 mp = hh->mp;
95 hh->error = 0;
96 if (hh->post) {
97 g_free(hh);
98 hh = NULL;
99 }
100 g_trace(G_T_TOPOLOGY, "g_load_class(%s)", mp->name);
101 KASSERT(mp->name != NULL && *mp->name != '\0',
102 ("GEOM class has no name"));
103 LIST_FOREACH(mp2, &g_classes, class) {
104 if (mp2 == mp) {
105 printf("The GEOM class %s is already loaded.\n",
106 mp2->name);
107 if (hh != NULL)
108 hh->error = EEXIST;
109 return;
110 } else if (strcmp(mp2->name, mp->name) == 0) {
111 printf("A GEOM class %s is already loaded.\n",
112 mp2->name);
113 if (hh != NULL)
114 hh->error = EEXIST;
115 return;
116 }
117 }
118
119 LIST_INIT(&mp->geom);
120 LIST_INSERT_HEAD(&g_classes, mp, class);
121 if (mp->init != NULL)
122 mp->init(mp);
123 if (mp->taste == NULL)
124 return;
125 LIST_FOREACH(mp2, &g_classes, class) {
126 if (mp == mp2)
127 continue;
128 LIST_FOREACH(gp, &mp2->geom, geom) {
129 LIST_FOREACH(pp, &gp->provider, provider) {
130 mp->taste(mp, pp, 0);
131 g_topology_assert();
132 }
133 }
134 }
135 }
136
137 static int
138 g_unload_class(struct g_class *mp)
139 {
140 struct g_geom *gp;
141 struct g_provider *pp;
142 struct g_consumer *cp;
143 int error;
144
145 g_topology_lock();
146 g_trace(G_T_TOPOLOGY, "g_unload_class(%s)", mp->name);
147 retry:
148 G_VALID_CLASS(mp);
149 LIST_FOREACH(gp, &mp->geom, geom) {
150 /* We refuse to unload if anything is open */
151 LIST_FOREACH(pp, &gp->provider, provider)
152 if (pp->acr || pp->acw || pp->ace) {
153 g_topology_unlock();
154 return (EBUSY);
155 }
156 LIST_FOREACH(cp, &gp->consumer, consumer)
157 if (cp->acr || cp->acw || cp->ace) {
158 g_topology_unlock();
159 return (EBUSY);
160 }
161 /* If the geom is withering, wait for it to finish. */
162 if (gp->flags & G_GEOM_WITHER) {
163 g_topology_sleep(mp, 1);
164 goto retry;
165 }
166 }
167
168 /*
169 * We allow unloading if we have no geoms, or a class
170 * method we can use to get rid of them.
171 */
172 if (!LIST_EMPTY(&mp->geom) && mp->destroy_geom == NULL) {
173 g_topology_unlock();
174 return (EOPNOTSUPP);
175 }
176
177 /* Bar new entries */
178 mp->taste = NULL;
179 mp->config = NULL;
180
181 LIST_FOREACH(gp, &mp->geom, geom) {
182 error = mp->destroy_geom(NULL, mp, gp);
183 if (error != 0) {
184 g_topology_unlock();
185 return (error);
186 }
187 }
188 /* Wait for withering to finish. */
189 for (;;) {
190 gp = LIST_FIRST(&mp->geom);
191 if (gp == NULL)
192 break;
193 KASSERT(gp->flags & G_GEOM_WITHER,
194 ("Non-withering geom in class %s", mp->name));
195 g_topology_sleep(mp, 1);
196 }
197 G_VALID_CLASS(mp);
198 if (mp->fini != NULL)
199 mp->fini(mp);
200 LIST_REMOVE(mp, class);
201 g_topology_unlock();
202
203 return (0);
204 }
205
206 int
207 g_modevent(module_t mod, int type, void *data)
208 {
209 struct g_hh00 *hh;
210 int error;
211 static int g_ignition;
212 struct g_class *mp;
213
214 mp = data;
215 if (mp->version != G_VERSION) {
216 printf("GEOM class %s has Wrong version %x\n",
217 mp->name, mp->version);
218 return (EINVAL);
219 }
220 if (!g_ignition) {
221 g_ignition++;
222 g_init();
223 }
224 error = EOPNOTSUPP;
225 switch (type) {
226 case MOD_LOAD:
227 g_trace(G_T_TOPOLOGY, "g_modevent(%s, LOAD)", mp->name);
228 hh = g_malloc(sizeof *hh, M_WAITOK | M_ZERO);
229 hh->mp = mp;
230 /*
231 * Once the system is not cold, MOD_LOAD calls will be
232 * from the userland and the g_event thread will be able
233 * to acknowledge their completion.
234 */
235 if (cold) {
236 hh->post = 1;
237 error = g_post_event(g_load_class, hh, M_WAITOK, NULL);
238 } else {
239 error = g_waitfor_event(g_load_class, hh, M_WAITOK,
240 NULL);
241 if (error == 0)
242 error = hh->error;
243 g_free(hh);
244 }
245 break;
246 case MOD_UNLOAD:
247 g_trace(G_T_TOPOLOGY, "g_modevent(%s, UNLOAD)", mp->name);
248 DROP_GIANT();
249 error = g_unload_class(mp);
250 PICKUP_GIANT();
251 if (error == 0) {
252 KASSERT(LIST_EMPTY(&mp->geom),
253 ("Unloaded class (%s) still has geom", mp->name));
254 }
255 break;
256 }
257 return (error);
258 }
259
260 static void
261 g_retaste_event(void *arg, int flag)
262 {
263 struct g_class *cp, *mp;
264 struct g_geom *gp, *gp2;
265 struct g_hh00 *hh;
266 struct g_provider *pp;
267
268 g_topology_assert();
269 if (flag == EV_CANCEL) /* XXX: can't happen ? */
270 return;
271 if (g_shutdown)
272 return;
273
274 hh = arg;
275 mp = hh->mp;
276 hh->error = 0;
277 if (hh->post) {
278 g_free(hh);
279 hh = NULL;
280 }
281 g_trace(G_T_TOPOLOGY, "g_retaste(%s)", mp->name);
282
283 LIST_FOREACH(cp, &g_classes, class) {
284 LIST_FOREACH(gp, &cp->geom, geom) {
285 LIST_FOREACH(pp, &gp->provider, provider) {
286 if (pp->acr || pp->acw || pp->ace)
287 continue;
288 LIST_FOREACH(gp2, &mp->geom, geom) {
289 if (!strcmp(pp->name, gp2->name))
290 break;
291 }
292 if (gp2 != NULL)
293 g_wither_geom(gp2, ENXIO);
294 mp->taste(mp, pp, 0);
295 g_topology_assert();
296 }
297 }
298 }
299 }
300
301 int
302 g_retaste(struct g_class *mp)
303 {
304 struct g_hh00 *hh;
305 int error;
306
307 if (mp->taste == NULL)
308 return (EINVAL);
309
310 hh = g_malloc(sizeof *hh, M_WAITOK | M_ZERO);
311 hh->mp = mp;
312
313 if (cold) {
314 hh->post = 1;
315 error = g_post_event(g_retaste_event, hh, M_WAITOK, NULL);
316 } else {
317 error = g_waitfor_event(g_retaste_event, hh, M_WAITOK, NULL);
318 if (error == 0)
319 error = hh->error;
320 g_free(hh);
321 }
322
323 return (error);
324 }
325
326 struct g_geom *
327 g_new_geomf(struct g_class *mp, const char *fmt, ...)
328 {
329 struct g_geom *gp;
330 va_list ap;
331 struct sbuf *sb;
332
333 g_topology_assert();
334 G_VALID_CLASS(mp);
335 sb = sbuf_new_auto();
336 va_start(ap, fmt);
337 sbuf_vprintf(sb, fmt, ap);
338 va_end(ap);
339 sbuf_finish(sb);
340 gp = g_malloc(sizeof *gp, M_WAITOK | M_ZERO);
341 gp->name = g_malloc(sbuf_len(sb) + 1, M_WAITOK | M_ZERO);
342 gp->class = mp;
343 gp->rank = 1;
344 LIST_INIT(&gp->consumer);
345 LIST_INIT(&gp->provider);
346 LIST_INSERT_HEAD(&mp->geom, gp, geom);
347 TAILQ_INSERT_HEAD(&geoms, gp, geoms);
348 strcpy(gp->name, sbuf_data(sb));
349 sbuf_delete(sb);
350 /* Fill in defaults from class */
351 gp->start = mp->start;
352 gp->spoiled = mp->spoiled;
353 gp->dumpconf = mp->dumpconf;
354 gp->access = mp->access;
355 gp->orphan = mp->orphan;
356 gp->ioctl = mp->ioctl;
357 return (gp);
358 }
359
360 void
361 g_destroy_geom(struct g_geom *gp)
362 {
363
364 g_topology_assert();
365 G_VALID_GEOM(gp);
366 g_trace(G_T_TOPOLOGY, "g_destroy_geom(%p(%s))", gp, gp->name);
367 KASSERT(LIST_EMPTY(&gp->consumer),
368 ("g_destroy_geom(%s) with consumer(s) [%p]",
369 gp->name, LIST_FIRST(&gp->consumer)));
370 KASSERT(LIST_EMPTY(&gp->provider),
371 ("g_destroy_geom(%s) with provider(s) [%p]",
372 gp->name, LIST_FIRST(&gp->provider)));
373 g_cancel_event(gp);
374 LIST_REMOVE(gp, geom);
375 TAILQ_REMOVE(&geoms, gp, geoms);
376 g_free(gp->name);
377 g_free(gp);
378 }
379
380 /*
381 * This function is called (repeatedly) until the geom has withered away.
382 */
383 void
384 g_wither_geom(struct g_geom *gp, int error)
385 {
386 struct g_provider *pp;
387
388 g_topology_assert();
389 G_VALID_GEOM(gp);
390 g_trace(G_T_TOPOLOGY, "g_wither_geom(%p(%s))", gp, gp->name);
391 if (!(gp->flags & G_GEOM_WITHER)) {
392 gp->flags |= G_GEOM_WITHER;
393 LIST_FOREACH(pp, &gp->provider, provider)
394 if (!(pp->flags & G_PF_ORPHAN))
395 g_orphan_provider(pp, error);
396 }
397 g_do_wither();
398 }
399
400 /*
401 * Convenience function to destroy a particular provider.
402 */
403 void
404 g_wither_provider(struct g_provider *pp, int error)
405 {
406
407 pp->flags |= G_PF_WITHER;
408 if (!(pp->flags & G_PF_ORPHAN))
409 g_orphan_provider(pp, error);
410 }
411
412 /*
413 * This function is called (repeatedly) until the has withered away.
414 */
415 void
416 g_wither_geom_close(struct g_geom *gp, int error)
417 {
418 struct g_consumer *cp;
419
420 g_topology_assert();
421 G_VALID_GEOM(gp);
422 g_trace(G_T_TOPOLOGY, "g_wither_geom_close(%p(%s))", gp, gp->name);
423 LIST_FOREACH(cp, &gp->consumer, consumer)
424 if (cp->acr || cp->acw || cp->ace)
425 g_access(cp, -cp->acr, -cp->acw, -cp->ace);
426 g_wither_geom(gp, error);
427 }
428
429 /*
430 * This function is called (repeatedly) until we cant wash away more
431 * withered bits at present. Return value contains two bits. Bit 0
432 * set means "withering stuff we can't wash now", bit 1 means "call
433 * me again, there may be stuff I didn't get the first time around.
434 */
435 int
436 g_wither_washer()
437 {
438 struct g_class *mp;
439 struct g_geom *gp, *gp2;
440 struct g_provider *pp, *pp2;
441 struct g_consumer *cp, *cp2;
442 int result;
443
444 result = 0;
445 g_topology_assert();
446 LIST_FOREACH(mp, &g_classes, class) {
447 LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
448 LIST_FOREACH_SAFE(pp, &gp->provider, provider, pp2) {
449 if (!(pp->flags & G_PF_WITHER))
450 continue;
451 if (LIST_EMPTY(&pp->consumers))
452 g_destroy_provider(pp);
453 else
454 result |= 1;
455 }
456 if (!(gp->flags & G_GEOM_WITHER))
457 continue;
458 LIST_FOREACH_SAFE(pp, &gp->provider, provider, pp2) {
459 if (LIST_EMPTY(&pp->consumers))
460 g_destroy_provider(pp);
461 else
462 result |= 1;
463 }
464 LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp2) {
465 if (cp->acr || cp->acw || cp->ace) {
466 result |= 1;
467 continue;
468 }
469 if (cp->provider != NULL)
470 g_detach(cp);
471 g_destroy_consumer(cp);
472 result |= 2;
473 }
474 if (LIST_EMPTY(&gp->provider) &&
475 LIST_EMPTY(&gp->consumer))
476 g_destroy_geom(gp);
477 else
478 result |= 1;
479 }
480 }
481 return (result);
482 }
483
484 struct g_consumer *
485 g_new_consumer(struct g_geom *gp)
486 {
487 struct g_consumer *cp;
488
489 g_topology_assert();
490 G_VALID_GEOM(gp);
491 KASSERT(!(gp->flags & G_GEOM_WITHER),
492 ("g_new_consumer on WITHERing geom(%s) (class %s)",
493 gp->name, gp->class->name));
494 KASSERT(gp->orphan != NULL,
495 ("g_new_consumer on geom(%s) (class %s) without orphan",
496 gp->name, gp->class->name));
497
498 cp = g_malloc(sizeof *cp, M_WAITOK | M_ZERO);
499 cp->geom = gp;
500 cp->stat = devstat_new_entry(cp, -1, 0, DEVSTAT_ALL_SUPPORTED,
501 DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX);
502 LIST_INSERT_HEAD(&gp->consumer, cp, consumer);
503 return(cp);
504 }
505
506 void
507 g_destroy_consumer(struct g_consumer *cp)
508 {
509 struct g_geom *gp;
510
511 g_topology_assert();
512 G_VALID_CONSUMER(cp);
513 g_trace(G_T_TOPOLOGY, "g_destroy_consumer(%p)", cp);
514 KASSERT (cp->provider == NULL, ("g_destroy_consumer but attached"));
515 KASSERT (cp->acr == 0, ("g_destroy_consumer with acr"));
516 KASSERT (cp->acw == 0, ("g_destroy_consumer with acw"));
517 KASSERT (cp->ace == 0, ("g_destroy_consumer with ace"));
518 g_cancel_event(cp);
519 gp = cp->geom;
520 LIST_REMOVE(cp, consumer);
521 devstat_remove_entry(cp->stat);
522 g_free(cp);
523 if (gp->flags & G_GEOM_WITHER)
524 g_do_wither();
525 }
526
527 static void
528 g_new_provider_event(void *arg, int flag)
529 {
530 struct g_class *mp;
531 struct g_provider *pp;
532 struct g_consumer *cp;
533
534 g_topology_assert();
535 if (flag == EV_CANCEL)
536 return;
537 if (g_shutdown)
538 return;
539 pp = arg;
540 G_VALID_PROVIDER(pp);
541 KASSERT(!(pp->flags & G_PF_WITHER),
542 ("g_new_provider_event but withered"));
543 LIST_FOREACH(mp, &g_classes, class) {
544 if (mp->taste == NULL)
545 continue;
546 LIST_FOREACH(cp, &pp->consumers, consumers)
547 if (cp->geom->class == mp)
548 break;
549 if (cp != NULL)
550 continue;
551 mp->taste(mp, pp, 0);
552 g_topology_assert();
553 }
554 }
555
556
557 struct g_provider *
558 g_new_providerf(struct g_geom *gp, const char *fmt, ...)
559 {
560 struct g_provider *pp;
561 struct sbuf *sb;
562 va_list ap;
563
564 g_topology_assert();
565 G_VALID_GEOM(gp);
566 KASSERT(gp->access != NULL,
567 ("new provider on geom(%s) without ->access (class %s)",
568 gp->name, gp->class->name));
569 KASSERT(gp->start != NULL,
570 ("new provider on geom(%s) without ->start (class %s)",
571 gp->name, gp->class->name));
572 KASSERT(!(gp->flags & G_GEOM_WITHER),
573 ("new provider on WITHERing geom(%s) (class %s)",
574 gp->name, gp->class->name));
575 sb = sbuf_new_auto();
576 va_start(ap, fmt);
577 sbuf_vprintf(sb, fmt, ap);
578 va_end(ap);
579 sbuf_finish(sb);
580 pp = g_malloc(sizeof *pp + sbuf_len(sb) + 1, M_WAITOK | M_ZERO);
581 pp->name = (char *)(pp + 1);
582 strcpy(pp->name, sbuf_data(sb));
583 sbuf_delete(sb);
584 LIST_INIT(&pp->consumers);
585 pp->error = ENXIO;
586 pp->geom = gp;
587 pp->stat = devstat_new_entry(pp, -1, 0, DEVSTAT_ALL_SUPPORTED,
588 DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX);
589 LIST_INSERT_HEAD(&gp->provider, pp, provider);
590 g_post_event(g_new_provider_event, pp, M_WAITOK, pp, gp, NULL);
591 return (pp);
592 }
593
594 void
595 g_error_provider(struct g_provider *pp, int error)
596 {
597
598 /* G_VALID_PROVIDER(pp); We may not have g_topology */
599 pp->error = error;
600 }
601
602 struct g_provider *
603 g_provider_by_name(char const *arg)
604 {
605 struct g_class *cp;
606 struct g_geom *gp;
607 struct g_provider *pp;
608
609 LIST_FOREACH(cp, &g_classes, class) {
610 LIST_FOREACH(gp, &cp->geom, geom) {
611 LIST_FOREACH(pp, &gp->provider, provider) {
612 if (!strcmp(arg, pp->name))
613 return (pp);
614 }
615 }
616 }
617 return (NULL);
618 }
619
620 void
621 g_destroy_provider(struct g_provider *pp)
622 {
623 struct g_geom *gp;
624
625 g_topology_assert();
626 G_VALID_PROVIDER(pp);
627 KASSERT(LIST_EMPTY(&pp->consumers),
628 ("g_destroy_provider but attached"));
629 KASSERT (pp->acr == 0, ("g_destroy_provider with acr"));
630 KASSERT (pp->acw == 0, ("g_destroy_provider with acw"));
631 KASSERT (pp->ace == 0, ("g_destroy_provider with ace"));
632 g_cancel_event(pp);
633 LIST_REMOVE(pp, provider);
634 gp = pp->geom;
635 devstat_remove_entry(pp->stat);
636 g_free(pp);
637 if ((gp->flags & G_GEOM_WITHER))
638 g_do_wither();
639 }
640
641 /*
642 * We keep the "geoms" list sorted by topological order (== increasing
643 * numerical rank) at all times.
644 * When an attach is done, the attaching geoms rank is invalidated
645 * and it is moved to the tail of the list.
646 * All geoms later in the sequence has their ranks reevaluated in
647 * sequence. If we cannot assign rank to a geom because it's
648 * prerequisites do not have rank, we move that element to the tail
649 * of the sequence with invalid rank as well.
650 * At some point we encounter our original geom and if we stil fail
651 * to assign it a rank, there must be a loop and we fail back to
652 * g_attach() which detach again and calls redo_rank again
653 * to fix up the damage.
654 * It would be much simpler code wise to do it recursively, but we
655 * can't risk that on the kernel stack.
656 */
657
658 static int
659 redo_rank(struct g_geom *gp)
660 {
661 struct g_consumer *cp;
662 struct g_geom *gp1, *gp2;
663 int n, m;
664
665 g_topology_assert();
666 G_VALID_GEOM(gp);
667
668 /* Invalidate this geoms rank and move it to the tail */
669 gp1 = TAILQ_NEXT(gp, geoms);
670 if (gp1 != NULL) {
671 gp->rank = 0;
672 TAILQ_REMOVE(&geoms, gp, geoms);
673 TAILQ_INSERT_TAIL(&geoms, gp, geoms);
674 } else {
675 gp1 = gp;
676 }
677
678 /* re-rank the rest of the sequence */
679 for (; gp1 != NULL; gp1 = gp2) {
680 gp1->rank = 0;
681 m = 1;
682 LIST_FOREACH(cp, &gp1->consumer, consumer) {
683 if (cp->provider == NULL)
684 continue;
685 n = cp->provider->geom->rank;
686 if (n == 0) {
687 m = 0;
688 break;
689 } else if (n >= m)
690 m = n + 1;
691 }
692 gp1->rank = m;
693 gp2 = TAILQ_NEXT(gp1, geoms);
694
695 /* got a rank, moving on */
696 if (m != 0)
697 continue;
698
699 /* no rank to original geom means loop */
700 if (gp == gp1)
701 return (ELOOP);
702
703 /* no rank, put it at the end move on */
704 TAILQ_REMOVE(&geoms, gp1, geoms);
705 TAILQ_INSERT_TAIL(&geoms, gp1, geoms);
706 }
707 return (0);
708 }
709
710 int
711 g_attach(struct g_consumer *cp, struct g_provider *pp)
712 {
713 int error;
714
715 g_topology_assert();
716 G_VALID_CONSUMER(cp);
717 G_VALID_PROVIDER(pp);
718 KASSERT(cp->provider == NULL, ("attach but attached"));
719 cp->provider = pp;
720 LIST_INSERT_HEAD(&pp->consumers, cp, consumers);
721 error = redo_rank(cp->geom);
722 if (error) {
723 LIST_REMOVE(cp, consumers);
724 cp->provider = NULL;
725 redo_rank(cp->geom);
726 }
727 return (error);
728 }
729
730 void
731 g_detach(struct g_consumer *cp)
732 {
733 struct g_provider *pp;
734
735 g_topology_assert();
736 G_VALID_CONSUMER(cp);
737 g_trace(G_T_TOPOLOGY, "g_detach(%p)", cp);
738 KASSERT(cp->provider != NULL, ("detach but not attached"));
739 KASSERT(cp->acr == 0, ("detach but nonzero acr"));
740 KASSERT(cp->acw == 0, ("detach but nonzero acw"));
741 KASSERT(cp->ace == 0, ("detach but nonzero ace"));
742 KASSERT(cp->nstart == cp->nend,
743 ("detach with active requests"));
744 pp = cp->provider;
745 LIST_REMOVE(cp, consumers);
746 cp->provider = NULL;
747 if (pp->geom->flags & G_GEOM_WITHER)
748 g_do_wither();
749 else if (pp->flags & G_PF_WITHER)
750 g_do_wither();
751 redo_rank(cp->geom);
752 }
753
754 /*
755 * g_access()
756 *
757 * Access-check with delta values. The question asked is "can provider
758 * "cp" change the access counters by the relative amounts dc[rwe] ?"
759 */
760
761 int
762 g_access(struct g_consumer *cp, int dcr, int dcw, int dce)
763 {
764 struct g_provider *pp;
765 int pr,pw,pe;
766 int error;
767
768 g_topology_assert();
769 G_VALID_CONSUMER(cp);
770 pp = cp->provider;
771 KASSERT(pp != NULL, ("access but not attached"));
772 G_VALID_PROVIDER(pp);
773
774 g_trace(G_T_ACCESS, "g_access(%p(%s), %d, %d, %d)",
775 cp, pp->name, dcr, dcw, dce);
776
777 KASSERT(cp->acr + dcr >= 0, ("access resulting in negative acr"));
778 KASSERT(cp->acw + dcw >= 0, ("access resulting in negative acw"));
779 KASSERT(cp->ace + dce >= 0, ("access resulting in negative ace"));
780 KASSERT(dcr != 0 || dcw != 0 || dce != 0, ("NOP access request"));
781 KASSERT(pp->geom->access != NULL, ("NULL geom->access"));
782
783 /*
784 * If our class cares about being spoiled, and we have been, we
785 * are probably just ahead of the event telling us that. Fail
786 * now rather than having to unravel this later.
787 */
788 if (cp->geom->spoiled != NULL && cp->spoiled &&
789 (dcr > 0 || dcw > 0 || dce > 0))
790 return (ENXIO);
791
792 /*
793 * Figure out what counts the provider would have had, if this
794 * consumer had (r0w0e0) at this time.
795 */
796 pr = pp->acr - cp->acr;
797 pw = pp->acw - cp->acw;
798 pe = pp->ace - cp->ace;
799
800 g_trace(G_T_ACCESS,
801 "open delta:[r%dw%de%d] old:[r%dw%de%d] provider:[r%dw%de%d] %p(%s)",
802 dcr, dcw, dce,
803 cp->acr, cp->acw, cp->ace,
804 pp->acr, pp->acw, pp->ace,
805 pp, pp->name);
806
807 /* If foot-shooting is enabled, any open on rank#1 is OK */
808 if ((g_debugflags & 16) && pp->geom->rank == 1)
809 ;
810 /* If we try exclusive but already write: fail */
811 else if (dce > 0 && pw > 0)
812 return (EPERM);
813 /* If we try write but already exclusive: fail */
814 else if (dcw > 0 && pe > 0)
815 return (EPERM);
816 /* If we try to open more but provider is error'ed: fail */
817 else if ((dcr > 0 || dcw > 0 || dce > 0) && pp->error != 0)
818 return (pp->error);
819
820 /* Ok then... */
821
822 error = pp->geom->access(pp, dcr, dcw, dce);
823 KASSERT(dcr > 0 || dcw > 0 || dce > 0 || error == 0,
824 ("Geom provider %s::%s failed closing ->access()",
825 pp->geom->class->name, pp->name));
826 if (!error) {
827 /*
828 * If we open first write, spoil any partner consumers.
829 * If we close last write and provider is not errored,
830 * trigger re-taste.
831 */
832 if (pp->acw == 0 && dcw != 0)
833 g_spoil(pp, cp);
834 else if (pp->acw != 0 && pp->acw == -dcw && pp->error == 0 &&
835 !(pp->geom->flags & G_GEOM_WITHER))
836 g_post_event(g_new_provider_event, pp, M_WAITOK,
837 pp, NULL);
838
839 pp->acr += dcr;
840 pp->acw += dcw;
841 pp->ace += dce;
842 cp->acr += dcr;
843 cp->acw += dcw;
844 cp->ace += dce;
845 if (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)
846 KASSERT(pp->sectorsize > 0,
847 ("Provider %s lacks sectorsize", pp->name));
848 }
849 return (error);
850 }
851
852 int
853 g_handleattr_int(struct bio *bp, const char *attribute, int val)
854 {
855
856 return (g_handleattr(bp, attribute, &val, sizeof val));
857 }
858
859 int
860 g_handleattr_off_t(struct bio *bp, const char *attribute, off_t val)
861 {
862
863 return (g_handleattr(bp, attribute, &val, sizeof val));
864 }
865
866 int
867 g_handleattr_str(struct bio *bp, const char *attribute, const char *str)
868 {
869
870 return (g_handleattr(bp, attribute, str, 0));
871 }
872
873 int
874 g_handleattr(struct bio *bp, const char *attribute, const void *val, int len)
875 {
876 int error = 0;
877
878 if (strcmp(bp->bio_attribute, attribute))
879 return (0);
880 if (len == 0) {
881 bzero(bp->bio_data, bp->bio_length);
882 if (strlcpy(bp->bio_data, val, bp->bio_length) >=
883 bp->bio_length) {
884 printf("%s: %s bio_length %jd len %zu -> EFAULT\n",
885 __func__, bp->bio_to->name,
886 (intmax_t)bp->bio_length, strlen(val));
887 error = EFAULT;
888 }
889 } else if (bp->bio_length == len) {
890 bcopy(val, bp->bio_data, len);
891 } else {
892 printf("%s: %s bio_length %jd len %d -> EFAULT\n", __func__,
893 bp->bio_to->name, (intmax_t)bp->bio_length, len);
894 error = EFAULT;
895 }
896 if (error == 0)
897 bp->bio_completed = bp->bio_length;
898 g_io_deliver(bp, error);
899 return (1);
900 }
901
902 int
903 g_std_access(struct g_provider *pp,
904 int dr __unused, int dw __unused, int de __unused)
905 {
906
907 g_topology_assert();
908 G_VALID_PROVIDER(pp);
909 return (0);
910 }
911
912 void
913 g_std_done(struct bio *bp)
914 {
915 struct bio *bp2;
916
917 bp2 = bp->bio_parent;
918 if (bp2->bio_error == 0)
919 bp2->bio_error = bp->bio_error;
920 bp2->bio_completed += bp->bio_completed;
921 g_destroy_bio(bp);
922 bp2->bio_inbed++;
923 if (bp2->bio_children == bp2->bio_inbed)
924 g_io_deliver(bp2, bp2->bio_error);
925 }
926
927 /* XXX: maybe this is only g_slice_spoiled */
928
929 void
930 g_std_spoiled(struct g_consumer *cp)
931 {
932 struct g_geom *gp;
933 struct g_provider *pp;
934
935 g_topology_assert();
936 G_VALID_CONSUMER(cp);
937 g_trace(G_T_TOPOLOGY, "g_std_spoiled(%p)", cp);
938 g_detach(cp);
939 gp = cp->geom;
940 LIST_FOREACH(pp, &gp->provider, provider)
941 g_orphan_provider(pp, ENXIO);
942 g_destroy_consumer(cp);
943 if (LIST_EMPTY(&gp->provider) && LIST_EMPTY(&gp->consumer))
944 g_destroy_geom(gp);
945 else
946 gp->flags |= G_GEOM_WITHER;
947 }
948
949 /*
950 * Spoiling happens when a provider is opened for writing, but consumers
951 * which are configured by in-band data are attached (slicers for instance).
952 * Since the write might potentially change the in-band data, such consumers
953 * need to re-evaluate their existence after the writing session closes.
954 * We do this by (offering to) tear them down when the open for write happens
955 * in return for a re-taste when it closes again.
956 * Together with the fact that such consumers grab an 'e' bit whenever they
957 * are open, regardless of mode, this ends up DTRT.
958 */
959
960 static void
961 g_spoil_event(void *arg, int flag)
962 {
963 struct g_provider *pp;
964 struct g_consumer *cp, *cp2;
965
966 g_topology_assert();
967 if (flag == EV_CANCEL)
968 return;
969 pp = arg;
970 G_VALID_PROVIDER(pp);
971 for (cp = LIST_FIRST(&pp->consumers); cp != NULL; cp = cp2) {
972 cp2 = LIST_NEXT(cp, consumers);
973 if (!cp->spoiled)
974 continue;
975 cp->spoiled = 0;
976 if (cp->geom->spoiled == NULL)
977 continue;
978 cp->geom->spoiled(cp);
979 g_topology_assert();
980 }
981 }
982
983 void
984 g_spoil(struct g_provider *pp, struct g_consumer *cp)
985 {
986 struct g_consumer *cp2;
987
988 g_topology_assert();
989 G_VALID_PROVIDER(pp);
990 G_VALID_CONSUMER(cp);
991
992 LIST_FOREACH(cp2, &pp->consumers, consumers) {
993 if (cp2 == cp)
994 continue;
995 /*
996 KASSERT(cp2->acr == 0, ("spoiling cp->acr = %d", cp2->acr));
997 KASSERT(cp2->acw == 0, ("spoiling cp->acw = %d", cp2->acw));
998 */
999 KASSERT(cp2->ace == 0, ("spoiling cp->ace = %d", cp2->ace));
1000 cp2->spoiled++;
1001 }
1002 g_post_event(g_spoil_event, pp, M_WAITOK, pp, NULL);
1003 }
1004
1005 int
1006 g_getattr__(const char *attr, struct g_consumer *cp, void *var, int len)
1007 {
1008 int error, i;
1009
1010 i = len;
1011 error = g_io_getattr(attr, cp, &i, var);
1012 if (error)
1013 return (error);
1014 if (i != len)
1015 return (EINVAL);
1016 return (0);
1017 }
1018
1019 static int
1020 g_get_device_prefix_len(const char *name)
1021 {
1022 int len;
1023
1024 if (strncmp(name, "ada", 3) == 0)
1025 len = 3;
1026 else if (strncmp(name, "ad", 2) == 0)
1027 len = 2;
1028 else
1029 return (0);
1030 if (name[len] < '' || name[len] > '9')
1031 return (0);
1032 do {
1033 len++;
1034 } while (name[len] >= '' && name[len] <= '9');
1035 return (len);
1036 }
1037
1038 int
1039 g_compare_names(const char *namea, const char *nameb)
1040 {
1041 int deva, devb;
1042
1043 if (strcmp(namea, nameb) == 0)
1044 return (1);
1045 deva = g_get_device_prefix_len(namea);
1046 if (deva == 0)
1047 return (0);
1048 devb = g_get_device_prefix_len(nameb);
1049 if (devb == 0)
1050 return (0);
1051 if (strcmp(namea + deva, nameb + devb) == 0)
1052 return (1);
1053 return (0);
1054 }
1055
1056 #if defined(DIAGNOSTIC) || defined(DDB)
1057 /*
1058 * This function walks the mesh and returns a non-zero integer if it
1059 * finds the argument pointer is an object. The return value indicates
1060 * which type of object it is believed to be. If topology is not locked,
1061 * this function is potentially dangerous, but we don't assert that the
1062 * topology lock is held when called from debugger.
1063 */
1064 int
1065 g_valid_obj(void const *ptr)
1066 {
1067 struct g_class *mp;
1068 struct g_geom *gp;
1069 struct g_consumer *cp;
1070 struct g_provider *pp;
1071
1072 #ifdef KDB
1073 if (kdb_active == 0)
1074 #endif
1075 g_topology_assert();
1076
1077 LIST_FOREACH(mp, &g_classes, class) {
1078 if (ptr == mp)
1079 return (1);
1080 LIST_FOREACH(gp, &mp->geom, geom) {
1081 if (ptr == gp)
1082 return (2);
1083 LIST_FOREACH(cp, &gp->consumer, consumer)
1084 if (ptr == cp)
1085 return (3);
1086 LIST_FOREACH(pp, &gp->provider, provider)
1087 if (ptr == pp)
1088 return (4);
1089 }
1090 }
1091 return(0);
1092 }
1093 #endif
1094
1095 #ifdef DDB
1096
1097 #define gprintf(...) do { \
1098 db_printf("%*s", indent, ""); \
1099 db_printf(__VA_ARGS__); \
1100 } while (0)
1101 #define gprintln(...) do { \
1102 gprintf(__VA_ARGS__); \
1103 db_printf("\n"); \
1104 } while (0)
1105
1106 #define ADDFLAG(obj, flag, sflag) do { \
1107 if ((obj)->flags & (flag)) { \
1108 if (comma) \
1109 strlcat(str, ",", size); \
1110 strlcat(str, (sflag), size); \
1111 comma = 1; \
1112 } \
1113 } while (0)
1114
1115 static char *
1116 provider_flags_to_string(struct g_provider *pp, char *str, size_t size)
1117 {
1118 int comma = 0;
1119
1120 bzero(str, size);
1121 if (pp->flags == 0) {
1122 strlcpy(str, "NONE", size);
1123 return (str);
1124 }
1125 ADDFLAG(pp, G_PF_CANDELETE, "G_PF_CANDELETE");
1126 ADDFLAG(pp, G_PF_WITHER, "G_PF_WITHER");
1127 ADDFLAG(pp, G_PF_ORPHAN, "G_PF_ORPHAN");
1128 return (str);
1129 }
1130
1131 static char *
1132 geom_flags_to_string(struct g_geom *gp, char *str, size_t size)
1133 {
1134 int comma = 0;
1135
1136 bzero(str, size);
1137 if (gp->flags == 0) {
1138 strlcpy(str, "NONE", size);
1139 return (str);
1140 }
1141 ADDFLAG(gp, G_GEOM_WITHER, "G_GEOM_WITHER");
1142 return (str);
1143 }
1144 static void
1145 db_show_geom_consumer(int indent, struct g_consumer *cp)
1146 {
1147
1148 if (indent == 0) {
1149 gprintln("consumer: %p", cp);
1150 gprintln(" class: %s (%p)", cp->geom->class->name,
1151 cp->geom->class);
1152 gprintln(" geom: %s (%p)", cp->geom->name, cp->geom);
1153 if (cp->provider == NULL)
1154 gprintln(" provider: none");
1155 else {
1156 gprintln(" provider: %s (%p)", cp->provider->name,
1157 cp->provider);
1158 }
1159 gprintln(" access: r%dw%de%d", cp->acr, cp->acw, cp->ace);
1160 gprintln(" spoiled: %d", cp->spoiled);
1161 gprintln(" nstart: %u", cp->nstart);
1162 gprintln(" nend: %u", cp->nend);
1163 } else {
1164 gprintf("consumer: %p (%s), access=r%dw%de%d", cp,
1165 cp->provider != NULL ? cp->provider->name : "none",
1166 cp->acr, cp->acw, cp->ace);
1167 if (cp->spoiled)
1168 db_printf(", spoiled=%d", cp->spoiled);
1169 db_printf("\n");
1170 }
1171 }
1172
1173 static void
1174 db_show_geom_provider(int indent, struct g_provider *pp)
1175 {
1176 struct g_consumer *cp;
1177 char flags[64];
1178
1179 if (indent == 0) {
1180 gprintln("provider: %s (%p)", pp->name, pp);
1181 gprintln(" class: %s (%p)", pp->geom->class->name,
1182 pp->geom->class);
1183 gprintln(" geom: %s (%p)", pp->geom->name, pp->geom);
1184 gprintln(" mediasize: %jd", (intmax_t)pp->mediasize);
1185 gprintln(" sectorsize: %u", pp->sectorsize);
1186 gprintln(" stripesize: %u", pp->stripesize);
1187 gprintln(" stripeoffset: %u", pp->stripeoffset);
1188 gprintln(" access: r%dw%de%d", pp->acr, pp->acw,
1189 pp->ace);
1190 gprintln(" flags: %s (0x%04x)",
1191 provider_flags_to_string(pp, flags, sizeof(flags)),
1192 pp->flags);
1193 gprintln(" error: %d", pp->error);
1194 gprintln(" nstart: %u", pp->nstart);
1195 gprintln(" nend: %u", pp->nend);
1196 if (LIST_EMPTY(&pp->consumers))
1197 gprintln(" consumers: none");
1198 } else {
1199 gprintf("provider: %s (%p), access=r%dw%de%d",
1200 pp->name, pp, pp->acr, pp->acw, pp->ace);
1201 if (pp->flags != 0) {
1202 db_printf(", flags=%s (0x%04x)",
1203 provider_flags_to_string(pp, flags, sizeof(flags)),
1204 pp->flags);
1205 }
1206 db_printf("\n");
1207 }
1208 if (!LIST_EMPTY(&pp->consumers)) {
1209 LIST_FOREACH(cp, &pp->consumers, consumers) {
1210 db_show_geom_consumer(indent + 2, cp);
1211 if (db_pager_quit)
1212 break;
1213 }
1214 }
1215 }
1216
1217 static void
1218 db_show_geom_geom(int indent, struct g_geom *gp)
1219 {
1220 struct g_provider *pp;
1221 struct g_consumer *cp;
1222 char flags[64];
1223
1224 if (indent == 0) {
1225 gprintln("geom: %s (%p)", gp->name, gp);
1226 gprintln(" class: %s (%p)", gp->class->name, gp->class);
1227 gprintln(" flags: %s (0x%04x)",
1228 geom_flags_to_string(gp, flags, sizeof(flags)), gp->flags);
1229 gprintln(" rank: %d", gp->rank);
1230 if (LIST_EMPTY(&gp->provider))
1231 gprintln(" providers: none");
1232 if (LIST_EMPTY(&gp->consumer))
1233 gprintln(" consumers: none");
1234 } else {
1235 gprintf("geom: %s (%p), rank=%d", gp->name, gp, gp->rank);
1236 if (gp->flags != 0) {
1237 db_printf(", flags=%s (0x%04x)",
1238 geom_flags_to_string(gp, flags, sizeof(flags)),
1239 gp->flags);
1240 }
1241 db_printf("\n");
1242 }
1243 if (!LIST_EMPTY(&gp->provider)) {
1244 LIST_FOREACH(pp, &gp->provider, provider) {
1245 db_show_geom_provider(indent + 2, pp);
1246 if (db_pager_quit)
1247 break;
1248 }
1249 }
1250 if (!LIST_EMPTY(&gp->consumer)) {
1251 LIST_FOREACH(cp, &gp->consumer, consumer) {
1252 db_show_geom_consumer(indent + 2, cp);
1253 if (db_pager_quit)
1254 break;
1255 }
1256 }
1257 }
1258
1259 static void
1260 db_show_geom_class(struct g_class *mp)
1261 {
1262 struct g_geom *gp;
1263
1264 db_printf("class: %s (%p)\n", mp->name, mp);
1265 LIST_FOREACH(gp, &mp->geom, geom) {
1266 db_show_geom_geom(2, gp);
1267 if (db_pager_quit)
1268 break;
1269 }
1270 }
1271
1272 /*
1273 * Print the GEOM topology or the given object.
1274 */
1275 DB_SHOW_COMMAND(geom, db_show_geom)
1276 {
1277 struct g_class *mp;
1278
1279 if (!have_addr) {
1280 /* No address given, print the entire topology. */
1281 LIST_FOREACH(mp, &g_classes, class) {
1282 db_show_geom_class(mp);
1283 db_printf("\n");
1284 if (db_pager_quit)
1285 break;
1286 }
1287 } else {
1288 switch (g_valid_obj((void *)addr)) {
1289 case 1:
1290 db_show_geom_class((struct g_class *)addr);
1291 break;
1292 case 2:
1293 db_show_geom_geom(0, (struct g_geom *)addr);
1294 break;
1295 case 3:
1296 db_show_geom_consumer(0, (struct g_consumer *)addr);
1297 break;
1298 case 4:
1299 db_show_geom_provider(0, (struct g_provider *)addr);
1300 break;
1301 default:
1302 db_printf("Not a GEOM object.\n");
1303 break;
1304 }
1305 }
1306 }
1307
1308 static void
1309 db_print_bio_cmd(struct bio *bp)
1310 {
1311 db_printf(" cmd: ");
1312 switch (bp->bio_cmd) {
1313 case BIO_READ: db_printf("BIO_READ"); break;
1314 case BIO_WRITE: db_printf("BIO_WRITE"); break;
1315 case BIO_DELETE: db_printf("BIO_DELETE"); break;
1316 case BIO_GETATTR: db_printf("BIO_GETATTR"); break;
1317 case BIO_FLUSH: db_printf("BIO_FLUSH"); break;
1318 case BIO_CMD0: db_printf("BIO_CMD0"); break;
1319 case BIO_CMD1: db_printf("BIO_CMD1"); break;
1320 case BIO_CMD2: db_printf("BIO_CMD2"); break;
1321 default: db_printf("UNKNOWN"); break;
1322 }
1323 db_printf("\n");
1324 }
1325
1326 static void
1327 db_print_bio_flags(struct bio *bp)
1328 {
1329 int comma;
1330
1331 comma = 0;
1332 db_printf(" flags: ");
1333 if (bp->bio_flags & BIO_ERROR) {
1334 db_printf("BIO_ERROR");
1335 comma = 1;
1336 }
1337 if (bp->bio_flags & BIO_DONE) {
1338 db_printf("%sBIO_DONE", (comma ? ", " : ""));
1339 comma = 1;
1340 }
1341 if (bp->bio_flags & BIO_ONQUEUE)
1342 db_printf("%sBIO_ONQUEUE", (comma ? ", " : ""));
1343 db_printf("\n");
1344 }
1345
1346 /*
1347 * Print useful information in a BIO
1348 */
1349 DB_SHOW_COMMAND(bio, db_show_bio)
1350 {
1351 struct bio *bp;
1352
1353 if (have_addr) {
1354 bp = (struct bio *)addr;
1355 db_printf("BIO %p\n", bp);
1356 db_print_bio_cmd(bp);
1357 db_print_bio_flags(bp);
1358 db_printf(" cflags: 0x%hhx\n", bp->bio_cflags);
1359 db_printf(" pflags: 0x%hhx\n", bp->bio_pflags);
1360 db_printf(" offset: %jd\n", (intmax_t)bp->bio_offset);
1361 db_printf(" length: %jd\n", (intmax_t)bp->bio_length);
1362 db_printf(" bcount: %ld\n", bp->bio_bcount);
1363 db_printf(" resid: %ld\n", bp->bio_resid);
1364 db_printf(" completed: %jd\n", (intmax_t)bp->bio_completed);
1365 db_printf(" children: %u\n", bp->bio_children);
1366 db_printf(" inbed: %u\n", bp->bio_inbed);
1367 db_printf(" error: %d\n", bp->bio_error);
1368 db_printf(" parent: %p\n", bp->bio_parent);
1369 db_printf(" driver1: %p\n", bp->bio_driver1);
1370 db_printf(" driver2: %p\n", bp->bio_driver2);
1371 db_printf(" caller1: %p\n", bp->bio_caller1);
1372 db_printf(" caller2: %p\n", bp->bio_caller2);
1373 db_printf(" bio_from: %p\n", bp->bio_from);
1374 db_printf(" bio_to: %p\n", bp->bio_to);
1375 }
1376 }
1377
1378 #undef gprintf
1379 #undef gprintln
1380 #undef ADDFLAG
1381
1382 #endif /* DDB */
Cache object: 83225b02fff76f1ffcf355b01aa5adb4
|