1 /* $NetBSD: rf_dagfuncs.c,v 1.20 2004/03/04 00:54:30 oster Exp $ */
2 /*
3 * Copyright (c) 1995 Carnegie-Mellon University.
4 * All rights reserved.
5 *
6 * Author: Mark Holland, William V. Courtright II
7 *
8 * Permission to use, copy, modify and distribute this software and
9 * its documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 *
18 * Carnegie Mellon requests users of this software to return to
19 *
20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
21 * School of Computer Science
22 * Carnegie Mellon University
23 * Pittsburgh PA 15213-3890
24 *
25 * any improvements or extensions that they make and grant Carnegie the
26 * rights to redistribute these changes.
27 */
28
29 /*
30 * dagfuncs.c -- DAG node execution routines
31 *
32 * Rules:
33 * 1. Every DAG execution function must eventually cause node->status to
34 * get set to "good" or "bad", and "FinishNode" to be called. In the
35 * case of nodes that complete immediately (xor, NullNodeFunc, etc),
36 * the node execution function can do these two things directly. In
37 * the case of nodes that have to wait for some event (a disk read to
38 * complete, a lock to be released, etc) to occur before they can
39 * complete, this is typically achieved by having whatever module
40 * is doing the operation call GenericWakeupFunc upon completion.
41 * 2. DAG execution functions should check the status in the DAG header
42 * and NOP out their operations if the status is not "enable". However,
43 * execution functions that release resources must be sure to release
44 * them even when they NOP out the function that would use them.
45 * Functions that acquire resources should go ahead and acquire them
46 * even when they NOP, so that a downstream release node will not have
47 * to check to find out whether or not the acquire was suppressed.
48 */
49
50 #include <sys/cdefs.h>
51 __KERNEL_RCSID(0, "$NetBSD: rf_dagfuncs.c,v 1.20 2004/03/04 00:54:30 oster Exp $");
52
53 #include <sys/param.h>
54 #include <sys/ioctl.h>
55
56 #include "rf_archs.h"
57 #include "rf_raid.h"
58 #include "rf_dag.h"
59 #include "rf_layout.h"
60 #include "rf_etimer.h"
61 #include "rf_acctrace.h"
62 #include "rf_diskqueue.h"
63 #include "rf_dagfuncs.h"
64 #include "rf_general.h"
65 #include "rf_engine.h"
66 #include "rf_dagutils.h"
67
68 #include "rf_kintf.h"
69
70 #if RF_INCLUDE_PARITYLOGGING > 0
71 #include "rf_paritylog.h"
72 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
73
74 int (*rf_DiskReadFunc) (RF_DagNode_t *);
75 int (*rf_DiskWriteFunc) (RF_DagNode_t *);
76 int (*rf_DiskReadUndoFunc) (RF_DagNode_t *);
77 int (*rf_DiskWriteUndoFunc) (RF_DagNode_t *);
78 int (*rf_DiskUnlockFunc) (RF_DagNode_t *);
79 int (*rf_DiskUnlockUndoFunc) (RF_DagNode_t *);
80 int (*rf_RegularXorUndoFunc) (RF_DagNode_t *);
81 int (*rf_SimpleXorUndoFunc) (RF_DagNode_t *);
82 int (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *);
83
84 /*****************************************************************************
85 * main (only) configuration routine for this module
86 ****************************************************************************/
87 int
88 rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp)
89 {
90 RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) ||
91 ((sizeof(long) == 4) && RF_LONGSHIFT == 2));
92 rf_DiskReadFunc = rf_DiskReadFuncForThreads;
93 rf_DiskReadUndoFunc = rf_DiskUndoFunc;
94 rf_DiskWriteFunc = rf_DiskWriteFuncForThreads;
95 rf_DiskWriteUndoFunc = rf_DiskUndoFunc;
96 rf_DiskUnlockFunc = rf_DiskUnlockFuncForThreads;
97 rf_DiskUnlockUndoFunc = rf_NullNodeUndoFunc;
98 rf_RegularXorUndoFunc = rf_NullNodeUndoFunc;
99 rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc;
100 rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc;
101 return (0);
102 }
103
104
105
106 /*****************************************************************************
107 * the execution function associated with a terminate node
108 ****************************************************************************/
109 int
110 rf_TerminateFunc(RF_DagNode_t *node)
111 {
112 RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes);
113 node->status = rf_good;
114 return (rf_FinishNode(node, RF_THREAD_CONTEXT));
115 }
116
117 int
118 rf_TerminateUndoFunc(RF_DagNode_t *node)
119 {
120 return (0);
121 }
122
123
124 /*****************************************************************************
125 * execution functions associated with a mirror node
126 *
127 * parameters:
128 *
129 * 0 - physical disk addres of data
130 * 1 - buffer for holding read data
131 * 2 - parity stripe ID
132 * 3 - flags
133 * 4 - physical disk address of mirror (parity)
134 *
135 ****************************************************************************/
136
137 int
138 rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node)
139 {
140 /* select the mirror copy with the shortest queue and fill in node
141 * parameters with physical disk address */
142
143 rf_SelectMirrorDiskIdle(node);
144 return (rf_DiskReadFunc(node));
145 }
146
147 #if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) || (RF_DEBUG_VALIDATE_DAG > 0)
148 int
149 rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node)
150 {
151 /* select the mirror copy with the shortest queue and fill in node
152 * parameters with physical disk address */
153
154 rf_SelectMirrorDiskPartition(node);
155 return (rf_DiskReadFunc(node));
156 }
157 #endif
158
159 int
160 rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node)
161 {
162 return (0);
163 }
164
165
166
167 #if RF_INCLUDE_PARITYLOGGING > 0
168 /*****************************************************************************
169 * the execution function associated with a parity log update node
170 ****************************************************************************/
171 int
172 rf_ParityLogUpdateFunc(RF_DagNode_t *node)
173 {
174 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
175 caddr_t buf = (caddr_t) node->params[1].p;
176 RF_ParityLogData_t *logData;
177 #if RF_ACC_TRACE > 0
178 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
179 RF_Etimer_t timer;
180 #endif
181
182 if (node->dagHdr->status == rf_enable) {
183 #if RF_ACC_TRACE > 0
184 RF_ETIMER_START(timer);
185 #endif
186 logData = rf_CreateParityLogData(RF_UPDATE, pda, buf,
187 (RF_Raid_t *) (node->dagHdr->raidPtr),
188 node->wakeFunc, (void *) node,
189 node->dagHdr->tracerec, timer);
190 if (logData)
191 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
192 else {
193 #if RF_ACC_TRACE > 0
194 RF_ETIMER_STOP(timer);
195 RF_ETIMER_EVAL(timer);
196 tracerec->plog_us += RF_ETIMER_VAL_US(timer);
197 #endif
198 (node->wakeFunc) (node, ENOMEM);
199 }
200 }
201 return (0);
202 }
203
204
205 /*****************************************************************************
206 * the execution function associated with a parity log overwrite node
207 ****************************************************************************/
208 int
209 rf_ParityLogOverwriteFunc(RF_DagNode_t *node)
210 {
211 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
212 caddr_t buf = (caddr_t) node->params[1].p;
213 RF_ParityLogData_t *logData;
214 #if RF_ACC_TRACE > 0
215 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
216 RF_Etimer_t timer;
217 #endif
218
219 if (node->dagHdr->status == rf_enable) {
220 #if RF_ACC_TRACE > 0
221 RF_ETIMER_START(timer);
222 #endif
223 logData = rf_CreateParityLogData(RF_OVERWRITE, pda, buf,
224 (RF_Raid_t *) (node->dagHdr->raidPtr),
225 node->wakeFunc, (void *) node, node->dagHdr->tracerec, timer);
226 if (logData)
227 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
228 else {
229 #if RF_ACC_TRACE > 0
230 RF_ETIMER_STOP(timer);
231 RF_ETIMER_EVAL(timer);
232 tracerec->plog_us += RF_ETIMER_VAL_US(timer);
233 #endif
234 (node->wakeFunc) (node, ENOMEM);
235 }
236 }
237 return (0);
238 }
239
240 int
241 rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node)
242 {
243 return (0);
244 }
245
246 int
247 rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node)
248 {
249 return (0);
250 }
251 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
252
253 /*****************************************************************************
254 * the execution function associated with a NOP node
255 ****************************************************************************/
256 int
257 rf_NullNodeFunc(RF_DagNode_t *node)
258 {
259 node->status = rf_good;
260 return (rf_FinishNode(node, RF_THREAD_CONTEXT));
261 }
262
263 int
264 rf_NullNodeUndoFunc(RF_DagNode_t *node)
265 {
266 node->status = rf_undone;
267 return (rf_FinishNode(node, RF_THREAD_CONTEXT));
268 }
269
270
271 /*****************************************************************************
272 * the execution function associated with a disk-read node
273 ****************************************************************************/
274 int
275 rf_DiskReadFuncForThreads(RF_DagNode_t *node)
276 {
277 RF_DiskQueueData_t *req;
278 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
279 caddr_t buf = (caddr_t) node->params[1].p;
280 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
281 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
282 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
283 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_READ : RF_IO_TYPE_NOP;
284 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
285 void *b_proc = NULL;
286
287 if (node->dagHdr->bp)
288 b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
289
290 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
291 buf, parityStripeID, which_ru,
292 (int (*) (void *, int)) node->wakeFunc,
293 node, NULL,
294 #if RF_ACC_TRACE > 0
295 node->dagHdr->tracerec,
296 #else
297 NULL,
298 #endif
299 (void *) (node->dagHdr->raidPtr), 0, b_proc);
300 if (!req) {
301 (node->wakeFunc) (node, ENOMEM);
302 } else {
303 node->dagFuncData = (void *) req;
304 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
305 }
306 return (0);
307 }
308
309
310 /*****************************************************************************
311 * the execution function associated with a disk-write node
312 ****************************************************************************/
313 int
314 rf_DiskWriteFuncForThreads(RF_DagNode_t *node)
315 {
316 RF_DiskQueueData_t *req;
317 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
318 caddr_t buf = (caddr_t) node->params[1].p;
319 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
320 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
321 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
322 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP;
323 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
324 void *b_proc = NULL;
325
326 if (node->dagHdr->bp)
327 b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
328
329 /* normal processing (rollaway or forward recovery) begins here */
330 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
331 buf, parityStripeID, which_ru,
332 (int (*) (void *, int)) node->wakeFunc,
333 (void *) node, NULL,
334 #if RF_ACC_TRACE > 0
335 node->dagHdr->tracerec,
336 #else
337 NULL,
338 #endif
339 (void *) (node->dagHdr->raidPtr),
340 0, b_proc);
341
342 if (!req) {
343 (node->wakeFunc) (node, ENOMEM);
344 } else {
345 node->dagFuncData = (void *) req;
346 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
347 }
348
349 return (0);
350 }
351 /*****************************************************************************
352 * the undo function for disk nodes
353 * Note: this is not a proper undo of a write node, only locks are released.
354 * old data is not restored to disk!
355 ****************************************************************************/
356 int
357 rf_DiskUndoFunc(RF_DagNode_t *node)
358 {
359 RF_DiskQueueData_t *req;
360 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
361 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
362
363 req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP,
364 0L, 0, NULL, 0L, 0,
365 (int (*) (void *, int)) node->wakeFunc,
366 (void *) node,
367 NULL,
368 #if RF_ACC_TRACE > 0
369 node->dagHdr->tracerec,
370 #else
371 NULL,
372 #endif
373 (void *) (node->dagHdr->raidPtr),
374 RF_UNLOCK_DISK_QUEUE, NULL);
375 if (!req)
376 (node->wakeFunc) (node, ENOMEM);
377 else {
378 node->dagFuncData = (void *) req;
379 rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY);
380 }
381
382 return (0);
383 }
384 /*****************************************************************************
385 * the execution function associated with an "unlock disk queue" node
386 ****************************************************************************/
387 int
388 rf_DiskUnlockFuncForThreads(RF_DagNode_t *node)
389 {
390 RF_DiskQueueData_t *req;
391 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
392 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
393
394 req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP,
395 0L, 0, NULL, 0L, 0,
396 (int (*) (void *, int)) node->wakeFunc,
397 (void *) node,
398 NULL,
399 #if RF_ACC_TRACE > 0
400 node->dagHdr->tracerec,
401 #else
402 NULL,
403 #endif
404 (void *) (node->dagHdr->raidPtr),
405 RF_UNLOCK_DISK_QUEUE, NULL);
406 if (!req)
407 (node->wakeFunc) (node, ENOMEM);
408 else {
409 node->dagFuncData = (void *) req;
410 rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY);
411 }
412
413 return (0);
414 }
415 /*****************************************************************************
416 * Callback routine for DiskRead and DiskWrite nodes. When the disk
417 * op completes, the routine is called to set the node status and
418 * inform the execution engine that the node has fired.
419 ****************************************************************************/
420 int
421 rf_GenericWakeupFunc(RF_DagNode_t *node, int status)
422 {
423
424 switch (node->status) {
425 case rf_fired:
426 if (status)
427 node->status = rf_bad;
428 else
429 node->status = rf_good;
430 break;
431 case rf_recover:
432 /* probably should never reach this case */
433 if (status)
434 node->status = rf_panic;
435 else
436 node->status = rf_undone;
437 break;
438 default:
439 printf("rf_GenericWakeupFunc:");
440 printf("node->status is %d,", node->status);
441 printf("status is %d \n", status);
442 RF_PANIC();
443 break;
444 }
445 if (node->dagFuncData)
446 rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData);
447 return (rf_FinishNode(node, RF_INTR_CONTEXT));
448 }
449
450
451 /*****************************************************************************
452 * there are three distinct types of xor nodes:
453
454 * A "regular xor" is used in the fault-free case where the access
455 * spans a complete stripe unit. It assumes that the result buffer is
456 * one full stripe unit in size, and uses the stripe-unit-offset
457 * values that it computes from the PDAs to determine where within the
458 * stripe unit to XOR each argument buffer.
459 *
460 * A "simple xor" is used in the fault-free case where the access
461 * touches only a portion of one (or two, in some cases) stripe
462 * unit(s). It assumes that all the argument buffers are of the same
463 * size and have the same stripe unit offset.
464 *
465 * A "recovery xor" is used in the degraded-mode case. It's similar
466 * to the regular xor function except that it takes the failed PDA as
467 * an additional parameter, and uses it to determine what portions of
468 * the argument buffers need to be xor'd into the result buffer, and
469 * where in the result buffer they should go.
470 ****************************************************************************/
471
472 /* xor the params together and store the result in the result field.
473 * assume the result field points to a buffer that is the size of one
474 * SU, and use the pda params to determine where within the buffer to
475 * XOR the input buffers. */
476 int
477 rf_RegularXorFunc(RF_DagNode_t *node)
478 {
479 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
480 #if RF_ACC_TRACE > 0
481 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
482 RF_Etimer_t timer;
483 #endif
484 int i, retcode;
485
486 retcode = 0;
487 if (node->dagHdr->status == rf_enable) {
488 /* don't do the XOR if the input is the same as the output */
489 #if RF_ACC_TRACE > 0
490 RF_ETIMER_START(timer);
491 #endif
492 for (i = 0; i < node->numParams - 1; i += 2)
493 if (node->params[i + 1].p != node->results[0]) {
494 retcode = rf_XorIntoBuffer(raidPtr, (RF_PhysDiskAddr_t *) node->params[i].p,
495 (char *) node->params[i + 1].p, (char *) node->results[0]);
496 }
497 #if RF_ACC_TRACE > 0
498 RF_ETIMER_STOP(timer);
499 RF_ETIMER_EVAL(timer);
500 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
501 #endif
502 }
503 return (rf_GenericWakeupFunc(node, retcode)); /* call wake func
504 * explicitly since no
505 * I/O in this node */
506 }
507 /* xor the inputs into the result buffer, ignoring placement issues */
508 int
509 rf_SimpleXorFunc(RF_DagNode_t *node)
510 {
511 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
512 int i, retcode = 0;
513 #if RF_ACC_TRACE > 0
514 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
515 RF_Etimer_t timer;
516 #endif
517
518 if (node->dagHdr->status == rf_enable) {
519 #if RF_ACC_TRACE > 0
520 RF_ETIMER_START(timer);
521 #endif
522 /* don't do the XOR if the input is the same as the output */
523 for (i = 0; i < node->numParams - 1; i += 2)
524 if (node->params[i + 1].p != node->results[0]) {
525 retcode = rf_bxor((char *) node->params[i + 1].p, (char *) node->results[0],
526 rf_RaidAddressToByte(raidPtr, ((RF_PhysDiskAddr_t *) node->params[i].p)->numSector));
527 }
528 #if RF_ACC_TRACE > 0
529 RF_ETIMER_STOP(timer);
530 RF_ETIMER_EVAL(timer);
531 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
532 #endif
533 }
534 return (rf_GenericWakeupFunc(node, retcode)); /* call wake func
535 * explicitly since no
536 * I/O in this node */
537 }
538 /* this xor is used by the degraded-mode dag functions to recover lost
539 * data. the second-to-last parameter is the PDA for the failed
540 * portion of the access. the code here looks at this PDA and assumes
541 * that the xor target buffer is equal in size to the number of
542 * sectors in the failed PDA. It then uses the other PDAs in the
543 * parameter list to determine where within the target buffer the
544 * corresponding data should be xored. */
545 int
546 rf_RecoveryXorFunc(RF_DagNode_t *node)
547 {
548 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
549 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout;
550 RF_PhysDiskAddr_t *failedPDA = (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p;
551 int i, retcode = 0;
552 RF_PhysDiskAddr_t *pda;
553 int suoffset, failedSUOffset = rf_StripeUnitOffset(layoutPtr, failedPDA->startSector);
554 char *srcbuf, *destbuf;
555 #if RF_ACC_TRACE > 0
556 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
557 RF_Etimer_t timer;
558 #endif
559
560 if (node->dagHdr->status == rf_enable) {
561 #if RF_ACC_TRACE > 0
562 RF_ETIMER_START(timer);
563 #endif
564 for (i = 0; i < node->numParams - 2; i += 2)
565 if (node->params[i + 1].p != node->results[0]) {
566 pda = (RF_PhysDiskAddr_t *) node->params[i].p;
567 srcbuf = (char *) node->params[i + 1].p;
568 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector);
569 destbuf = ((char *) node->results[0]) + rf_RaidAddressToByte(raidPtr, suoffset - failedSUOffset);
570 retcode = rf_bxor(srcbuf, destbuf, rf_RaidAddressToByte(raidPtr, pda->numSector));
571 }
572 #if RF_ACC_TRACE > 0
573 RF_ETIMER_STOP(timer);
574 RF_ETIMER_EVAL(timer);
575 tracerec->xor_us += RF_ETIMER_VAL_US(timer);
576 #endif
577 }
578 return (rf_GenericWakeupFunc(node, retcode));
579 }
580 /*****************************************************************************
581 * The next three functions are utilities used by the above
582 * xor-execution functions.
583 ****************************************************************************/
584
585
586 /*
587 * this is just a glorified buffer xor. targbuf points to a buffer
588 * that is one full stripe unit in size. srcbuf points to a buffer
589 * that may be less than 1 SU, but never more. When the access
590 * described by pda is one SU in size (which by implication means it's
591 * SU-aligned), all that happens is (targbuf) <- (srcbuf ^ targbuf).
592 * When the access is less than one SU in size the XOR occurs on only
593 * the portion of targbuf identified in the pda. */
594
595 int
596 rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda,
597 char *srcbuf, char *targbuf)
598 {
599 char *targptr;
600 int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit;
601 int SUOffset = pda->startSector % sectPerSU;
602 int length, retcode = 0;
603
604 RF_ASSERT(pda->numSector <= sectPerSU);
605
606 targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset);
607 length = rf_RaidAddressToByte(raidPtr, pda->numSector);
608 retcode = rf_bxor(srcbuf, targptr, length);
609 return (retcode);
610 }
611 /* it really should be the case that the buffer pointers (returned by
612 * malloc) are aligned to the natural word size of the machine, so
613 * this is the only case we optimize for. The length should always be
614 * a multiple of the sector size, so there should be no problem with
615 * leftover bytes at the end. */
616 int
617 rf_bxor(char *src, char *dest, int len)
618 {
619 unsigned mask = sizeof(long) - 1, retcode = 0;
620
621 if (!(((unsigned long) src) & mask) &&
622 !(((unsigned long) dest) & mask) && !(len & mask)) {
623 retcode = rf_longword_bxor((unsigned long *) src,
624 (unsigned long *) dest,
625 len >> RF_LONGSHIFT);
626 } else {
627 RF_ASSERT(0);
628 }
629 return (retcode);
630 }
631
632 /* When XORing in kernel mode, we need to map each user page to kernel
633 * space before we can access it. We don't want to assume anything
634 * about which input buffers are in kernel/user space, nor about their
635 * alignment, so in each loop we compute the maximum number of bytes
636 * that we can xor without crossing any page boundaries, and do only
637 * this many bytes before the next remap.
638 *
639 * len - is in longwords
640 */
641 int
642 rf_longword_bxor(unsigned long *src, unsigned long *dest, int len)
643 {
644 unsigned long *end = src + len;
645 unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */
646 unsigned long *pg_src, *pg_dest; /* per-page source/dest pointers */
647 int longs_this_time;/* # longwords to xor in the current iteration */
648
649 pg_src = src;
650 pg_dest = dest;
651 if (!pg_src || !pg_dest)
652 return (EFAULT);
653
654 while (len >= 4) {
655 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src), RF_BLIP(pg_dest)) >> RF_LONGSHIFT); /* note len in longwords */
656 src += longs_this_time;
657 dest += longs_this_time;
658 len -= longs_this_time;
659 while (longs_this_time >= 4) {
660 d0 = pg_dest[0];
661 d1 = pg_dest[1];
662 d2 = pg_dest[2];
663 d3 = pg_dest[3];
664 s0 = pg_src[0];
665 s1 = pg_src[1];
666 s2 = pg_src[2];
667 s3 = pg_src[3];
668 pg_dest[0] = d0 ^ s0;
669 pg_dest[1] = d1 ^ s1;
670 pg_dest[2] = d2 ^ s2;
671 pg_dest[3] = d3 ^ s3;
672 pg_src += 4;
673 pg_dest += 4;
674 longs_this_time -= 4;
675 }
676 while (longs_this_time > 0) { /* cannot cross any page
677 * boundaries here */
678 *pg_dest++ ^= *pg_src++;
679 longs_this_time--;
680 }
681
682 /* either we're done, or we've reached a page boundary on one
683 * (or possibly both) of the pointers */
684 if (len) {
685 if (RF_PAGE_ALIGNED(src))
686 pg_src = src;
687 if (RF_PAGE_ALIGNED(dest))
688 pg_dest = dest;
689 if (!pg_src || !pg_dest)
690 return (EFAULT);
691 }
692 }
693 while (src < end) {
694 *pg_dest++ ^= *pg_src++;
695 src++;
696 dest++;
697 len--;
698 if (RF_PAGE_ALIGNED(src))
699 pg_src = src;
700 if (RF_PAGE_ALIGNED(dest))
701 pg_dest = dest;
702 }
703 RF_ASSERT(len == 0);
704 return (0);
705 }
706
707 #if 0
708 /*
709 dst = a ^ b ^ c;
710 a may equal dst
711 see comment above longword_bxor
712 len is length in longwords
713 */
714 int
715 rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b,
716 unsigned long *c, int len, void *bp)
717 {
718 unsigned long a0, a1, a2, a3, b0, b1, b2, b3;
719 unsigned long *pg_a, *pg_b, *pg_c, *pg_dst; /* per-page source/dest
720 * pointers */
721 int longs_this_time;/* # longs to xor in the current iteration */
722 char dst_is_a = 0;
723
724 pg_a = a;
725 pg_b = b;
726 pg_c = c;
727 if (a == dst) {
728 pg_dst = pg_a;
729 dst_is_a = 1;
730 } else {
731 pg_dst = dst;
732 }
733
734 /* align dest to cache line. Can't cross a pg boundary on dst here. */
735 while ((((unsigned long) pg_dst) & 0x1f)) {
736 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
737 dst++;
738 a++;
739 b++;
740 c++;
741 if (RF_PAGE_ALIGNED(a)) {
742 pg_a = a;
743 if (!pg_a)
744 return (EFAULT);
745 }
746 if (RF_PAGE_ALIGNED(b)) {
747 pg_b = a;
748 if (!pg_b)
749 return (EFAULT);
750 }
751 if (RF_PAGE_ALIGNED(c)) {
752 pg_c = a;
753 if (!pg_c)
754 return (EFAULT);
755 }
756 len--;
757 }
758
759 while (len > 4) {
760 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a), RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >> RF_LONGSHIFT);
761 a += longs_this_time;
762 b += longs_this_time;
763 c += longs_this_time;
764 dst += longs_this_time;
765 len -= longs_this_time;
766 while (longs_this_time >= 4) {
767 a0 = pg_a[0];
768 longs_this_time -= 4;
769
770 a1 = pg_a[1];
771 a2 = pg_a[2];
772
773 a3 = pg_a[3];
774 pg_a += 4;
775
776 b0 = pg_b[0];
777 b1 = pg_b[1];
778
779 b2 = pg_b[2];
780 b3 = pg_b[3];
781 /* start dual issue */
782 a0 ^= b0;
783 b0 = pg_c[0];
784
785 pg_b += 4;
786 a1 ^= b1;
787
788 a2 ^= b2;
789 a3 ^= b3;
790
791 b1 = pg_c[1];
792 a0 ^= b0;
793
794 b2 = pg_c[2];
795 a1 ^= b1;
796
797 b3 = pg_c[3];
798 a2 ^= b2;
799
800 pg_dst[0] = a0;
801 a3 ^= b3;
802 pg_dst[1] = a1;
803 pg_c += 4;
804 pg_dst[2] = a2;
805 pg_dst[3] = a3;
806 pg_dst += 4;
807 }
808 while (longs_this_time > 0) { /* cannot cross any page
809 * boundaries here */
810 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
811 longs_this_time--;
812 }
813
814 if (len) {
815 if (RF_PAGE_ALIGNED(a)) {
816 pg_a = a;
817 if (!pg_a)
818 return (EFAULT);
819 if (dst_is_a)
820 pg_dst = pg_a;
821 }
822 if (RF_PAGE_ALIGNED(b)) {
823 pg_b = b;
824 if (!pg_b)
825 return (EFAULT);
826 }
827 if (RF_PAGE_ALIGNED(c)) {
828 pg_c = c;
829 if (!pg_c)
830 return (EFAULT);
831 }
832 if (!dst_is_a)
833 if (RF_PAGE_ALIGNED(dst)) {
834 pg_dst = dst;
835 if (!pg_dst)
836 return (EFAULT);
837 }
838 }
839 }
840 while (len) {
841 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
842 dst++;
843 a++;
844 b++;
845 c++;
846 if (RF_PAGE_ALIGNED(a)) {
847 pg_a = a;
848 if (!pg_a)
849 return (EFAULT);
850 if (dst_is_a)
851 pg_dst = pg_a;
852 }
853 if (RF_PAGE_ALIGNED(b)) {
854 pg_b = b;
855 if (!pg_b)
856 return (EFAULT);
857 }
858 if (RF_PAGE_ALIGNED(c)) {
859 pg_c = c;
860 if (!pg_c)
861 return (EFAULT);
862 }
863 if (!dst_is_a)
864 if (RF_PAGE_ALIGNED(dst)) {
865 pg_dst = dst;
866 if (!pg_dst)
867 return (EFAULT);
868 }
869 len--;
870 }
871 return (0);
872 }
873
874 int
875 rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b,
876 unsigned char *c, unsigned long len, void *bp)
877 {
878 RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7) == 0);
879
880 return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a,
881 (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT, bp));
882 }
883 #endif
Cache object: 01861325bb0422f220e9c4da0d6a2b78
|