zfs_fletcher_superscalar4.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*
    2  * Implement fast Fletcher4 using superscalar pipelines.
    3  *
    4  * Use regular C code to compute
    5  * Fletcher4 in four incremental 64-bit parallel accumulator streams,
    6  * and then combine the streams to form the final four checksum words.
    7  * This implementation is a derivative of the AVX SIMD implementation by
    8  * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
    9  *
   10  * Copyright (C) 2016 Romain Dolbeau.
   11  *
   12  * Authors:
   13  *      Romain Dolbeau <romain.dolbeau@atos.net>
   14  *
   15  * This software is available to you under a choice of one of two
   16  * licenses.  You may choose to be licensed under the terms of the GNU
   17  * General Public License (GPL) Version 2, available from the file
   18  * COPYING in the main directory of this source tree, or the
   19  * OpenIB.org BSD license below:
   20  *
   21  *     Redistribution and use in source and binary forms, with or
   22  *     without modification, are permitted provided that the following
   23  *     conditions are met:
   24  *
   25  *      - Redistributions of source code must retain the above
   26  *        copyright notice, this list of conditions and the following
   27  *        disclaimer.
   28  *
   29  *      - Redistributions in binary form must reproduce the above
   30  *        copyright notice, this list of conditions and the following
   31  *        disclaimer in the documentation and/or other materials
   32  *        provided with the distribution.
   33  *
   34  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   35  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   36  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   37  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   38  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   39  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   40  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   41  * SOFTWARE.
   42  */
   43 
   44 #include <sys/param.h>
   45 #include <sys/byteorder.h>
   46 #include <sys/spa_checksum.h>
   47 #include <sys/string.h>
   48 #include <zfs_fletcher.h>
   49 
   50 ZFS_NO_SANITIZE_UNDEFINED
   51 static void
   52 fletcher_4_superscalar4_init(fletcher_4_ctx_t *ctx)
   53 {
   54         memset(ctx->superscalar, 0, 4 * sizeof (zfs_fletcher_superscalar_t));
   55 }
   56 
   57 ZFS_NO_SANITIZE_UNDEFINED
   58 static void
   59 fletcher_4_superscalar4_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
   60 {
   61         uint64_t A, B, C, D;
   62 
   63         A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1] +
   64             ctx->superscalar[0].v[2] + ctx->superscalar[0].v[3];
   65         B = 0 - ctx->superscalar[0].v[1] - 2 * ctx->superscalar[0].v[2] -
   66             3 * ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
   67             4 * ctx->superscalar[1].v[1] + 4 * ctx->superscalar[1].v[2] +
   68             4 * ctx->superscalar[1].v[3];
   69 
   70         C = ctx->superscalar[0].v[2] + 3 * ctx->superscalar[0].v[3] -
   71             6 * ctx->superscalar[1].v[0] - 10 * ctx->superscalar[1].v[1] -
   72             14 * ctx->superscalar[1].v[2] - 18 * ctx->superscalar[1].v[3] +
   73             16 * ctx->superscalar[2].v[0] + 16 * ctx->superscalar[2].v[1] +
   74             16 * ctx->superscalar[2].v[2] + 16 * ctx->superscalar[2].v[3];
   75 
   76         D = 0 - ctx->superscalar[0].v[3] + 4 * ctx->superscalar[1].v[0] +
   77             10 * ctx->superscalar[1].v[1] + 20 * ctx->superscalar[1].v[2] +
   78             34 * ctx->superscalar[1].v[3] - 48 * ctx->superscalar[2].v[0] -
   79             64 * ctx->superscalar[2].v[1] - 80 * ctx->superscalar[2].v[2] -
   80             96 * ctx->superscalar[2].v[3] + 64 * ctx->superscalar[3].v[0] +
   81             64 * ctx->superscalar[3].v[1] + 64 * ctx->superscalar[3].v[2] +
   82             64 * ctx->superscalar[3].v[3];
   83 
   84         ZIO_SET_CHECKSUM(zcp, A, B, C, D);
   85 }
   86 
   87 ZFS_NO_SANITIZE_UNDEFINED
   88 static void
   89 fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx,
   90     const void *buf, uint64_t size)
   91 {
   92         const uint32_t *ip = buf;
   93         const uint32_t *ipend = ip + (size / sizeof (uint32_t));
   94         uint64_t a, b, c, d;
   95         uint64_t a2, b2, c2, d2;
   96         uint64_t a3, b3, c3, d3;
   97         uint64_t a4, b4, c4, d4;
   98 
   99         a = ctx->superscalar[0].v[0];
  100         b = ctx->superscalar[1].v[0];
  101         c = ctx->superscalar[2].v[0];
  102         d = ctx->superscalar[3].v[0];
  103         a2 = ctx->superscalar[0].v[1];
  104         b2 = ctx->superscalar[1].v[1];
  105         c2 = ctx->superscalar[2].v[1];
  106         d2 = ctx->superscalar[3].v[1];
  107         a3 = ctx->superscalar[0].v[2];
  108         b3 = ctx->superscalar[1].v[2];
  109         c3 = ctx->superscalar[2].v[2];
  110         d3 = ctx->superscalar[3].v[2];
  111         a4 = ctx->superscalar[0].v[3];
  112         b4 = ctx->superscalar[1].v[3];
  113         c4 = ctx->superscalar[2].v[3];
  114         d4 = ctx->superscalar[3].v[3];
  115 
  116         do {
  117                 a += ip[0];
  118                 a2 += ip[1];
  119                 a3 += ip[2];
  120                 a4 += ip[3];
  121                 b += a;
  122                 b2 += a2;
  123                 b3 += a3;
  124                 b4 += a4;
  125                 c += b;
  126                 c2 += b2;
  127                 c3 += b3;
  128                 c4 += b4;
  129                 d += c;
  130                 d2 += c2;
  131                 d3 += c3;
  132                 d4 += c4;
  133         } while ((ip += 4) < ipend);
  134 
  135         ctx->superscalar[0].v[0] = a;
  136         ctx->superscalar[1].v[0] = b;
  137         ctx->superscalar[2].v[0] = c;
  138         ctx->superscalar[3].v[0] = d;
  139         ctx->superscalar[0].v[1] = a2;
  140         ctx->superscalar[1].v[1] = b2;
  141         ctx->superscalar[2].v[1] = c2;
  142         ctx->superscalar[3].v[1] = d2;
  143         ctx->superscalar[0].v[2] = a3;
  144         ctx->superscalar[1].v[2] = b3;
  145         ctx->superscalar[2].v[2] = c3;
  146         ctx->superscalar[3].v[2] = d3;
  147         ctx->superscalar[0].v[3] = a4;
  148         ctx->superscalar[1].v[3] = b4;
  149         ctx->superscalar[2].v[3] = c4;
  150         ctx->superscalar[3].v[3] = d4;
  151 }
  152 
  153 ZFS_NO_SANITIZE_UNDEFINED
  154 static void
  155 fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx,
  156     const void *buf, uint64_t size)
  157 {
  158         const uint32_t *ip = buf;
  159         const uint32_t *ipend = ip + (size / sizeof (uint32_t));
  160         uint64_t a, b, c, d;
  161         uint64_t a2, b2, c2, d2;
  162         uint64_t a3, b3, c3, d3;
  163         uint64_t a4, b4, c4, d4;
  164 
  165         a = ctx->superscalar[0].v[0];
  166         b = ctx->superscalar[1].v[0];
  167         c = ctx->superscalar[2].v[0];
  168         d = ctx->superscalar[3].v[0];
  169         a2 = ctx->superscalar[0].v[1];
  170         b2 = ctx->superscalar[1].v[1];
  171         c2 = ctx->superscalar[2].v[1];
  172         d2 = ctx->superscalar[3].v[1];
  173         a3 = ctx->superscalar[0].v[2];
  174         b3 = ctx->superscalar[1].v[2];
  175         c3 = ctx->superscalar[2].v[2];
  176         d3 = ctx->superscalar[3].v[2];
  177         a4 = ctx->superscalar[0].v[3];
  178         b4 = ctx->superscalar[1].v[3];
  179         c4 = ctx->superscalar[2].v[3];
  180         d4 = ctx->superscalar[3].v[3];
  181 
  182         do {
  183                 a += BSWAP_32(ip[0]);
  184                 a2 += BSWAP_32(ip[1]);
  185                 a3 += BSWAP_32(ip[2]);
  186                 a4 += BSWAP_32(ip[3]);
  187                 b += a;
  188                 b2 += a2;
  189                 b3 += a3;
  190                 b4 += a4;
  191                 c += b;
  192                 c2 += b2;
  193                 c3 += b3;
  194                 c4 += b4;
  195                 d += c;
  196                 d2 += c2;
  197                 d3 += c3;
  198                 d4 += c4;
  199         } while ((ip += 4) < ipend);
  200 
  201         ctx->superscalar[0].v[0] = a;
  202         ctx->superscalar[1].v[0] = b;
  203         ctx->superscalar[2].v[0] = c;
  204         ctx->superscalar[3].v[0] = d;
  205         ctx->superscalar[0].v[1] = a2;
  206         ctx->superscalar[1].v[1] = b2;
  207         ctx->superscalar[2].v[1] = c2;
  208         ctx->superscalar[3].v[1] = d2;
  209         ctx->superscalar[0].v[2] = a3;
  210         ctx->superscalar[1].v[2] = b3;
  211         ctx->superscalar[2].v[2] = c3;
  212         ctx->superscalar[3].v[2] = d3;
  213         ctx->superscalar[0].v[3] = a4;
  214         ctx->superscalar[1].v[3] = b4;
  215         ctx->superscalar[2].v[3] = c4;
  216         ctx->superscalar[3].v[3] = d4;
  217 }
  218 
  219 static boolean_t fletcher_4_superscalar4_valid(void)
  220 {
  221         return (B_TRUE);
  222 }
  223 
  224 const fletcher_4_ops_t fletcher_4_superscalar4_ops = {
  225         .init_native = fletcher_4_superscalar4_init,
  226         .compute_native = fletcher_4_superscalar4_native,
  227         .fini_native = fletcher_4_superscalar4_fini,
  228         .init_byteswap = fletcher_4_superscalar4_init,
  229         .compute_byteswap = fletcher_4_superscalar4_byteswap,
  230         .fini_byteswap = fletcher_4_superscalar4_fini,
  231         .valid = fletcher_4_superscalar4_valid,
  232         .name = "superscalar4"
  233 };

Cache object: 581cdb0a674a2df4d7abe6c9a3ce20c3

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/contrib/openzfs/module/zcommon/zfs_fletcher_superscalar4.c

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/zcommon/zfs_fletcher_superscalar4.c