zfs_fletcher_superscalar.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*
    2  * Implement fast Fletcher4 using superscalar pipelines.
    3  *
    4  * Use regular C code to compute
    5  * Fletcher4 in two incremental 64-bit parallel accumulator streams,
    6  * and then combine the streams to form the final four checksum words.
    7  * This implementation is a derivative of the AVX SIMD implementation by
    8  * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c).
    9  *
   10  * Copyright (C) 2016 Romain Dolbeau.
   11  *
   12  * Authors:
   13  *      Romain Dolbeau <romain.dolbeau@atos.net>
   14  *
   15  * This software is available to you under a choice of one of two
   16  * licenses.  You may choose to be licensed under the terms of the GNU
   17  * General Public License (GPL) Version 2, available from the file
   18  * COPYING in the main directory of this source tree, or the
   19  * OpenIB.org BSD license below:
   20  *
   21  *     Redistribution and use in source and binary forms, with or
   22  *     without modification, are permitted provided that the following
   23  *     conditions are met:
   24  *
   25  *      - Redistributions of source code must retain the above
   26  *        copyright notice, this list of conditions and the following
   27  *        disclaimer.
   28  *
   29  *      - Redistributions in binary form must reproduce the above
   30  *        copyright notice, this list of conditions and the following
   31  *        disclaimer in the documentation and/or other materials
   32  *        provided with the distribution.
   33  *
   34  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   35  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   36  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   37  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   38  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   39  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   40  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   41  * SOFTWARE.
   42  */
   43 
   44 #include <sys/param.h>
   45 #include <sys/byteorder.h>
   46 #include <sys/spa_checksum.h>
   47 #include <sys/string.h>
   48 #include <zfs_fletcher.h>
   49 
   50 ZFS_NO_SANITIZE_UNDEFINED
   51 static void
   52 fletcher_4_superscalar_init(fletcher_4_ctx_t *ctx)
   53 {
   54         memset(ctx->superscalar, 0, 4 * sizeof (zfs_fletcher_superscalar_t));
   55 }
   56 
   57 ZFS_NO_SANITIZE_UNDEFINED
   58 static void
   59 fletcher_4_superscalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
   60 {
   61         uint64_t A, B, C, D;
   62         A = ctx->superscalar[0].v[0] + ctx->superscalar[0].v[1];
   63         B = 2 * ctx->superscalar[1].v[0] + 2 * ctx->superscalar[1].v[1] -
   64             ctx->superscalar[0].v[1];
   65         C = 4 * ctx->superscalar[2].v[0] - ctx->superscalar[1].v[0] +
   66             4 * ctx->superscalar[2].v[1] - 3 * ctx->superscalar[1].v[1];
   67         D = 8 * ctx->superscalar[3].v[0] - 4 * ctx->superscalar[2].v[0] +
   68             8 * ctx->superscalar[3].v[1] - 8 * ctx->superscalar[2].v[1] +
   69             ctx->superscalar[1].v[1];
   70         ZIO_SET_CHECKSUM(zcp, A, B, C, D);
   71 }
   72 
   73 ZFS_NO_SANITIZE_UNDEFINED
   74 static void
   75 fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx,
   76     const void *buf, uint64_t size)
   77 {
   78         const uint32_t *ip = buf;
   79         const uint32_t *ipend = ip + (size / sizeof (uint32_t));
   80         uint64_t a, b, c, d;
   81         uint64_t a2, b2, c2, d2;
   82 
   83         a = ctx->superscalar[0].v[0];
   84         b = ctx->superscalar[1].v[0];
   85         c = ctx->superscalar[2].v[0];
   86         d = ctx->superscalar[3].v[0];
   87         a2 = ctx->superscalar[0].v[1];
   88         b2 = ctx->superscalar[1].v[1];
   89         c2 = ctx->superscalar[2].v[1];
   90         d2 = ctx->superscalar[3].v[1];
   91 
   92         do {
   93                 a += ip[0];
   94                 a2 += ip[1];
   95                 b += a;
   96                 b2 += a2;
   97                 c += b;
   98                 c2 += b2;
   99                 d += c;
  100                 d2 += c2;
  101         } while ((ip += 2) < ipend);
  102 
  103         ctx->superscalar[0].v[0] = a;
  104         ctx->superscalar[1].v[0] = b;
  105         ctx->superscalar[2].v[0] = c;
  106         ctx->superscalar[3].v[0] = d;
  107         ctx->superscalar[0].v[1] = a2;
  108         ctx->superscalar[1].v[1] = b2;
  109         ctx->superscalar[2].v[1] = c2;
  110         ctx->superscalar[3].v[1] = d2;
  111 }
  112 
  113 ZFS_NO_SANITIZE_UNDEFINED
  114 static void
  115 fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx,
  116     const void *buf, uint64_t size)
  117 {
  118         const uint32_t *ip = buf;
  119         const uint32_t *ipend = ip + (size / sizeof (uint32_t));
  120         uint64_t a, b, c, d;
  121         uint64_t a2, b2, c2, d2;
  122 
  123         a = ctx->superscalar[0].v[0];
  124         b = ctx->superscalar[1].v[0];
  125         c = ctx->superscalar[2].v[0];
  126         d = ctx->superscalar[3].v[0];
  127         a2 = ctx->superscalar[0].v[1];
  128         b2 = ctx->superscalar[1].v[1];
  129         c2 = ctx->superscalar[2].v[1];
  130         d2 = ctx->superscalar[3].v[1];
  131 
  132         do {
  133                 a += BSWAP_32(ip[0]);
  134                 a2 += BSWAP_32(ip[1]);
  135                 b += a;
  136                 b2 += a2;
  137                 c += b;
  138                 c2 += b2;
  139                 d += c;
  140                 d2 += c2;
  141         } while ((ip += 2) < ipend);
  142 
  143         ctx->superscalar[0].v[0] = a;
  144         ctx->superscalar[1].v[0] = b;
  145         ctx->superscalar[2].v[0] = c;
  146         ctx->superscalar[3].v[0] = d;
  147         ctx->superscalar[0].v[1] = a2;
  148         ctx->superscalar[1].v[1] = b2;
  149         ctx->superscalar[2].v[1] = c2;
  150         ctx->superscalar[3].v[1] = d2;
  151 }
  152 
  153 static boolean_t fletcher_4_superscalar_valid(void)
  154 {
  155         return (B_TRUE);
  156 }
  157 
  158 const fletcher_4_ops_t fletcher_4_superscalar_ops = {
  159         .init_native = fletcher_4_superscalar_init,
  160         .compute_native = fletcher_4_superscalar_native,
  161         .fini_native = fletcher_4_superscalar_fini,
  162         .init_byteswap = fletcher_4_superscalar_init,
  163         .compute_byteswap = fletcher_4_superscalar_byteswap,
  164         .fini_byteswap = fletcher_4_superscalar_fini,
  165         .valid = fletcher_4_superscalar_valid,
  166         .name = "superscalar"
  167 };

Cache object: b52ea039c9dc1e91706732cb814075ac

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/contrib/openzfs/module/zcommon/zfs_fletcher_superscalar.c

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/zcommon/zfs_fletcher_superscalar.c