mlx5_fs_tcp.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2020-2021, Mellanox Technologies, Ltd.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  *
   13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
   14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
   17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   23  * SUCH DAMAGE.
   24  */
   25 
   26 #include "opt_inet.h"
   27 #include "opt_inet6.h"
   28 
   29 #include <dev/mlx5/mlx5_en/en.h>
   30 
   31 #include <dev/mlx5/mlx5_core/fs_core.h>
   32 #include <dev/mlx5/mlx5_core/fs_tcp.h>
   33 #include <dev/mlx5/device.h>
   34 
   35 #include <sys/domain.h>
   36 
   37 #include <netinet/in_pcb.h>
   38 
   39 #if defined(INET) || defined(INET6)
   40 static void
   41 accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
   42 {
   43         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
   44         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
   45         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
   46         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
   47         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
   48             outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
   49             &inp->inp_faddr, 4);
   50         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
   51             outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
   52             &inp->inp_laddr, 4);
   53         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
   54             outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
   55         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
   56             outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
   57 }
   58 #endif
   59 
   60 #ifdef INET6
   61 static void
   62 accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
   63 {
   64         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
   65         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
   66         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
   67         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
   68         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
   69             outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
   70             &inp->in6p_faddr, 16);
   71         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
   72             outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
   73             &inp->in6p_laddr, 16);
   74         memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
   75             outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
   76             0xff, 16);
   77         memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
   78             outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
   79             0xff, 16);
   80 }
   81 #endif
   82 
   83 void
   84 mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule)
   85 {
   86         mlx5_del_flow_rule(rule);
   87 }
   88 
   89 struct mlx5_flow_rule *
   90 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
   91     struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
   92     uint16_t vlan_id)
   93 {
   94         struct mlx5_flow_destination dest = {};
   95         struct mlx5e_flow_table *ft = NULL;
   96 #if defined(INET) || defined(INET6)
   97         struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
   98 #endif
   99         struct mlx5_flow_rule *flow;
  100         struct mlx5_flow_spec *spec;
  101 
  102         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
  103         if (!spec)
  104                 return (ERR_PTR(-ENOMEM));
  105 
  106         spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
  107 
  108         INP_RLOCK(inp);
  109         /* Set VLAN ID to match, if any. */
  110         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
  111         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
  112         if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) {
  113                 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
  114                 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id);
  115         }
  116 
  117         /* Set TCP port numbers. */
  118         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
  119             outer_headers.tcp_dport);
  120         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
  121             outer_headers.tcp_sport);
  122         MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
  123             ntohs(inp->inp_lport));
  124         MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
  125             ntohs(inp->inp_fport));
  126 
  127         /* Set IP addresses. */
  128         switch (INP_SOCKAF(inp->inp_socket)) {
  129 #ifdef INET
  130         case AF_INET:
  131                 accel_fs_tcp_set_ipv4_flow(spec, inp);
  132                 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
  133                 break;
  134 #endif
  135 #ifdef INET6
  136         case AF_INET6:
  137                 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
  138                     IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) {
  139                         accel_fs_tcp_set_ipv4_flow(spec, inp);
  140                         ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
  141                 } else {
  142                         accel_fs_tcp_set_ipv6_flow(spec, inp);
  143                         ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP];
  144                 }
  145                 break;
  146 #endif
  147         default:
  148                 break;
  149         }
  150         INP_RUNLOCK(inp);
  151 
  152         if (!ft) {
  153                 flow = ERR_PTR(-EINVAL);
  154                 goto out;
  155         }
  156 
  157         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
  158         dest.tir_num = tirn;
  159 
  160         flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable,
  161             spec->match_criteria,
  162             spec->match_value,
  163             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
  164             flow_tag,
  165             &dest);
  166 out:
  167         kvfree(spec);
  168         return (flow);
  169 }
  170 
  171 static int
  172 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
  173 {
  174         static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
  175         static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
  176         struct mlx5_flow_destination dest = {};
  177         struct mlx5e_accel_fs_tcp *fs_tcp;
  178         struct mlx5_flow_rule *rule;
  179 
  180         fs_tcp = &priv->fts.accel_tcp;
  181 
  182         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
  183 
  184         /*
  185          * Traffic not matched by flow table rules should be forwarded
  186          * to the next flow table in order to not be dropped by the
  187          * default action. Refer to the diagram in
  188          * mlx5_en_flow_table.c for more information about the order
  189          * of flow tables.
  190          */
  191         dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
  192             priv->fts.vlan.t : fs_tcp->tables[type + 1].t;
  193 
  194         rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value,
  195             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_DEFAULT_FLOW_TAG, &dest);
  196         if (IS_ERR(rule))
  197                 return (PTR_ERR(rule));
  198 
  199         fs_tcp->default_rules[type] = rule;
  200         return (0);
  201 }
  202 
  203 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS   (2)
  204 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE  (BIT(16) - 1)
  205 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE  (BIT(0))
  206 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE   (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
  207                                          MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
  208 static int
  209 accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type)
  210 {
  211         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
  212         void *outer_headers_c;
  213         int ix = 0;
  214         u32 *in;
  215         int err;
  216         u8 *mc;
  217 
  218         ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
  219         in = kvzalloc(inlen, GFP_KERNEL);
  220         if (!in || !ft->g) {
  221                 kfree(ft->g);
  222                 kvfree(in);
  223                 return (-ENOMEM);
  224         }
  225 
  226         mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
  227         outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
  228         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
  229         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
  230         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag);
  231         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid);
  232 
  233         switch (type) {
  234         case MLX5E_ACCEL_FS_IPV4_TCP:
  235         case MLX5E_ACCEL_FS_IPV6_TCP:
  236                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
  237                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
  238                 break;
  239         default:
  240                 err = -EINVAL;
  241                 goto out;
  242         }
  243 
  244         switch (type) {
  245         case MLX5E_ACCEL_FS_IPV4_TCP:
  246                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
  247                     src_ipv4_src_ipv6.ipv4_layout.ipv4);
  248                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
  249                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
  250                 break;
  251         case MLX5E_ACCEL_FS_IPV6_TCP:
  252                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
  253                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
  254                     0xff, 16);
  255                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
  256                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
  257                     0xff, 16);
  258                 break;
  259         default:
  260                 err = -EINVAL;
  261                 goto out;
  262         }
  263 
  264         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
  265         MLX5_SET_CFG(in, start_flow_index, ix);
  266         ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
  267         MLX5_SET_CFG(in, end_flow_index, ix - 1);
  268         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
  269         if (IS_ERR(ft->g[ft->num_groups]))
  270                 goto err;
  271         ft->num_groups++;
  272 
  273         /* Default Flow Group */
  274         memset(in, 0, inlen);
  275         MLX5_SET_CFG(in, start_flow_index, ix);
  276         ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
  277         MLX5_SET_CFG(in, end_flow_index, ix - 1);
  278         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
  279         if (IS_ERR(ft->g[ft->num_groups]))
  280                 goto err;
  281         ft->num_groups++;
  282 
  283         kvfree(in);
  284         return (0);
  285 
  286 err:
  287         err = PTR_ERR(ft->g[ft->num_groups]);
  288         ft->g[ft->num_groups] = NULL;
  289 out:
  290         kvfree(in);
  291 
  292         return (err);
  293 }
  294 
  295 static void
  296 accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft)
  297 {
  298         int i;
  299 
  300         for (i = ft->num_groups - 1; i >= 0; i--) {
  301                 if (!IS_ERR_OR_NULL(ft->g[i]))
  302                         mlx5_destroy_flow_group(ft->g[i]);
  303                 ft->g[i] = NULL;
  304         }
  305         ft->num_groups = 0;
  306 }
  307 
  308 static int
  309 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
  310 {
  311         struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
  312         int err;
  313 
  314         ft->num_groups = 0;
  315         ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp",
  316             MLX5E_ACCEL_FS_TCP_TABLE_SIZE);
  317         if (IS_ERR(ft->t)) {
  318                 err = PTR_ERR(ft->t);
  319                 ft->t = NULL;
  320                 return (err);
  321         }
  322 
  323         err = accel_fs_tcp_create_groups(ft, type);
  324         if (err)
  325                 goto err_destroy_flow_table;
  326 
  327         return (0);
  328 
  329 err_destroy_flow_table:
  330         mlx5_destroy_flow_table(ft->t);
  331         ft->t = NULL;
  332         return (err);
  333 }
  334 
  335 static void
  336 accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
  337 {
  338         struct mlx5e_accel_fs_tcp *fs_tcp;
  339         struct mlx5e_flow_table *ft;
  340 
  341         fs_tcp = &priv->fts.accel_tcp;
  342         ft = fs_tcp->tables + i;
  343 
  344         mlx5_del_flow_rule(fs_tcp->default_rules[i]);
  345 
  346         accel_fs_tcp_destroy_groups(ft);
  347         kfree(ft->g);
  348         ft->g = NULL;
  349         mlx5_destroy_flow_table(ft->t);
  350         ft->t = NULL;
  351 }
  352 
  353 void
  354 mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
  355 {
  356         int i;
  357 
  358         if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
  359                 return;
  360 
  361         for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++)
  362                 accel_fs_tcp_destroy_table(priv, i);
  363 }
  364 
  365 int
  366 mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
  367 {
  368         int i, err;
  369 
  370         if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
  371                 return (0);
  372 
  373         /* Setup namespace pointer. */
  374         priv->fts.accel_tcp.ns = mlx5_get_flow_namespace(
  375             priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS);
  376 
  377         /*
  378          * Create flow tables first, because the priority level is
  379          * assigned at allocation time.
  380          */
  381         for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
  382                 err = accel_fs_tcp_create_table(priv, i);
  383                 if (err)
  384                         goto err_destroy_tables;
  385         }
  386 
  387         /* Create default rules last. */
  388         for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
  389                 err = accel_fs_tcp_add_default_rule(priv, i);
  390                 if (err)
  391                         goto err_destroy_rules;
  392         }
  393         return (0);
  394 
  395 err_destroy_rules:
  396         while (i--)
  397                 mlx5_del_flow_rule(priv->fts.accel_tcp.default_rules[i]);
  398         i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;
  399 
  400 err_destroy_tables:
  401         while (i--)
  402                 accel_fs_tcp_destroy_table(priv, i);
  403         return (err);
  404 }
Cache object: 6d9418e8b0e79a104a9769bcd60284d5
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
