net/mlx5: Use flow steering infrastructure for mlx5_en

Expose the new flow steering API and remove the old
one.

Few changes are required:

1. The Ethernet flow steering follows the existing implementation, but uses
the new steering API. The old flow steering implementation is removed.

2. Move the E-switch FDB management to use the new API.

3. When driver is loaded call to mlx5_init_fs which initialize
the flow steering tree structure, open namespaces for NIC receive
and for E-switch FDB.

4. Call to mlx5_cleanup_fs when the driver is unloaded.
Signed-off-by: NMaor Gottlieb <maorg@mellanox.com>
Signed-off-by: NMoni Shoua <monis@mellanox.com>
Signed-off-by: NMatan Barak <matanb@mellanox.com>
Signed-off-by: NSaeed Mahameed <saeedm@mellanox.com>
Signed-off-by: NDavid S. Miller <davem@davemloft.net>

drivers/net/ethernet/mellanox/mlx5/core/Makefile

@@ -3,6 +3,6 @@ obj-$(CONFIG_MLX5_CORE)		+= mlx5_core.o
 mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		health.o mcg.o cq.o srq.o alloc.o qp.o port.o mr.o pd.o   \
 		mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o
-mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o flow_table.o eswitch.o \
+mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o \
 		en_main.o en_flow_table.o en_ethtool.o en_tx.o en_rx.o \
 		en_txrx.o

drivers/net/ethernet/mellanox/mlx5/core/en.h

@@ -64,6 +64,8 @@
 #define MLX5E_UPDATE_STATS_INTERVAL    200 /* msecs */
 #define MLX5E_SQ_BF_BUDGET             16
 
+#define MLX5E_NUM_MAIN_GROUPS 9
+
 static const char vport_strings[][ETH_GSTRING_LEN] = {
 	/* vport statistics */
 	"rx_packets",
@@ -442,7 +444,7 @@ enum mlx5e_rqt_ix {
 struct mlx5e_eth_addr_info {
 	u8  addr[ETH_ALEN + 2];
 	u32 tt_vec;
-	u32 ft_ix[MLX5E_NUM_TT]; /* flow table index per traffic type */
+	struct mlx5_flow_rule *ft_rule[MLX5E_NUM_TT];
 };
 
 #define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)
@@ -466,15 +468,22 @@ enum {
 
 struct mlx5e_vlan_db {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
-	u32           active_vlans_ft_ix[VLAN_N_VID];
-	u32           untagged_rule_ft_ix;
-	u32           any_vlan_rule_ft_ix;
+	struct mlx5_flow_rule	*active_vlans_rule[VLAN_N_VID];
+	struct mlx5_flow_rule	*untagged_rule;
+	struct mlx5_flow_rule	*any_vlan_rule;
 	bool          filter_disabled;
 };
 
 struct mlx5e_flow_table {
-	void *vlan;
-	void *main;
+	int num_groups;
+	struct mlx5_flow_table		*t;
+	struct mlx5_flow_group		**g;
+};
+
+struct mlx5e_flow_tables {
+	struct mlx5_flow_namespace	*ns;
+	struct mlx5e_flow_table		vlan;
+	struct mlx5e_flow_table		main;
 };
 
 struct mlx5e_priv {
@@ -497,7 +506,7 @@ struct mlx5e_priv {
 	u32                        rqtn[MLX5E_NUM_RQT];
 	u32                        tirn[MLX5E_NUM_TT];
 
-	struct mlx5e_flow_table    ft;
+	struct mlx5e_flow_tables   fts;
 	struct mlx5e_eth_addr_db   eth_addr;
 	struct mlx5e_vlan_db       vlan;
 

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

@@ -30,7 +30,7 @@
  * SOFTWARE.
  */
 
-#include <linux/mlx5/flow_table.h>
+#include <linux/mlx5/fs.h>
 #include "en.h"
 #include "eswitch.h"
 

drivers/net/ethernet/mellanox/mlx5/core/eswitch.c

@@ -34,7 +34,7 @@
 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/mlx5_ifc.h>
 #include <linux/mlx5/vport.h>
-#include <linux/mlx5/flow_table.h>
+#include <linux/mlx5/fs.h>
 #include "mlx5_core.h"
 #include "eswitch.h"
 
@@ -321,220 +321,6 @@ static void del_l2_table_entry(struct mlx5_core_dev *dev, u32 index)
 	free_l2_table_index(l2_table, index);
 }
 
-/* E-Switch FDB flow steering */
-struct dest_node {
-	struct list_head list;
-	struct mlx5_flow_destination dest;
-};
-
-static int _mlx5_flow_rule_apply(struct mlx5_flow_rule *fr)
-{
-	bool was_valid = fr->valid;
-	struct dest_node *dest_n;
-	u32 dest_list_size = 0;
-	void *in_match_value;
-	u32 *flow_context;
-	u32 flow_index;
-	int err;
-	int i;
-
-	if (list_empty(&fr->dest_list)) {
-		if (fr->valid)
-			mlx5_del_flow_table_entry(fr->ft, fr->fi);
-		fr->valid = false;
-		return 0;
-	}
-
-	list_for_each_entry(dest_n, &fr->dest_list, list)
-		dest_list_size++;
-
-	flow_context = mlx5_vzalloc(MLX5_ST_SZ_BYTES(flow_context) +
-				    MLX5_ST_SZ_BYTES(dest_format_struct) *
-				    dest_list_size);
-	if (!flow_context)
-		return -ENOMEM;
-
-	MLX5_SET(flow_context, flow_context, flow_tag, fr->flow_tag);
-	MLX5_SET(flow_context, flow_context, action, fr->action);
-	MLX5_SET(flow_context, flow_context, destination_list_size,
-		 dest_list_size);
-
-	i = 0;
-	list_for_each_entry(dest_n, &fr->dest_list, list) {
-		void *dest_addr = MLX5_ADDR_OF(flow_context, flow_context,
-					       destination[i++]);
-
-		MLX5_SET(dest_format_struct, dest_addr, destination_type,
-			 dest_n->dest.type);
-		MLX5_SET(dest_format_struct, dest_addr, destination_id,
-			 dest_n->dest.vport_num);
-	}
-
-	in_match_value = MLX5_ADDR_OF(flow_context, flow_context, match_value);
-	memcpy(in_match_value, fr->match_value, MLX5_ST_SZ_BYTES(fte_match_param));
-
-	err = mlx5_add_flow_table_entry(fr->ft, fr->match_criteria_enable,
-					fr->match_criteria, flow_context,
-					&flow_index);
-	if (!err) {
-		if (was_valid)
-			mlx5_del_flow_table_entry(fr->ft, fr->fi);
-		fr->fi = flow_index;
-		fr->valid = true;
-	}
-	kfree(flow_context);
-	return err;
-}
-
-static int mlx5_flow_rule_add_dest(struct mlx5_flow_rule *fr,
-				   struct mlx5_flow_destination *new_dest)
-{
-	struct dest_node *dest_n;
-	int err;
-
-	dest_n = kzalloc(sizeof(*dest_n), GFP_KERNEL);
-	if (!dest_n)
-		return -ENOMEM;
-
-	memcpy(&dest_n->dest, new_dest, sizeof(dest_n->dest));
-	mutex_lock(&fr->mutex);
-	list_add(&dest_n->list, &fr->dest_list);
-	err = _mlx5_flow_rule_apply(fr);
-	if (err) {
-		list_del(&dest_n->list);
-		kfree(dest_n);
-	}
-	mutex_unlock(&fr->mutex);
-	return err;
-}
-
-static int mlx5_flow_rule_del_dest(struct mlx5_flow_rule *fr,
-				   struct mlx5_flow_destination *dest)
-{
-	struct dest_node *dest_n;
-	struct dest_node *n;
-	int err;
-
-	mutex_lock(&fr->mutex);
-	list_for_each_entry_safe(dest_n, n, &fr->dest_list, list) {
-		if (dest->vport_num == dest_n->dest.vport_num)
-			goto found;
-	}
-	mutex_unlock(&fr->mutex);
-	return -ENOENT;
-
-found:
-	list_del(&dest_n->list);
-	err = _mlx5_flow_rule_apply(fr);
-	mutex_unlock(&fr->mutex);
-	kfree(dest_n);
-
-	return err;
-}
-
-static struct mlx5_flow_rule *find_fr(struct mlx5_eswitch *esw,
-				      u8 match_criteria_enable,
-				      u32 *match_value)
-{
-	struct hlist_head *hash = esw->mc_table;
-	struct esw_mc_addr *esw_mc;
-	u8 *dmac_v;
-
-	dmac_v = MLX5_ADDR_OF(fte_match_param, match_value,
-			      outer_headers.dmac_47_16);
-
-	/* UNICAST FULL MATCH */
-	if (!is_multicast_ether_addr(dmac_v))
-		return NULL;
-
-	/* MULTICAST FULL MATCH */
-	esw_mc = l2addr_hash_find(hash, dmac_v, struct esw_mc_addr);
-
-	return esw_mc ? esw_mc->uplink_rule : NULL;
-}
-
-static struct mlx5_flow_rule *alloc_fr(void *ft,
-				       u8 match_criteria_enable,
-				       u32 *match_criteria,
-				       u32 *match_value,
-				       u32 action,
-				       u32 flow_tag)
-{
-	struct mlx5_flow_rule *fr = kzalloc(sizeof(*fr), GFP_KERNEL);
-
-	if (!fr)
-		return NULL;
-
-	fr->match_criteria = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	fr->match_value = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
-	if (!fr->match_criteria || !fr->match_value) {
-		kfree(fr->match_criteria);
-		kfree(fr->match_value);
-		kfree(fr);
-		return NULL;
-	}
-
-	memcpy(fr->match_criteria, match_criteria, MLX5_ST_SZ_BYTES(fte_match_param));
-	memcpy(fr->match_value, match_value, MLX5_ST_SZ_BYTES(fte_match_param));
-	fr->match_criteria_enable = match_criteria_enable;
-	fr->flow_tag = flow_tag;
-	fr->action = action;
-
-	mutex_init(&fr->mutex);
-	INIT_LIST_HEAD(&fr->dest_list);
-	atomic_set(&fr->refcount, 0);
-	fr->ft = ft;
-	return fr;
-}
-
-static void deref_fr(struct mlx5_flow_rule *fr)
-{
-	if (!atomic_dec_and_test(&fr->refcount))
-		return;
-
-	kfree(fr->match_criteria);
-	kfree(fr->match_value);
-	kfree(fr);
-}
-
-static struct mlx5_flow_rule *
-mlx5_add_flow_rule(struct mlx5_eswitch *esw,
-		   u8 match_criteria_enable,
-		   u32 *match_criteria,
-		   u32 *match_value,
-		   u32 action,
-		   u32 flow_tag,
-		   struct mlx5_flow_destination *dest)
-{
-	struct mlx5_flow_rule *fr;
-	int err;
-
-	fr = find_fr(esw, match_criteria_enable, match_value);
-	fr = fr ? fr : alloc_fr(esw->fdb_table.fdb, match_criteria_enable, match_criteria,
-				match_value, action, flow_tag);
-	if (!fr)
-		return NULL;
-
-	atomic_inc(&fr->refcount);
-
-	err = mlx5_flow_rule_add_dest(fr, dest);
-	if (err) {
-		deref_fr(fr);
-		return NULL;
-	}
-
-	return fr;
-}
-
-static void mlx5_del_flow_rule(struct mlx5_flow_rule *fr, u32 vport)
-{
-	struct mlx5_flow_destination dest;
-
-	dest.vport_num = vport;
-	mlx5_flow_rule_del_dest(fr, &dest);
-	deref_fr(fr);
-}
-
 /* E-Switch FDB */
 static struct mlx5_flow_rule *
 esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
@@ -569,7 +355,7 @@ esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
 		  "\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
 		  dmac_v, dmac_c, vport);
 	flow_rule =
-		mlx5_add_flow_rule(esw,
+		mlx5_add_flow_rule(esw->fdb_table.fdb,
 				   match_header,
 				   match_c,
 				   match_v,
@@ -589,33 +375,61 @@ esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
 
 static int esw_create_fdb_table(struct mlx5_eswitch *esw, int nvports)
 {
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	struct mlx5_core_dev *dev = esw->dev;
-	struct mlx5_flow_table_group g;
+	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *fdb;
+	struct mlx5_flow_group *g;
+	void *match_criteria;
+	int table_size;
+	u32 *flow_group_in;
 	u8 *dmac;
+	int err = 0;
 
 	esw_debug(dev, "Create FDB log_max_size(%d)\n",
 		  MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size));
 
-	memset(&g, 0, sizeof(g));
-	/* UC MC Full match rules*/
-	g.log_sz = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
-	g.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
-	dmac = MLX5_ADDR_OF(fte_match_param, g.match_criteria,
-			    outer_headers.dmac_47_16);
-	/* Match criteria mask */
-	memset(dmac, 0xff, 6);
+	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
+	if (!root_ns) {
+		esw_warn(dev, "Failed to get FDB flow namespace\n");
+		return -ENOMEM;
+	}
 
-	fdb = mlx5_create_flow_table(dev, 0,
-				     MLX5_FLOW_TABLE_TYPE_ESWITCH,
-				     1, &g);
-	if (fdb)
-		esw_debug(dev, "ESW: FDB Table created fdb->id %d\n", mlx5_get_flow_table_id(fdb));
-	else
-		esw_warn(dev, "ESW: Failed to create FDB Table\n");
+	flow_group_in = mlx5_vzalloc(inlen);
+	if (!flow_group_in)
+		return -ENOMEM;
+	memset(flow_group_in, 0, inlen);
+
+	table_size = BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size));
+	fdb = mlx5_create_flow_table(root_ns, 0, table_size);
+	if (IS_ERR_OR_NULL(fdb)) {
+		err = PTR_ERR(fdb);
+		esw_warn(dev, "Failed to create FDB Table err %d\n", err);
+		goto out;
+	}
 
+	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
+		 MLX5_MATCH_OUTER_HEADERS);
+	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
+	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, outer_headers.dmac_47_16);
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, table_size - 1);
+	eth_broadcast_addr(dmac);
+
+	g = mlx5_create_flow_group(fdb, flow_group_in);
+	if (IS_ERR_OR_NULL(g)) {
+		err = PTR_ERR(g);
+		esw_warn(dev, "Failed to create flow group err(%d)\n", err);
+		goto out;
+	}
+
+	esw->fdb_table.addr_grp = g;
 	esw->fdb_table.fdb = fdb;
-	return fdb ? 0 : -ENOMEM;
+out:
+	kfree(flow_group_in);
+	if (err && !IS_ERR_OR_NULL(fdb))
+		mlx5_destroy_flow_table(fdb);
+	return err;
 }
 
 static void esw_destroy_fdb_table(struct mlx5_eswitch *esw)
@@ -623,10 +437,11 @@ static void esw_destroy_fdb_table(struct mlx5_eswitch *esw)
 	if (!esw->fdb_table.fdb)
 		return;
 
-	esw_debug(esw->dev, "Destroy FDB Table fdb(%d)\n",
-		  mlx5_get_flow_table_id(esw->fdb_table.fdb));
+	esw_debug(esw->dev, "Destroy FDB Table\n");
+	mlx5_destroy_flow_group(esw->fdb_table.addr_grp);
 	mlx5_destroy_flow_table(esw->fdb_table.fdb);
 	esw->fdb_table.fdb = NULL;
+	esw->fdb_table.addr_grp = NULL;
 }
 
 /* E-Switch vport UC/MC lists management */
@@ -689,7 +504,7 @@ static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 	del_l2_table_entry(esw->dev, esw_uc->table_index);
 
 	if (vaddr->flow_rule)
-		mlx5_del_flow_rule(vaddr->flow_rule, vport);
+		mlx5_del_flow_rule(vaddr->flow_rule);
 	vaddr->flow_rule = NULL;
 
 	l2addr_hash_del(esw_uc);
@@ -750,14 +565,14 @@ static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 		  esw_mc->uplink_rule);
 
 	if (vaddr->flow_rule)
-		mlx5_del_flow_rule(vaddr->flow_rule, vport);
+		mlx5_del_flow_rule(vaddr->flow_rule);
 	vaddr->flow_rule = NULL;
 
 	if (--esw_mc->refcnt)
 		return 0;
 
 	if (esw_mc->uplink_rule)
-		mlx5_del_flow_rule(esw_mc->uplink_rule, UPLINK_VPORT);
+		mlx5_del_flow_rule(esw_mc->uplink_rule);
 
 	l2addr_hash_del(esw_mc);
 	return 0;

drivers/net/ethernet/mellanox/mlx5/core/eswitch.h

@@ -88,20 +88,6 @@ struct l2addr_node {
 	kfree(ptr);                                         \
 })
 
-struct mlx5_flow_rule {
-	void             *ft;
-	u32              fi;
-	u8               match_criteria_enable;
-	u32              *match_criteria;
-	u32              *match_value;
-	u32              action;
-	u32              flow_tag;
-	bool             valid;
-	atomic_t         refcount;
-	struct mutex     mutex; /* protect flow rule updates */
-	struct list_head dest_list;
-};
-
 struct mlx5_vport {
 	struct mlx5_core_dev    *dev;
 	int                     vport;
@@ -126,6 +112,7 @@ struct mlx5_l2_table {
 
 struct mlx5_eswitch_fdb {
 	void *fdb;
+	struct mlx5_flow_group *addr_grp;
 };
 
 struct mlx5_eswitch {

drivers/net/ethernet/mellanox/mlx5/core/flow_table.c

-/*
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/export.h>
-#include <linux/mlx5/driver.h>
-#include <linux/mlx5/flow_table.h>
-#include "mlx5_core.h"
-
-struct mlx5_ftg {
-	struct mlx5_flow_table_group    g;
-	u32				id;
-	u32				start_ix;
-};
-
-struct mlx5_flow_table {
-	struct mlx5_core_dev	*dev;
-	u8			level;
-	u8			type;
-	u32			id;
-	struct mutex		mutex; /* sync bitmap alloc */
-	u16			num_groups;
-	struct mlx5_ftg		*group;
-	unsigned long		*bitmap;
-	u32			size;
-};
-
-static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix,
-				   u32 flow_index, void *flow_context)
-{
-	u32 out[MLX5_ST_SZ_DW(set_fte_out)];
-	u32 *in;
-	void *in_flow_context;
-	int fcdls =
-		MLX5_GET(flow_context, flow_context, destination_list_size) *
-		MLX5_ST_SZ_BYTES(dest_format_struct);
-	int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls;
-	int err;
-
-	in = mlx5_vzalloc(inlen);
-	if (!in) {
-		mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
-		return -ENOMEM;
-	}
-
-	MLX5_SET(set_fte_in, in, table_type, ft->type);
-	MLX5_SET(set_fte_in, in, table_id,   ft->id);
-	MLX5_SET(set_fte_in, in, flow_index, flow_index);
-	MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
-
-	in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
-	memcpy(in_flow_context, flow_context,
-	       MLX5_ST_SZ_BYTES(flow_context) + fcdls);
-
-	MLX5_SET(flow_context, in_flow_context, group_id,
-		 ft->group[group_ix].id);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
-					 sizeof(out));
-	kvfree(in);
-
-	return err;
-}
-
-static void mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index)
-{
-	u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
-	u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
-#define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
-	MLX5_SET_DFTEI(in, table_type, ft->type);
-	MLX5_SET_DFTEI(in, table_id,   ft->id);
-	MLX5_SET_DFTEI(in, flow_index, flow_index);
-	MLX5_SET_DFTEI(in, opcode,     MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
-
-	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
-}
-
-static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i)
-{
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
-#define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
-	MLX5_SET_DFGI(in, table_type, ft->type);
-	MLX5_SET_DFGI(in, table_id,   ft->id);
-	MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP);
-	MLX5_SET_DFGI(in, group_id, ft->group[i].id);
-	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
-}
-
-static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i)
-{
-	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
-	u32 *in;
-	void *in_match_criteria;
-	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
-	struct mlx5_flow_table_group *g = &ft->group[i].g;
-	u32 start_ix = ft->group[i].start_ix;
-	u32 end_ix = start_ix + (1 << g->log_sz) - 1;
-	int err;
-
-	in = mlx5_vzalloc(inlen);
-	if (!in) {
-		mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
-		return -ENOMEM;
-	}
-	in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in,
-					 match_criteria);
-
-	memset(out, 0, sizeof(out));
-
-#define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
-	MLX5_SET_CFGI(in, table_type,            ft->type);
-	MLX5_SET_CFGI(in, table_id,              ft->id);
-	MLX5_SET_CFGI(in, opcode,                MLX5_CMD_OP_CREATE_FLOW_GROUP);
-	MLX5_SET_CFGI(in, start_flow_index,      start_ix);
-	MLX5_SET_CFGI(in, end_flow_index,        end_ix);
-	MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable);
-
-	memcpy(in_match_criteria, g->match_criteria,
-	       MLX5_ST_SZ_BYTES(fte_match_param));
-
-	err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
-					 sizeof(out));
-	if (!err)
-		ft->group[i].id = MLX5_GET(create_flow_group_out, out,
-					   group_id);
-
-	kvfree(in);
-
-	return err;
-}
-
-static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft)
-{
-	int i;
-
-	for (i = 0; i < ft->num_groups; i++)
-		mlx5_destroy_flow_group_cmd(ft, i);
-}
-
-static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft)
-{
-	int err;
-	int i;
-
-	for (i = 0; i < ft->num_groups; i++) {
-		err = mlx5_create_flow_group_cmd(ft, i);
-		if (err)
-			goto err_destroy_flow_table_groups;
-	}
-
-	return 0;
-
-err_destroy_flow_table_groups:
-	for (i--; i >= 0; i--)
-		mlx5_destroy_flow_group_cmd(ft, i);
-
-	return err;
-}
-
-static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft)
-{
-	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
-	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
-	int err;
-
-	memset(in, 0, sizeof(in));
-
-	MLX5_SET(create_flow_table_in, in, table_type, ft->type);
-	MLX5_SET(create_flow_table_in, in, level,      ft->level);
-	MLX5_SET(create_flow_table_in, in, log_size,   order_base_2(ft->size));
-
-	MLX5_SET(create_flow_table_in, in, opcode,
-		 MLX5_CMD_OP_CREATE_FLOW_TABLE);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
-					 sizeof(out));
-	if (err)
-		return err;
-
-	ft->id = MLX5_GET(create_flow_table_out, out, table_id);
-
-	return 0;
-}
-
-static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft)
-{
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
-#define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
-	MLX5_SET_DFTI(in, table_type, ft->type);
-	MLX5_SET_DFTI(in, table_id,   ft->id);
-	MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE);
-
-	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
-}
-
-static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable,
-			   u32 *match_criteria, int *group_ix)
-{
-	void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria,
-				      outer_headers);
-	void *mc_misc  = MLX5_ADDR_OF(fte_match_param, match_criteria,
-				      misc_parameters);
-	void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
-				      inner_headers);
-	int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
-	int mc_misc_sz  = MLX5_ST_SZ_BYTES(fte_match_set_misc);
-	int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
-	int i;
-
-	for (i = 0; i < ft->num_groups; i++) {
-		struct mlx5_flow_table_group *g = &ft->group[i].g;
-		void *gmc_outer = MLX5_ADDR_OF(fte_match_param,
-					       g->match_criteria,
-					       outer_headers);
-		void *gmc_misc  = MLX5_ADDR_OF(fte_match_param,
-					       g->match_criteria,
-					       misc_parameters);
-		void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
-					       g->match_criteria,
-					       inner_headers);
-
-		if (g->match_criteria_enable != match_criteria_enable)
-			continue;
-
-		if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS)
-			if (memcmp(mc_outer, gmc_outer, mc_outer_sz))
-				continue;
-
-		if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS)
-			if (memcmp(mc_misc, gmc_misc, mc_misc_sz))
-				continue;
-
-		if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS)
-			if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
-				continue;
-
-		*group_ix = i;
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix)
-{
-	struct mlx5_ftg *g = &ft->group[group_ix];
-	int err = 0;
-
-	mutex_lock(&ft->mutex);
-
-	*ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix);
-	if (*ix >= (g->start_ix + (1 << g->g.log_sz)))
-		err = -ENOSPC;
-	else
-		__set_bit(*ix, ft->bitmap);
-
-	mutex_unlock(&ft->mutex);
-
-	return err;
-}
-
-static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix)
-{
-	__clear_bit(ix, ft->bitmap);
-}
-
-int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
-			      void *match_criteria, void *flow_context,
-			      u32 *flow_index)
-{
-	struct mlx5_flow_table *ft = flow_table;
-	int group_ix;
-	int err;
-
-	err = mlx5_find_group(ft, match_criteria_enable, match_criteria,
-			      &group_ix);
-	if (err) {
-		mlx5_core_warn(ft->dev, "mlx5_find_group failed\n");
-		return err;
-	}
-
-	err = alloc_flow_index(ft, group_ix, flow_index);
-	if (err) {
-		mlx5_core_warn(ft->dev, "alloc_flow_index failed\n");
-		return err;
-	}
-
-	return mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context);
-}
-EXPORT_SYMBOL(mlx5_add_flow_table_entry);
-
-void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index)
-{
-	struct mlx5_flow_table *ft = flow_table;
-
-	mlx5_del_flow_entry_cmd(ft, flow_index);
-	mlx5_free_flow_index(ft, flow_index);
-}
-EXPORT_SYMBOL(mlx5_del_flow_table_entry);
-
-void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
-			     u16 num_groups,
-			     struct mlx5_flow_table_group *group)
-{
-	struct mlx5_flow_table *ft;
-	u32 start_ix = 0;
-	u32 ft_size = 0;
-	void *gr;
-	void *bm;
-	int err;
-	int i;
-
-	for (i = 0; i < num_groups; i++)
-		ft_size += (1 << group[i].log_sz);
-
-	ft = kzalloc(sizeof(*ft), GFP_KERNEL);
-	gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL);
-	bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL);
-	if (!ft || !gr || !bm)
-		goto err_free_ft;
-
-	ft->group	= gr;
-	ft->bitmap	= bm;
-	ft->num_groups	= num_groups;
-	ft->level	= level;
-	ft->type	= table_type;
-	ft->size	= ft_size;
-	ft->dev		= dev;
-	mutex_init(&ft->mutex);
-
-	for (i = 0; i < ft->num_groups; i++) {
-		memcpy(&ft->group[i].g, &group[i], sizeof(*group));
-		ft->group[i].start_ix = start_ix;
-		start_ix += 1 << group[i].log_sz;
-	}
-
-	err = mlx5_create_flow_table_cmd(ft);
-	if (err)
-		goto err_free_ft;
-
-	err = mlx5_create_flow_table_groups(ft);
-	if (err)
-		goto err_destroy_flow_table_cmd;
-
-	return ft;
-
-err_destroy_flow_table_cmd:
-	mlx5_destroy_flow_table_cmd(ft);
-
-err_free_ft:
-	mlx5_core_warn(dev, "failed to alloc flow table\n");
-	kfree(bm);
-	kfree(gr);
-	kfree(ft);
-
-	return NULL;
-}
-EXPORT_SYMBOL(mlx5_create_flow_table);
-
-void mlx5_destroy_flow_table(void *flow_table)
-{
-	struct mlx5_flow_table *ft = flow_table;
-
-	mlx5_destroy_flow_table_groups(ft);
-	mlx5_destroy_flow_table_cmd(ft);
-	kfree(ft->bitmap);
-	kfree(ft->group);
-	kfree(ft);
-}
-EXPORT_SYMBOL(mlx5_destroy_flow_table);
-
-u32 mlx5_get_flow_table_id(void *flow_table)
-{
-	struct mlx5_flow_table *ft = flow_table;
-
-	return ft->id;
-}
-EXPORT_SYMBOL(mlx5_get_flow_table_id);

drivers/net/ethernet/mellanox/mlx5/core/fs_core.c

@@ -436,7 +436,7 @@ static struct mlx5_flow_table *alloc_flow_table(int level, int max_fte,
 	return ft;
 }
 
-static struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
 					       int prio,
 					       int max_fte)
 {
@@ -491,7 +491,7 @@ static struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace
 	return ERR_PTR(err);
 }
 
-static struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
+struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
 					       u32 *fg_in)
 {
 	struct mlx5_flow_group *fg;
@@ -669,7 +669,7 @@ static struct mlx5_flow_rule *add_rule_fg(struct mlx5_flow_group *fg,
 	return rule;
 }
 
-static struct mlx5_flow_rule *
+struct mlx5_flow_rule *
 mlx5_add_flow_rule(struct mlx5_flow_table *ft,
 		   u8 match_criteria_enable,
 		   u32 *match_criteria,
@@ -699,12 +699,12 @@ mlx5_add_flow_rule(struct mlx5_flow_table *ft,
 	return rule;
 }
 
-static void mlx5_del_flow_rule(struct mlx5_flow_rule *rule)
+void mlx5_del_flow_rule(struct mlx5_flow_rule *rule)
 {
 	tree_remove_node(&rule->node);
 }
 
-static int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
+int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
 {
 	if (tree_remove_node(&ft->node))
 		mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n",
@@ -713,14 +713,14 @@ static int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
 	return 0;
 }
 
-static void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
+void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
 {
 	if (tree_remove_node(&fg->node))
 		mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
 			       fg->id);
 }
 
-static struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
+struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 						    enum mlx5_flow_namespace_type type)
 {
 	struct mlx5_flow_root_namespace *root_ns = dev->priv.root_ns;
@@ -867,7 +867,7 @@ static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev
 	struct mlx5_flow_root_namespace *root_ns;
 	struct mlx5_flow_namespace *ns;
 
-	/* create the root namespace */
+	/* Create the root namespace */
 	root_ns = mlx5_vzalloc(sizeof(*root_ns));
 	if (!root_ns)
 		return NULL;
@@ -1018,7 +1018,7 @@ static int init_fdb_root_ns(struct mlx5_core_dev *dev)
 	if (!dev->priv.fdb_root_ns)
 		return -ENOMEM;
 
-	/* create 1 prio*/
+	/* Create single prio */
 	prio = fs_create_prio(&dev->priv.fdb_root_ns->ns, 0, 1, 0);
 	if (IS_ERR(prio)) {
 		cleanup_single_prio_root_ns(dev, dev->priv.fdb_root_ns);

drivers/net/ethernet/mellanox/mlx5/core/main.c

@@ -49,6 +49,7 @@
 #include <linux/delay.h>
 #include <linux/mlx5/mlx5_ifc.h>
 #include "mlx5_core.h"
+#include "fs_core.h"
 #ifdef CONFIG_MLX5_CORE_EN
 #include "eswitch.h"
 #endif
@@ -1055,6 +1056,11 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	mlx5_init_srq_table(dev);
 	mlx5_init_mr_table(dev);
 
+	err = mlx5_init_fs(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init flow steering\n");
+		goto err_fs;
+	}
 #ifdef CONFIG_MLX5_CORE_EN
 	err = mlx5_eswitch_init(dev);
 	if (err) {
@@ -1093,6 +1099,8 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
 #endif
 err_reg_dev:
+	mlx5_cleanup_fs(dev);
+err_fs:
 	mlx5_cleanup_mr_table(dev);
 	mlx5_cleanup_srq_table(dev);
 	mlx5_cleanup_qp_table(dev);
@@ -1165,6 +1173,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
 #endif
 
+	mlx5_cleanup_fs(dev);
 	mlx5_cleanup_mr_table(dev);
 	mlx5_cleanup_srq_table(dev);
 	mlx5_cleanup_qp_table(dev);

include/linux/mlx5/flow_table.h

-/*
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef MLX5_FLOW_TABLE_H
-#define MLX5_FLOW_TABLE_H
-
-#include <linux/mlx5/driver.h>
-
-struct mlx5_flow_table_group {
-	u8	log_sz;
-	u8	match_criteria_enable;
-	u32	match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
-};
-
-struct mlx5_flow_destination {
-	enum mlx5_flow_destination_type	type;
-	union {
-		u32			tir_num;
-		void			*ft;
-		u32			vport_num;
-	};
-};
-
-void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
-			     u16 num_groups,
-			     struct mlx5_flow_table_group *group);
-void mlx5_destroy_flow_table(void *flow_table);
-int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
-			      void *match_criteria, void *flow_context,
-			      u32 *flow_index);
-void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index);
-u32 mlx5_get_flow_table_id(void *flow_table);
-
-#endif /* MLX5_FLOW_TABLE_H */

include/linux/mlx5/fs.h

@@ -33,6 +33,7 @@
 #ifndef _MLX5_FS_
 #define _MLX5_FS_
 
+#include <linux/mlx5/driver.h>
 #include <linux/mlx5/mlx5_ifc.h>
 
 #define MLX5_FS_DEFAULT_FLOW_TAG 0x0
@@ -43,6 +44,9 @@ enum mlx5_flow_namespace_type {
 };
 
 struct mlx5_flow_table;
+struct mlx5_flow_group;
+struct mlx5_flow_rule;
+struct mlx5_flow_namespace;
 
 struct mlx5_flow_destination {
 	enum mlx5_flow_destination_type	type;
@@ -52,4 +56,38 @@ struct mlx5_flow_destination {
 		u32			vport_num;
 	};
 };
+
+struct mlx5_flow_namespace *
+mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
+			enum mlx5_flow_namespace_type type);
+
+struct mlx5_flow_table *
+mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+		       int prio,
+		       int num_flow_table_entries);
+int mlx5_destroy_flow_table(struct mlx5_flow_table *ft);
+
+/* inbox should be set with the following values:
+ * start_flow_index
+ * end_flow_index
+ * match_criteria_enable
+ * match_criteria
+ */
+struct mlx5_flow_group *
+mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in);
+void mlx5_destroy_flow_group(struct mlx5_flow_group *fg);
+
+/* Single destination per rule.
+ * Group ID is implied by the match criteria.
+ */
+struct mlx5_flow_rule *
+mlx5_add_flow_rule(struct mlx5_flow_table *ft,
+		   u8 match_criteria_enable,
+		   u32 *match_criteria,
+		   u32 *match_value,
+		   u32 action,
+		   u32 flow_tag,
+		   struct mlx5_flow_destination *dest);
+void mlx5_del_flow_rule(struct mlx5_flow_rule *fr);
+
 #endif