Merge branch 'nfp-devlink-port-implementation'

Jakub Kicinski says:

====================
nfp: devlink port implementation

This series adds basic devlink support.  The operations we can perform
are port show and port split/unsplit.

v2:
Register devlink first, and then register all the ports.  Port {,un}split
searches the port list, which is protected by a mutex.  If port split
is requested before ports are registered we will simply not find the port
and return -EINVAL.
====================
Signed-off-by: NDavid S. Miller <davem@davemloft.net>

drivers/net/ethernet/netronome/nfp/Makefile

@@ -15,6 +15,7 @@ nfp-objs := \
 	    nfpcore/nfp_rtsym.o \
 	    nfpcore/nfp_target.o \
 	    nfp_app.o \
+	    nfp_devlink.o \
 	    nfp_main.o \
 	    nfp_net_common.o \
 	    nfp_net_ethtool.o \

drivers/net/ethernet/netronome/nfp/nfp_devlink.c

+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/rtnetlink.h>
+#include <net/devlink.h>
+
+#include "nfpcore/nfp_nsp.h"
+#include "nfp_app.h"
+#include "nfp_main.h"
+#include "nfp_port.h"
+
+static int
+nfp_devlink_fill_eth_port(struct nfp_port *port,
+			  struct nfp_eth_table_port *copy)
+{
+	struct nfp_eth_table_port *eth_port;
+
+	eth_port = __nfp_port_get_eth_port(port);
+	if (!eth_port)
+		return -EINVAL;
+
+	memcpy(copy, eth_port, sizeof(*eth_port));
+
+	return 0;
+}
+
+static int
+nfp_devlink_fill_eth_port_from_id(struct nfp_pf *pf, unsigned int port_index,
+				  struct nfp_eth_table_port *copy)
+{
+	struct nfp_port *port;
+
+	port = nfp_port_from_id(pf, NFP_PORT_PHYS_PORT, port_index);
+
+	return nfp_devlink_fill_eth_port(port, copy);
+}
+
+static int
+nfp_devlink_set_lanes(struct nfp_pf *pf, unsigned int idx, unsigned int lanes)
+{
+	struct nfp_nsp *nsp;
+	int ret;
+
+	nsp = nfp_eth_config_start(pf->cpp, idx);
+	if (IS_ERR(nsp))
+		return PTR_ERR(nsp);
+
+	ret = __nfp_eth_set_split(nsp, lanes);
+	if (ret) {
+		nfp_eth_config_cleanup_end(nsp);
+		return ret;
+	}
+
+	ret = nfp_eth_config_commit_end(nsp);
+	if (ret < 0)
+		return ret;
+	if (ret) /* no change */
+		return 0;
+
+	return nfp_net_refresh_port_table_sync(pf);
+}
+
+static int
+nfp_devlink_port_split(struct devlink *devlink, unsigned int port_index,
+		       unsigned int count)
+{
+	struct nfp_pf *pf = devlink_priv(devlink);
+	struct nfp_eth_table_port eth_port;
+	int ret;
+
+	if (count < 2)
+		return -EINVAL;
+
+	mutex_lock(&pf->lock);
+
+	rtnl_lock();
+	ret = nfp_devlink_fill_eth_port_from_id(pf, port_index, &eth_port);
+	rtnl_unlock();
+	if (ret)
+		goto out;
+
+	if (eth_port.is_split || eth_port.port_lanes % count) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = nfp_devlink_set_lanes(pf, eth_port.index,
+				    eth_port.port_lanes / count);
+out:
+	mutex_unlock(&pf->lock);
+
+	return ret;
+}
+
+static int
+nfp_devlink_port_unsplit(struct devlink *devlink, unsigned int port_index)
+{
+	struct nfp_pf *pf = devlink_priv(devlink);
+	struct nfp_eth_table_port eth_port;
+	int ret;
+
+	mutex_lock(&pf->lock);
+
+	rtnl_lock();
+	ret = nfp_devlink_fill_eth_port_from_id(pf, port_index, &eth_port);
+	rtnl_unlock();
+	if (ret)
+		goto out;
+
+	if (!eth_port.is_split) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = nfp_devlink_set_lanes(pf, eth_port.index, eth_port.port_lanes);
+out:
+	mutex_unlock(&pf->lock);
+
+	return ret;
+}
+
+const struct devlink_ops nfp_devlink_ops = {
+	.port_split		= nfp_devlink_port_split,
+	.port_unsplit		= nfp_devlink_port_unsplit,
+};
+
+int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port)
+{
+	struct nfp_eth_table_port eth_port;
+	struct devlink *devlink;
+	int ret;
+
+	rtnl_lock();
+	ret = nfp_devlink_fill_eth_port(port, &eth_port);
+	rtnl_unlock();
+	if (ret)
+		return ret;
+
+	devlink_port_type_eth_set(&port->dl_port, port->netdev);
+	if (eth_port.is_split)
+		devlink_port_split_set(&port->dl_port, eth_port.label_port);
+
+	devlink = priv_to_devlink(app->pf);
+
+	return devlink_port_register(devlink, &port->dl_port, port->eth_id);
+}
+
+void nfp_devlink_port_unregister(struct nfp_port *port)
+{
+	devlink_port_unregister(&port->dl_port);
+}

drivers/net/ethernet/netronome/nfp/nfp_main.c

@@ -41,9 +41,11 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/firmware.h>
 #include <linux/vermagic.h>
+#include <net/devlink.h>
 
 #include "nfpcore/nfp.h"
 #include "nfpcore/nfp_cpp.h"
@@ -315,6 +317,7 @@ static void nfp_fw_unload(struct nfp_pf *pf)
 static int nfp_pci_probe(struct pci_dev *pdev,
 			 const struct pci_device_id *pci_id)
 {
+	struct devlink *devlink;
 	struct nfp_pf *pf;
 	int err;
 
@@ -335,13 +338,15 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		goto err_pci_disable;
 	}
 
-	pf = kzalloc(sizeof(*pf), GFP_KERNEL);
-	if (!pf) {
+	devlink = devlink_alloc(&nfp_devlink_ops, sizeof(*pf));
+	if (!devlink) {
 		err = -ENOMEM;
 		goto err_rel_regions;
 	}
+	pf = devlink_priv(devlink);
 	INIT_LIST_HEAD(&pf->vnics);
 	INIT_LIST_HEAD(&pf->ports);
+	mutex_init(&pf->lock);
 	pci_set_drvdata(pdev, pf);
 	pf->pdev = pdev;
 
@@ -360,10 +365,14 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		 nfp_hwinfo_lookup(pf->cpp, "assembly.revision"),
 		 nfp_hwinfo_lookup(pf->cpp, "cpld.version"));
 
-	err = nfp_nsp_init(pdev, pf);
+	err = devlink_register(devlink, &pdev->dev);
 	if (err)
 		goto err_cpp_free;
 
+	err = nfp_nsp_init(pdev, pf);
+	if (err)
+		goto err_devlink_unreg;
+
 	nfp_pcie_sriov_read_nfd_limit(pf);
 
 	err = nfp_net_pci_probe(pf);
@@ -376,11 +385,14 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 	if (pf->fw_loaded)
 		nfp_fw_unload(pf);
 	kfree(pf->eth_tbl);
+err_devlink_unreg:
+	devlink_unregister(devlink);
 err_cpp_free:
 	nfp_cpp_free(pf->cpp);
 err_disable_msix:
 	pci_set_drvdata(pdev, NULL);
-	kfree(pf);
+	mutex_destroy(&pf->lock);
+	devlink_free(devlink);
 err_rel_regions:
 	pci_release_regions(pdev);
 err_pci_disable:
@@ -392,11 +404,16 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 static void nfp_pci_remove(struct pci_dev *pdev)
 {
 	struct nfp_pf *pf = pci_get_drvdata(pdev);
+	struct devlink *devlink;
+
+	devlink = priv_to_devlink(pf);
 
 	nfp_net_pci_remove(pf);
 
 	nfp_pcie_sriov_disable(pdev);
 
+	devlink_unregister(devlink);
+
 	if (pf->fw_loaded)
 		nfp_fw_unload(pf);
 
@@ -404,7 +421,8 @@ static void nfp_pci_remove(struct pci_dev *pdev)
 	nfp_cpp_free(pf->cpp);
 
 	kfree(pf->eth_tbl);
-	kfree(pf);
+	mutex_destroy(&pf->lock);
+	devlink_free(devlink);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 }

drivers/net/ethernet/netronome/nfp/nfp_main.h

@@ -47,6 +47,7 @@
 #include <linux/workqueue.h>
 
 struct dentry;
+struct devlink_ops;
 struct pci_dev;
 
 struct nfp_cpp;
@@ -107,6 +108,8 @@ struct nfp_pf {
 
 extern struct pci_driver nfp_netvf_pci_driver;
 
+extern const struct devlink_ops nfp_devlink_ops;
+
 int nfp_net_pci_probe(struct nfp_pf *pf);
 void nfp_net_pci_remove(struct nfp_pf *pf);
 

drivers/net/ethernet/netronome/nfp/nfp_net_main.c

@@ -43,6 +43,7 @@
 #include <linux/etherdevice.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
+#include <linux/lockdep.h>
 #include <linux/pci.h>
 #include <linux/pci_regs.h>
 #include <linux/msi.h>
@@ -354,9 +355,20 @@ nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id)
 
 	nfp_net_debugfs_vnic_add(nn, pf->ddir, id);
 
+	if (nn->port) {
+		err = nfp_devlink_port_register(pf->app, nn->port);
+		if (err)
+			goto err_dfs_clean;
+	}
+
 	nfp_net_info(nn);
 
 	return 0;
+
+err_dfs_clean:
+	nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
+	nfp_net_clean(nn);
+	return err;
 }
 
 static int
@@ -418,6 +430,14 @@ nfp_net_pf_alloc_vnics(struct nfp_pf *pf, void __iomem *ctrl_bar,
 	return err;
 }
 
+static void nfp_net_pf_clean_vnic(struct nfp_pf *pf, struct nfp_net *nn)
+{
+	if (nn->port)
+		nfp_devlink_port_unregister(nn->port);
+	nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
+	nfp_net_clean(nn);
+}
+
 static int
 nfp_net_pf_spawn_vnics(struct nfp_pf *pf,
 		       void __iomem *ctrl_bar, void __iomem *tx_bar,
@@ -480,10 +500,8 @@ nfp_net_pf_spawn_vnics(struct nfp_pf *pf,
 	return 0;
 
 err_prev_deinit:
-	list_for_each_entry_continue_reverse(nn, &pf->vnics, vnic_list) {
-		nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
-		nfp_net_clean(nn);
-	}
+	list_for_each_entry_continue_reverse(nn, &pf->vnics, vnic_list)
+		nfp_net_pf_clean_vnic(pf, nn);
 	nfp_net_irqs_disable(pf->pdev);
 err_vec_free:
 	kfree(pf->irq_entries);
@@ -502,6 +520,7 @@ static int nfp_net_pf_app_init(struct nfp_pf *pf)
 static void nfp_net_pf_app_clean(struct nfp_pf *pf)
 {
 	nfp_app_free(pf->app);
+	pf->app = NULL;
 }
 
 static void nfp_net_pci_remove_finish(struct nfp_pf *pf)
@@ -543,19 +562,17 @@ nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port,
 	return 0;
 }
 
-static void nfp_net_refresh_vnics(struct work_struct *work)
+int nfp_net_refresh_port_table_sync(struct nfp_pf *pf)
 {
-	struct nfp_pf *pf = container_of(work, struct nfp_pf,
-					 port_refresh_work);
 	struct nfp_eth_table *eth_table;
 	struct nfp_net *nn, *next;
 	struct nfp_port *port;
 
-	mutex_lock(&pf->lock);
+	lockdep_assert_held(&pf->lock);
 
 	/* Check for nfp_net_pci_remove() racing against us */
 	if (list_empty(&pf->vnics))
-		goto out;
+		return 0;
 
 	/* Update state of all ports */
 	rtnl_lock();
@@ -569,7 +586,7 @@ static void nfp_net_refresh_vnics(struct work_struct *work)
 				set_bit(NFP_PORT_CHANGED, &port->flags);
 		rtnl_unlock();
 		nfp_err(pf->cpp, "Error refreshing port config!\n");
-		goto out;
+		return -EIO;
 	}
 
 	list_for_each_entry(port, &pf->ports, port_list)
@@ -584,15 +601,23 @@ static void nfp_net_refresh_vnics(struct work_struct *work)
 		if (!nn->port || nn->port->type != NFP_PORT_INVALID)
 			continue;
 
-		nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
-		nfp_net_clean(nn);
-
+		nfp_net_pf_clean_vnic(pf, nn);
 		nfp_net_pf_free_vnic(pf, nn);
 	}
 
 	if (list_empty(&pf->vnics))
 		nfp_net_pci_remove_finish(pf);
-out:
+
+	return 0;
+}
+
+static void nfp_net_refresh_vnics(struct work_struct *work)
+{
+	struct nfp_pf *pf = container_of(work, struct nfp_pf,
+					 port_refresh_work);
+
+	mutex_lock(&pf->lock);
+	nfp_net_refresh_port_table_sync(pf);
 	mutex_unlock(&pf->lock);
 }
 
@@ -641,7 +666,6 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 	int err;
 
 	INIT_WORK(&pf->port_refresh_work, nfp_net_refresh_vnics);
-	mutex_init(&pf->lock);
 
 	/* Verify that the board has completed initialization */
 	if (!nfp_is_ready(pf->cpp)) {
@@ -760,11 +784,8 @@ void nfp_net_pci_remove(struct nfp_pf *pf)
 	if (list_empty(&pf->vnics))
 		goto out;
 
-	list_for_each_entry(nn, &pf->vnics, vnic_list) {
-		nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
-
-		nfp_net_clean(nn);
-	}
+	list_for_each_entry(nn, &pf->vnics, vnic_list)
+		nfp_net_pf_clean_vnic(pf, nn);
 
 	nfp_net_pf_free_vnics(pf);
 

drivers/net/ethernet/netronome/nfp/nfp_port.c

@@ -31,6 +31,8 @@
  * SOFTWARE.
  */
 
+#include <linux/lockdep.h>
+
 #include "nfpcore/nfp_nsp.h"
 #include "nfp_app.h"
 #include "nfp_main.h"
@@ -48,6 +50,23 @@ struct nfp_port *nfp_port_from_netdev(struct net_device *netdev)
 	return nn->port;
 }
 
+struct nfp_port *
+nfp_port_from_id(struct nfp_pf *pf, enum nfp_port_type type, unsigned int id)
+{
+	struct nfp_port *port;
+
+	lockdep_assert_held(&pf->lock);
+
+	if (type != NFP_PORT_PHYS_PORT)
+		return NULL;
+
+	list_for_each_entry(port, &pf->ports, port_list)
+		if (port->eth_id == id)
+			return port;
+
+	return NULL;
+}
+
 struct nfp_eth_table_port *__nfp_port_get_eth_port(struct nfp_port *port)
 {
 	if (!port)

drivers/net/ethernet/netronome/nfp/nfp_port.h

@@ -34,8 +34,11 @@
 #ifndef _NFP_PORT_H_
 #define _NFP_PORT_H_
 
+#include <net/devlink.h>
+
 struct net_device;
 struct nfp_app;
+struct nfp_pf;
 struct nfp_port;
 
 /**
@@ -66,6 +69,7 @@ enum nfp_port_flags {
  * @type:	what port type does the entity represent
  * @flags:	port flags
  * @app:	backpointer to the app structure
+ * @dl_port:	devlink port structure
  * @eth_id:	for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme
  * @eth_port:	for %NFP_PORT_PHYS_PORT translated ETH Table port entry
  * @port_list:	entry on pf's list of ports
@@ -78,6 +82,8 @@ struct nfp_port {
 
 	struct nfp_app *app;
 
+	struct devlink_port dl_port;
+
 	unsigned int eth_id;
 	struct nfp_eth_table_port *eth_port;
 
@@ -85,6 +91,8 @@ struct nfp_port {
 };
 
 struct nfp_port *nfp_port_from_netdev(struct net_device *netdev);
+struct nfp_port *
+nfp_port_from_id(struct nfp_pf *pf, enum nfp_port_type type, unsigned int id);
 struct nfp_eth_table_port *__nfp_port_get_eth_port(struct nfp_port *port);
 struct nfp_eth_table_port *nfp_port_get_eth_port(struct nfp_port *port);
 
@@ -98,5 +106,9 @@ void nfp_port_free(struct nfp_port *port);
 
 int nfp_net_refresh_eth_port(struct nfp_port *port);
 void nfp_net_refresh_port_table(struct nfp_port *port);
+int nfp_net_refresh_port_table_sync(struct nfp_pf *pf);
+
+int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port);
+void nfp_devlink_port_unregister(struct nfp_port *port);
 
 #endif

drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h

@@ -96,6 +96,7 @@ enum nfp_eth_aneg {
  * @override_changed: is media reconfig pending?
  *
  * @port_type:	one of %PORT_* defines for ethtool
+ * @port_lanes:	total number of lanes on the port (sum of lanes of all subports)
  * @is_split:	is interface part of a split port
  */
 struct nfp_eth_table {
@@ -127,6 +128,8 @@ struct nfp_eth_table {
 		/* Computed fields */
 		u8 port_type;
 
+		unsigned int port_lanes;
+
 		bool is_split;
 	} ports[0];
 };

drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c

@@ -186,17 +186,19 @@ nfp_eth_port_translate(struct nfp_nsp *nsp, const union eth_table_entry *src,
 }
 
 static void
-nfp_eth_mark_split_ports(struct nfp_cpp *cpp, struct nfp_eth_table *table)
+nfp_eth_calc_port_geometry(struct nfp_cpp *cpp, struct nfp_eth_table *table)
 {
 	unsigned int i, j;
 
 	for (i = 0; i < table->count; i++)
 		for (j = 0; j < table->count; j++) {
-			if (i == j)
-				continue;
 			if (table->ports[i].label_port !=
 			    table->ports[j].label_port)
 				continue;
+			table->ports[i].port_lanes += table->ports[j].lanes;
+
+			if (i == j)
+				continue;
 			if (table->ports[i].label_subport ==
 			    table->ports[j].label_subport)
 				nfp_warn(cpp,
@@ -205,7 +207,6 @@ nfp_eth_mark_split_ports(struct nfp_cpp *cpp, struct nfp_eth_table *table)
 					 table->ports[i].label_subport);
 
 			table->ports[i].is_split = true;
-			break;
 		}
 }
 
@@ -289,7 +290,7 @@ __nfp_eth_read_ports(struct nfp_cpp *cpp, struct nfp_nsp *nsp)
 			nfp_eth_port_translate(nsp, &entries[i], i,
 					       &table->ports[j++]);
 
-	nfp_eth_mark_split_ports(cpp, table);
+	nfp_eth_calc_port_geometry(cpp, table);
 	for (i = 0; i < table->count; i++)
 		nfp_eth_calc_port_type(cpp, &table->ports[i]);