bcmsysport.c 66.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * Broadcom BCM7xxx System Port Ethernet MAC driver
 *
 * Copyright (C) 2014 Broadcom Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_net.h>
#include <linux/of_mdio.h>
#include <linux/phy.h>
#include <linux/phy_fixed.h>
25
#include <net/dsa.h>
26 27 28 29 30 31 32 33 34
#include <net/ip.h>
#include <net/ipv6.h>

#include "bcmsysport.h"

/* I/O accessors register helpers */
#define BCM_SYSPORT_IO_MACRO(name, offset) \
static inline u32 name##_readl(struct bcm_sysport_priv *priv, u32 off)	\
{									\
35
	u32 reg = readl_relaxed(priv->base + offset + off);		\
36 37 38 39 40
	return reg;							\
}									\
static inline void name##_writel(struct bcm_sysport_priv *priv,		\
				  u32 val, u32 off)			\
{									\
41
	writel_relaxed(val, priv->base + offset + off);			\
42 43 44 45 46
}									\

BCM_SYSPORT_IO_MACRO(intrl2_0, SYS_PORT_INTRL2_0_OFFSET);
BCM_SYSPORT_IO_MACRO(intrl2_1, SYS_PORT_INTRL2_1_OFFSET);
BCM_SYSPORT_IO_MACRO(umac, SYS_PORT_UMAC_OFFSET);
47
BCM_SYSPORT_IO_MACRO(gib, SYS_PORT_GIB_OFFSET);
48 49 50 51 52 53 54
BCM_SYSPORT_IO_MACRO(tdma, SYS_PORT_TDMA_OFFSET);
BCM_SYSPORT_IO_MACRO(rxchk, SYS_PORT_RXCHK_OFFSET);
BCM_SYSPORT_IO_MACRO(txchk, SYS_PORT_TXCHK_OFFSET);
BCM_SYSPORT_IO_MACRO(rbuf, SYS_PORT_RBUF_OFFSET);
BCM_SYSPORT_IO_MACRO(tbuf, SYS_PORT_TBUF_OFFSET);
BCM_SYSPORT_IO_MACRO(topctrl, SYS_PORT_TOPCTRL_OFFSET);

55 56 57 58 59 60 61
/* On SYSTEMPORT Lite, any register after RDMA_STATUS has the exact
 * same layout, except it has been moved by 4 bytes up, *sigh*
 */
static inline u32 rdma_readl(struct bcm_sysport_priv *priv, u32 off)
{
	if (priv->is_lite && off >= RDMA_STATUS)
		off += 4;
62
	return readl_relaxed(priv->base + SYS_PORT_RDMA_OFFSET + off);
63 64 65 66 67 68
}

static inline void rdma_writel(struct bcm_sysport_priv *priv, u32 val, u32 off)
{
	if (priv->is_lite && off >= RDMA_STATUS)
		off += 4;
69
	writel_relaxed(val, priv->base + SYS_PORT_RDMA_OFFSET + off);
70 71 72 73 74 75 76 77 78 79 80 81 82 83
}

static inline u32 tdma_control_bit(struct bcm_sysport_priv *priv, u32 bit)
{
	if (!priv->is_lite) {
		return BIT(bit);
	} else {
		if (bit >= ACB_ALGO)
			return BIT(bit + 1);
		else
			return BIT(bit);
	}
}

84 85 86 87 88 89 90 91
/* L2-interrupt masking/unmasking helpers, does automatic saving of the applied
 * mask in a software copy to avoid CPU_MASK_STATUS reads in hot-paths.
  */
#define BCM_SYSPORT_INTR_L2(which)	\
static inline void intrl2_##which##_mask_clear(struct bcm_sysport_priv *priv, \
						u32 mask)		\
{									\
	priv->irq##which##_mask &= ~(mask);				\
92
	intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);	\
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
}									\
static inline void intrl2_##which##_mask_set(struct bcm_sysport_priv *priv, \
						u32 mask)		\
{									\
	intrl2_## which##_writel(priv, mask, INTRL2_CPU_MASK_SET);	\
	priv->irq##which##_mask |= (mask);				\
}									\

BCM_SYSPORT_INTR_L2(0)
BCM_SYSPORT_INTR_L2(1)

/* Register accesses to GISB/RBUS registers are expensive (few hundred
 * nanoseconds), so keep the check for 64-bits explicit here to save
 * one register write per-packet on 32-bits platforms.
 */
static inline void dma_desc_set_addr(struct bcm_sysport_priv *priv,
				     void __iomem *d,
				     dma_addr_t addr)
{
#ifdef CONFIG_PHYS_ADDR_T_64BIT
113
	writel_relaxed(upper_32_bits(addr) & DESC_ADDR_HI_MASK,
114
		     d + DESC_ADDR_HI_STATUS_LEN);
115
#endif
116
	writel_relaxed(lower_32_bits(addr), d + DESC_ADDR_LO);
117 118 119
}

static inline void tdma_port_write_desc_addr(struct bcm_sysport_priv *priv,
120 121
					     struct dma_desc *desc,
					     unsigned int port)
122 123 124 125 126 127 128 129
{
	/* Ports are latched, so write upper address first */
	tdma_writel(priv, desc->addr_status_len, TDMA_WRITE_PORT_HI(port));
	tdma_writel(priv, desc->addr_lo, TDMA_WRITE_PORT_LO(port));
}

/* Ethtool operations */
static int bcm_sysport_set_rx_csum(struct net_device *dev,
130
				   netdev_features_t wanted)
131 132 133 134
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u32 reg;

135
	priv->rx_chk_en = !!(wanted & NETIF_F_RXCSUM);
136
	reg = rxchk_readl(priv, RXCHK_CONTROL);
137
	if (priv->rx_chk_en)
138 139 140 141 142 143 144
		reg |= RXCHK_EN;
	else
		reg &= ~RXCHK_EN;

	/* If UniMAC forwards CRC, we need to skip over it to get
	 * a valid CHK bit to be set in the per-packet status word
	 */
145
	if (priv->rx_chk_en && priv->crc_fwd)
146 147 148 149
		reg |= RXCHK_SKIP_FCS;
	else
		reg &= ~RXCHK_SKIP_FCS;

150 151 152 153 154 155 156 157 158
	/* If Broadcom tags are enabled (e.g: using a switch), make
	 * sure we tell the RXCHK hardware to expect a 4-bytes Broadcom
	 * tag after the Ethernet MAC Source Address.
	 */
	if (netdev_uses_dsa(dev))
		reg |= RXCHK_BRCM_TAG_EN;
	else
		reg &= ~RXCHK_BRCM_TAG_EN;

159 160 161 162 163 164
	rxchk_writel(priv, reg, RXCHK_CONTROL);

	return 0;
}

static int bcm_sysport_set_tx_csum(struct net_device *dev,
165
				   netdev_features_t wanted)
166 167 168 169 170 171 172 173 174 175
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u32 reg;

	/* Hardware transmit checksum requires us to enable the Transmit status
	 * block prepended to the packet contents
	 */
	priv->tsb_en = !!(wanted & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
	reg = tdma_readl(priv, TDMA_CONTROL);
	if (priv->tsb_en)
176
		reg |= tdma_control_bit(priv, TSB_EN);
177
	else
178
		reg &= ~tdma_control_bit(priv, TSB_EN);
179 180 181 182 183 184
	tdma_writel(priv, reg, TDMA_CONTROL);

	return 0;
}

static int bcm_sysport_set_features(struct net_device *dev,
185
				    netdev_features_t features)
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
{
	netdev_features_t changed = features ^ dev->features;
	netdev_features_t wanted = dev->wanted_features;
	int ret = 0;

	if (changed & NETIF_F_RXCSUM)
		ret = bcm_sysport_set_rx_csum(dev, wanted);
	if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
		ret = bcm_sysport_set_tx_csum(dev, wanted);

	return ret;
}

/* Hardware counters must be kept in sync because the order/offset
 * is important here (order in structure declaration = order in hardware)
 */
static const struct bcm_sysport_stats bcm_sysport_gstrings_stats[] = {
	/* general stats */
204 205 206 207
	STAT_NETDEV64(rx_packets),
	STAT_NETDEV64(tx_packets),
	STAT_NETDEV64(rx_bytes),
	STAT_NETDEV64(tx_bytes),
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
	STAT_NETDEV(rx_errors),
	STAT_NETDEV(tx_errors),
	STAT_NETDEV(rx_dropped),
	STAT_NETDEV(tx_dropped),
	STAT_NETDEV(multicast),
	/* UniMAC RSV counters */
	STAT_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
	STAT_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
	STAT_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
	STAT_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
	STAT_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
	STAT_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
	STAT_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
	STAT_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
	STAT_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
	STAT_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
	STAT_MIB_RX("rx_pkts", mib.rx.pkt),
	STAT_MIB_RX("rx_bytes", mib.rx.bytes),
	STAT_MIB_RX("rx_multicast", mib.rx.mca),
	STAT_MIB_RX("rx_broadcast", mib.rx.bca),
	STAT_MIB_RX("rx_fcs", mib.rx.fcs),
	STAT_MIB_RX("rx_control", mib.rx.cf),
	STAT_MIB_RX("rx_pause", mib.rx.pf),
	STAT_MIB_RX("rx_unknown", mib.rx.uo),
	STAT_MIB_RX("rx_align", mib.rx.aln),
	STAT_MIB_RX("rx_outrange", mib.rx.flr),
	STAT_MIB_RX("rx_code", mib.rx.cde),
	STAT_MIB_RX("rx_carrier", mib.rx.fcr),
	STAT_MIB_RX("rx_oversize", mib.rx.ovr),
	STAT_MIB_RX("rx_jabber", mib.rx.jbr),
	STAT_MIB_RX("rx_mtu_err", mib.rx.mtue),
	STAT_MIB_RX("rx_good_pkts", mib.rx.pok),
	STAT_MIB_RX("rx_unicast", mib.rx.uc),
	STAT_MIB_RX("rx_ppp", mib.rx.ppp),
	STAT_MIB_RX("rx_crc", mib.rx.rcrc),
	/* UniMAC TSV counters */
	STAT_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
	STAT_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
	STAT_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
	STAT_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
	STAT_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
	STAT_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
	STAT_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
	STAT_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
	STAT_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
	STAT_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
	STAT_MIB_TX("tx_pkts", mib.tx.pkts),
	STAT_MIB_TX("tx_multicast", mib.tx.mca),
	STAT_MIB_TX("tx_broadcast", mib.tx.bca),
	STAT_MIB_TX("tx_pause", mib.tx.pf),
	STAT_MIB_TX("tx_control", mib.tx.cf),
	STAT_MIB_TX("tx_fcs_err", mib.tx.fcs),
	STAT_MIB_TX("tx_oversize", mib.tx.ovr),
	STAT_MIB_TX("tx_defer", mib.tx.drf),
	STAT_MIB_TX("tx_excess_defer", mib.tx.edf),
	STAT_MIB_TX("tx_single_col", mib.tx.scl),
	STAT_MIB_TX("tx_multi_col", mib.tx.mcl),
	STAT_MIB_TX("tx_late_col", mib.tx.lcl),
	STAT_MIB_TX("tx_excess_col", mib.tx.ecl),
	STAT_MIB_TX("tx_frags", mib.tx.frg),
	STAT_MIB_TX("tx_total_col", mib.tx.ncl),
	STAT_MIB_TX("tx_jabber", mib.tx.jbr),
	STAT_MIB_TX("tx_bytes", mib.tx.bytes),
	STAT_MIB_TX("tx_good_pkts", mib.tx.pok),
	STAT_MIB_TX("tx_unicast", mib.tx.uc),
	/* UniMAC RUNT counters */
	STAT_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
	STAT_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
	STAT_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
	STAT_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
	/* RXCHK misc statistics */
	STAT_RXCHK("rxchk_bad_csum", mib.rxchk_bad_csum, RXCHK_BAD_CSUM_CNTR),
	STAT_RXCHK("rxchk_other_pkt_disc", mib.rxchk_other_pkt_disc,
281
		   RXCHK_OTHER_DISC_CNTR),
282 283 284
	/* RBUF misc statistics */
	STAT_RBUF("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt, RBUF_OVFL_DISC_CNTR),
	STAT_RBUF("rbuf_err_cnt", mib.rbuf_err_cnt, RBUF_ERR_PKT_CNTR),
285 286 287
	STAT_MIB_SOFT("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
	STAT_MIB_SOFT("rx_dma_failed", mib.rx_dma_failed),
	STAT_MIB_SOFT("tx_dma_failed", mib.tx_dma_failed),
288
	/* Per TX-queue statistics are dynamically appended */
289 290 291 292 293
};

#define BCM_SYSPORT_STATS_LEN	ARRAY_SIZE(bcm_sysport_gstrings_stats)

static void bcm_sysport_get_drvinfo(struct net_device *dev,
294
				    struct ethtool_drvinfo *info)
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
{
	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->version, "0.1", sizeof(info->version));
	strlcpy(info->bus_info, "platform", sizeof(info->bus_info));
}

static u32 bcm_sysport_get_msglvl(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);

	return priv->msg_enable;
}

static void bcm_sysport_set_msglvl(struct net_device *dev, u32 enable)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);

	priv->msg_enable = enable;
}

315 316 317 318
static inline bool bcm_sysport_lite_stat_valid(enum bcm_sysport_stat_type type)
{
	switch (type) {
	case BCM_SYSPORT_STAT_NETDEV:
319
	case BCM_SYSPORT_STAT_NETDEV64:
320 321 322 323 324 325 326 327 328
	case BCM_SYSPORT_STAT_RXCHK:
	case BCM_SYSPORT_STAT_RBUF:
	case BCM_SYSPORT_STAT_SOFT:
		return true;
	default:
		return false;
	}
}

329 330
static int bcm_sysport_get_sset_count(struct net_device *dev, int string_set)
{
331 332 333 334
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	const struct bcm_sysport_stats *s;
	unsigned int i, j;

335 336
	switch (string_set) {
	case ETH_SS_STATS:
337 338 339 340 341 342 343
		for (i = 0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
			s = &bcm_sysport_gstrings_stats[i];
			if (priv->is_lite &&
			    !bcm_sysport_lite_stat_valid(s->type))
				continue;
			j++;
		}
344 345
		/* Include per-queue statistics */
		return j + dev->num_tx_queues * NUM_SYSPORT_TXQ_STAT;
346 347 348 349 350 351
	default:
		return -EOPNOTSUPP;
	}
}

static void bcm_sysport_get_strings(struct net_device *dev,
352
				    u32 stringset, u8 *data)
353
{
354 355
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	const struct bcm_sysport_stats *s;
356
	char buf[128];
357
	int i, j;
358 359 360

	switch (stringset) {
	case ETH_SS_STATS:
361 362 363 364 365 366 367
		for (i = 0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
			s = &bcm_sysport_gstrings_stats[i];
			if (priv->is_lite &&
			    !bcm_sysport_lite_stat_valid(s->type))
				continue;

			memcpy(data + j * ETH_GSTRING_LEN, s->stat_string,
368
			       ETH_GSTRING_LEN);
369
			j++;
370
		}
371 372 373 374 375 376 377 378 379 380 381 382

		for (i = 0; i < dev->num_tx_queues; i++) {
			snprintf(buf, sizeof(buf), "txq%d_packets", i);
			memcpy(data + j * ETH_GSTRING_LEN, buf,
			       ETH_GSTRING_LEN);
			j++;

			snprintf(buf, sizeof(buf), "txq%d_bytes", i);
			memcpy(data + j * ETH_GSTRING_LEN, buf,
			       ETH_GSTRING_LEN);
			j++;
		}
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
		break;
	default:
		break;
	}
}

static void bcm_sysport_update_mib_counters(struct bcm_sysport_priv *priv)
{
	int i, j = 0;

	for (i = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
		const struct bcm_sysport_stats *s;
		u8 offset = 0;
		u32 val = 0;
		char *p;

		s = &bcm_sysport_gstrings_stats[i];
		switch (s->type) {
		case BCM_SYSPORT_STAT_NETDEV:
402
		case BCM_SYSPORT_STAT_NETDEV64:
403
		case BCM_SYSPORT_STAT_SOFT:
404 405 406 407
			continue;
		case BCM_SYSPORT_STAT_MIB_RX:
		case BCM_SYSPORT_STAT_MIB_TX:
		case BCM_SYSPORT_STAT_RUNT:
408 409 410
			if (priv->is_lite)
				continue;

411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
			if (s->type != BCM_SYSPORT_STAT_MIB_RX)
				offset = UMAC_MIB_STAT_OFFSET;
			val = umac_readl(priv, UMAC_MIB_START + j + offset);
			break;
		case BCM_SYSPORT_STAT_RXCHK:
			val = rxchk_readl(priv, s->reg_offset);
			if (val == ~0)
				rxchk_writel(priv, 0, s->reg_offset);
			break;
		case BCM_SYSPORT_STAT_RBUF:
			val = rbuf_readl(priv, s->reg_offset);
			if (val == ~0)
				rbuf_writel(priv, 0, s->reg_offset);
			break;
		}

		j += s->stat_sizeof;
		p = (char *)priv + s->stat_offset;
		*(u32 *)p = val;
	}

	netif_dbg(priv, hw, priv->netdev, "updated MIB counters\n");
}

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
static void bcm_sysport_update_tx_stats(struct bcm_sysport_priv *priv,
					u64 *tx_bytes, u64 *tx_packets)
{
	struct bcm_sysport_tx_ring *ring;
	u64 bytes = 0, packets = 0;
	unsigned int start;
	unsigned int q;

	for (q = 0; q < priv->netdev->num_tx_queues; q++) {
		ring = &priv->tx_rings[q];
		do {
			start = u64_stats_fetch_begin_irq(&priv->syncp);
			bytes = ring->bytes;
			packets = ring->packets;
		} while (u64_stats_fetch_retry_irq(&priv->syncp, start));

		*tx_bytes += bytes;
		*tx_packets += packets;
	}
}

456
static void bcm_sysport_get_stats(struct net_device *dev,
457
				  struct ethtool_stats *stats, u64 *data)
458 459
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
460 461
	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
	struct u64_stats_sync *syncp = &priv->syncp;
462
	struct bcm_sysport_tx_ring *ring;
463
	u64 tx_bytes = 0, tx_packets = 0;
464
	unsigned int start;
465
	int i, j;
466

467
	if (netif_running(dev)) {
468
		bcm_sysport_update_mib_counters(priv);
469 470 471 472
		bcm_sysport_update_tx_stats(priv, &tx_bytes, &tx_packets);
		stats64->tx_bytes = tx_bytes;
		stats64->tx_packets = tx_packets;
	}
473

474
	for (i =  0, j = 0; i < BCM_SYSPORT_STATS_LEN; i++) {
475 476 477 478 479 480
		const struct bcm_sysport_stats *s;
		char *p;

		s = &bcm_sysport_gstrings_stats[i];
		if (s->type == BCM_SYSPORT_STAT_NETDEV)
			p = (char *)&dev->stats;
481 482
		else if (s->type == BCM_SYSPORT_STAT_NETDEV64)
			p = (char *)stats64;
483 484
		else
			p = (char *)priv;
485

486 487
		if (priv->is_lite && !bcm_sysport_lite_stat_valid(s->type))
			continue;
488
		p += s->stat_offset;
489

490 491
		if (s->stat_sizeof == sizeof(u64) &&
		    s->type == BCM_SYSPORT_STAT_NETDEV64) {
492 493 494 495
			do {
				start = u64_stats_fetch_begin_irq(syncp);
				data[i] = *(u64 *)p;
			} while (u64_stats_fetch_retry_irq(syncp, start));
496
		} else
497
			data[i] = *(u32 *)p;
498
		j++;
499
	}
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515

	/* For SYSTEMPORT Lite since we have holes in our statistics, j would
	 * be equal to BCM_SYSPORT_STATS_LEN at the end of the loop, but it
	 * needs to point to how many total statistics we have minus the
	 * number of per TX queue statistics
	 */
	j = bcm_sysport_get_sset_count(dev, ETH_SS_STATS) -
	    dev->num_tx_queues * NUM_SYSPORT_TXQ_STAT;

	for (i = 0; i < dev->num_tx_queues; i++) {
		ring = &priv->tx_rings[i];
		data[j] = ring->packets;
		j++;
		data[j] = ring->bytes;
		j++;
	}
516 517
}

518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
static void bcm_sysport_get_wol(struct net_device *dev,
				struct ethtool_wolinfo *wol)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u32 reg;

	wol->supported = WAKE_MAGIC | WAKE_MAGICSECURE;
	wol->wolopts = priv->wolopts;

	if (!(priv->wolopts & WAKE_MAGICSECURE))
		return;

	/* Return the programmed SecureOn password */
	reg = umac_readl(priv, UMAC_PSW_MS);
	put_unaligned_be16(reg, &wol->sopass[0]);
	reg = umac_readl(priv, UMAC_PSW_LS);
	put_unaligned_be32(reg, &wol->sopass[2]);
}

static int bcm_sysport_set_wol(struct net_device *dev,
538
			       struct ethtool_wolinfo *wol)
539 540 541 542 543 544 545 546 547 548 549 550 551 552
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	struct device *kdev = &priv->pdev->dev;
	u32 supported = WAKE_MAGIC | WAKE_MAGICSECURE;

	if (!device_can_wakeup(kdev))
		return -ENOTSUPP;

	if (wol->wolopts & ~supported)
		return -EINVAL;

	/* Program the SecureOn password */
	if (wol->wolopts & WAKE_MAGICSECURE) {
		umac_writel(priv, get_unaligned_be16(&wol->sopass[0]),
553
			    UMAC_PSW_MS);
554
		umac_writel(priv, get_unaligned_be32(&wol->sopass[2]),
555
			    UMAC_PSW_LS);
556 557 558 559 560
	}

	/* Flag the device and relevant IRQ as wakeup capable */
	if (wol->wolopts) {
		device_set_wakeup_enable(kdev, 1);
561 562
		if (priv->wol_irq_disabled)
			enable_irq_wake(priv->wol_irq);
563 564 565 566 567 568 569 570 571 572 573 574 575 576
		priv->wol_irq_disabled = 0;
	} else {
		device_set_wakeup_enable(kdev, 0);
		/* Avoid unbalanced disable_irq_wake calls */
		if (!priv->wol_irq_disabled)
			disable_irq_wake(priv->wol_irq);
		priv->wol_irq_disabled = 1;
	}

	priv->wolopts = wol->wolopts;

	return 0;
}

577 578 579 580 581 582 583 584 585 586 587
static int bcm_sysport_get_coalesce(struct net_device *dev,
				    struct ethtool_coalesce *ec)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u32 reg;

	reg = tdma_readl(priv, TDMA_DESC_RING_INTR_CONTROL(0));

	ec->tx_coalesce_usecs = (reg >> RING_TIMEOUT_SHIFT) * 8192 / 1000;
	ec->tx_max_coalesced_frames = reg & RING_INTR_THRESH_MASK;

588 589 590 591 592
	reg = rdma_readl(priv, RDMA_MBDONE_INTR);

	ec->rx_coalesce_usecs = (reg >> RDMA_TIMEOUT_SHIFT) * 8192 / 1000;
	ec->rx_max_coalesced_frames = reg & RDMA_INTR_THRESH_MASK;

593 594 595 596 597 598 599 600 601 602
	return 0;
}

static int bcm_sysport_set_coalesce(struct net_device *dev,
				    struct ethtool_coalesce *ec)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	unsigned int i;
	u32 reg;

603 604 605
	/* Base system clock is 125Mhz, DMA timeout is this reference clock
	 * divided by 1024, which yield roughly 8.192 us, our maximum value has
	 * to fit in the RING_TIMEOUT_MASK (16 bits).
606 607
	 */
	if (ec->tx_max_coalesced_frames > RING_INTR_THRESH_MASK ||
608 609 610
	    ec->tx_coalesce_usecs > (RING_TIMEOUT_MASK * 8) + 1 ||
	    ec->rx_max_coalesced_frames > RDMA_INTR_THRESH_MASK ||
	    ec->rx_coalesce_usecs > (RDMA_TIMEOUT_MASK * 8) + 1)
611 612
		return -EINVAL;

613 614
	if ((ec->tx_coalesce_usecs == 0 && ec->tx_max_coalesced_frames == 0) ||
	    (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0))
615 616 617 618 619 620 621 622 623 624 625 626
		return -EINVAL;

	for (i = 0; i < dev->num_tx_queues; i++) {
		reg = tdma_readl(priv, TDMA_DESC_RING_INTR_CONTROL(i));
		reg &= ~(RING_INTR_THRESH_MASK |
			 RING_TIMEOUT_MASK << RING_TIMEOUT_SHIFT);
		reg |= ec->tx_max_coalesced_frames;
		reg |= DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000, 8192) <<
			 RING_TIMEOUT_SHIFT;
		tdma_writel(priv, reg, TDMA_DESC_RING_INTR_CONTROL(i));
	}

627 628 629 630 631 632 633 634
	reg = rdma_readl(priv, RDMA_MBDONE_INTR);
	reg &= ~(RDMA_INTR_THRESH_MASK |
		 RDMA_TIMEOUT_MASK << RDMA_TIMEOUT_SHIFT);
	reg |= ec->rx_max_coalesced_frames;
	reg |= DIV_ROUND_UP(ec->rx_coalesce_usecs * 1000, 8192) <<
			    RDMA_TIMEOUT_SHIFT;
	rdma_writel(priv, reg, RDMA_MBDONE_INTR);

635 636 637
	return 0;
}

638 639
static void bcm_sysport_free_cb(struct bcm_sysport_cb *cb)
{
640
	dev_consume_skb_any(cb->skb);
641 642 643 644
	cb->skb = NULL;
	dma_unmap_addr_set(cb, dma_addr, 0);
}

645 646
static struct sk_buff *bcm_sysport_rx_refill(struct bcm_sysport_priv *priv,
					     struct bcm_sysport_cb *cb)
647 648 649
{
	struct device *kdev = &priv->pdev->dev;
	struct net_device *ndev = priv->netdev;
650
	struct sk_buff *skb, *rx_skb;
651 652
	dma_addr_t mapping;

653 654 655 656
	/* Allocate a new SKB for a new packet */
	skb = netdev_alloc_skb(priv->netdev, RX_BUF_LENGTH);
	if (!skb) {
		priv->mib.alloc_rx_buff_failed++;
657
		netif_err(priv, rx_err, ndev, "SKB alloc failed\n");
658
		return NULL;
659 660
	}

661
	mapping = dma_map_single(kdev, skb->data,
662
				 RX_BUF_LENGTH, DMA_FROM_DEVICE);
663
	if (dma_mapping_error(kdev, mapping)) {
664
		priv->mib.rx_dma_failed++;
665
		dev_kfree_skb_any(skb);
666
		netif_err(priv, rx_err, ndev, "DMA mapping failure\n");
667
		return NULL;
668 669
	}

670 671 672 673 674 675 676 677
	/* Grab the current SKB on the ring */
	rx_skb = cb->skb;
	if (likely(rx_skb))
		dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
				 RX_BUF_LENGTH, DMA_FROM_DEVICE);

	/* Put the new SKB on the ring */
	cb->skb = skb;
678
	dma_unmap_addr_set(cb, dma_addr, mapping);
679
	dma_desc_set_addr(priv, cb->bd_addr, mapping);
680 681 682

	netif_dbg(priv, rx_status, ndev, "RX refill\n");

683 684
	/* Return the current SKB to the caller */
	return rx_skb;
685 686 687 688 689
}

static int bcm_sysport_alloc_rx_bufs(struct bcm_sysport_priv *priv)
{
	struct bcm_sysport_cb *cb;
690
	struct sk_buff *skb;
691 692 693
	unsigned int i;

	for (i = 0; i < priv->num_rx_bds; i++) {
694
		cb = &priv->rx_cbs[i];
695 696 697 698 699
		skb = bcm_sysport_rx_refill(priv, cb);
		if (skb)
			dev_kfree_skb(skb);
		if (!cb->skb)
			return -ENOMEM;
700 701
	}

702
	return 0;
703 704 705 706 707 708
}

/* Poll the hardware for up to budget packets to process */
static unsigned int bcm_sysport_desc_rx(struct bcm_sysport_priv *priv,
					unsigned int budget)
{
709
	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
710 711 712 713 714 715
	struct net_device *ndev = priv->netdev;
	unsigned int processed = 0, to_process;
	struct bcm_sysport_cb *cb;
	struct sk_buff *skb;
	unsigned int p_index;
	u16 len, status;
716
	struct bcm_rsb *rsb;
717

718 719 720
	/* Clear status before servicing to reduce spurious interrupts */
	intrl2_0_writel(priv, INTRL2_0_RDMA_MBDONE, INTRL2_CPU_CLEAR);

721 722 723 724 725 726 727 728
	/* Determine how much we should process since last call, SYSTEMPORT Lite
	 * groups the producer and consumer indexes into the same 32-bit
	 * which we access using RDMA_CONS_INDEX
	 */
	if (!priv->is_lite)
		p_index = rdma_readl(priv, RDMA_PROD_INDEX);
	else
		p_index = rdma_readl(priv, RDMA_CONS_INDEX);
729 730
	p_index &= RDMA_PROD_INDEX_MASK;

731
	to_process = (p_index - priv->rx_c_index) & RDMA_CONS_INDEX_MASK;
732 733

	netif_dbg(priv, rx_status, ndev,
734 735
		  "p_index=%d rx_c_index=%d to_process=%d\n",
		  p_index, priv->rx_c_index, to_process);
736

737
	while ((processed < to_process) && (processed < budget)) {
738
		cb = &priv->rx_cbs[priv->rx_read_ptr];
739
		skb = bcm_sysport_rx_refill(priv, cb);
740 741 742 743 744 745 746 747 748 749 750


		/* We do not have a backing SKB, so we do not a corresponding
		 * DMA mapping for this incoming packet since
		 * bcm_sysport_rx_refill always either has both skb and mapping
		 * or none.
		 */
		if (unlikely(!skb)) {
			netif_err(priv, rx_err, ndev, "out of memory!\n");
			ndev->stats.rx_dropped++;
			ndev->stats.rx_errors++;
751
			goto next;
752 753
		}

754
		/* Extract the Receive Status Block prepended */
755
		rsb = (struct bcm_rsb *)skb->data;
756 757
		len = (rsb->rx_status_len >> DESC_LEN_SHIFT) & DESC_LEN_MASK;
		status = (rsb->rx_status_len >> DESC_STATUS_SHIFT) &
758
			  DESC_STATUS_MASK;
759 760

		netif_dbg(priv, rx_status, ndev,
761 762 763
			  "p=%d, c=%d, rd_ptr=%d, len=%d, flag=0x%04x\n",
			  p_index, priv->rx_c_index, priv->rx_read_ptr,
			  len, status);
764

765 766 767 768 769 770 771 772
		if (unlikely(len > RX_BUF_LENGTH)) {
			netif_err(priv, rx_status, ndev, "oversized packet\n");
			ndev->stats.rx_length_errors++;
			ndev->stats.rx_errors++;
			dev_kfree_skb_any(skb);
			goto next;
		}

773 774 775 776
		if (unlikely(!(status & DESC_EOP) || !(status & DESC_SOP))) {
			netif_err(priv, rx_status, ndev, "fragmented packet!\n");
			ndev->stats.rx_dropped++;
			ndev->stats.rx_errors++;
777 778
			dev_kfree_skb_any(skb);
			goto next;
779 780 781 782
		}

		if (unlikely(status & (RX_STATUS_ERR | RX_STATUS_OVFLOW))) {
			netif_err(priv, rx_err, ndev, "error packet\n");
783
			if (status & RX_STATUS_OVFLOW)
784 785 786
				ndev->stats.rx_over_errors++;
			ndev->stats.rx_dropped++;
			ndev->stats.rx_errors++;
787 788
			dev_kfree_skb_any(skb);
			goto next;
789 790 791 792 793 794 795 796
		}

		skb_put(skb, len);

		/* Hardware validated our checksum */
		if (likely(status & DESC_L4_CSUM))
			skb->ip_summed = CHECKSUM_UNNECESSARY;

797 798 799
		/* Hardware pre-pends packets with 2bytes before Ethernet
		 * header plus we have the Receive Status Block, strip off all
		 * of this from the SKB.
800 801 802 803 804 805 806 807 808 809 810 811 812
		 */
		skb_pull(skb, sizeof(*rsb) + 2);
		len -= (sizeof(*rsb) + 2);

		/* UniMAC may forward CRC */
		if (priv->crc_fwd) {
			skb_trim(skb, len - ETH_FCS_LEN);
			len -= ETH_FCS_LEN;
		}

		skb->protocol = eth_type_trans(skb, ndev);
		ndev->stats.rx_packets++;
		ndev->stats.rx_bytes += len;
813 814 815 816
		u64_stats_update_begin(&priv->syncp);
		stats64->rx_packets++;
		stats64->rx_bytes += len;
		u64_stats_update_end(&priv->syncp);
817 818

		napi_gro_receive(&priv->napi, skb);
819 820 821 822 823 824
next:
		processed++;
		priv->rx_read_ptr++;

		if (priv->rx_read_ptr == priv->num_rx_bds)
			priv->rx_read_ptr = 0;
825 826 827 828 829
	}

	return processed;
}

830
static void bcm_sysport_tx_reclaim_one(struct bcm_sysport_tx_ring *ring,
831 832 833
				       struct bcm_sysport_cb *cb,
				       unsigned int *bytes_compl,
				       unsigned int *pkts_compl)
834
{
835
	struct bcm_sysport_priv *priv = ring->priv;
836 837 838 839 840
	struct device *kdev = &priv->pdev->dev;

	if (cb->skb) {
		*bytes_compl += cb->skb->len;
		dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
841 842
				 dma_unmap_len(cb, dma_len),
				 DMA_TO_DEVICE);
843 844 845 846
		(*pkts_compl)++;
		bcm_sysport_free_cb(cb);
	/* SKB fragment */
	} else if (dma_unmap_addr(cb, dma_addr)) {
847
		*bytes_compl += dma_unmap_len(cb, dma_len);
848
		dma_unmap_page(kdev, dma_unmap_addr(cb, dma_addr),
849
			       dma_unmap_len(cb, dma_len), DMA_TO_DEVICE);
850 851 852 853 854 855 856 857 858 859
		dma_unmap_addr_set(cb, dma_addr, 0);
	}
}

/* Reclaim queued SKBs for transmission completion, lockless version */
static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
					     struct bcm_sysport_tx_ring *ring)
{
	unsigned int c_index, last_c_index, last_tx_cn, num_tx_cbs;
	unsigned int pkts_compl = 0, bytes_compl = 0;
860
	struct net_device *ndev = priv->netdev;
861 862 863
	struct bcm_sysport_cb *cb;
	u32 hw_ind;

864 865 866 867 868 869 870
	/* Clear status before servicing to reduce spurious interrupts */
	if (!ring->priv->is_lite)
		intrl2_1_writel(ring->priv, BIT(ring->index), INTRL2_CPU_CLEAR);
	else
		intrl2_0_writel(ring->priv, BIT(ring->index +
				INTRL2_0_TDMA_MBDONE_SHIFT), INTRL2_CPU_CLEAR);

871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
	/* Compute how many descriptors have been processed since last call */
	hw_ind = tdma_readl(priv, TDMA_DESC_RING_PROD_CONS_INDEX(ring->index));
	c_index = (hw_ind >> RING_CONS_INDEX_SHIFT) & RING_CONS_INDEX_MASK;
	ring->p_index = (hw_ind & RING_PROD_INDEX_MASK);

	last_c_index = ring->c_index;
	num_tx_cbs = ring->size;

	c_index &= (num_tx_cbs - 1);

	if (c_index >= last_c_index)
		last_tx_cn = c_index - last_c_index;
	else
		last_tx_cn = num_tx_cbs - last_c_index + c_index;

	netif_dbg(priv, tx_done, ndev,
887 888
		  "ring=%d c_index=%d last_tx_cn=%d last_c_index=%d\n",
		  ring->index, c_index, last_tx_cn, last_c_index);
889 890 891

	while (last_tx_cn-- > 0) {
		cb = ring->cbs + last_c_index;
892
		bcm_sysport_tx_reclaim_one(ring, cb, &bytes_compl, &pkts_compl);
893 894 895 896 897 898

		ring->desc_count++;
		last_c_index++;
		last_c_index &= (num_tx_cbs - 1);
	}

899 900 901 902 903
	u64_stats_update_begin(&priv->syncp);
	ring->packets += pkts_compl;
	ring->bytes += bytes_compl;
	u64_stats_update_end(&priv->syncp);

904 905 906
	ring->c_index = c_index;

	netif_dbg(priv, tx_done, ndev,
907 908
		  "ring=%d c_index=%d pkts_compl=%d, bytes_compl=%d\n",
		  ring->index, ring->c_index, pkts_compl, bytes_compl);
909 910 911 912 913 914 915 916

	return pkts_compl;
}

/* Locked version of the per-ring TX reclaim routine */
static unsigned int bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
					   struct bcm_sysport_tx_ring *ring)
{
917
	struct netdev_queue *txq;
918
	unsigned int released;
919
	unsigned long flags;
920

921 922
	txq = netdev_get_tx_queue(priv->netdev, ring->index);

923
	spin_lock_irqsave(&ring->lock, flags);
924
	released = __bcm_sysport_tx_reclaim(priv, ring);
925 926 927
	if (released)
		netif_tx_wake_queue(txq);

928
	spin_unlock_irqrestore(&ring->lock, flags);
929 930 931 932

	return released;
}

933 934 935 936 937 938 939 940 941 942 943
/* Locked version of the per-ring TX reclaim, but does not wake the queue */
static void bcm_sysport_tx_clean(struct bcm_sysport_priv *priv,
				 struct bcm_sysport_tx_ring *ring)
{
	unsigned long flags;

	spin_lock_irqsave(&ring->lock, flags);
	__bcm_sysport_tx_reclaim(priv, ring);
	spin_unlock_irqrestore(&ring->lock, flags);
}

944 945 946 947 948 949 950 951
static int bcm_sysport_tx_poll(struct napi_struct *napi, int budget)
{
	struct bcm_sysport_tx_ring *ring =
		container_of(napi, struct bcm_sysport_tx_ring, napi);
	unsigned int work_done = 0;

	work_done = bcm_sysport_tx_reclaim(ring->priv, ring);

952
	if (work_done == 0) {
953 954
		napi_complete(napi);
		/* re-enable TX interrupt */
955 956 957 958 959
		if (!ring->priv->is_lite)
			intrl2_1_mask_clear(ring->priv, BIT(ring->index));
		else
			intrl2_0_mask_clear(ring->priv, BIT(ring->index +
					    INTRL2_0_TDMA_MBDONE_SHIFT));
960 961

		return 0;
962 963
	}

964
	return budget;
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
}

static void bcm_sysport_tx_reclaim_all(struct bcm_sysport_priv *priv)
{
	unsigned int q;

	for (q = 0; q < priv->netdev->num_tx_queues; q++)
		bcm_sysport_tx_reclaim(priv, &priv->tx_rings[q]);
}

static int bcm_sysport_poll(struct napi_struct *napi, int budget)
{
	struct bcm_sysport_priv *priv =
		container_of(napi, struct bcm_sysport_priv, napi);
	unsigned int work_done = 0;

	work_done = bcm_sysport_desc_rx(priv, budget);

	priv->rx_c_index += work_done;
	priv->rx_c_index &= RDMA_CONS_INDEX_MASK;
985 986 987 988 989 990 991 992 993

	/* SYSTEMPORT Lite groups the producer/consumer index, producer is
	 * maintained by HW, but writes to it will be ignore while RDMA
	 * is active
	 */
	if (!priv->is_lite)
		rdma_writel(priv, priv->rx_c_index, RDMA_CONS_INDEX);
	else
		rdma_writel(priv, priv->rx_c_index << 16, RDMA_CONS_INDEX);
994 995

	if (work_done < budget) {
996
		napi_complete_done(napi, work_done);
997 998 999 1000 1001 1002 1003
		/* re-enable RX interrupts */
		intrl2_0_mask_clear(priv, INTRL2_0_RDMA_MBDONE);
	}

	return work_done;
}

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
static void bcm_sysport_resume_from_wol(struct bcm_sysport_priv *priv)
{
	u32 reg;

	/* Stop monitoring MPD interrupt */
	intrl2_0_mask_set(priv, INTRL2_0_MPD);

	/* Clear the MagicPacket detection logic */
	reg = umac_readl(priv, UMAC_MPD_CTRL);
	reg &= ~MPD_EN;
	umac_writel(priv, reg, UMAC_MPD_CTRL);

	netif_dbg(priv, wol, priv->netdev, "resumed from WOL\n");
}
1018 1019 1020 1021 1022 1023

/* RX and misc interrupt routine */
static irqreturn_t bcm_sysport_rx_isr(int irq, void *dev_id)
{
	struct net_device *dev = dev_id;
	struct bcm_sysport_priv *priv = netdev_priv(dev);
1024 1025
	struct bcm_sysport_tx_ring *txr;
	unsigned int ring, ring_bit;
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039

	priv->irq0_stat = intrl2_0_readl(priv, INTRL2_CPU_STATUS) &
			  ~intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
	intrl2_0_writel(priv, priv->irq0_stat, INTRL2_CPU_CLEAR);

	if (unlikely(priv->irq0_stat == 0)) {
		netdev_warn(priv->netdev, "spurious RX interrupt\n");
		return IRQ_NONE;
	}

	if (priv->irq0_stat & INTRL2_0_RDMA_MBDONE) {
		if (likely(napi_schedule_prep(&priv->napi))) {
			/* disable RX interrupts */
			intrl2_0_mask_set(priv, INTRL2_0_RDMA_MBDONE);
1040
			__napi_schedule_irqoff(&priv->napi);
1041 1042 1043 1044 1045 1046 1047 1048 1049
		}
	}

	/* TX ring is full, perform a full reclaim since we do not know
	 * which one would trigger this interrupt
	 */
	if (priv->irq0_stat & INTRL2_0_TX_RING_FULL)
		bcm_sysport_tx_reclaim_all(priv);

1050 1051 1052 1053 1054
	if (priv->irq0_stat & INTRL2_0_MPD) {
		netdev_info(priv->netdev, "Wake-on-LAN interrupt!\n");
		bcm_sysport_resume_from_wol(priv);
	}

1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
	if (!priv->is_lite)
		goto out;

	for (ring = 0; ring < dev->num_tx_queues; ring++) {
		ring_bit = BIT(ring + INTRL2_0_TDMA_MBDONE_SHIFT);
		if (!(priv->irq0_stat & ring_bit))
			continue;

		txr = &priv->tx_rings[ring];

		if (likely(napi_schedule_prep(&txr->napi))) {
			intrl2_0_mask_set(priv, ring_bit);
			__napi_schedule(&txr->napi);
		}
	}
out:
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
	return IRQ_HANDLED;
}

/* TX interrupt service routine */
static irqreturn_t bcm_sysport_tx_isr(int irq, void *dev_id)
{
	struct net_device *dev = dev_id;
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	struct bcm_sysport_tx_ring *txr;
	unsigned int ring;

	priv->irq1_stat = intrl2_1_readl(priv, INTRL2_CPU_STATUS) &
				~intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
	intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);

	if (unlikely(priv->irq1_stat == 0)) {
		netdev_warn(priv->netdev, "spurious TX interrupt\n");
		return IRQ_NONE;
	}

	for (ring = 0; ring < dev->num_tx_queues; ring++) {
		if (!(priv->irq1_stat & BIT(ring)))
			continue;

		txr = &priv->tx_rings[ring];

		if (likely(napi_schedule_prep(&txr->napi))) {
			intrl2_1_mask_set(priv, BIT(ring));
1099
			__napi_schedule_irqoff(&txr->napi);
1100 1101 1102 1103 1104 1105
		}
	}

	return IRQ_HANDLED;
}

1106 1107 1108 1109 1110 1111 1112 1113 1114
static irqreturn_t bcm_sysport_wol_isr(int irq, void *dev_id)
{
	struct bcm_sysport_priv *priv = dev_id;

	pm_wakeup_event(&priv->pdev->dev, 0);

	return IRQ_HANDLED;
}

1115 1116 1117 1118 1119 1120 1121 1122 1123
#ifdef CONFIG_NET_POLL_CONTROLLER
static void bcm_sysport_poll_controller(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);

	disable_irq(priv->irq0);
	bcm_sysport_rx_isr(priv->irq0, priv);
	enable_irq(priv->irq0);

1124 1125 1126 1127 1128
	if (!priv->is_lite) {
		disable_irq(priv->irq1);
		bcm_sysport_tx_isr(priv->irq1, priv);
		enable_irq(priv->irq1);
	}
1129 1130 1131
}
#endif

1132 1133
static struct sk_buff *bcm_sysport_insert_tsb(struct sk_buff *skb,
					      struct net_device *dev)
1134 1135
{
	struct sk_buff *nskb;
1136
	struct bcm_tsb *tsb;
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
	u32 csum_info;
	u8 ip_proto;
	u16 csum_start;
	u16 ip_ver;

	/* Re-allocate SKB if needed */
	if (unlikely(skb_headroom(skb) < sizeof(*tsb))) {
		nskb = skb_realloc_headroom(skb, sizeof(*tsb));
		dev_kfree_skb(skb);
		if (!nskb) {
			dev->stats.tx_errors++;
			dev->stats.tx_dropped++;
1149
			return NULL;
1150 1151 1152 1153
		}
		skb = nskb;
	}

1154
	tsb = skb_push(skb, sizeof(*tsb));
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
	/* Zero-out TSB by default */
	memset(tsb, 0, sizeof(*tsb));

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
		ip_ver = htons(skb->protocol);
		switch (ip_ver) {
		case ETH_P_IP:
			ip_proto = ip_hdr(skb)->protocol;
			break;
		case ETH_P_IPV6:
			ip_proto = ipv6_hdr(skb)->nexthdr;
			break;
		default:
1168
			return skb;
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
		}

		/* Get the checksum offset and the L4 (transport) offset */
		csum_start = skb_checksum_start_offset(skb) - sizeof(*tsb);
		csum_info = (csum_start + skb->csum_offset) & L4_CSUM_PTR_MASK;
		csum_info |= (csum_start << L4_PTR_SHIFT);

		if (ip_proto == IPPROTO_TCP || ip_proto == IPPROTO_UDP) {
			csum_info |= L4_LENGTH_VALID;
			if (ip_proto == IPPROTO_UDP && ip_ver == ETH_P_IP)
				csum_info |= L4_UDP;
1180
		} else {
1181
			csum_info = 0;
1182
		}
1183 1184 1185 1186

		tsb->l4_ptr_dest_map = csum_info;
	}

1187
	return skb;
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
}

static netdev_tx_t bcm_sysport_xmit(struct sk_buff *skb,
				    struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	struct device *kdev = &priv->pdev->dev;
	struct bcm_sysport_tx_ring *ring;
	struct bcm_sysport_cb *cb;
	struct netdev_queue *txq;
	struct dma_desc *desc;
1199
	unsigned int skb_len;
1200
	unsigned long flags;
1201 1202 1203 1204 1205 1206 1207 1208 1209
	dma_addr_t mapping;
	u32 len_status;
	u16 queue;
	int ret;

	queue = skb_get_queue_mapping(skb);
	txq = netdev_get_tx_queue(dev, queue);
	ring = &priv->tx_rings[queue];

1210 1211
	/* lock against tx reclaim in BH context and TX ring full interrupt */
	spin_lock_irqsave(&ring->lock, flags);
1212 1213 1214 1215 1216 1217 1218
	if (unlikely(ring->desc_count == 0)) {
		netif_tx_stop_queue(txq);
		netdev_err(dev, "queue %d awake and ring full!\n", queue);
		ret = NETDEV_TX_BUSY;
		goto out;
	}

1219 1220 1221 1222 1223 1224 1225
	/* The Ethernet switch we are interfaced with needs packets to be at
	 * least 64 bytes (including FCS) otherwise they will be discarded when
	 * they enter the switch port logic. When Broadcom tags are enabled, we
	 * need to make sure that packets are at least 68 bytes
	 * (including FCS and tag) because the length verification is done after
	 * the Broadcom tag is stripped off the ingress packet.
	 */
1226
	if (skb_put_padto(skb, ETH_ZLEN + ENET_BRCM_TAG_LEN)) {
1227 1228 1229 1230
		ret = NETDEV_TX_OK;
		goto out;
	}

1231 1232 1233 1234 1235 1236 1237 1238 1239
	/* Insert TSB and checksum infos */
	if (priv->tsb_en) {
		skb = bcm_sysport_insert_tsb(skb, dev);
		if (!skb) {
			ret = NETDEV_TX_OK;
			goto out;
		}
	}

1240
	skb_len = skb->len;
1241 1242

	mapping = dma_map_single(kdev, skb->data, skb_len, DMA_TO_DEVICE);
1243
	if (dma_mapping_error(kdev, mapping)) {
1244
		priv->mib.tx_dma_failed++;
1245
		netif_err(priv, tx_err, dev, "DMA map failed at %p (len=%d)\n",
1246
			  skb->data, skb_len);
1247 1248 1249 1250 1251 1252 1253 1254
		ret = NETDEV_TX_OK;
		goto out;
	}

	/* Remember the SKB for future freeing */
	cb = &ring->cbs[ring->curr_desc];
	cb->skb = skb;
	dma_unmap_addr_set(cb, dma_addr, mapping);
1255
	dma_unmap_len_set(cb, dma_len, skb_len);
1256 1257 1258 1259 1260 1261

	/* Fetch a descriptor entry from our pool */
	desc = ring->desc_cpu;

	desc->addr_lo = lower_32_bits(mapping);
	len_status = upper_32_bits(mapping) & DESC_ADDR_HI_MASK;
1262
	len_status |= (skb_len << DESC_LEN_SHIFT);
1263
	len_status |= (DESC_SOP | DESC_EOP | TX_STATUS_APP_CRC) <<
1264
		       DESC_STATUS_SHIFT;
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	if (skb->ip_summed == CHECKSUM_PARTIAL)
		len_status |= (DESC_L4_CSUM << DESC_STATUS_SHIFT);

	ring->curr_desc++;
	if (ring->curr_desc == ring->size)
		ring->curr_desc = 0;
	ring->desc_count--;

	/* Ensure write completion of the descriptor status/length
	 * in DRAM before the System Port WRITE_PORT register latches
	 * the value
	 */
	wmb();
	desc->addr_status_len = len_status;
	wmb();

	/* Write this descriptor address to the RING write port */
	tdma_port_write_desc_addr(priv, desc, ring->index);

	/* Check ring space and update SW control flow */
	if (ring->desc_count == 0)
		netif_tx_stop_queue(txq);

	netif_dbg(priv, tx_queued, dev, "ring=%d desc_count=%d, curr_desc=%d\n",
1289
		  ring->index, ring->desc_count, ring->curr_desc);
1290 1291 1292

	ret = NETDEV_TX_OK;
out:
1293
	spin_unlock_irqrestore(&ring->lock, flags);
1294 1295 1296 1297 1298 1299 1300
	return ret;
}

static void bcm_sysport_tx_timeout(struct net_device *dev)
{
	netdev_warn(dev, "transmit timeout!\n");

1301
	netif_trans_update(dev);
1302 1303 1304 1305 1306 1307 1308 1309 1310
	dev->stats.tx_errors++;

	netif_tx_wake_all_queues(dev);
}

/* phylib adjust link callback */
static void bcm_sysport_adj_link(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
1311
	struct phy_device *phydev = dev->phydev;
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
	unsigned int changed = 0;
	u32 cmd_bits = 0, reg;

	if (priv->old_link != phydev->link) {
		changed = 1;
		priv->old_link = phydev->link;
	}

	if (priv->old_duplex != phydev->duplex) {
		changed = 1;
		priv->old_duplex = phydev->duplex;
	}

1325 1326 1327
	if (priv->is_lite)
		goto out;

1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	switch (phydev->speed) {
	case SPEED_2500:
		cmd_bits = CMD_SPEED_2500;
		break;
	case SPEED_1000:
		cmd_bits = CMD_SPEED_1000;
		break;
	case SPEED_100:
		cmd_bits = CMD_SPEED_100;
		break;
	case SPEED_10:
		cmd_bits = CMD_SPEED_10;
		break;
	default:
		break;
	}
	cmd_bits <<= CMD_SPEED_SHIFT;

	if (phydev->duplex == DUPLEX_HALF)
		cmd_bits |= CMD_HD_EN;

	if (priv->old_pause != phydev->pause) {
		changed = 1;
		priv->old_pause = phydev->pause;
	}

	if (!phydev->pause)
		cmd_bits |= CMD_RX_PAUSE_IGNORE | CMD_TX_PAUSE_IGNORE;

1357 1358 1359 1360
	if (!changed)
		return;

	if (phydev->link) {
1361 1362
		reg = umac_readl(priv, UMAC_CMD);
		reg &= ~((CMD_SPEED_MASK << CMD_SPEED_SHIFT) |
1363 1364
			CMD_HD_EN | CMD_RX_PAUSE_IGNORE |
			CMD_TX_PAUSE_IGNORE);
1365 1366 1367
		reg |= cmd_bits;
		umac_writel(priv, reg, UMAC_CMD);
	}
1368 1369 1370
out:
	if (changed)
		phy_print_status(phydev);
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
}

static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
				    unsigned int index)
{
	struct bcm_sysport_tx_ring *ring = &priv->tx_rings[index];
	struct device *kdev = &priv->pdev->dev;
	size_t size;
	void *p;
	u32 reg;

	/* Simple descriptors partitioning for now */
	size = 256;

	/* We just need one DMA descriptor which is DMA-able, since writing to
	 * the port will allocate a new descriptor in its internal linked-list
	 */
1388 1389
	p = dma_zalloc_coherent(kdev, sizeof(struct dma_desc), &ring->desc_dma,
				GFP_KERNEL);
1390 1391 1392 1393 1394
	if (!p) {
		netif_err(priv, hw, priv->netdev, "DMA alloc failed\n");
		return -ENOMEM;
	}

1395
	ring->cbs = kcalloc(size, sizeof(struct bcm_sysport_cb), GFP_KERNEL);
1396
	if (!ring->cbs) {
1397 1398
		dma_free_coherent(kdev, sizeof(struct dma_desc),
				  ring->desc_cpu, ring->desc_dma);
1399 1400 1401 1402 1403 1404 1405
		netif_err(priv, hw, priv->netdev, "CB allocation failed\n");
		return -ENOMEM;
	}

	/* Initialize SW view of the ring */
	spin_lock_init(&ring->lock);
	ring->priv = priv;
E
Eric Dumazet 已提交
1406
	netif_tx_napi_add(priv->netdev, &ring->napi, bcm_sysport_tx_poll, 64);
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
	ring->index = index;
	ring->size = size;
	ring->alloc_size = ring->size;
	ring->desc_cpu = p;
	ring->desc_count = ring->size;
	ring->curr_desc = 0;

	/* Initialize HW ring */
	tdma_writel(priv, RING_EN, TDMA_DESC_RING_HEAD_TAIL_PTR(index));
	tdma_writel(priv, 0, TDMA_DESC_RING_COUNT(index));
	tdma_writel(priv, 1, TDMA_DESC_RING_INTR_CONTROL(index));
	tdma_writel(priv, 0, TDMA_DESC_RING_PROD_CONS_INDEX(index));
1419 1420 1421 1422 1423 1424 1425

	/* Configure QID and port mapping */
	reg = tdma_readl(priv, TDMA_DESC_RING_MAPPING(index));
	reg &= ~(RING_QID_MASK | RING_PORT_ID_MASK << RING_PORT_ID_SHIFT);
	reg |= ring->switch_queue & RING_QID_MASK;
	reg |= ring->switch_port << RING_PORT_ID_SHIFT;
	tdma_writel(priv, reg, TDMA_DESC_RING_MAPPING(index));
1426 1427
	tdma_writel(priv, 0, TDMA_DESC_RING_PCP_DEI_VID(index));

1428 1429 1430 1431 1432
	/* Enable ACB algorithm 2 */
	reg = tdma_readl(priv, TDMA_CONTROL);
	reg |= tdma_control_bit(priv, ACB_ALGO);
	tdma_writel(priv, reg, TDMA_CONTROL);

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
	/* Do not use tdma_control_bit() here because TSB_SWAP1 collides
	 * with the original definition of ACB_ALGO
	 */
	reg = tdma_readl(priv, TDMA_CONTROL);
	if (priv->is_lite)
		reg &= ~BIT(TSB_SWAP1);
	/* Set a correct TSB format based on host endian */
	if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
		reg |= tdma_control_bit(priv, TSB_SWAP0);
	else
		reg &= ~tdma_control_bit(priv, TSB_SWAP0);
	tdma_writel(priv, reg, TDMA_CONTROL);

1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
	/* Program the number of descriptors as MAX_THRESHOLD and half of
	 * its size for the hysteresis trigger
	 */
	tdma_writel(priv, ring->size |
			1 << RING_HYST_THRESH_SHIFT,
			TDMA_DESC_RING_MAX_HYST(index));

	/* Enable the ring queue in the arbiter */
	reg = tdma_readl(priv, TDMA_TIER1_ARB_0_QUEUE_EN);
	reg |= (1 << index);
	tdma_writel(priv, reg, TDMA_TIER1_ARB_0_QUEUE_EN);

	napi_enable(&ring->napi);

	netif_dbg(priv, hw, priv->netdev,
1461 1462 1463
		  "TDMA cfg, size=%d, desc_cpu=%p switch q=%d,port=%d\n",
		  ring->size, ring->desc_cpu, ring->switch_queue,
		  ring->switch_port);
1464 1465 1466 1467 1468

	return 0;
}

static void bcm_sysport_fini_tx_ring(struct bcm_sysport_priv *priv,
1469
				     unsigned int index)
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
{
	struct bcm_sysport_tx_ring *ring = &priv->tx_rings[index];
	struct device *kdev = &priv->pdev->dev;
	u32 reg;

	/* Caller should stop the TDMA engine */
	reg = tdma_readl(priv, TDMA_STATUS);
	if (!(reg & TDMA_DISABLED))
		netdev_warn(priv->netdev, "TDMA not stopped!\n");

1480 1481 1482 1483 1484 1485 1486
	/* ring->cbs is the last part in bcm_sysport_init_tx_ring which could
	 * fail, so by checking this pointer we know whether the TX ring was
	 * fully initialized or not.
	 */
	if (!ring->cbs)
		return;

1487 1488 1489
	napi_disable(&ring->napi);
	netif_napi_del(&ring->napi);

1490
	bcm_sysport_tx_clean(priv, ring);
1491 1492 1493 1494 1495

	kfree(ring->cbs);
	ring->cbs = NULL;

	if (ring->desc_dma) {
1496 1497
		dma_free_coherent(kdev, sizeof(struct dma_desc),
				  ring->desc_cpu, ring->desc_dma);
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507
		ring->desc_dma = 0;
	}
	ring->size = 0;
	ring->alloc_size = 0;

	netif_dbg(priv, hw, priv->netdev, "TDMA fini done\n");
}

/* RDMA helper */
static inline int rdma_enable_set(struct bcm_sysport_priv *priv,
1508
				  unsigned int enable)
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
{
	unsigned int timeout = 1000;
	u32 reg;

	reg = rdma_readl(priv, RDMA_CONTROL);
	if (enable)
		reg |= RDMA_EN;
	else
		reg &= ~RDMA_EN;
	rdma_writel(priv, reg, RDMA_CONTROL);

	/* Poll for RMDA disabling completion */
	do {
		reg = rdma_readl(priv, RDMA_STATUS);
		if (!!(reg & RDMA_DISABLED) == !enable)
			return 0;
		usleep_range(1000, 2000);
	} while (timeout-- > 0);

	netdev_err(priv->netdev, "timeout waiting for RDMA to finish\n");

	return -ETIMEDOUT;
}

/* TDMA helper */
static inline int tdma_enable_set(struct bcm_sysport_priv *priv,
1535
				  unsigned int enable)
1536 1537 1538 1539 1540 1541
{
	unsigned int timeout = 1000;
	u32 reg;

	reg = tdma_readl(priv, TDMA_CONTROL);
	if (enable)
1542
		reg |= tdma_control_bit(priv, TDMA_EN);
1543
	else
1544
		reg &= ~tdma_control_bit(priv, TDMA_EN);
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
	tdma_writel(priv, reg, TDMA_CONTROL);

	/* Poll for TMDA disabling completion */
	do {
		reg = tdma_readl(priv, TDMA_STATUS);
		if (!!(reg & TDMA_DISABLED) == !enable)
			return 0;

		usleep_range(1000, 2000);
	} while (timeout-- > 0);

	netdev_err(priv->netdev, "timeout waiting for TDMA to finish\n");

	return -ETIMEDOUT;
}

static int bcm_sysport_init_rx_ring(struct bcm_sysport_priv *priv)
{
1563
	struct bcm_sysport_cb *cb;
1564 1565
	u32 reg;
	int ret;
1566
	int i;
1567 1568

	/* Initialize SW view of the RX ring */
1569
	priv->num_rx_bds = priv->num_rx_desc_words / WORDS_PER_DESC;
1570 1571 1572
	priv->rx_bds = priv->base + SYS_PORT_RDMA_OFFSET;
	priv->rx_c_index = 0;
	priv->rx_read_ptr = 0;
1573 1574
	priv->rx_cbs = kcalloc(priv->num_rx_bds, sizeof(struct bcm_sysport_cb),
				GFP_KERNEL);
1575 1576 1577 1578 1579
	if (!priv->rx_cbs) {
		netif_err(priv, hw, priv->netdev, "CB allocation failed\n");
		return -ENOMEM;
	}

1580 1581 1582 1583 1584
	for (i = 0; i < priv->num_rx_bds; i++) {
		cb = priv->rx_cbs + i;
		cb->bd_addr = priv->rx_bds + i * DESC_SIZE;
	}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
	ret = bcm_sysport_alloc_rx_bufs(priv);
	if (ret) {
		netif_err(priv, hw, priv->netdev, "SKB allocation failed\n");
		return ret;
	}

	/* Initialize HW, ensure RDMA is disabled */
	reg = rdma_readl(priv, RDMA_STATUS);
	if (!(reg & RDMA_DISABLED))
		rdma_enable_set(priv, 0);

	rdma_writel(priv, 0, RDMA_WRITE_PTR_LO);
	rdma_writel(priv, 0, RDMA_WRITE_PTR_HI);
	rdma_writel(priv, 0, RDMA_PROD_INDEX);
	rdma_writel(priv, 0, RDMA_CONS_INDEX);
	rdma_writel(priv, priv->num_rx_bds << RDMA_RING_SIZE_SHIFT |
			  RX_BUF_LENGTH, RDMA_RING_BUF_SIZE);
	/* Operate the queue in ring mode */
	rdma_writel(priv, 0, RDMA_START_ADDR_HI);
	rdma_writel(priv, 0, RDMA_START_ADDR_LO);
	rdma_writel(priv, 0, RDMA_END_ADDR_HI);
1606
	rdma_writel(priv, priv->num_rx_desc_words - 1, RDMA_END_ADDR_LO);
1607 1608 1609 1610

	rdma_writel(priv, 1, RDMA_MBDONE_INTR);

	netif_dbg(priv, hw, priv->netdev,
1611 1612
		  "RDMA cfg, num_rx_bds=%d, rx_bds=%p\n",
		  priv->num_rx_bds, priv->rx_bds);
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631

	return 0;
}

static void bcm_sysport_fini_rx_ring(struct bcm_sysport_priv *priv)
{
	struct bcm_sysport_cb *cb;
	unsigned int i;
	u32 reg;

	/* Caller should ensure RDMA is disabled */
	reg = rdma_readl(priv, RDMA_STATUS);
	if (!(reg & RDMA_DISABLED))
		netdev_warn(priv->netdev, "RDMA not stopped!\n");

	for (i = 0; i < priv->num_rx_bds; i++) {
		cb = &priv->rx_cbs[i];
		if (dma_unmap_addr(cb, dma_addr))
			dma_unmap_single(&priv->pdev->dev,
1632 1633
					 dma_unmap_addr(cb, dma_addr),
					 RX_BUF_LENGTH, DMA_FROM_DEVICE);
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647
		bcm_sysport_free_cb(cb);
	}

	kfree(priv->rx_cbs);
	priv->rx_cbs = NULL;

	netif_dbg(priv, hw, priv->netdev, "RDMA fini done\n");
}

static void bcm_sysport_set_rx_mode(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u32 reg;

1648 1649 1650
	if (priv->is_lite)
		return;

1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
	reg = umac_readl(priv, UMAC_CMD);
	if (dev->flags & IFF_PROMISC)
		reg |= CMD_PROMISC;
	else
		reg &= ~CMD_PROMISC;
	umac_writel(priv, reg, UMAC_CMD);

	/* No support for ALLMULTI */
	if (dev->flags & IFF_ALLMULTI)
		return;
}

static inline void umac_enable_set(struct bcm_sysport_priv *priv,
1664
				   u32 mask, unsigned int enable)
1665 1666 1667
{
	u32 reg;

1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
	if (!priv->is_lite) {
		reg = umac_readl(priv, UMAC_CMD);
		if (enable)
			reg |= mask;
		else
			reg &= ~mask;
		umac_writel(priv, reg, UMAC_CMD);
	} else {
		reg = gib_readl(priv, GIB_CONTROL);
		if (enable)
			reg |= mask;
		else
			reg &= ~mask;
		gib_writel(priv, reg, GIB_CONTROL);
	}
1683 1684 1685 1686 1687 1688

	/* UniMAC stops on a packet boundary, wait for a full-sized packet
	 * to be processed (1 msec).
	 */
	if (enable == 0)
		usleep_range(1000, 2000);
1689 1690
}

1691
static inline void umac_reset(struct bcm_sysport_priv *priv)
1692 1693 1694
{
	u32 reg;

1695 1696 1697
	if (priv->is_lite)
		return;

1698 1699 1700 1701 1702 1703 1704
	reg = umac_readl(priv, UMAC_CMD);
	reg |= CMD_SW_RESET;
	umac_writel(priv, reg, UMAC_CMD);
	udelay(10);
	reg = umac_readl(priv, UMAC_CMD);
	reg &= ~CMD_SW_RESET;
	umac_writel(priv, reg, UMAC_CMD);
1705 1706 1707
}

static void umac_set_hw_addr(struct bcm_sysport_priv *priv,
1708
			     unsigned char *addr)
1709
{
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
	u32 mac0 = (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) |
		    addr[3];
	u32 mac1 = (addr[4] << 8) | addr[5];

	if (!priv->is_lite) {
		umac_writel(priv, mac0, UMAC_MAC0);
		umac_writel(priv, mac1, UMAC_MAC1);
	} else {
		gib_writel(priv, mac0, GIB_MAC0);
		gib_writel(priv, mac1, GIB_MAC1);
	}
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
}

static void topctrl_flush(struct bcm_sysport_priv *priv)
{
	topctrl_writel(priv, RX_FLUSH, RX_FLUSH_CNTL);
	topctrl_writel(priv, TX_FLUSH, TX_FLUSH_CNTL);
	mdelay(1);
	topctrl_writel(priv, 0, RX_FLUSH_CNTL);
	topctrl_writel(priv, 0, TX_FLUSH_CNTL);
}

1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
static int bcm_sysport_change_mac(struct net_device *dev, void *p)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EINVAL;

	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);

	/* interface is disabled, changes to MAC will be reflected on next
	 * open call
	 */
	if (!netif_running(dev))
		return 0;

	umac_set_hw_addr(priv, dev->dev_addr);

	return 0;
}

1753 1754
static void bcm_sysport_get_stats64(struct net_device *dev,
				    struct rtnl_link_stats64 *stats)
1755 1756
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
1757 1758
	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
	unsigned int start;
1759

1760 1761
	netdev_stats_to_stats64(stats, &dev->stats);

1762 1763
	bcm_sysport_update_tx_stats(priv, &stats->tx_bytes,
				    &stats->tx_packets);
1764 1765 1766 1767 1768 1769

	do {
		start = u64_stats_fetch_begin_irq(&priv->syncp);
		stats->rx_packets = stats64->rx_packets;
		stats->rx_bytes = stats64->rx_bytes;
	} while (u64_stats_fetch_retry_irq(&priv->syncp, start));
1770 1771
}

1772 1773 1774 1775 1776 1777 1778
static void bcm_sysport_netif_start(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);

	/* Enable NAPI */
	napi_enable(&priv->napi);

1779 1780 1781
	/* Enable RX interrupt and TX ring full interrupt */
	intrl2_0_mask_clear(priv, INTRL2_0_RDMA_MBDONE | INTRL2_0_TX_RING_FULL);

1782
	phy_start(dev->phydev);
1783

1784 1785 1786 1787 1788
	/* Enable TX interrupts for the TXQs */
	if (!priv->is_lite)
		intrl2_1_mask_clear(priv, 0xffffffff);
	else
		intrl2_0_mask_clear(priv, INTRL2_0_TDMA_MBDONE_MASK);
1789 1790 1791 1792 1793

	/* Last call before we start the real business */
	netif_tx_start_all_queues(dev);
}

1794 1795 1796 1797 1798 1799
static void rbuf_init(struct bcm_sysport_priv *priv)
{
	u32 reg;

	reg = rbuf_readl(priv, RBUF_CONTROL);
	reg |= RBUF_4B_ALGN | RBUF_RSB_EN;
1800
	/* Set a correct RSB format on SYSTEMPORT Lite */
1801
	if (priv->is_lite)
1802
		reg &= ~RBUF_RSB_SWAP1;
1803 1804 1805

	/* Set a correct RSB format based on host endian */
	if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1806
		reg |= RBUF_RSB_SWAP0;
1807 1808
	else
		reg &= ~RBUF_RSB_SWAP0;
1809 1810 1811
	rbuf_writel(priv, reg, RBUF_CONTROL);
}

1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834
static inline void bcm_sysport_mask_all_intrs(struct bcm_sysport_priv *priv)
{
	intrl2_0_mask_set(priv, 0xffffffff);
	intrl2_0_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
	if (!priv->is_lite) {
		intrl2_1_mask_set(priv, 0xffffffff);
		intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
	}
}

static inline void gib_set_pad_extension(struct bcm_sysport_priv *priv)
{
	u32 __maybe_unused reg;

	/* Include Broadcom tag in pad extension */
	if (netdev_uses_dsa(priv->netdev)) {
		reg = gib_readl(priv, GIB_CONTROL);
		reg &= ~(GIB_PAD_EXTENSION_MASK << GIB_PAD_EXTENSION_SHIFT);
		reg |= ENET_BRCM_TAG_LEN << GIB_PAD_EXTENSION_SHIFT;
		gib_writel(priv, reg, GIB_CONTROL);
	}
}

1835 1836 1837
static int bcm_sysport_open(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
1838
	struct phy_device *phydev;
1839 1840 1841 1842
	unsigned int i;
	int ret;

	/* Reset UniMAC */
1843
	umac_reset(priv);
1844 1845 1846 1847 1848

	/* Flush TX and RX FIFOs at TOPCTRL level */
	topctrl_flush(priv);

	/* Disable the UniMAC RX/TX */
1849
	umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, 0);
1850 1851

	/* Enable RBUF 2bytes alignment and Receive Status Block */
1852
	rbuf_init(priv);
1853 1854

	/* Set maximum frame length */
1855 1856 1857 1858
	if (!priv->is_lite)
		umac_writel(priv, UMAC_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
	else
		gib_set_pad_extension(priv);
1859 1860 1861 1862 1863

	/* Set MAC address */
	umac_set_hw_addr(priv, dev->dev_addr);

	/* Read CRC forward */
1864 1865 1866 1867 1868
	if (!priv->is_lite)
		priv->crc_fwd = !!(umac_readl(priv, UMAC_CMD) & CMD_CRC_FWD);
	else
		priv->crc_fwd = !!(gib_readl(priv, GIB_CONTROL) &
				   GIB_FCS_STRIP);
1869

1870 1871 1872
	phydev = of_phy_connect(dev, priv->phy_dn, bcm_sysport_adj_link,
				0, priv->phy_interface);
	if (!phydev) {
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
		netdev_err(dev, "could not attach to PHY\n");
		return -ENODEV;
	}

	/* Reset house keeping link status */
	priv->old_duplex = -1;
	priv->old_link = -1;
	priv->old_pause = -1;

	/* mask all interrupts and request them */
1883
	bcm_sysport_mask_all_intrs(priv);
1884 1885 1886 1887 1888 1889 1890

	ret = request_irq(priv->irq0, bcm_sysport_rx_isr, 0, dev->name, dev);
	if (ret) {
		netdev_err(dev, "failed to request RX interrupt\n");
		goto out_phy_disconnect;
	}

1891 1892 1893 1894 1895 1896 1897
	if (!priv->is_lite) {
		ret = request_irq(priv->irq1, bcm_sysport_tx_isr, 0,
				  dev->name, dev);
		if (ret) {
			netdev_err(dev, "failed to request TX interrupt\n");
			goto out_free_irq0;
		}
1898 1899 1900 1901 1902 1903 1904
	}

	/* Initialize both hardware and software ring */
	for (i = 0; i < dev->num_tx_queues; i++) {
		ret = bcm_sysport_init_tx_ring(priv, i);
		if (ret) {
			netdev_err(dev, "failed to initialize TX ring %d\n",
1905
				   i);
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
			goto out_free_tx_ring;
		}
	}

	/* Initialize linked-list */
	tdma_writel(priv, TDMA_LL_RAM_INIT_BUSY, TDMA_STATUS);

	/* Initialize RX ring */
	ret = bcm_sysport_init_rx_ring(priv);
	if (ret) {
		netdev_err(dev, "failed to initialize RX ring\n");
		goto out_free_rx_ring;
	}

	/* Turn on RDMA */
	ret = rdma_enable_set(priv, 1);
	if (ret)
		goto out_free_rx_ring;

	/* Turn on TDMA */
	ret = tdma_enable_set(priv, 1);
	if (ret)
		goto out_clear_rx_int;

	/* Turn on UniMAC TX/RX */
1931
	umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, 1);
1932

1933
	bcm_sysport_netif_start(dev);
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943

	return 0;

out_clear_rx_int:
	intrl2_0_mask_set(priv, INTRL2_0_RDMA_MBDONE | INTRL2_0_TX_RING_FULL);
out_free_rx_ring:
	bcm_sysport_fini_rx_ring(priv);
out_free_tx_ring:
	for (i = 0; i < dev->num_tx_queues; i++)
		bcm_sysport_fini_tx_ring(priv, i);
1944 1945
	if (!priv->is_lite)
		free_irq(priv->irq1, dev);
1946 1947 1948
out_free_irq0:
	free_irq(priv->irq0, dev);
out_phy_disconnect:
1949
	phy_disconnect(phydev);
1950 1951 1952
	return ret;
}

1953
static void bcm_sysport_netif_stop(struct net_device *dev)
1954 1955 1956 1957 1958 1959
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);

	/* stop all software from updating hardware */
	netif_tx_stop_all_queues(dev);
	napi_disable(&priv->napi);
1960
	phy_stop(dev->phydev);
1961 1962

	/* mask all interrupts */
1963
	bcm_sysport_mask_all_intrs(priv);
1964 1965 1966 1967 1968 1969 1970 1971 1972
}

static int bcm_sysport_stop(struct net_device *dev)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	unsigned int i;
	int ret;

	bcm_sysport_netif_stop(dev);
1973 1974

	/* Disable UniMAC RX */
1975
	umac_enable_set(priv, CMD_RX_EN, 0);
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992

	ret = tdma_enable_set(priv, 0);
	if (ret) {
		netdev_err(dev, "timeout disabling RDMA\n");
		return ret;
	}

	/* Wait for a maximum packet size to be drained */
	usleep_range(2000, 3000);

	ret = rdma_enable_set(priv, 0);
	if (ret) {
		netdev_err(dev, "timeout disabling TDMA\n");
		return ret;
	}

	/* Disable UniMAC TX */
1993
	umac_enable_set(priv, CMD_TX_EN, 0);
1994 1995 1996 1997 1998 1999 2000

	/* Free RX/TX rings SW structures */
	for (i = 0; i < dev->num_tx_queues; i++)
		bcm_sysport_fini_tx_ring(priv, i);
	bcm_sysport_fini_rx_ring(priv);

	free_irq(priv->irq0, dev);
2001 2002
	if (!priv->is_lite)
		free_irq(priv->irq1, dev);
2003 2004

	/* Disconnect from PHY */
2005
	phy_disconnect(dev->phydev);
2006 2007 2008 2009

	return 0;
}

2010
static const struct ethtool_ops bcm_sysport_ethtool_ops = {
2011 2012 2013 2014 2015 2016 2017
	.get_drvinfo		= bcm_sysport_get_drvinfo,
	.get_msglevel		= bcm_sysport_get_msglvl,
	.set_msglevel		= bcm_sysport_set_msglvl,
	.get_link		= ethtool_op_get_link,
	.get_strings		= bcm_sysport_get_strings,
	.get_ethtool_stats	= bcm_sysport_get_stats,
	.get_sset_count		= bcm_sysport_get_sset_count,
2018 2019
	.get_wol		= bcm_sysport_get_wol,
	.set_wol		= bcm_sysport_set_wol,
2020 2021
	.get_coalesce		= bcm_sysport_get_coalesce,
	.set_coalesce		= bcm_sysport_set_coalesce,
2022 2023
	.get_link_ksettings     = phy_ethtool_get_link_ksettings,
	.set_link_ksettings     = phy_ethtool_set_link_ksettings,
2024 2025
};

2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111
static u16 bcm_sysport_select_queue(struct net_device *dev, struct sk_buff *skb,
				    void *accel_priv,
				    select_queue_fallback_t fallback)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	u16 queue = skb_get_queue_mapping(skb);
	struct bcm_sysport_tx_ring *tx_ring;
	unsigned int q, port;

	if (!netdev_uses_dsa(dev))
		return fallback(dev, skb);

	/* DSA tagging layer will have configured the correct queue */
	q = BRCM_TAG_GET_QUEUE(queue);
	port = BRCM_TAG_GET_PORT(queue);
	tx_ring = priv->ring_map[q + port * priv->per_port_num_tx_queues];

	return tx_ring->index;
}

static int bcm_sysport_map_queues(struct net_device *dev,
				  struct dsa_notifier_register_info *info)
{
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	struct bcm_sysport_tx_ring *ring;
	struct net_device *slave_dev;
	unsigned int num_tx_queues;
	unsigned int q, start, port;

	/* We can't be setting up queue inspection for non directly attached
	 * switches
	 */
	if (info->switch_number)
		return 0;

	port = info->port_number;
	slave_dev = info->info.dev;

	/* On SYSTEMPORT Lite we have twice as less queues, so we cannot do a
	 * 1:1 mapping, we can only do a 2:1 mapping. By reducing the number of
	 * per-port (slave_dev) network devices queue, we achieve just that.
	 * This need to happen now before any slave network device is used such
	 * it accurately reflects the number of real TX queues.
	 */
	if (priv->is_lite)
		netif_set_real_num_tx_queues(slave_dev,
					     slave_dev->num_tx_queues / 2);
	num_tx_queues = slave_dev->real_num_tx_queues;

	if (priv->per_port_num_tx_queues &&
	    priv->per_port_num_tx_queues != num_tx_queues)
		netdev_warn(slave_dev, "asymetric number of per-port queues\n");

	priv->per_port_num_tx_queues = num_tx_queues;

	start = find_first_zero_bit(&priv->queue_bitmap, dev->num_tx_queues);
	for (q = 0; q < num_tx_queues; q++) {
		ring = &priv->tx_rings[q + start];

		/* Just remember the mapping actual programming done
		 * during bcm_sysport_init_tx_ring
		 */
		ring->switch_queue = q;
		ring->switch_port = port;
		priv->ring_map[q + port * num_tx_queues] = ring;

		/* Set all queues as being used now */
		set_bit(q + start, &priv->queue_bitmap);
	}

	return 0;
}

static int bcm_sysport_dsa_notifier(struct notifier_block *unused,
				    unsigned long event, void *ptr)
{
	struct dsa_notifier_register_info *info;

	if (event != DSA_PORT_REGISTER)
		return NOTIFY_DONE;

	info = ptr;

	return notifier_from_errno(bcm_sysport_map_queues(info->master, info));
}

2112 2113 2114 2115 2116 2117 2118
static const struct net_device_ops bcm_sysport_netdev_ops = {
	.ndo_start_xmit		= bcm_sysport_xmit,
	.ndo_tx_timeout		= bcm_sysport_tx_timeout,
	.ndo_open		= bcm_sysport_open,
	.ndo_stop		= bcm_sysport_stop,
	.ndo_set_features	= bcm_sysport_set_features,
	.ndo_set_rx_mode	= bcm_sysport_set_rx_mode,
2119
	.ndo_set_mac_address	= bcm_sysport_change_mac,
2120 2121 2122
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= bcm_sysport_poll_controller,
#endif
2123
	.ndo_get_stats64	= bcm_sysport_get_stats64,
2124
	.ndo_select_queue	= bcm_sysport_select_queue,
2125 2126 2127 2128
};

#define REV_FMT	"v%2x.%02x"

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
static const struct bcm_sysport_hw_params bcm_sysport_params[] = {
	[SYSTEMPORT] = {
		.is_lite = false,
		.num_rx_desc_words = SP_NUM_HW_RX_DESC_WORDS,
	},
	[SYSTEMPORT_LITE] = {
		.is_lite = true,
		.num_rx_desc_words = SP_LT_NUM_HW_RX_DESC_WORDS,
	},
};

static const struct of_device_id bcm_sysport_of_match[] = {
	{ .compatible = "brcm,systemportlite-v1.00",
	  .data = &bcm_sysport_params[SYSTEMPORT_LITE] },
	{ .compatible = "brcm,systemport-v1.00",
	  .data = &bcm_sysport_params[SYSTEMPORT] },
	{ .compatible = "brcm,systemport",
	  .data = &bcm_sysport_params[SYSTEMPORT] },
	{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, bcm_sysport_of_match);

2151 2152
static int bcm_sysport_probe(struct platform_device *pdev)
{
2153 2154
	const struct bcm_sysport_hw_params *params;
	const struct of_device_id *of_id = NULL;
2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
	struct bcm_sysport_priv *priv;
	struct device_node *dn;
	struct net_device *dev;
	const void *macaddr;
	struct resource *r;
	u32 txq, rxq;
	int ret;

	dn = pdev->dev.of_node;
	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2165 2166 2167 2168 2169 2170
	of_id = of_match_node(bcm_sysport_of_match, dn);
	if (!of_id || !of_id->data)
		return -EINVAL;

	/* Fairly quickly we need to know the type of adapter we have */
	params = of_id->data;
2171 2172 2173 2174 2175 2176 2177

	/* Read the Transmit/Receive Queue properties */
	if (of_property_read_u32(dn, "systemport,num-txq", &txq))
		txq = TDMA_NUM_RINGS;
	if (of_property_read_u32(dn, "systemport,num-rxq", &rxq))
		rxq = 1;

2178 2179 2180 2181
	/* Sanity check the number of transmit queues */
	if (!txq || txq > TDMA_NUM_RINGS)
		return -EINVAL;

2182 2183 2184 2185 2186 2187 2188
	dev = alloc_etherdev_mqs(sizeof(*priv), txq, rxq);
	if (!dev)
		return -ENOMEM;

	/* Initialize private members */
	priv = netdev_priv(dev);

2189 2190 2191 2192 2193 2194 2195
	/* Allocate number of TX rings */
	priv->tx_rings = devm_kcalloc(&pdev->dev, txq,
				      sizeof(struct bcm_sysport_tx_ring),
				      GFP_KERNEL);
	if (!priv->tx_rings)
		return -ENOMEM;

2196 2197 2198
	priv->is_lite = params->is_lite;
	priv->num_rx_desc_words = params->num_rx_desc_words;

2199
	priv->irq0 = platform_get_irq(pdev, 0);
2200
	if (!priv->is_lite) {
2201
		priv->irq1 = platform_get_irq(pdev, 1);
2202 2203 2204 2205
		priv->wol_irq = platform_get_irq(pdev, 2);
	} else {
		priv->wol_irq = platform_get_irq(pdev, 1);
	}
2206
	if (priv->irq0 <= 0 || (priv->irq1 <= 0 && !priv->is_lite)) {
2207 2208
		dev_err(&pdev->dev, "invalid interrupts\n");
		ret = -EINVAL;
2209
		goto err_free_netdev;
2210 2211
	}

2212 2213 2214
	priv->base = devm_ioremap_resource(&pdev->dev, r);
	if (IS_ERR(priv->base)) {
		ret = PTR_ERR(priv->base);
2215
		goto err_free_netdev;
2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
	}

	priv->netdev = dev;
	priv->pdev = pdev;

	priv->phy_interface = of_get_phy_mode(dn);
	/* Default to GMII interface mode */
	if (priv->phy_interface < 0)
		priv->phy_interface = PHY_INTERFACE_MODE_GMII;

2226 2227 2228 2229 2230 2231 2232
	/* In the case of a fixed PHY, the DT node associated
	 * to the PHY is the Ethernet MAC DT node.
	 */
	if (of_phy_is_fixed_link(dn)) {
		ret = of_phy_register_fixed_link(dn);
		if (ret) {
			dev_err(&pdev->dev, "failed to register fixed PHY\n");
2233
			goto err_free_netdev;
2234 2235 2236 2237 2238
		}

		priv->phy_dn = dn;
	}

2239 2240 2241 2242
	/* Initialize netdevice members */
	macaddr = of_get_mac_address(dn);
	if (!macaddr || !is_valid_ether_addr(macaddr)) {
		dev_warn(&pdev->dev, "using random Ethernet MAC\n");
2243
		eth_hw_addr_random(dev);
2244 2245 2246 2247 2248 2249
	} else {
		ether_addr_copy(dev->dev_addr, macaddr);
	}

	SET_NETDEV_DEV(dev, &pdev->dev);
	dev_set_drvdata(&pdev->dev, dev);
2250
	dev->ethtool_ops = &bcm_sysport_ethtool_ops;
2251 2252 2253 2254 2255 2256 2257
	dev->netdev_ops = &bcm_sysport_netdev_ops;
	netif_napi_add(dev, &priv->napi, bcm_sysport_poll, 64);

	/* HW supported features, none enabled by default */
	dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_HIGHDMA |
				NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;

2258 2259 2260
	/* Request the WOL interrupt and advertise suspend if available */
	priv->wol_irq_disabled = 1;
	ret = devm_request_irq(&pdev->dev, priv->wol_irq,
2261
			       bcm_sysport_wol_isr, 0, dev->name, priv);
2262 2263 2264
	if (!ret)
		device_set_wakeup_capable(&pdev->dev, 1);

2265
	/* Set the needed headroom once and for all */
2266 2267
	BUILD_BUG_ON(sizeof(struct bcm_tsb) != 8);
	dev->needed_headroom += sizeof(struct bcm_tsb);
2268

2269 2270 2271
	/* libphy will adjust the link state accordingly */
	netif_carrier_off(dev);

2272 2273
	u64_stats_init(&priv->syncp);

2274 2275 2276 2277 2278 2279 2280 2281
	priv->dsa_notifier.notifier_call = bcm_sysport_dsa_notifier;

	ret = register_dsa_notifier(&priv->dsa_notifier);
	if (ret) {
		dev_err(&pdev->dev, "failed to register DSA notifier\n");
		goto err_deregister_fixed_link;
	}

2282 2283 2284
	ret = register_netdev(dev);
	if (ret) {
		dev_err(&pdev->dev, "failed to register net_device\n");
2285
		goto err_deregister_notifier;
2286 2287 2288 2289
	}

	priv->rev = topctrl_readl(priv, REV_CNTL) & REV_MASK;
	dev_info(&pdev->dev,
2290
		 "Broadcom SYSTEMPORT%s" REV_FMT
2291
		 " at 0x%p (irqs: %d, %d, TXQs: %d, RXQs: %d)\n",
2292
		 priv->is_lite ? " Lite" : "",
2293 2294
		 (priv->rev >> 8) & 0xff, priv->rev & 0xff,
		 priv->base, priv->irq0, priv->irq1, txq, rxq);
2295 2296

	return 0;
2297

2298 2299
err_deregister_notifier:
	unregister_dsa_notifier(&priv->dsa_notifier);
2300 2301 2302 2303
err_deregister_fixed_link:
	if (of_phy_is_fixed_link(dn))
		of_phy_deregister_fixed_link(dn);
err_free_netdev:
2304 2305 2306 2307 2308 2309 2310
	free_netdev(dev);
	return ret;
}

static int bcm_sysport_remove(struct platform_device *pdev)
{
	struct net_device *dev = dev_get_drvdata(&pdev->dev);
2311
	struct bcm_sysport_priv *priv = netdev_priv(dev);
2312
	struct device_node *dn = pdev->dev.of_node;
2313 2314 2315 2316

	/* Not much to do, ndo_close has been called
	 * and we use managed allocations
	 */
2317
	unregister_dsa_notifier(&priv->dsa_notifier);
2318
	unregister_netdev(dev);
2319 2320
	if (of_phy_is_fixed_link(dn))
		of_phy_deregister_fixed_link(dn);
2321 2322 2323 2324 2325 2326
	free_netdev(dev);
	dev_set_drvdata(&pdev->dev, NULL);

	return 0;
}

2327
#ifdef CONFIG_PM_SLEEP
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370
static int bcm_sysport_suspend_to_wol(struct bcm_sysport_priv *priv)
{
	struct net_device *ndev = priv->netdev;
	unsigned int timeout = 1000;
	u32 reg;

	/* Password has already been programmed */
	reg = umac_readl(priv, UMAC_MPD_CTRL);
	reg |= MPD_EN;
	reg &= ~PSW_EN;
	if (priv->wolopts & WAKE_MAGICSECURE)
		reg |= PSW_EN;
	umac_writel(priv, reg, UMAC_MPD_CTRL);

	/* Make sure RBUF entered WoL mode as result */
	do {
		reg = rbuf_readl(priv, RBUF_STATUS);
		if (reg & RBUF_WOL_MODE)
			break;

		udelay(10);
	} while (timeout-- > 0);

	/* Do not leave the UniMAC RBUF matching only MPD packets */
	if (!timeout) {
		reg = umac_readl(priv, UMAC_MPD_CTRL);
		reg &= ~MPD_EN;
		umac_writel(priv, reg, UMAC_MPD_CTRL);
		netif_err(priv, wol, ndev, "failed to enter WOL mode\n");
		return -ETIMEDOUT;
	}

	/* UniMAC receive needs to be turned on */
	umac_enable_set(priv, CMD_RX_EN, 1);

	/* Enable the interrupt wake-up source */
	intrl2_0_mask_clear(priv, INTRL2_0_MPD);

	netif_dbg(priv, wol, ndev, "entered WOL mode\n");

	return 0;
}

2371 2372 2373 2374 2375
static int bcm_sysport_suspend(struct device *d)
{
	struct net_device *dev = dev_get_drvdata(d);
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	unsigned int i;
2376
	int ret = 0;
2377 2378 2379 2380 2381 2382 2383
	u32 reg;

	if (!netif_running(dev))
		return 0;

	bcm_sysport_netif_stop(dev);

2384
	phy_suspend(dev->phydev);
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397

	netif_device_detach(dev);

	/* Disable UniMAC RX */
	umac_enable_set(priv, CMD_RX_EN, 0);

	ret = rdma_enable_set(priv, 0);
	if (ret) {
		netdev_err(dev, "RDMA timeout!\n");
		return ret;
	}

	/* Disable RXCHK if enabled */
2398
	if (priv->rx_chk_en) {
2399 2400 2401 2402 2403 2404
		reg = rxchk_readl(priv, RXCHK_CONTROL);
		reg &= ~RXCHK_EN;
		rxchk_writel(priv, reg, RXCHK_CONTROL);
	}

	/* Flush RX pipe */
2405 2406
	if (!priv->wolopts)
		topctrl_writel(priv, RX_FLUSH, RX_FLUSH_CNTL);
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425

	ret = tdma_enable_set(priv, 0);
	if (ret) {
		netdev_err(dev, "TDMA timeout!\n");
		return ret;
	}

	/* Wait for a packet boundary */
	usleep_range(2000, 3000);

	umac_enable_set(priv, CMD_TX_EN, 0);

	topctrl_writel(priv, TX_FLUSH, TX_FLUSH_CNTL);

	/* Free RX/TX rings SW structures */
	for (i = 0; i < dev->num_tx_queues; i++)
		bcm_sysport_fini_tx_ring(priv, i);
	bcm_sysport_fini_rx_ring(priv);

2426 2427 2428 2429 2430
	/* Get prepared for Wake-on-LAN */
	if (device_may_wakeup(d) && priv->wolopts)
		ret = bcm_sysport_suspend_to_wol(priv);

	return ret;
2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443
}

static int bcm_sysport_resume(struct device *d)
{
	struct net_device *dev = dev_get_drvdata(d);
	struct bcm_sysport_priv *priv = netdev_priv(dev);
	unsigned int i;
	u32 reg;
	int ret;

	if (!netif_running(dev))
		return 0;

2444 2445
	umac_reset(priv);

2446 2447 2448 2449 2450
	/* We may have been suspended and never received a WOL event that
	 * would turn off MPD detection, take care of that now
	 */
	bcm_sysport_resume_from_wol(priv);

2451 2452 2453 2454 2455
	/* Initialize both hardware and software ring */
	for (i = 0; i < dev->num_tx_queues; i++) {
		ret = bcm_sysport_init_tx_ring(priv, i);
		if (ret) {
			netdev_err(dev, "failed to initialize TX ring %d\n",
2456
				   i);
2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
			goto out_free_tx_rings;
		}
	}

	/* Initialize linked-list */
	tdma_writel(priv, TDMA_LL_RAM_INIT_BUSY, TDMA_STATUS);

	/* Initialize RX ring */
	ret = bcm_sysport_init_rx_ring(priv);
	if (ret) {
		netdev_err(dev, "failed to initialize RX ring\n");
		goto out_free_rx_ring;
	}

	netif_device_attach(dev);

	/* RX pipe enable */
	topctrl_writel(priv, 0, RX_FLUSH_CNTL);

	ret = rdma_enable_set(priv, 1);
	if (ret) {
		netdev_err(dev, "failed to enable RDMA\n");
		goto out_free_rx_ring;
	}

	/* Enable rxhck */
2483
	if (priv->rx_chk_en) {
2484 2485 2486 2487 2488 2489 2490 2491
		reg = rxchk_readl(priv, RXCHK_CONTROL);
		reg |= RXCHK_EN;
		rxchk_writel(priv, reg, RXCHK_CONTROL);
	}

	rbuf_init(priv);

	/* Set maximum frame length */
2492 2493 2494 2495
	if (!priv->is_lite)
		umac_writel(priv, UMAC_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
	else
		gib_set_pad_extension(priv);
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512

	/* Set MAC address */
	umac_set_hw_addr(priv, dev->dev_addr);

	umac_enable_set(priv, CMD_RX_EN, 1);

	/* TX pipe enable */
	topctrl_writel(priv, 0, TX_FLUSH_CNTL);

	umac_enable_set(priv, CMD_TX_EN, 1);

	ret = tdma_enable_set(priv, 1);
	if (ret) {
		netdev_err(dev, "TDMA timeout!\n");
		goto out_free_rx_ring;
	}

2513
	phy_resume(dev->phydev);
2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530

	bcm_sysport_netif_start(dev);

	return 0;

out_free_rx_ring:
	bcm_sysport_fini_rx_ring(priv);
out_free_tx_rings:
	for (i = 0; i < dev->num_tx_queues; i++)
		bcm_sysport_fini_tx_ring(priv, i);
	return ret;
}
#endif

static SIMPLE_DEV_PM_OPS(bcm_sysport_pm_ops,
		bcm_sysport_suspend, bcm_sysport_resume);

2531 2532 2533 2534 2535 2536
static struct platform_driver bcm_sysport_driver = {
	.probe	= bcm_sysport_probe,
	.remove	= bcm_sysport_remove,
	.driver =  {
		.name = "brcm-systemport",
		.of_match_table = bcm_sysport_of_match,
2537
		.pm = &bcm_sysport_pm_ops,
2538 2539 2540 2541 2542 2543 2544 2545
	},
};
module_platform_driver(bcm_sysport_driver);

MODULE_AUTHOR("Broadcom Corporation");
MODULE_DESCRIPTION("Broadcom System Port Ethernet MAC driver");
MODULE_ALIAS("platform:brcm-systemport");
MODULE_LICENSE("GPL");