stmmac_main.c 100.8 KB
Newer Older
1 2 3 4
/*******************************************************************************
  This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
  ST Ethernet IPs are built around a Synopsys IP Core.

5
	Copyright(C) 2007-2011 STMicroelectronics Ltd
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>

  Documentation available at:
	http://www.stlinux.com
  Support available at:
	https://bugzilla.stlinux.com/
*******************************************************************************/

31
#include <linux/clk.h>
32 33 34 35 36 37 38 39 40
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/skbuff.h>
#include <linux/ethtool.h>
#include <linux/if_ether.h>
#include <linux/crc32.h>
#include <linux/mii.h>
41
#include <linux/if.h>
42 43
#include <linux/if_vlan.h>
#include <linux/dma-mapping.h>
44
#include <linux/slab.h>
45
#include <linux/prefetch.h>
46
#include <linux/pinctrl/consumer.h>
47
#ifdef CONFIG_DEBUG_FS
48 49
#include <linux/debugfs.h>
#include <linux/seq_file.h>
50
#endif /* CONFIG_DEBUG_FS */
51 52
#include <linux/net_tstamp.h>
#include "stmmac_ptp.h"
53
#include "stmmac.h"
54
#include <linux/reset.h>
55
#include <linux/of_mdio.h>
56
#include "dwmac1000.h"
57 58

#define STMMAC_ALIGN(x)	L1_CACHE_ALIGN(x)
A
Alexandre TORGUE 已提交
59
#define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
60 61

/* Module parameters */
62
#define TX_TIMEO	5000
63 64
static int watchdog = TX_TIMEO;
module_param(watchdog, int, S_IRUGO | S_IWUSR);
65
MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
66

67
static int debug = -1;
68
module_param(debug, int, S_IRUGO | S_IWUSR);
69
MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
70

71
static int phyaddr = -1;
72 73 74
module_param(phyaddr, int, S_IRUGO);
MODULE_PARM_DESC(phyaddr, "Physical device address");

75
#define STMMAC_TX_THRESH	(DMA_TX_SIZE / 4)
76
#define STMMAC_RX_THRESH	(DMA_RX_SIZE / 4)
77 78 79 80 81 82 83 84 85 86 87 88 89 90

static int flow_ctrl = FLOW_OFF;
module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");

static int pause = PAUSE_TIME;
module_param(pause, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(pause, "Flow Control Pause Time");

#define TC_DEFAULT 64
static int tc = TC_DEFAULT;
module_param(tc, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(tc, "DMA threshold control value");

91 92
#define	DEFAULT_BUFSIZE	1536
static int buf_sz = DEFAULT_BUFSIZE;
93 94 95
module_param(buf_sz, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(buf_sz, "DMA buffer size");

96 97
#define	STMMAC_RX_COPYBREAK	256

98 99 100 101
static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);

102 103 104 105
#define STMMAC_DEFAULT_LPI_TIMER	1000
static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
module_param(eee_timer, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
G
Giuseppe CAVALLARO 已提交
106
#define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
107

108 109
/* By default the driver will use the ring mode to manage tx and rx descriptors,
 * but allow user to force to use the chain instead of the ring
110 111 112 113 114
 */
static unsigned int chain_mode;
module_param(chain_mode, int, S_IRUGO);
MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");

115 116
static irqreturn_t stmmac_interrupt(int irq, void *dev_id);

117
#ifdef CONFIG_DEBUG_FS
118
static int stmmac_init_fs(struct net_device *dev);
119
static void stmmac_exit_fs(struct net_device *dev);
120 121
#endif

122 123
#define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))

124 125
/**
 * stmmac_verify_args - verify the driver parameters.
126 127
 * Description: it checks the driver parameters and set a default in case of
 * errors.
128 129 130 131 132
 */
static void stmmac_verify_args(void)
{
	if (unlikely(watchdog < 0))
		watchdog = TX_TIMEO;
133 134
	if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
		buf_sz = DEFAULT_BUFSIZE;
135 136 137 138 139 140
	if (unlikely(flow_ctrl > 1))
		flow_ctrl = FLOW_AUTO;
	else if (likely(flow_ctrl < 0))
		flow_ctrl = FLOW_OFF;
	if (unlikely((pause < 0) || (pause > 0xffff)))
		pause = PAUSE_TIME;
141 142
	if (eee_timer < 0)
		eee_timer = STMMAC_DEFAULT_LPI_TIMER;
143 144
}

145 146 147 148 149 150 151 152 153 154 155 156
/**
 * stmmac_clk_csr_set - dynamically set the MDC clock
 * @priv: driver private structure
 * Description: this is to dynamically set the MDC clock according to the csr
 * clock input.
 * Note:
 *	If a specific clk_csr value is passed from the platform
 *	this means that the CSR Clock Range selection cannot be
 *	changed at run-time and it is fixed (as reported in the driver
 *	documentation). Viceversa the driver will try to set the MDC
 *	clock dynamically according to the actual clock input.
 */
157 158 159 160 161 162 163
static void stmmac_clk_csr_set(struct stmmac_priv *priv)
{
	u32 clk_rate;

	clk_rate = clk_get_rate(priv->stmmac_clk);

	/* Platform provided default clk_csr would be assumed valid
G
Giuseppe CAVALLARO 已提交
164 165 166 167 168 169
	 * for all other cases except for the below mentioned ones.
	 * For values higher than the IEEE 802.3 specified frequency
	 * we can not estimate the proper divider as it is not known
	 * the frequency of clk_csr_i. So we do not change the default
	 * divider.
	 */
170 171 172 173 174 175 176 177 178 179 180
	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
		if (clk_rate < CSR_F_35M)
			priv->clk_csr = STMMAC_CSR_20_35M;
		else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
			priv->clk_csr = STMMAC_CSR_35_60M;
		else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
			priv->clk_csr = STMMAC_CSR_60_100M;
		else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
			priv->clk_csr = STMMAC_CSR_100_150M;
		else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
			priv->clk_csr = STMMAC_CSR_150_250M;
181
		else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
182
			priv->clk_csr = STMMAC_CSR_250_300M;
G
Giuseppe CAVALLARO 已提交
183
	}
184 185
}

186 187
static void print_pkt(unsigned char *buf, int len)
{
188 189
	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
190 191 192 193
}

static inline u32 stmmac_tx_avail(struct stmmac_priv *priv)
{
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
	unsigned avail;

	if (priv->dirty_tx > priv->cur_tx)
		avail = priv->dirty_tx - priv->cur_tx - 1;
	else
		avail = DMA_TX_SIZE - priv->cur_tx + priv->dirty_tx - 1;

	return avail;
}

static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv)
{
	unsigned dirty;

	if (priv->dirty_rx <= priv->cur_rx)
		dirty = priv->cur_rx - priv->dirty_rx;
	else
		dirty = DMA_RX_SIZE - priv->dirty_rx + priv->cur_rx;

	return dirty;
214 215
}

216
/**
217
 * stmmac_hw_fix_mac_speed - callback for speed selection
218 219 220
 * @priv: driver private structure
 * Description: on some platforms (e.g. ST), some HW system configuraton
 * registers have to be set according to the link speed negotiated.
221 222 223
 */
static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
{
224 225
	struct net_device *ndev = priv->dev;
	struct phy_device *phydev = ndev->phydev;
226 227

	if (likely(priv->plat->fix_mac_speed))
G
Giuseppe CAVALLARO 已提交
228
		priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
229 230
}

231
/**
232
 * stmmac_enable_eee_mode - check and enter in LPI mode
233
 * @priv: driver private structure
234 235
 * Description: this function is to verify and enter in LPI mode in case of
 * EEE.
236
 */
237 238 239 240 241
static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
{
	/* Check and enter in LPI mode */
	if ((priv->dirty_tx == priv->cur_tx) &&
	    (priv->tx_path_in_lpi_mode == false))
242 243
		priv->hw->mac->set_eee_mode(priv->hw,
					    priv->plat->en_tx_lpi_clockgating);
244 245
}

246
/**
247
 * stmmac_disable_eee_mode - disable and exit from LPI mode
248 249 250 251
 * @priv: driver private structure
 * Description: this function is to exit and disable EEE in case of
 * LPI state is true. This is called by the xmit.
 */
252 253
void stmmac_disable_eee_mode(struct stmmac_priv *priv)
{
254
	priv->hw->mac->reset_eee_mode(priv->hw);
255 256 257 258 259
	del_timer_sync(&priv->eee_ctrl_timer);
	priv->tx_path_in_lpi_mode = false;
}

/**
260
 * stmmac_eee_ctrl_timer - EEE TX SW timer.
261 262
 * @arg : data hook
 * Description:
263
 *  if there is no data transfer and if we are not in LPI state,
264 265 266 267 268 269 270
 *  then MAC Transmitter can be moved to LPI state.
 */
static void stmmac_eee_ctrl_timer(unsigned long arg)
{
	struct stmmac_priv *priv = (struct stmmac_priv *)arg;

	stmmac_enable_eee_mode(priv);
G
Giuseppe CAVALLARO 已提交
271
	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
272 273 274
}

/**
275
 * stmmac_eee_init - init EEE
276
 * @priv: driver private structure
277
 * Description:
278 279 280
 *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 *  can also manage EEE, this function enable the LPI state and start related
 *  timer.
281 282 283
 */
bool stmmac_eee_init(struct stmmac_priv *priv)
{
284
	struct net_device *ndev = priv->dev;
285
	unsigned long flags;
286 287
	bool ret = false;

G
Giuseppe CAVALLARO 已提交
288 289 290
	/* Using PCS we cannot dial with the phy registers at this stage
	 * so we do not support extra feature like EEE.
	 */
291 292 293
	if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
	    (priv->hw->pcs == STMMAC_PCS_TBI) ||
	    (priv->hw->pcs == STMMAC_PCS_RTBI))
G
Giuseppe CAVALLARO 已提交
294 295
		goto out;

296 297
	/* MAC core supports the EEE feature. */
	if (priv->dma_cap.eee) {
298 299
		int tx_lpi_timer = priv->tx_lpi_timer;

300
		/* Check if the PHY supports EEE */
301
		if (phy_init_eee(ndev->phydev, 1)) {
302 303 304 305 306
			/* To manage at run-time if the EEE cannot be supported
			 * anymore (for example because the lp caps have been
			 * changed).
			 * In that case the driver disable own timers.
			 */
307
			spin_lock_irqsave(&priv->lock, flags);
308
			if (priv->eee_active) {
309
				netdev_dbg(priv->dev, "disable EEE\n");
310
				del_timer_sync(&priv->eee_ctrl_timer);
311
				priv->hw->mac->set_eee_timer(priv->hw, 0,
312 313 314
							     tx_lpi_timer);
			}
			priv->eee_active = 0;
315
			spin_unlock_irqrestore(&priv->lock, flags);
316
			goto out;
317 318
		}
		/* Activate the EEE and start timers */
319
		spin_lock_irqsave(&priv->lock, flags);
G
Giuseppe CAVALLARO 已提交
320 321
		if (!priv->eee_active) {
			priv->eee_active = 1;
322 323 324 325 326
			setup_timer(&priv->eee_ctrl_timer,
				    stmmac_eee_ctrl_timer,
				    (unsigned long)priv);
			mod_timer(&priv->eee_ctrl_timer,
				  STMMAC_LPI_T(eee_timer));
G
Giuseppe CAVALLARO 已提交
327

328
			priv->hw->mac->set_eee_timer(priv->hw,
G
Giuseppe CAVALLARO 已提交
329
						     STMMAC_DEFAULT_LIT_LS,
330
						     tx_lpi_timer);
331 332
		}
		/* Set HW EEE according to the speed */
333
		priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
334 335

		ret = true;
336 337
		spin_unlock_irqrestore(&priv->lock, flags);

338
		netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
339 340 341 342 343
	}
out:
	return ret;
}

344
/* stmmac_get_tx_hwtstamp - get HW TX timestamps
345
 * @priv: driver private structure
346
 * @p : descriptor pointer
347 348 349 350 351 352
 * @skb : the socket buffer
 * Description :
 * This function will read timestamp from the descriptor & pass it to stack.
 * and also perform some sanity checks.
 */
static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
353
				   struct dma_desc *p, struct sk_buff *skb)
354 355 356 357 358 359 360
{
	struct skb_shared_hwtstamps shhwtstamp;
	u64 ns;

	if (!priv->hwts_tx_en)
		return;

G
Giuseppe CAVALLARO 已提交
361
	/* exit if skb doesn't support hw tstamp */
362
	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
363 364 365
		return;

	/* check tx tstamp status */
366 367 368
	if (!priv->hw->desc->get_tx_timestamp_status(p)) {
		/* get the valid tstamp */
		ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
369

370 371
		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
		shhwtstamp.hwtstamp = ns_to_ktime(ns);
372

373 374 375 376
		netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns);
		/* pass tstamp to stack */
		skb_tstamp_tx(skb, &shhwtstamp);
	}
377 378 379 380

	return;
}

381
/* stmmac_get_rx_hwtstamp - get HW RX timestamps
382
 * @priv: driver private structure
383 384
 * @p : descriptor pointer
 * @np : next descriptor pointer
385 386 387 388 389
 * @skb : the socket buffer
 * Description :
 * This function will read received packet's timestamp from the descriptor
 * and pass it to stack. It also perform some sanity checks.
 */
390 391
static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
				   struct dma_desc *np, struct sk_buff *skb)
392 393 394 395 396 397 398
{
	struct skb_shared_hwtstamps *shhwtstamp = NULL;
	u64 ns;

	if (!priv->hwts_rx_en)
		return;

399 400 401 402 403 404 405
	/* Check if timestamp is available */
	if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
		/* For GMAC4, the valid timestamp is from CTX next desc. */
		if (priv->plat->has_gmac4)
			ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
		else
			ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
406

407 408 409 410 411 412 413
		netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns);
		shhwtstamp = skb_hwtstamps(skb);
		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
		shhwtstamp->hwtstamp = ns_to_ktime(ns);
	} else  {
		netdev_err(priv->dev, "cannot get RX hw timestamp\n");
	}
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
}

/**
 *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 *  @dev: device pointer.
 *  @ifr: An IOCTL specefic structure, that can contain a pointer to
 *  a proprietary structure used to pass information to the driver.
 *  Description:
 *  This function configures the MAC to enable/disable both outgoing(TX)
 *  and incoming(RX) packets time stamping based on user input.
 *  Return Value:
 *  0 on success and an appropriate -ve integer on failure.
 */
static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	struct hwtstamp_config config;
A
Arnd Bergmann 已提交
431
	struct timespec64 now;
432 433 434 435 436 437 438 439 440 441
	u64 temp = 0;
	u32 ptp_v2 = 0;
	u32 tstamp_all = 0;
	u32 ptp_over_ipv4_udp = 0;
	u32 ptp_over_ipv6_udp = 0;
	u32 ptp_over_ethernet = 0;
	u32 snap_type_sel = 0;
	u32 ts_master_en = 0;
	u32 ts_event_en = 0;
	u32 value = 0;
442
	u32 sec_inc;
443 444 445 446 447 448 449 450 451 452

	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
		netdev_alert(priv->dev, "No support for HW time stamping\n");
		priv->hwts_tx_en = 0;
		priv->hwts_rx_en = 0;

		return -EOPNOTSUPP;
	}

	if (copy_from_user(&config, ifr->ifr_data,
G
Giuseppe CAVALLARO 已提交
453
			   sizeof(struct hwtstamp_config)))
454 455
		return -EFAULT;

456 457
	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
		   __func__, config.flags, config.tx_type, config.rx_filter);
458 459 460 461 462

	/* reserved for future extensions */
	if (config.flags)
		return -EINVAL;

463 464
	if (config.tx_type != HWTSTAMP_TX_OFF &&
	    config.tx_type != HWTSTAMP_TX_ON)
465 466 467 468 469
		return -ERANGE;

	if (priv->adv_ts) {
		switch (config.rx_filter) {
		case HWTSTAMP_FILTER_NONE:
G
Giuseppe CAVALLARO 已提交
470
			/* time stamp no incoming packet at all */
471 472 473 474
			config.rx_filter = HWTSTAMP_FILTER_NONE;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
G
Giuseppe CAVALLARO 已提交
475
			/* PTP v1, UDP, any kind of event packet */
476 477 478 479 480 481 482 483 484
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
			/* take time stamp for all event messages */
			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
G
Giuseppe CAVALLARO 已提交
485
			/* PTP v1, UDP, Sync packet */
486 487 488 489 490 491 492 493 494
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
495
			/* PTP v1, UDP, Delay_req packet */
496 497 498 499 500 501 502 503 504 505
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
G
Giuseppe CAVALLARO 已提交
506
			/* PTP v2, UDP, any kind of event packet */
507 508 509 510 511 512 513 514 515 516
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for all event messages */
			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
G
Giuseppe CAVALLARO 已提交
517
			/* PTP v2, UDP, Sync packet */
518 519 520 521 522 523 524 525 526 527
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
528
			/* PTP v2, UDP, Delay_req packet */
529 530 531 532 533 534 535 536 537 538 539
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_EVENT:
G
Giuseppe CAVALLARO 已提交
540
			/* PTP v2/802.AS1 any layer, any kind of event packet */
541 542 543 544 545 546 547 548 549 550 551
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for all event messages */
			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_SYNC:
G
Giuseppe CAVALLARO 已提交
552
			/* PTP v2/802.AS1, any layer, Sync packet */
553 554 555 556 557 558 559 560 561 562 563
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
564
			/* PTP v2/802.AS1, any layer, Delay_req packet */
565 566 567 568 569 570 571 572 573 574 575 576
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

		case HWTSTAMP_FILTER_ALL:
G
Giuseppe CAVALLARO 已提交
577
			/* time stamp any incoming packet */
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
			config.rx_filter = HWTSTAMP_FILTER_ALL;
			tstamp_all = PTP_TCR_TSENALL;
			break;

		default:
			return -ERANGE;
		}
	} else {
		switch (config.rx_filter) {
		case HWTSTAMP_FILTER_NONE:
			config.rx_filter = HWTSTAMP_FILTER_NONE;
			break;
		default:
			/* PTP v1, UDP, any kind of event packet */
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
			break;
		}
	}
	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
597
	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
598 599

	if (!priv->hwts_tx_en && !priv->hwts_rx_en)
600
		priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
601 602
	else {
		value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
G
Giuseppe CAVALLARO 已提交
603 604 605
			 tstamp_all | ptp_v2 | ptp_over_ethernet |
			 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
			 ts_master_en | snap_type_sel);
606
		priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
607 608

		/* program Sub Second Increment reg */
609
		sec_inc = priv->hw->ptp->config_sub_second_increment(
610 611
			priv->ptpaddr, priv->clk_ptp_rate,
			priv->plat->has_gmac4);
612
		temp = div_u64(1000000000ULL, sec_inc);
613 614 615 616

		/* calculate default added value:
		 * formula is :
		 * addend = (2^32)/freq_div_ratio;
617
		 * where, freq_div_ratio = 1e9ns/sec_inc
618
		 */
619
		temp = (u64)(temp << 32);
620
		priv->default_addend = div_u64(temp, priv->clk_ptp_rate);
621
		priv->hw->ptp->config_addend(priv->ptpaddr,
622 623 624
					     priv->default_addend);

		/* initialize system time */
A
Arnd Bergmann 已提交
625 626 627
		ktime_get_real_ts64(&now);

		/* lower 32 bits of tv_sec are safe until y2106 */
628
		priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
629 630 631 632 633 634 635
					    now.tv_nsec);
	}

	return copy_to_user(ifr->ifr_data, &config,
			    sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
}

636
/**
637
 * stmmac_init_ptp - init PTP
638
 * @priv: driver private structure
639
 * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
640
 * This is done by looking at the HW cap. register.
641
 * This function also registers the ptp driver.
642
 */
643
static int stmmac_init_ptp(struct stmmac_priv *priv)
644
{
645 646 647
	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
		return -EOPNOTSUPP;

648 649 650 651 652
	/* Fall-back to main clock in case of no PTP ref is passed */
	priv->clk_ptp_ref = devm_clk_get(priv->device, "clk_ptp_ref");
	if (IS_ERR(priv->clk_ptp_ref)) {
		priv->clk_ptp_rate = clk_get_rate(priv->stmmac_clk);
		priv->clk_ptp_ref = NULL;
653
		netdev_dbg(priv->dev, "PTP uses main clock\n");
654 655 656
	} else {
		clk_prepare_enable(priv->clk_ptp_ref);
		priv->clk_ptp_rate = clk_get_rate(priv->clk_ptp_ref);
657
		netdev_dbg(priv->dev, "PTP rate %d\n", priv->clk_ptp_rate);
658 659
	}

660
	priv->adv_ts = 0;
661 662 663 664 665
	/* Check if adv_ts can be enabled for dwmac 4.x core */
	if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
		priv->adv_ts = 1;
	/* Dwmac 3.x core with extend_desc can support adv_ts */
	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
666 667
		priv->adv_ts = 1;

668 669
	if (priv->dma_cap.time_stamp)
		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
670

671 672 673
	if (priv->adv_ts)
		netdev_info(priv->dev,
			    "IEEE 1588-2008 Advanced Timestamp supported\n");
674 675 676 677

	priv->hw->ptp = &stmmac_ptp;
	priv->hwts_tx_en = 0;
	priv->hwts_rx_en = 0;
678

679 680 681
	stmmac_ptp_register(priv);

	return 0;
682 683 684 685
}

static void stmmac_release_ptp(struct stmmac_priv *priv)
{
686 687
	if (priv->clk_ptp_ref)
		clk_disable_unprepare(priv->clk_ptp_ref);
688
	stmmac_ptp_unregister(priv);
689 690
}

691
/**
692
 * stmmac_adjust_link - adjusts the link parameters
693
 * @dev: net device structure
694 695 696 697 698
 * Description: this is the helper called by the physical abstraction layer
 * drivers to communicate the phy link status. According the speed and duplex
 * this driver can invoke registered glue-logic as well.
 * It also invoke the eee initialization because it could happen when switch
 * on different networks (that are eee capable).
699 700 701 702
 */
static void stmmac_adjust_link(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
703
	struct phy_device *phydev = dev->phydev;
704 705 706 707 708 709 710 711
	unsigned long flags;
	int new_state = 0;
	unsigned int fc = priv->flow_ctrl, pause_time = priv->pause;

	if (phydev == NULL)
		return;

	spin_lock_irqsave(&priv->lock, flags);
712

713
	if (phydev->link) {
714
		u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
715 716 717 718 719 720

		/* Now we make sure that we can be in full duplex mode.
		 * If not, we operate in half-duplex mode. */
		if (phydev->duplex != priv->oldduplex) {
			new_state = 1;
			if (!(phydev->duplex))
721
				ctrl &= ~priv->hw->link.duplex;
722
			else
723
				ctrl |= priv->hw->link.duplex;
724 725 726 727
			priv->oldduplex = phydev->duplex;
		}
		/* Flow Control operation */
		if (phydev->pause)
728
			priv->hw->mac->flow_ctrl(priv->hw, phydev->duplex,
729
						 fc, pause_time);
730 731 732 733 734

		if (phydev->speed != priv->speed) {
			new_state = 1;
			switch (phydev->speed) {
			case 1000:
A
Alexandre TORGUE 已提交
735 736
				if (likely((priv->plat->has_gmac) ||
					   (priv->plat->has_gmac4)))
737
					ctrl &= ~priv->hw->link.port;
G
Giuseppe CAVALLARO 已提交
738
				stmmac_hw_fix_mac_speed(priv);
739 740 741
				break;
			case 100:
			case 10:
A
Alexandre TORGUE 已提交
742 743
				if (likely((priv->plat->has_gmac) ||
					   (priv->plat->has_gmac4))) {
744
					ctrl |= priv->hw->link.port;
745
					if (phydev->speed == SPEED_100) {
746
						ctrl |= priv->hw->link.speed;
747
					} else {
748
						ctrl &= ~(priv->hw->link.speed);
749 750
					}
				} else {
751
					ctrl &= ~priv->hw->link.port;
752
				}
753
				stmmac_hw_fix_mac_speed(priv);
754 755
				break;
			default:
756 757 758
				netif_warn(priv, link, priv->dev,
					   "Speed (%d) not 10/100\n",
					   phydev->speed);
759 760 761 762 763 764
				break;
			}

			priv->speed = phydev->speed;
		}

765
		writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780

		if (!priv->oldlink) {
			new_state = 1;
			priv->oldlink = 1;
		}
	} else if (priv->oldlink) {
		new_state = 1;
		priv->oldlink = 0;
		priv->speed = 0;
		priv->oldduplex = -1;
	}

	if (new_state && netif_msg_link(priv))
		phy_print_status(phydev);

781 782
	spin_unlock_irqrestore(&priv->lock, flags);

783 784 785 786 787 788 789 790 791 792
	if (phydev->is_pseudo_fixed_link)
		/* Stop PHY layer to call the hook to adjust the link in case
		 * of a switch is attached to the stmmac driver.
		 */
		phydev->irq = PHY_IGNORE_INTERRUPT;
	else
		/* At this stage, init the EEE if supported.
		 * Never called in case of fixed_link.
		 */
		priv->eee_enabled = stmmac_eee_init(priv);
793 794
}

795
/**
796
 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
797 798 799 800 801
 * @priv: driver private structure
 * Description: this is to verify if the HW supports the PCS.
 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 * configured for the TBI, RTBI, or SGMII PHY interface.
 */
802 803 804 805 806
static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
{
	int interface = priv->plat->interface;

	if (priv->dma_cap.pcs) {
B
Byungho An 已提交
807 808 809 810
		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
811
			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
812
			priv->hw->pcs = STMMAC_PCS_RGMII;
B
Byungho An 已提交
813
		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
814
			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
815
			priv->hw->pcs = STMMAC_PCS_SGMII;
816 817 818 819
		}
	}
}

820 821 822 823 824 825 826 827 828 829 830 831
/**
 * stmmac_init_phy - PHY initialization
 * @dev: net device structure
 * Description: it initializes the driver's PHY state, and attaches the PHY
 * to the mac driver.
 *  Return value:
 *  0 on success
 */
static int stmmac_init_phy(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	struct phy_device *phydev;
832
	char phy_id_fmt[MII_BUS_ID_SIZE + 3];
833
	char bus_id[MII_BUS_ID_SIZE];
834
	int interface = priv->plat->interface;
835
	int max_speed = priv->plat->max_speed;
836 837 838 839
	priv->oldlink = 0;
	priv->speed = 0;
	priv->oldduplex = -1;

840 841 842 843
	if (priv->plat->phy_node) {
		phydev = of_phy_connect(dev, priv->plat->phy_node,
					&stmmac_adjust_link, 0, interface);
	} else {
G
Giuseppe CAVALLARO 已提交
844 845
		snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
			 priv->plat->bus_id);
846 847 848

		snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
			 priv->plat->phy_addr);
849
		netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
850
			   phy_id_fmt);
851 852 853 854

		phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
				     interface);
	}
855

856
	if (IS_ERR_OR_NULL(phydev)) {
857
		netdev_err(priv->dev, "Could not attach to PHY\n");
858 859 860
		if (!phydev)
			return -ENODEV;

861 862 863
		return PTR_ERR(phydev);
	}

864
	/* Stop Advertising 1000BASE Capability if interface is not GMII */
865
	if ((interface == PHY_INTERFACE_MODE_MII) ||
866
	    (interface == PHY_INTERFACE_MODE_RMII) ||
P
Pavel Machek 已提交
867
		(max_speed < 1000 && max_speed > 0))
868 869
		phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
					 SUPPORTED_1000baseT_Full);
870

871 872 873 874 875 876 877
	/*
	 * Broken HW is sometimes missing the pull-up resistor on the
	 * MDIO line, which results in reads to non-existent devices returning
	 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
	 * device as well.
	 * Note: phydev->phy_id is the result of reading the UID PHY registers.
	 */
878
	if (!priv->plat->phy_node && phydev->phy_id == 0) {
879 880 881
		phy_disconnect(phydev);
		return -ENODEV;
	}
882

883 884 885 886 887 888 889
	/* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
	 * subsequent PHY polling, make sure we force a link transition if
	 * we have a UP/DOWN/UP transition
	 */
	if (phydev->is_pseudo_fixed_link)
		phydev->irq = PHY_POLL;

890 891
	netdev_dbg(priv->dev, "%s: attached to PHY (UID 0x%x) Link = %d\n",
		   __func__, phydev->phy_id, phydev->link);
892 893 894 895

	return 0;
}

896 897
static void stmmac_display_rings(struct stmmac_priv *priv)
{
898 899
	void *head_rx, *head_tx;

900
	if (priv->extend_desc) {
901 902
		head_rx = (void *)priv->dma_erx;
		head_tx = (void *)priv->dma_etx;
903
	} else {
904 905
		head_rx = (void *)priv->dma_rx;
		head_tx = (void *)priv->dma_tx;
906
	}
907 908 909 910 911

	/* Display Rx ring */
	priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
	/* Display Tx ring */
	priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
912 913
}

914 915 916 917 918 919 920 921
static int stmmac_set_bfsize(int mtu, int bufsize)
{
	int ret = bufsize;

	if (mtu >= BUF_SIZE_4KiB)
		ret = BUF_SIZE_8KiB;
	else if (mtu >= BUF_SIZE_2KiB)
		ret = BUF_SIZE_4KiB;
922
	else if (mtu > DEFAULT_BUFSIZE)
923 924
		ret = BUF_SIZE_2KiB;
	else
925
		ret = DEFAULT_BUFSIZE;
926 927 928 929

	return ret;
}

930
/**
931
 * stmmac_clear_descriptors - clear descriptors
932 933 934 935
 * @priv: driver private structure
 * Description: this function is called to clear the tx and rx descriptors
 * in case of both basic and extended descriptors are used.
 */
936 937 938 939 940
static void stmmac_clear_descriptors(struct stmmac_priv *priv)
{
	int i;

	/* Clear the Rx/Tx descriptors */
941
	for (i = 0; i < DMA_RX_SIZE; i++)
942 943 944
		if (priv->extend_desc)
			priv->hw->desc->init_rx_desc(&priv->dma_erx[i].basic,
						     priv->use_riwt, priv->mode,
945
						     (i == DMA_RX_SIZE - 1));
946 947 948
		else
			priv->hw->desc->init_rx_desc(&priv->dma_rx[i],
						     priv->use_riwt, priv->mode,
949 950
						     (i == DMA_RX_SIZE - 1));
	for (i = 0; i < DMA_TX_SIZE; i++)
951 952 953
		if (priv->extend_desc)
			priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic,
						     priv->mode,
954
						     (i == DMA_TX_SIZE - 1));
955 956 957
		else
			priv->hw->desc->init_tx_desc(&priv->dma_tx[i],
						     priv->mode,
958
						     (i == DMA_TX_SIZE - 1));
959 960
}

961 962 963 964 965 966 967 968 969
/**
 * stmmac_init_rx_buffers - init the RX descriptor buffer.
 * @priv: driver private structure
 * @p: descriptor pointer
 * @i: descriptor index
 * @flags: gfp flag.
 * Description: this function is called to allocate a receive buffer, perform
 * the DMA mapping and init the descriptor.
 */
970
static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
971
				  int i, gfp_t flags)
972 973 974
{
	struct sk_buff *skb;

975
	skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
976
	if (!skb) {
977 978
		netdev_err(priv->dev,
			   "%s: Rx init fails; skb is NULL\n", __func__);
979
		return -ENOMEM;
980 981 982 983 984
	}
	priv->rx_skbuff[i] = skb;
	priv->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
						priv->dma_buf_sz,
						DMA_FROM_DEVICE);
985
	if (dma_mapping_error(priv->device, priv->rx_skbuff_dma[i])) {
986
		netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
987 988 989
		dev_kfree_skb_any(skb);
		return -EINVAL;
	}
990

A
Alexandre TORGUE 已提交
991
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
992
		p->des0 = cpu_to_le32(priv->rx_skbuff_dma[i]);
A
Alexandre TORGUE 已提交
993
	else
994
		p->des2 = cpu_to_le32(priv->rx_skbuff_dma[i]);
995

G
Giuseppe CAVALLARO 已提交
996
	if ((priv->hw->mode->init_desc3) &&
997
	    (priv->dma_buf_sz == BUF_SIZE_16KiB))
G
Giuseppe CAVALLARO 已提交
998
		priv->hw->mode->init_desc3(p);
999 1000 1001 1002

	return 0;
}

1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
static void stmmac_free_rx_buffers(struct stmmac_priv *priv, int i)
{
	if (priv->rx_skbuff[i]) {
		dma_unmap_single(priv->device, priv->rx_skbuff_dma[i],
				 priv->dma_buf_sz, DMA_FROM_DEVICE);
		dev_kfree_skb_any(priv->rx_skbuff[i]);
	}
	priv->rx_skbuff[i] = NULL;
}

1013 1014 1015
/**
 * init_dma_desc_rings - init the RX/TX descriptor rings
 * @dev: net device structure
1016 1017
 * @flags: gfp flag.
 * Description: this function initializes the DMA RX/TX descriptors
1018 1019
 * and allocates the socket buffers. It suppors the chained and ring
 * modes.
1020
 */
1021
static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1022 1023 1024
{
	int i;
	struct stmmac_priv *priv = netdev_priv(dev);
1025
	unsigned int bfsize = 0;
1026
	int ret = -ENOMEM;
1027

G
Giuseppe CAVALLARO 已提交
1028 1029
	if (priv->hw->mode->set_16kib_bfsize)
		bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1030

1031
	if (bfsize < BUF_SIZE_16KiB)
1032
		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1033

1034 1035
	priv->dma_buf_sz = bfsize;

1036 1037 1038 1039 1040 1041 1042
	netif_dbg(priv, probe, priv->dev,
		  "(%s) dma_rx_phy=0x%08x dma_tx_phy=0x%08x\n",
		  __func__, (u32)priv->dma_rx_phy, (u32)priv->dma_tx_phy);

	/* RX INITIALIZATION */
	netif_dbg(priv, probe, priv->dev,
		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
1043

1044
	for (i = 0; i < DMA_RX_SIZE; i++) {
1045 1046 1047 1048 1049
		struct dma_desc *p;
		if (priv->extend_desc)
			p = &((priv->dma_erx + i)->basic);
		else
			p = priv->dma_rx + i;
1050

1051
		ret = stmmac_init_rx_buffers(priv, p, i, flags);
1052 1053
		if (ret)
			goto err_init_rx_buffers;
1054

1055 1056 1057
		netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
			  priv->rx_skbuff[i], priv->rx_skbuff[i]->data,
			  (unsigned int)priv->rx_skbuff_dma[i]);
1058 1059
	}
	priv->cur_rx = 0;
1060
	priv->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1061 1062
	buf_sz = bfsize;

1063 1064 1065
	/* Setup the chained descriptor addresses */
	if (priv->mode == STMMAC_CHAIN_MODE) {
		if (priv->extend_desc) {
G
Giuseppe CAVALLARO 已提交
1066
			priv->hw->mode->init(priv->dma_erx, priv->dma_rx_phy,
1067
					     DMA_RX_SIZE, 1);
G
Giuseppe CAVALLARO 已提交
1068
			priv->hw->mode->init(priv->dma_etx, priv->dma_tx_phy,
1069
					     DMA_TX_SIZE, 1);
1070
		} else {
G
Giuseppe CAVALLARO 已提交
1071
			priv->hw->mode->init(priv->dma_rx, priv->dma_rx_phy,
1072
					     DMA_RX_SIZE, 0);
G
Giuseppe CAVALLARO 已提交
1073
			priv->hw->mode->init(priv->dma_tx, priv->dma_tx_phy,
1074
					     DMA_TX_SIZE, 0);
1075 1076 1077
		}
	}

1078
	/* TX INITIALIZATION */
1079
	for (i = 0; i < DMA_TX_SIZE; i++) {
1080 1081 1082 1083 1084
		struct dma_desc *p;
		if (priv->extend_desc)
			p = &((priv->dma_etx + i)->basic);
		else
			p = priv->dma_tx + i;
A
Alexandre TORGUE 已提交
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094

		if (priv->synopsys_id >= DWMAC_CORE_4_00) {
			p->des0 = 0;
			p->des1 = 0;
			p->des2 = 0;
			p->des3 = 0;
		} else {
			p->des2 = 0;
		}

G
Giuseppe CAVALLARO 已提交
1095 1096
		priv->tx_skbuff_dma[i].buf = 0;
		priv->tx_skbuff_dma[i].map_as_page = false;
1097
		priv->tx_skbuff_dma[i].len = 0;
1098
		priv->tx_skbuff_dma[i].last_segment = false;
1099 1100
		priv->tx_skbuff[i] = NULL;
	}
1101

1102 1103
	priv->dirty_tx = 0;
	priv->cur_tx = 0;
B
Beniamino Galvani 已提交
1104
	netdev_reset_queue(priv->dev);
1105

1106
	stmmac_clear_descriptors(priv);
1107

1108 1109
	if (netif_msg_hw(priv))
		stmmac_display_rings(priv);
1110 1111 1112 1113 1114 1115

	return 0;
err_init_rx_buffers:
	while (--i >= 0)
		stmmac_free_rx_buffers(priv, i);
	return ret;
1116 1117 1118 1119 1120 1121
}

static void dma_free_rx_skbufs(struct stmmac_priv *priv)
{
	int i;

1122
	for (i = 0; i < DMA_RX_SIZE; i++)
1123
		stmmac_free_rx_buffers(priv, i);
1124 1125 1126 1127 1128 1129
}

static void dma_free_tx_skbufs(struct stmmac_priv *priv)
{
	int i;

1130
	for (i = 0; i < DMA_TX_SIZE; i++) {
1131 1132 1133 1134 1135 1136 1137
		struct dma_desc *p;

		if (priv->extend_desc)
			p = &((priv->dma_etx + i)->basic);
		else
			p = priv->dma_tx + i;

G
Giuseppe CAVALLARO 已提交
1138 1139 1140 1141
		if (priv->tx_skbuff_dma[i].buf) {
			if (priv->tx_skbuff_dma[i].map_as_page)
				dma_unmap_page(priv->device,
					       priv->tx_skbuff_dma[i].buf,
1142
					       priv->tx_skbuff_dma[i].len,
G
Giuseppe CAVALLARO 已提交
1143 1144 1145 1146
					       DMA_TO_DEVICE);
			else
				dma_unmap_single(priv->device,
						 priv->tx_skbuff_dma[i].buf,
1147
						 priv->tx_skbuff_dma[i].len,
G
Giuseppe CAVALLARO 已提交
1148
						 DMA_TO_DEVICE);
1149
		}
1150

1151
		if (priv->tx_skbuff[i] != NULL) {
1152 1153
			dev_kfree_skb_any(priv->tx_skbuff[i]);
			priv->tx_skbuff[i] = NULL;
G
Giuseppe CAVALLARO 已提交
1154 1155
			priv->tx_skbuff_dma[i].buf = 0;
			priv->tx_skbuff_dma[i].map_as_page = false;
1156 1157 1158 1159
		}
	}
}

1160 1161 1162 1163 1164 1165 1166 1167
/**
 * alloc_dma_desc_resources - alloc TX/RX resources.
 * @priv: private structure
 * Description: according to which descriptor can be used (extend or basic)
 * this function allocates the resources for TX and RX paths. In case of
 * reception, for example, it pre-allocated the RX socket buffer in order to
 * allow zero-copy mechanism.
 */
1168 1169 1170 1171
static int alloc_dma_desc_resources(struct stmmac_priv *priv)
{
	int ret = -ENOMEM;

1172
	priv->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, sizeof(dma_addr_t),
1173 1174 1175 1176
					    GFP_KERNEL);
	if (!priv->rx_skbuff_dma)
		return -ENOMEM;

1177
	priv->rx_skbuff = kmalloc_array(DMA_RX_SIZE, sizeof(struct sk_buff *),
1178 1179 1180 1181
					GFP_KERNEL);
	if (!priv->rx_skbuff)
		goto err_rx_skbuff;

1182
	priv->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
G
Giuseppe CAVALLARO 已提交
1183
					    sizeof(*priv->tx_skbuff_dma),
1184 1185 1186 1187
					    GFP_KERNEL);
	if (!priv->tx_skbuff_dma)
		goto err_tx_skbuff_dma;

1188
	priv->tx_skbuff = kmalloc_array(DMA_TX_SIZE, sizeof(struct sk_buff *),
1189 1190 1191 1192 1193
					GFP_KERNEL);
	if (!priv->tx_skbuff)
		goto err_tx_skbuff;

	if (priv->extend_desc) {
1194
		priv->dma_erx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE *
1195 1196 1197 1198
						    sizeof(struct
							   dma_extended_desc),
						    &priv->dma_rx_phy,
						    GFP_KERNEL);
1199 1200 1201
		if (!priv->dma_erx)
			goto err_dma;

1202
		priv->dma_etx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE *
1203 1204 1205 1206
						    sizeof(struct
							   dma_extended_desc),
						    &priv->dma_tx_phy,
						    GFP_KERNEL);
1207
		if (!priv->dma_etx) {
1208
			dma_free_coherent(priv->device, DMA_RX_SIZE *
1209 1210
					  sizeof(struct dma_extended_desc),
					  priv->dma_erx, priv->dma_rx_phy);
1211 1212 1213
			goto err_dma;
		}
	} else {
1214
		priv->dma_rx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE *
1215 1216 1217
						   sizeof(struct dma_desc),
						   &priv->dma_rx_phy,
						   GFP_KERNEL);
1218 1219 1220
		if (!priv->dma_rx)
			goto err_dma;

1221
		priv->dma_tx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE *
1222 1223 1224
						   sizeof(struct dma_desc),
						   &priv->dma_tx_phy,
						   GFP_KERNEL);
1225
		if (!priv->dma_tx) {
1226
			dma_free_coherent(priv->device, DMA_RX_SIZE *
1227 1228
					  sizeof(struct dma_desc),
					  priv->dma_rx, priv->dma_rx_phy);
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
			goto err_dma;
		}
	}

	return 0;

err_dma:
	kfree(priv->tx_skbuff);
err_tx_skbuff:
	kfree(priv->tx_skbuff_dma);
err_tx_skbuff_dma:
	kfree(priv->rx_skbuff);
err_rx_skbuff:
	kfree(priv->rx_skbuff_dma);
	return ret;
}

1246 1247 1248 1249 1250 1251
static void free_dma_desc_resources(struct stmmac_priv *priv)
{
	/* Release the DMA TX/RX socket buffers */
	dma_free_rx_skbufs(priv);
	dma_free_tx_skbufs(priv);

G
Giuseppe CAVALLARO 已提交
1252
	/* Free DMA regions of consistent memory previously allocated */
1253 1254
	if (!priv->extend_desc) {
		dma_free_coherent(priv->device,
1255
				  DMA_TX_SIZE * sizeof(struct dma_desc),
1256 1257
				  priv->dma_tx, priv->dma_tx_phy);
		dma_free_coherent(priv->device,
1258
				  DMA_RX_SIZE * sizeof(struct dma_desc),
1259 1260
				  priv->dma_rx, priv->dma_rx_phy);
	} else {
1261
		dma_free_coherent(priv->device, DMA_TX_SIZE *
1262 1263
				  sizeof(struct dma_extended_desc),
				  priv->dma_etx, priv->dma_tx_phy);
1264
		dma_free_coherent(priv->device, DMA_RX_SIZE *
1265 1266 1267
				  sizeof(struct dma_extended_desc),
				  priv->dma_erx, priv->dma_rx_phy);
	}
1268 1269
	kfree(priv->rx_skbuff_dma);
	kfree(priv->rx_skbuff);
1270
	kfree(priv->tx_skbuff_dma);
1271 1272 1273
	kfree(priv->tx_skbuff);
}

J
jpinto 已提交
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
/**
 *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
 *  @priv: driver private structure
 *  Description: It is used for enabling the rx queues in the MAC
 */
static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
{
	int rx_count = priv->dma_cap.number_rx_queues;
	int queue = 0;

	/* If GMAC does not have multiple queues, then this is not necessary*/
	if (rx_count == 1)
		return;

	/**
	 *  If the core is synthesized with multiple rx queues / multiple
	 *  dma channels, then rx queues will be disabled by default.
	 *  For now only rx queue 0 is enabled.
	 */
	priv->hw->mac->rx_queue_enable(priv->hw, queue);
}

1296 1297
/**
 *  stmmac_dma_operation_mode - HW DMA operation mode
1298
 *  @priv: driver private structure
1299 1300
 *  Description: it is used for configuring the DMA operation mode register in
 *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1301 1302 1303
 */
static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
{
1304 1305
	int rxfifosz = priv->plat->rx_fifo_size;

1306
	if (priv->plat->force_thresh_dma_mode)
1307
		priv->hw->dma->dma_mode(priv->ioaddr, tc, tc, rxfifosz);
1308
	else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1309 1310 1311
		/*
		 * In case of GMAC, SF mode can be enabled
		 * to perform the TX COE in HW. This depends on:
1312 1313 1314 1315
		 * 1) TX COE if actually supported
		 * 2) There is no bugged Jumbo frame support
		 *    that needs to not insert csum in the TDES.
		 */
1316 1317
		priv->hw->dma->dma_mode(priv->ioaddr, SF_DMA_MODE, SF_DMA_MODE,
					rxfifosz);
1318
		priv->xstats.threshold = SF_DMA_MODE;
1319
	} else
1320 1321
		priv->hw->dma->dma_mode(priv->ioaddr, tc, SF_DMA_MODE,
					rxfifosz);
1322 1323 1324
}

/**
1325
 * stmmac_tx_clean - to manage the transmission completion
1326
 * @priv: driver private structure
1327
 * Description: it reclaims the transmit resources after transmission completes.
1328
 */
1329
static void stmmac_tx_clean(struct stmmac_priv *priv)
1330
{
B
Beniamino Galvani 已提交
1331
	unsigned int bytes_compl = 0, pkts_compl = 0;
1332
	unsigned int entry = priv->dirty_tx;
1333

1334
	netif_tx_lock(priv->dev);
1335

1336 1337
	priv->xstats.tx_clean++;

1338
	while (entry != priv->cur_tx) {
1339
		struct sk_buff *skb = priv->tx_skbuff[entry];
1340
		struct dma_desc *p;
1341
		int status;
1342 1343

		if (priv->extend_desc)
G
Giuseppe CAVALLARO 已提交
1344
			p = (struct dma_desc *)(priv->dma_etx + entry);
1345 1346
		else
			p = priv->dma_tx + entry;
1347

1348
		status = priv->hw->desc->tx_status(&priv->dev->stats,
G
Giuseppe CAVALLARO 已提交
1349 1350
						      &priv->xstats, p,
						      priv->ioaddr);
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
		/* Check if the descriptor is owned by the DMA */
		if (unlikely(status & tx_dma_own))
			break;

		/* Just consider the last segment and ...*/
		if (likely(!(status & tx_not_ls))) {
			/* ... verify the status error condition */
			if (unlikely(status & tx_err)) {
				priv->dev->stats.tx_errors++;
			} else {
1361 1362
				priv->dev->stats.tx_packets++;
				priv->xstats.tx_pkt_n++;
1363
			}
1364
			stmmac_get_tx_hwtstamp(priv, p, skb);
1365 1366
		}

G
Giuseppe CAVALLARO 已提交
1367 1368 1369 1370
		if (likely(priv->tx_skbuff_dma[entry].buf)) {
			if (priv->tx_skbuff_dma[entry].map_as_page)
				dma_unmap_page(priv->device,
					       priv->tx_skbuff_dma[entry].buf,
1371
					       priv->tx_skbuff_dma[entry].len,
G
Giuseppe CAVALLARO 已提交
1372 1373 1374 1375
					       DMA_TO_DEVICE);
			else
				dma_unmap_single(priv->device,
						 priv->tx_skbuff_dma[entry].buf,
1376
						 priv->tx_skbuff_dma[entry].len,
G
Giuseppe CAVALLARO 已提交
1377 1378
						 DMA_TO_DEVICE);
			priv->tx_skbuff_dma[entry].buf = 0;
A
Alexandre TORGUE 已提交
1379
			priv->tx_skbuff_dma[entry].len = 0;
G
Giuseppe CAVALLARO 已提交
1380
			priv->tx_skbuff_dma[entry].map_as_page = false;
1381
		}
A
Alexandre TORGUE 已提交
1382 1383 1384 1385

		if (priv->hw->mode->clean_desc3)
			priv->hw->mode->clean_desc3(priv, p);

1386
		priv->tx_skbuff_dma[entry].last_segment = false;
1387
		priv->tx_skbuff_dma[entry].is_jumbo = false;
1388 1389

		if (likely(skb != NULL)) {
B
Beniamino Galvani 已提交
1390 1391
			pkts_compl++;
			bytes_compl += skb->len;
1392
			dev_consume_skb_any(skb);
1393 1394 1395
			priv->tx_skbuff[entry] = NULL;
		}

1396
		priv->hw->desc->release_tx_desc(p, priv->mode);
1397

1398
		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1399
	}
1400
	priv->dirty_tx = entry;
B
Beniamino Galvani 已提交
1401 1402 1403

	netdev_completed_queue(priv->dev, pkts_compl, bytes_compl);

1404
	if (unlikely(netif_queue_stopped(priv->dev) &&
1405 1406 1407 1408
	    stmmac_tx_avail(priv) > STMMAC_TX_THRESH)) {
		netif_dbg(priv, tx_done, priv->dev,
			  "%s: restart transmit\n", __func__);
		netif_wake_queue(priv->dev);
1409
	}
1410 1411 1412

	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
		stmmac_enable_eee_mode(priv);
G
Giuseppe CAVALLARO 已提交
1413
		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1414
	}
1415
	netif_tx_unlock(priv->dev);
1416 1417
}

1418
static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv)
1419
{
1420
	priv->hw->dma->enable_dma_irq(priv->ioaddr);
1421 1422
}

1423
static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv)
1424
{
1425
	priv->hw->dma->disable_dma_irq(priv->ioaddr);
1426 1427 1428
}

/**
1429
 * stmmac_tx_err - to manage the tx error
1430
 * @priv: driver private structure
1431
 * Description: it cleans the descriptors and restarts the transmission
1432
 * in case of transmission errors.
1433 1434 1435
 */
static void stmmac_tx_err(struct stmmac_priv *priv)
{
1436
	int i;
1437 1438
	netif_stop_queue(priv->dev);

1439
	priv->hw->dma->stop_tx(priv->ioaddr);
1440
	dma_free_tx_skbufs(priv);
1441
	for (i = 0; i < DMA_TX_SIZE; i++)
1442 1443 1444
		if (priv->extend_desc)
			priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic,
						     priv->mode,
1445
						     (i == DMA_TX_SIZE - 1));
1446 1447 1448
		else
			priv->hw->desc->init_tx_desc(&priv->dma_tx[i],
						     priv->mode,
1449
						     (i == DMA_TX_SIZE - 1));
1450 1451
	priv->dirty_tx = 0;
	priv->cur_tx = 0;
B
Beniamino Galvani 已提交
1452
	netdev_reset_queue(priv->dev);
1453
	priv->hw->dma->start_tx(priv->ioaddr);
1454 1455 1456 1457 1458

	priv->dev->stats.tx_errors++;
	netif_wake_queue(priv->dev);
}

1459
/**
1460
 * stmmac_dma_interrupt - DMA ISR
1461 1462
 * @priv: driver private structure
 * Description: this is the DMA ISR. It is called by the main ISR.
1463 1464
 * It calls the dwmac dma routine and schedule poll method in case of some
 * work can be done.
1465
 */
1466 1467 1468
static void stmmac_dma_interrupt(struct stmmac_priv *priv)
{
	int status;
1469
	int rxfifosz = priv->plat->rx_fifo_size;
1470

1471
	status = priv->hw->dma->dma_interrupt(priv->ioaddr, &priv->xstats);
1472 1473 1474 1475 1476 1477 1478
	if (likely((status & handle_rx)) || (status & handle_tx)) {
		if (likely(napi_schedule_prep(&priv->napi))) {
			stmmac_disable_dma_irq(priv);
			__napi_schedule(&priv->napi);
		}
	}
	if (unlikely(status & tx_hard_error_bump_tc)) {
1479
		/* Try to bump up the dma threshold on this failure */
1480 1481
		if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
		    (tc <= 256)) {
1482
			tc += 64;
1483
			if (priv->plat->force_thresh_dma_mode)
1484 1485
				priv->hw->dma->dma_mode(priv->ioaddr, tc, tc,
							rxfifosz);
1486 1487
			else
				priv->hw->dma->dma_mode(priv->ioaddr, tc,
1488
							SF_DMA_MODE, rxfifosz);
1489
			priv->xstats.threshold = tc;
1490
		}
1491 1492
	} else if (unlikely(status == tx_hard_error))
		stmmac_tx_err(priv);
1493 1494
}

1495 1496 1497 1498 1499
/**
 * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
 * @priv: driver private structure
 * Description: this masks the MMC irq, in fact, the counters are managed in SW.
 */
1500 1501 1502
static void stmmac_mmc_setup(struct stmmac_priv *priv)
{
	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
1503
			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
1504

1505 1506
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
A
Alexandre TORGUE 已提交
1507
		priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
1508 1509
	} else {
		priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
A
Alexandre TORGUE 已提交
1510
		priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
1511
	}
1512 1513

	dwmac_mmc_intr_all_mask(priv->mmcaddr);
G
Giuseppe CAVALLARO 已提交
1514 1515

	if (priv->dma_cap.rmon) {
1516
		dwmac_mmc_ctrl(priv->mmcaddr, mode);
G
Giuseppe CAVALLARO 已提交
1517 1518
		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
	} else
1519
		netdev_info(priv->dev, "No MAC Management Counters available\n");
1520 1521
}

1522
/**
1523
 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
1524 1525
 * @priv: driver private structure
 * Description: select the Enhanced/Alternate or Normal descriptors.
1526 1527
 * In case of Enhanced/Alternate, it checks if the extended descriptors are
 * supported by the HW capability register.
1528
 */
1529 1530 1531
static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
{
	if (priv->plat->enh_desc) {
1532
		dev_info(priv->device, "Enhanced/Alternate descriptors\n");
1533 1534 1535

		/* GMAC older than 3.50 has no extended descriptors */
		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
1536
			dev_info(priv->device, "Enabled extended descriptors\n");
1537 1538
			priv->extend_desc = 1;
		} else
1539
			dev_warn(priv->device, "Extended descriptors not supported\n");
1540

1541 1542
		priv->hw->desc = &enh_desc_ops;
	} else {
1543
		dev_info(priv->device, "Normal descriptors\n");
1544 1545 1546 1547 1548
		priv->hw->desc = &ndesc_ops;
	}
}

/**
1549
 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
1550
 * @priv: driver private structure
1551 1552 1553 1554 1555
 * Description:
 *  new GMAC chip generations have a new register to indicate the
 *  presence of the optional feature/functions.
 *  This can be also used to override the value passed through the
 *  platform and necessary for old MAC10/100 and GMAC chips.
1556 1557 1558
 */
static int stmmac_get_hw_features(struct stmmac_priv *priv)
{
1559
	u32 ret = 0;
1560

1561
	if (priv->hw->dma->get_hw_feature) {
1562 1563 1564
		priv->hw->dma->get_hw_feature(priv->ioaddr,
					      &priv->dma_cap);
		ret = 1;
1565
	}
1566

1567
	return ret;
1568 1569
}

1570
/**
1571
 * stmmac_check_ether_addr - check if the MAC addr is valid
1572 1573 1574 1575 1576
 * @priv: driver private structure
 * Description:
 * it is to verify if the MAC address is valid, in case of failures it
 * generates a random MAC address
 */
1577 1578 1579
static void stmmac_check_ether_addr(struct stmmac_priv *priv)
{
	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
1580
		priv->hw->mac->get_umac_addr(priv->hw,
1581
					     priv->dev->dev_addr, 0);
G
Giuseppe CAVALLARO 已提交
1582
		if (!is_valid_ether_addr(priv->dev->dev_addr))
1583
			eth_hw_addr_random(priv->dev);
1584 1585
		netdev_info(priv->dev, "device MAC address %pM\n",
			    priv->dev->dev_addr);
1586 1587 1588
	}
}

1589
/**
1590
 * stmmac_init_dma_engine - DMA init.
1591 1592 1593 1594 1595 1596
 * @priv: driver private structure
 * Description:
 * It inits the DMA invoking the specific MAC/GMAC callback.
 * Some DMA parameters can be passed from the platform;
 * in case of these are not passed a default is kept for the MAC or GMAC.
 */
1597 1598
static int stmmac_init_dma_engine(struct stmmac_priv *priv)
{
1599
	int atds = 0;
1600
	int ret = 0;
1601

1602 1603
	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
		dev_err(priv->device, "Invalid DMA configuration\n");
1604
		return -EINVAL;
1605 1606
	}

1607 1608 1609
	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
		atds = 1;

1610 1611 1612 1613 1614 1615
	ret = priv->hw->dma->reset(priv->ioaddr);
	if (ret) {
		dev_err(priv->device, "Failed to reset the dma\n");
		return ret;
	}

1616
	priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
1617
			    priv->dma_tx_phy, priv->dma_rx_phy, atds);
1618

A
Alexandre TORGUE 已提交
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		priv->rx_tail_addr = priv->dma_rx_phy +
			    (DMA_RX_SIZE * sizeof(struct dma_desc));
		priv->hw->dma->set_rx_tail_ptr(priv->ioaddr, priv->rx_tail_addr,
					       STMMAC_CHAN0);

		priv->tx_tail_addr = priv->dma_tx_phy +
			    (DMA_TX_SIZE * sizeof(struct dma_desc));
		priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr,
					       STMMAC_CHAN0);
	}

	if (priv->plat->axi && priv->hw->dma->axi)
1632 1633
		priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);

1634
	return ret;
1635 1636
}

1637
/**
1638
 * stmmac_tx_timer - mitigation sw timer for tx.
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
 * @data: data pointer
 * Description:
 * This is the timer handler to directly invoke the stmmac_tx_clean.
 */
static void stmmac_tx_timer(unsigned long data)
{
	struct stmmac_priv *priv = (struct stmmac_priv *)data;

	stmmac_tx_clean(priv);
}

/**
1651
 * stmmac_init_tx_coalesce - init tx mitigation options.
1652
 * @priv: driver private structure
1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
 * Description:
 * This inits the transmit coalesce parameters: i.e. timer rate,
 * timer handler and default threshold used for enabling the
 * interrupt on completion bit.
 */
static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
{
	priv->tx_coal_frames = STMMAC_TX_FRAMES;
	priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
	init_timer(&priv->txtimer);
	priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
	priv->txtimer.data = (unsigned long)priv;
	priv->txtimer.function = stmmac_tx_timer;
	add_timer(&priv->txtimer);
}

1669
/**
1670
 * stmmac_hw_setup - setup mac in a usable state.
1671 1672
 *  @dev : pointer to the device structure.
 *  Description:
1673 1674 1675 1676
 *  this is the main function to setup the HW in a usable state because the
 *  dma engine is reset, the core registers are configured (e.g. AXI,
 *  Checksum features, timers). The DMA is ready to start receiving and
 *  transmitting.
1677 1678 1679 1680
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
1681
static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
1682 1683 1684 1685 1686 1687 1688
{
	struct stmmac_priv *priv = netdev_priv(dev);
	int ret;

	/* DMA initialization and SW reset */
	ret = stmmac_init_dma_engine(priv);
	if (ret < 0) {
1689 1690
		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
			   __func__);
1691 1692 1693 1694
		return ret;
	}

	/* Copy the MAC addr into the HW  */
1695
	priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
1696 1697 1698 1699 1700

	/* If required, perform hw setup of the bus. */
	if (priv->plat->bus_setup)
		priv->plat->bus_setup(priv->ioaddr);

1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
	/* PS and related bits will be programmed according to the speed */
	if (priv->hw->pcs) {
		int speed = priv->plat->mac_port_sel_speed;

		if ((speed == SPEED_10) || (speed == SPEED_100) ||
		    (speed == SPEED_1000)) {
			priv->hw->ps = speed;
		} else {
			dev_warn(priv->device, "invalid port speed\n");
			priv->hw->ps = 0;
		}
	}

1714
	/* Initialize the MAC Core */
1715
	priv->hw->mac->core_init(priv->hw, dev->mtu);
1716

J
jpinto 已提交
1717 1718 1719 1720
	/* Initialize MAC RX Queues */
	if (priv->hw->mac->rx_queue_enable)
		stmmac_mac_enable_rx_queues(priv);

1721 1722
	ret = priv->hw->mac->rx_ipc(priv->hw);
	if (!ret) {
1723
		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
1724
		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
1725
		priv->hw->rx_csum = 0;
1726 1727
	}

1728
	/* Enable the MAC Rx/Tx */
A
Alexandre TORGUE 已提交
1729 1730 1731 1732
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
		stmmac_dwmac4_set_mac(priv->ioaddr, true);
	else
		stmmac_set_mac(priv->ioaddr, true);
1733 1734 1735 1736 1737 1738

	/* Set the HW DMA mode and the COE */
	stmmac_dma_operation_mode(priv);

	stmmac_mmc_setup(priv);

1739 1740
	if (init_ptp) {
		ret = stmmac_init_ptp(priv);
1741
		if (ret)
1742
			netdev_warn(priv->dev, "fail to init PTP.\n");
1743
	}
1744

1745
#ifdef CONFIG_DEBUG_FS
1746 1747
	ret = stmmac_init_fs(dev);
	if (ret < 0)
1748 1749
		netdev_warn(priv->dev, "%s: failed debugFS registration\n",
			    __func__);
1750 1751
#endif
	/* Start the ball rolling... */
1752
	netdev_dbg(priv->dev, "DMA RX/TX processes started...\n");
1753 1754 1755 1756 1757
	priv->hw->dma->start_tx(priv->ioaddr);
	priv->hw->dma->start_rx(priv->ioaddr);

	/* Dump DMA/MAC registers */
	if (netif_msg_hw(priv)) {
1758
		priv->hw->mac->dump_regs(priv->hw);
1759 1760 1761 1762 1763 1764 1765 1766 1767
		priv->hw->dma->dump_regs(priv->ioaddr);
	}
	priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;

	if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
		priv->rx_riwt = MAX_DMA_RIWT;
		priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT);
	}

1768
	if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
1769
		priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
1770

A
Alexandre TORGUE 已提交
1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
	/*  set TX ring length */
	if (priv->hw->dma->set_tx_ring_len)
		priv->hw->dma->set_tx_ring_len(priv->ioaddr,
					       (DMA_TX_SIZE - 1));
	/*  set RX ring length */
	if (priv->hw->dma->set_rx_ring_len)
		priv->hw->dma->set_rx_ring_len(priv->ioaddr,
					       (DMA_RX_SIZE - 1));
	/* Enable TSO */
	if (priv->tso)
		priv->hw->dma->enable_tso(priv->ioaddr, 1, STMMAC_CHAN0);

1783 1784 1785
	return 0;
}

1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
/**
 *  stmmac_open - open entry point of the driver
 *  @dev : pointer to the device structure.
 *  Description:
 *  This function is the open entry point of the driver.
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
static int stmmac_open(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	int ret;

1800 1801
	stmmac_check_ether_addr(priv);

1802 1803 1804
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI) {
1805 1806
		ret = stmmac_init_phy(dev);
		if (ret) {
1807 1808 1809
			netdev_err(priv->dev,
				   "%s: Cannot attach to PHY (error: %d)\n",
				   __func__, ret);
1810
			return ret;
1811
		}
1812
	}
1813

1814 1815 1816 1817
	/* Extra statistics */
	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
	priv->xstats.threshold = tc;

1818
	priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
1819
	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
1820

1821
	ret = alloc_dma_desc_resources(priv);
1822
	if (ret < 0) {
1823 1824
		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
			   __func__);
1825 1826 1827
		goto dma_desc_error;
	}

1828 1829
	ret = init_dma_desc_rings(dev, GFP_KERNEL);
	if (ret < 0) {
1830 1831
		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
			   __func__);
1832 1833 1834
		goto init_error;
	}

1835
	ret = stmmac_hw_setup(dev, true);
1836
	if (ret < 0) {
1837
		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
1838
		goto init_error;
1839 1840
	}

1841 1842
	stmmac_init_tx_coalesce(priv);

1843 1844
	if (dev->phydev)
		phy_start(dev->phydev);
1845

1846 1847
	/* Request the IRQ lines */
	ret = request_irq(dev->irq, stmmac_interrupt,
G
Giuseppe CAVALLARO 已提交
1848
			  IRQF_SHARED, dev->name, dev);
1849
	if (unlikely(ret < 0)) {
1850 1851 1852
		netdev_err(priv->dev,
			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
			   __func__, dev->irq, ret);
1853
		goto init_error;
1854 1855
	}

1856 1857 1858 1859 1860
	/* Request the Wake IRQ in case of another line is used for WoL */
	if (priv->wol_irq != dev->irq) {
		ret = request_irq(priv->wol_irq, stmmac_interrupt,
				  IRQF_SHARED, dev->name, dev);
		if (unlikely(ret < 0)) {
1861 1862 1863
			netdev_err(priv->dev,
				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
				   __func__, priv->wol_irq, ret);
1864
			goto wolirq_error;
1865 1866 1867
		}
	}

1868
	/* Request the IRQ lines */
1869
	if (priv->lpi_irq > 0) {
1870 1871 1872
		ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
				  dev->name, dev);
		if (unlikely(ret < 0)) {
1873 1874 1875
			netdev_err(priv->dev,
				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
				   __func__, priv->lpi_irq, ret);
1876
			goto lpiirq_error;
1877 1878 1879
		}
	}

1880 1881
	napi_enable(&priv->napi);
	netif_start_queue(dev);
1882

1883
	return 0;
1884

1885
lpiirq_error:
1886 1887
	if (priv->wol_irq != dev->irq)
		free_irq(priv->wol_irq, dev);
1888
wolirq_error:
1889 1890
	free_irq(dev->irq, dev);

1891 1892
init_error:
	free_dma_desc_resources(priv);
1893
dma_desc_error:
1894 1895
	if (dev->phydev)
		phy_disconnect(dev->phydev);
1896

1897
	return ret;
1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
}

/**
 *  stmmac_release - close entry point of the driver
 *  @dev : device pointer.
 *  Description:
 *  This is the stop entry point of the driver.
 */
static int stmmac_release(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);

1910 1911 1912
	if (priv->eee_enabled)
		del_timer_sync(&priv->eee_ctrl_timer);

1913
	/* Stop and disconnect the PHY */
1914 1915 1916
	if (dev->phydev) {
		phy_stop(dev->phydev);
		phy_disconnect(dev->phydev);
1917 1918 1919 1920 1921 1922
	}

	netif_stop_queue(dev);

	napi_disable(&priv->napi);

1923 1924
	del_timer_sync(&priv->txtimer);

1925 1926
	/* Free the IRQ lines */
	free_irq(dev->irq, dev);
1927 1928
	if (priv->wol_irq != dev->irq)
		free_irq(priv->wol_irq, dev);
1929
	if (priv->lpi_irq > 0)
1930
		free_irq(priv->lpi_irq, dev);
1931 1932

	/* Stop TX/RX DMA and clear the descriptors */
1933 1934
	priv->hw->dma->stop_tx(priv->ioaddr);
	priv->hw->dma->stop_rx(priv->ioaddr);
1935 1936 1937 1938

	/* Release and free the Rx/Tx resources */
	free_dma_desc_resources(priv);

1939
	/* Disable the MAC Rx/Tx */
1940
	stmmac_set_mac(priv->ioaddr, false);
1941 1942 1943

	netif_carrier_off(dev);

1944
#ifdef CONFIG_DEBUG_FS
1945
	stmmac_exit_fs(dev);
1946 1947
#endif

1948 1949
	stmmac_release_ptp(priv);

1950 1951 1952
	return 0;
}

A
Alexandre TORGUE 已提交
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975
/**
 *  stmmac_tso_allocator - close entry point of the driver
 *  @priv: driver private structure
 *  @des: buffer start address
 *  @total_len: total length to fill in descriptors
 *  @last_segmant: condition for the last descriptor
 *  Description:
 *  This function fills descriptor and request new descriptors according to
 *  buffer length to fill
 */
static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
				 int total_len, bool last_segment)
{
	struct dma_desc *desc;
	int tmp_len;
	u32 buff_size;

	tmp_len = total_len;

	while (tmp_len > 0) {
		priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);
		desc = priv->dma_tx + priv->cur_tx;

1976
		desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
A
Alexandre TORGUE 已提交
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035
		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
			    TSO_MAX_BUFF_SIZE : tmp_len;

		priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
			0, 1,
			(last_segment) && (buff_size < TSO_MAX_BUFF_SIZE),
			0, 0);

		tmp_len -= TSO_MAX_BUFF_SIZE;
	}
}

/**
 *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
 *  @skb : the socket buffer
 *  @dev : device pointer
 *  Description: this is the transmit function that is called on TSO frames
 *  (support available on GMAC4 and newer chips).
 *  Diagram below show the ring programming in case of TSO frames:
 *
 *  First Descriptor
 *   --------
 *   | DES0 |---> buffer1 = L2/L3/L4 header
 *   | DES1 |---> TCP Payload (can continue on next descr...)
 *   | DES2 |---> buffer 1 and 2 len
 *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
 *   --------
 *	|
 *     ...
 *	|
 *   --------
 *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
 *   | DES1 | --|
 *   | DES2 | --> buffer 1 and 2 len
 *   | DES3 |
 *   --------
 *
 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
 */
static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
{
	u32 pay_len, mss;
	int tmp_pay_len = 0;
	struct stmmac_priv *priv = netdev_priv(dev);
	int nfrags = skb_shinfo(skb)->nr_frags;
	unsigned int first_entry, des;
	struct dma_desc *desc, *first, *mss_desc = NULL;
	u8 proto_hdr_len;
	int i;

	/* Compute header lengths */
	proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);

	/* Desc availability based on threshold should be enough safe */
	if (unlikely(stmmac_tx_avail(priv) <
		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
		if (!netif_queue_stopped(dev)) {
			netif_stop_queue(dev);
			/* This is a hard error, log it. */
2036 2037 2038
			netdev_err(priv->dev,
				   "%s: Tx Ring full when queue awake\n",
				   __func__);
A
Alexandre TORGUE 已提交
2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
		}
		return NETDEV_TX_BUSY;
	}

	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */

	mss = skb_shinfo(skb)->gso_size;

	/* set new MSS value if needed */
	if (mss != priv->mss) {
		mss_desc = priv->dma_tx + priv->cur_tx;
		priv->hw->desc->set_mss(mss_desc, mss);
		priv->mss = mss;
		priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);
	}

	if (netif_msg_tx_queued(priv)) {
		pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
			__func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
			skb->data_len);
	}

	first_entry = priv->cur_tx;

	desc = priv->dma_tx + first_entry;
	first = desc;

	/* first descriptor: fill Headers on Buf1 */
	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
			     DMA_TO_DEVICE);
	if (dma_mapping_error(priv->device, des))
		goto dma_map_err;

	priv->tx_skbuff_dma[first_entry].buf = des;
	priv->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
	priv->tx_skbuff[first_entry] = skb;

2077
	first->des0 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
2078 2079 2080

	/* Fill start of payload in buff2 of first descriptor */
	if (pay_len)
2081
		first->des1 = cpu_to_le32(des + proto_hdr_len);
A
Alexandre TORGUE 已提交
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109

	/* If needed take extra descriptors to fill the remaining payload */
	tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;

	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0));

	/* Prepare fragments */
	for (i = 0; i < nfrags; i++) {
		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

		des = skb_frag_dma_map(priv->device, frag, 0,
				       skb_frag_size(frag),
				       DMA_TO_DEVICE);

		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
				     (i == nfrags - 1));

		priv->tx_skbuff_dma[priv->cur_tx].buf = des;
		priv->tx_skbuff_dma[priv->cur_tx].len = skb_frag_size(frag);
		priv->tx_skbuff[priv->cur_tx] = NULL;
		priv->tx_skbuff_dma[priv->cur_tx].map_as_page = true;
	}

	priv->tx_skbuff_dma[priv->cur_tx].last_segment = true;

	priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);

	if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) {
2110 2111
		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
			  __func__);
A
Alexandre TORGUE 已提交
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
		netif_stop_queue(dev);
	}

	dev->stats.tx_bytes += skb->len;
	priv->xstats.tx_tso_frames++;
	priv->xstats.tx_tso_nfrags += nfrags;

	/* Manage tx mitigation */
	priv->tx_count_frames += nfrags + 1;
	if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
		mod_timer(&priv->txtimer,
			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
	} else {
		priv->tx_count_frames = 0;
		priv->hw->desc->set_tx_ic(desc);
		priv->xstats.tx_set_ic_bit++;
	}

	if (!priv->hwts_tx_en)
		skb_tx_timestamp(skb);

	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
		     priv->hwts_tx_en)) {
		/* declare that device is doing timestamping */
		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
		priv->hw->desc->enable_tx_timestamp(first);
	}

	/* Complete the first descriptor before granting the DMA */
	priv->hw->desc->prepare_tso_tx_desc(first, 1,
			proto_hdr_len,
			pay_len,
			1, priv->tx_skbuff_dma[first_entry].last_segment,
			tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));

	/* If context desc is used to change MSS */
	if (mss_desc)
		priv->hw->desc->set_tx_owner(mss_desc);

	/* The own bit must be the latest setting done when prepare the
	 * descriptor and then barrier is needed to make sure that
	 * all is coherent before granting the DMA engine.
	 */
P
Pavel Machek 已提交
2155
	dma_wmb();
A
Alexandre TORGUE 已提交
2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182

	if (netif_msg_pktdata(priv)) {
		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
			__func__, priv->cur_tx, priv->dirty_tx, first_entry,
			priv->cur_tx, first, nfrags);

		priv->hw->desc->display_ring((void *)priv->dma_tx, DMA_TX_SIZE,
					     0);

		pr_info(">>> frame to be transmitted: ");
		print_pkt(skb->data, skb_headlen(skb));
	}

	netdev_sent_queue(dev, skb->len);

	priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr,
				       STMMAC_CHAN0);

	return NETDEV_TX_OK;

dma_map_err:
	dev_err(priv->device, "Tx dma map failed\n");
	dev_kfree_skb(skb);
	priv->dev->stats.tx_dropped++;
	return NETDEV_TX_OK;
}

2183
/**
2184
 *  stmmac_xmit - Tx entry point of the driver
2185 2186
 *  @skb : the socket buffer
 *  @dev : device pointer
2187 2188 2189
 *  Description : this is the tx entry point of the driver.
 *  It programs the chain or the ring and supports oversized frames
 *  and SG feature.
2190 2191 2192 2193
 */
static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
2194
	unsigned int nopaged_len = skb_headlen(skb);
2195
	int i, csum_insertion = 0, is_jumbo = 0;
2196
	int nfrags = skb_shinfo(skb)->nr_frags;
2197
	unsigned int entry, first_entry;
2198
	struct dma_desc *desc, *first;
2199
	unsigned int enh_desc;
A
Alexandre TORGUE 已提交
2200 2201 2202 2203 2204 2205 2206
	unsigned int des;

	/* Manage oversized TCP frames for GMAC4 device */
	if (skb_is_gso(skb) && priv->tso) {
		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
			return stmmac_tso_xmit(skb, dev);
	}
2207 2208 2209 2210 2211

	if (unlikely(stmmac_tx_avail(priv) < nfrags + 1)) {
		if (!netif_queue_stopped(dev)) {
			netif_stop_queue(dev);
			/* This is a hard error, log it. */
2212 2213 2214
			netdev_err(priv->dev,
				   "%s: Tx Ring full when queue awake\n",
				   __func__);
2215 2216 2217 2218
		}
		return NETDEV_TX_BUSY;
	}

2219 2220 2221
	if (priv->tx_path_in_lpi_mode)
		stmmac_disable_eee_mode(priv);

2222
	entry = priv->cur_tx;
2223
	first_entry = entry;
2224

2225
	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2226

2227
	if (likely(priv->extend_desc))
G
Giuseppe CAVALLARO 已提交
2228
		desc = (struct dma_desc *)(priv->dma_etx + entry);
2229 2230 2231
	else
		desc = priv->dma_tx + entry;

2232 2233
	first = desc;

2234 2235 2236
	priv->tx_skbuff[first_entry] = skb;

	enh_desc = priv->plat->enh_desc;
2237
	/* To program the descriptors according to the size of the frame */
G
Giuseppe CAVALLARO 已提交
2238 2239 2240
	if (enh_desc)
		is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);

A
Alexandre TORGUE 已提交
2241 2242
	if (unlikely(is_jumbo) && likely(priv->synopsys_id <
					 DWMAC_CORE_4_00)) {
G
Giuseppe CAVALLARO 已提交
2243
		entry = priv->hw->mode->jumbo_frm(priv, skb, csum_insertion);
G
Giuseppe CAVALLARO 已提交
2244 2245
		if (unlikely(entry < 0))
			goto dma_map_err;
G
Giuseppe CAVALLARO 已提交
2246
	}
2247 2248

	for (i = 0; i < nfrags; i++) {
E
Eric Dumazet 已提交
2249 2250
		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
		int len = skb_frag_size(frag);
2251
		bool last_segment = (i == (nfrags - 1));
2252

2253 2254
		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);

2255
		if (likely(priv->extend_desc))
G
Giuseppe CAVALLARO 已提交
2256
			desc = (struct dma_desc *)(priv->dma_etx + entry);
2257 2258
		else
			desc = priv->dma_tx + entry;
2259

A
Alexandre TORGUE 已提交
2260 2261 2262
		des = skb_frag_dma_map(priv->device, frag, 0, len,
				       DMA_TO_DEVICE);
		if (dma_mapping_error(priv->device, des))
G
Giuseppe CAVALLARO 已提交
2263 2264
			goto dma_map_err; /* should reuse desc w/o issues */

2265
		priv->tx_skbuff[entry] = NULL;
A
Alexandre TORGUE 已提交
2266

2267 2268 2269 2270 2271
		priv->tx_skbuff_dma[entry].buf = des;
		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
			desc->des0 = cpu_to_le32(des);
		else
			desc->des2 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
2272

G
Giuseppe CAVALLARO 已提交
2273
		priv->tx_skbuff_dma[entry].map_as_page = true;
2274
		priv->tx_skbuff_dma[entry].len = len;
2275 2276 2277
		priv->tx_skbuff_dma[entry].last_segment = last_segment;

		/* Prepare the descriptor and set the own bit too */
2278
		priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
2279
						priv->mode, 1, last_segment);
2280 2281
	}

2282 2283 2284
	entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);

	priv->cur_tx = entry;
2285 2286

	if (netif_msg_pktdata(priv)) {
2287 2288
		void *tx_head;

2289 2290 2291 2292
		netdev_dbg(priv->dev,
			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
			   __func__, priv->cur_tx, priv->dirty_tx, first_entry,
			   entry, first, nfrags);
2293

2294
		if (priv->extend_desc)
2295
			tx_head = (void *)priv->dma_etx;
2296
		else
2297 2298 2299
			tx_head = (void *)priv->dma_tx;

		priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
2300

2301
		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
2302 2303
		print_pkt(skb->data, skb->len);
	}
2304

2305
	if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) {
2306 2307
		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
			  __func__);
2308 2309 2310 2311 2312
		netif_stop_queue(dev);
	}

	dev->stats.tx_bytes += skb->len;

2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
	/* According to the coalesce parameter the IC bit for the latest
	 * segment is reset and the timer re-started to clean the tx status.
	 * This approach takes care about the fragments: desc is the first
	 * element in case of no SG.
	 */
	priv->tx_count_frames += nfrags + 1;
	if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
		mod_timer(&priv->txtimer,
			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
	} else {
		priv->tx_count_frames = 0;
		priv->hw->desc->set_tx_ic(desc);
		priv->xstats.tx_set_ic_bit++;
2326 2327 2328 2329
	}

	if (!priv->hwts_tx_en)
		skb_tx_timestamp(skb);
2330

2331 2332 2333 2334 2335 2336 2337
	/* Ready to fill the first descriptor and set the OWN bit w/o any
	 * problems because all the descriptors are actually ready to be
	 * passed to the DMA engine.
	 */
	if (likely(!is_jumbo)) {
		bool last_segment = (nfrags == 0);

A
Alexandre TORGUE 已提交
2338 2339 2340
		des = dma_map_single(priv->device, skb->data,
				     nopaged_len, DMA_TO_DEVICE);
		if (dma_mapping_error(priv->device, des))
2341 2342
			goto dma_map_err;

2343 2344 2345 2346 2347
		priv->tx_skbuff_dma[first_entry].buf = des;
		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
			first->des0 = cpu_to_le32(des);
		else
			first->des2 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
2348

2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
		priv->tx_skbuff_dma[first_entry].len = nopaged_len;
		priv->tx_skbuff_dma[first_entry].last_segment = last_segment;

		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
			     priv->hwts_tx_en)) {
			/* declare that device is doing timestamping */
			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
			priv->hw->desc->enable_tx_timestamp(first);
		}

		/* Prepare the first descriptor setting the OWN bit too */
		priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
						csum_insertion, priv->mode, 1,
						last_segment);

		/* The own bit must be the latest setting done when prepare the
		 * descriptor and then barrier is needed to make sure that
		 * all is coherent before granting the DMA engine.
		 */
P
Pavel Machek 已提交
2368
		dma_wmb();
2369 2370
	}

B
Beniamino Galvani 已提交
2371
	netdev_sent_queue(dev, skb->len);
A
Alexandre TORGUE 已提交
2372 2373 2374 2375 2376 2377

	if (priv->synopsys_id < DWMAC_CORE_4_00)
		priv->hw->dma->enable_dma_transmission(priv->ioaddr);
	else
		priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr,
					       STMMAC_CHAN0);
2378

G
Giuseppe CAVALLARO 已提交
2379
	return NETDEV_TX_OK;
2380

G
Giuseppe CAVALLARO 已提交
2381
dma_map_err:
2382
	netdev_err(priv->dev, "Tx DMA map failed\n");
G
Giuseppe CAVALLARO 已提交
2383 2384
	dev_kfree_skb(skb);
	priv->dev->stats.tx_dropped++;
2385 2386 2387
	return NETDEV_TX_OK;
}

2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404
static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
{
	struct ethhdr *ehdr;
	u16 vlanid;

	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
	    NETIF_F_HW_VLAN_CTAG_RX &&
	    !__vlan_get_tag(skb, &vlanid)) {
		/* pop the vlan tag */
		ehdr = (struct ethhdr *)skb->data;
		memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
		skb_pull(skb, VLAN_HLEN);
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
	}
}


2405 2406 2407 2408 2409 2410 2411 2412
static inline int stmmac_rx_threshold_count(struct stmmac_priv *priv)
{
	if (priv->rx_zeroc_thresh < STMMAC_RX_THRESH)
		return 0;

	return 1;
}

2413
/**
2414
 * stmmac_rx_refill - refill used skb preallocated buffers
2415 2416 2417 2418
 * @priv: driver private structure
 * Description : this is to reallocate the skb for the reception process
 * that is based on zero-copy.
 */
2419 2420 2421
static inline void stmmac_rx_refill(struct stmmac_priv *priv)
{
	int bfsize = priv->dma_buf_sz;
2422 2423
	unsigned int entry = priv->dirty_rx;
	int dirty = stmmac_rx_dirty(priv);
2424

2425
	while (dirty-- > 0) {
2426 2427 2428
		struct dma_desc *p;

		if (priv->extend_desc)
G
Giuseppe CAVALLARO 已提交
2429
			p = (struct dma_desc *)(priv->dma_erx + entry);
2430 2431 2432
		else
			p = priv->dma_rx + entry;

2433 2434 2435
		if (likely(priv->rx_skbuff[entry] == NULL)) {
			struct sk_buff *skb;

E
Eric Dumazet 已提交
2436
			skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
2437 2438 2439 2440 2441 2442 2443
			if (unlikely(!skb)) {
				/* so for a while no zero-copy! */
				priv->rx_zeroc_thresh = STMMAC_RX_THRESH;
				if (unlikely(net_ratelimit()))
					dev_err(priv->device,
						"fail to alloc skb entry %d\n",
						entry);
2444
				break;
2445
			}
2446 2447 2448 2449 2450

			priv->rx_skbuff[entry] = skb;
			priv->rx_skbuff_dma[entry] =
			    dma_map_single(priv->device, skb->data, bfsize,
					   DMA_FROM_DEVICE);
G
Giuseppe CAVALLARO 已提交
2451 2452
			if (dma_mapping_error(priv->device,
					      priv->rx_skbuff_dma[entry])) {
2453
				netdev_err(priv->dev, "Rx DMA map failed\n");
G
Giuseppe CAVALLARO 已提交
2454 2455 2456
				dev_kfree_skb(skb);
				break;
			}
2457

A
Alexandre TORGUE 已提交
2458
			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
2459
				p->des0 = cpu_to_le32(priv->rx_skbuff_dma[entry]);
A
Alexandre TORGUE 已提交
2460 2461
				p->des1 = 0;
			} else {
2462
				p->des2 = cpu_to_le32(priv->rx_skbuff_dma[entry]);
A
Alexandre TORGUE 已提交
2463 2464 2465
			}
			if (priv->hw->mode->refill_desc3)
				priv->hw->mode->refill_desc3(priv, p);
2466

2467 2468 2469
			if (priv->rx_zeroc_thresh > 0)
				priv->rx_zeroc_thresh--;

2470 2471
			netif_dbg(priv, rx_status, priv->dev,
				  "refill entry #%d\n", entry);
2472
		}
P
Pavel Machek 已提交
2473
		dma_wmb();
A
Alexandre TORGUE 已提交
2474 2475 2476 2477 2478 2479

		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
			priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
		else
			priv->hw->desc->set_rx_owner(p);

P
Pavel Machek 已提交
2480
		dma_wmb();
2481 2482

		entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
2483
	}
2484
	priv->dirty_rx = entry;
2485 2486
}

2487
/**
2488
 * stmmac_rx - manage the receive process
2489 2490 2491 2492 2493
 * @priv: driver private structure
 * @limit: napi bugget.
 * Description :  this the function called by the napi poll method.
 * It gets all the frames inside the ring.
 */
2494 2495
static int stmmac_rx(struct stmmac_priv *priv, int limit)
{
2496
	unsigned int entry = priv->cur_rx;
2497 2498
	unsigned int next_entry;
	unsigned int count = 0;
2499
	int coe = priv->hw->rx_csum;
2500

2501
	if (netif_msg_rx_status(priv)) {
2502 2503
		void *rx_head;

2504
		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
2505
		if (priv->extend_desc)
2506
			rx_head = (void *)priv->dma_erx;
2507
		else
2508 2509 2510
			rx_head = (void *)priv->dma_rx;

		priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
2511
	}
2512
	while (count < limit) {
2513
		int status;
2514
		struct dma_desc *p;
2515
		struct dma_desc *np;
2516

2517
		if (priv->extend_desc)
G
Giuseppe CAVALLARO 已提交
2518
			p = (struct dma_desc *)(priv->dma_erx + entry);
2519
		else
G
Giuseppe CAVALLARO 已提交
2520
			p = priv->dma_rx + entry;
2521

2522 2523 2524 2525 2526
		/* read the status of the incoming frame */
		status = priv->hw->desc->rx_status(&priv->dev->stats,
						   &priv->xstats, p);
		/* check if managed by the DMA otherwise go ahead */
		if (unlikely(status & dma_own))
2527 2528 2529 2530
			break;

		count++;

2531 2532 2533
		priv->cur_rx = STMMAC_GET_ENTRY(priv->cur_rx, DMA_RX_SIZE);
		next_entry = priv->cur_rx;

2534
		if (priv->extend_desc)
2535
			np = (struct dma_desc *)(priv->dma_erx + next_entry);
2536
		else
2537 2538 2539
			np = priv->dma_rx + next_entry;

		prefetch(np);
2540

2541 2542 2543 2544 2545
		if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
			priv->hw->desc->rx_extended_status(&priv->dev->stats,
							   &priv->xstats,
							   priv->dma_erx +
							   entry);
2546
		if (unlikely(status == discard_frame)) {
2547
			priv->dev->stats.rx_errors++;
2548 2549 2550 2551 2552 2553 2554 2555
			if (priv->hwts_rx_en && !priv->extend_desc) {
				/* DESC2 & DESC3 will be overwitten by device
				 * with timestamp value, hence reinitialize
				 * them in stmmac_rx_refill() function so that
				 * device can reuse it.
				 */
				priv->rx_skbuff[entry] = NULL;
				dma_unmap_single(priv->device,
G
Giuseppe CAVALLARO 已提交
2556 2557 2558
						 priv->rx_skbuff_dma[entry],
						 priv->dma_buf_sz,
						 DMA_FROM_DEVICE);
2559 2560
			}
		} else {
2561
			struct sk_buff *skb;
2562
			int frame_len;
A
Alexandre TORGUE 已提交
2563 2564 2565
			unsigned int des;

			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2566
				des = le32_to_cpu(p->des0);
A
Alexandre TORGUE 已提交
2567
			else
2568
				des = le32_to_cpu(p->des2);
2569

G
Giuseppe CAVALLARO 已提交
2570 2571
			frame_len = priv->hw->desc->get_rx_frame_len(p, coe);

A
Alexandre TORGUE 已提交
2572 2573 2574 2575
			/*  If frame length is greather than skb buffer size
			 *  (preallocated during init) then the packet is
			 *  ignored
			 */
2576
			if (frame_len > priv->dma_buf_sz) {
2577 2578 2579
				netdev_err(priv->dev,
					   "len %d larger than size (%d)\n",
					   frame_len, priv->dma_buf_sz);
2580 2581 2582 2583
				priv->dev->stats.rx_length_errors++;
				break;
			}

2584
			/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
G
Giuseppe CAVALLARO 已提交
2585 2586
			 * Type frames (LLC/LLC-SNAP)
			 */
2587 2588
			if (unlikely(status != llc_snap))
				frame_len -= ETH_FCS_LEN;
2589

2590
			if (netif_msg_rx_status(priv)) {
2591 2592
				netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
					   p, entry, des);
2593
				if (frame_len > ETH_FRAME_LEN)
2594 2595
					netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
						   frame_len, status);
2596
			}
2597

A
Alexandre TORGUE 已提交
2598 2599 2600 2601 2602 2603 2604
			/* The zero-copy is always used for all the sizes
			 * in case of GMAC4 because it needs
			 * to refill the used descriptors, always.
			 */
			if (unlikely(!priv->plat->has_gmac4 &&
				     ((frame_len < priv->rx_copybreak) ||
				     stmmac_rx_threshold_count(priv)))) {
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631
				skb = netdev_alloc_skb_ip_align(priv->dev,
								frame_len);
				if (unlikely(!skb)) {
					if (net_ratelimit())
						dev_warn(priv->device,
							 "packet dropped\n");
					priv->dev->stats.rx_dropped++;
					break;
				}

				dma_sync_single_for_cpu(priv->device,
							priv->rx_skbuff_dma
							[entry], frame_len,
							DMA_FROM_DEVICE);
				skb_copy_to_linear_data(skb,
							priv->
							rx_skbuff[entry]->data,
							frame_len);

				skb_put(skb, frame_len);
				dma_sync_single_for_device(priv->device,
							   priv->rx_skbuff_dma
							   [entry], frame_len,
							   DMA_FROM_DEVICE);
			} else {
				skb = priv->rx_skbuff[entry];
				if (unlikely(!skb)) {
2632 2633 2634
					netdev_err(priv->dev,
						   "%s: Inconsistent Rx chain\n",
						   priv->dev->name);
2635 2636 2637 2638 2639
					priv->dev->stats.rx_dropped++;
					break;
				}
				prefetch(skb->data - NET_IP_ALIGN);
				priv->rx_skbuff[entry] = NULL;
2640
				priv->rx_zeroc_thresh++;
2641 2642 2643 2644 2645 2646

				skb_put(skb, frame_len);
				dma_unmap_single(priv->device,
						 priv->rx_skbuff_dma[entry],
						 priv->dma_buf_sz,
						 DMA_FROM_DEVICE);
2647 2648 2649
			}

			if (netif_msg_pktdata(priv)) {
2650 2651
				netdev_dbg(priv->dev, "frame received (%dbytes)",
					   frame_len);
2652 2653
				print_pkt(skb->data, frame_len);
			}
2654

2655 2656
			stmmac_get_rx_hwtstamp(priv, p, np, skb);

2657 2658
			stmmac_rx_vlan(priv->dev, skb);

2659 2660
			skb->protocol = eth_type_trans(skb, priv->dev);

G
Giuseppe CAVALLARO 已提交
2661
			if (unlikely(!coe))
2662
				skb_checksum_none_assert(skb);
2663
			else
2664
				skb->ip_summed = CHECKSUM_UNNECESSARY;
2665 2666

			napi_gro_receive(&priv->napi, skb);
2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686

			priv->dev->stats.rx_packets++;
			priv->dev->stats.rx_bytes += frame_len;
		}
		entry = next_entry;
	}

	stmmac_rx_refill(priv);

	priv->xstats.rx_pkt_n += count;

	return count;
}

/**
 *  stmmac_poll - stmmac poll method (NAPI)
 *  @napi : pointer to the napi structure.
 *  @budget : maximum number of packets that the current CPU can receive from
 *	      all interfaces.
 *  Description :
2687
 *  To look at the incoming frames and clear the tx resources.
2688 2689 2690 2691 2692 2693
 */
static int stmmac_poll(struct napi_struct *napi, int budget)
{
	struct stmmac_priv *priv = container_of(napi, struct stmmac_priv, napi);
	int work_done = 0;

2694 2695
	priv->xstats.napi_poll++;
	stmmac_tx_clean(priv);
2696

2697
	work_done = stmmac_rx(priv, budget);
2698 2699
	if (work_done < budget) {
		napi_complete(napi);
2700
		stmmac_enable_dma_irq(priv);
2701 2702 2703 2704 2705 2706 2707 2708
	}
	return work_done;
}

/**
 *  stmmac_tx_timeout
 *  @dev : Pointer to net device structure
 *  Description: this function is called when a packet transmission fails to
2709
 *   complete within a reasonable time. The driver will mark the error in the
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721
 *   netdev structure and arrange for the device to be reset to a sane state
 *   in order to transmit a new packet.
 */
static void stmmac_tx_timeout(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);

	/* Clear Tx resources and restart transmitting again */
	stmmac_tx_err(priv);
}

/**
2722
 *  stmmac_set_rx_mode - entry point for multicast addressing
2723 2724 2725 2726 2727 2728 2729
 *  @dev : pointer to the device structure
 *  Description:
 *  This function is a driver entry point which gets called by the kernel
 *  whenever multicast addresses must be enabled/disabled.
 *  Return value:
 *  void.
 */
2730
static void stmmac_set_rx_mode(struct net_device *dev)
2731 2732 2733
{
	struct stmmac_priv *priv = netdev_priv(dev);

2734
	priv->hw->mac->set_filter(priv->hw, dev);
2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749
}

/**
 *  stmmac_change_mtu - entry point to change MTU size for the device.
 *  @dev : device pointer.
 *  @new_mtu : the new MTU size for the device.
 *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
 *  to drive packet transmission. Ethernet has an MTU of 1500 octets
 *  (ETH_DATA_LEN). This value can be changed with ifconfig.
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
{
2750 2751
	struct stmmac_priv *priv = netdev_priv(dev);

2752
	if (netif_running(dev)) {
2753
		netdev_err(priv->dev, "must be stopped to change its MTU\n");
2754 2755 2756
		return -EBUSY;
	}

2757
	dev->mtu = new_mtu;
A
Alexandre TORGUE 已提交
2758

2759 2760 2761 2762 2763
	netdev_update_features(dev);

	return 0;
}

2764
static netdev_features_t stmmac_fix_features(struct net_device *dev,
G
Giuseppe CAVALLARO 已提交
2765
					     netdev_features_t features)
2766 2767 2768
{
	struct stmmac_priv *priv = netdev_priv(dev);

2769
	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
2770
		features &= ~NETIF_F_RXCSUM;
2771

2772
	if (!priv->plat->tx_coe)
2773
		features &= ~NETIF_F_CSUM_MASK;
2774

2775 2776 2777
	/* Some GMAC devices have a bugged Jumbo frame support that
	 * needs to have the Tx COE disabled for oversized frames
	 * (due to limited buffer sizes). In this case we disable
G
Giuseppe CAVALLARO 已提交
2778 2779
	 * the TX csum insertionin the TDES and not use SF.
	 */
2780
	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
2781
		features &= ~NETIF_F_CSUM_MASK;
2782

A
Alexandre TORGUE 已提交
2783 2784 2785 2786 2787 2788 2789 2790
	/* Disable tso if asked by ethtool */
	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
		if (features & NETIF_F_TSO)
			priv->tso = true;
		else
			priv->tso = false;
	}

2791
	return features;
2792 2793
}

2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
static int stmmac_set_features(struct net_device *netdev,
			       netdev_features_t features)
{
	struct stmmac_priv *priv = netdev_priv(netdev);

	/* Keep the COE Type in case of csum is supporting */
	if (features & NETIF_F_RXCSUM)
		priv->hw->rx_csum = priv->plat->rx_coe;
	else
		priv->hw->rx_csum = 0;
	/* No check needed because rx_coe has been set before and it will be
	 * fixed in case of issue.
	 */
	priv->hw->mac->rx_ipc(priv->hw);

	return 0;
}

2812 2813 2814 2815 2816
/**
 *  stmmac_interrupt - main ISR
 *  @irq: interrupt number.
 *  @dev_id: to pass the net device pointer.
 *  Description: this is the main driver interrupt service routine.
2817 2818 2819 2820 2821
 *  It can call:
 *  o DMA service routine (to manage incoming frame reception and transmission
 *    status)
 *  o Core interrupts to manage: remote wake-up, management counter, LPI
 *    interrupts.
2822
 */
2823 2824 2825 2826 2827
static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
{
	struct net_device *dev = (struct net_device *)dev_id;
	struct stmmac_priv *priv = netdev_priv(dev);

2828 2829 2830
	if (priv->irq_wake)
		pm_wakeup_event(priv->device, 0);

2831
	if (unlikely(!dev)) {
2832
		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
2833 2834 2835
		return IRQ_NONE;
	}

2836
	/* To handle GMAC own interrupts */
A
Alexandre TORGUE 已提交
2837
	if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
2838
		int status = priv->hw->mac->host_irq_status(priv->hw,
2839
							    &priv->xstats);
2840 2841
		if (unlikely(status)) {
			/* For LPI we need to save the tx status */
2842
			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
2843
				priv->tx_path_in_lpi_mode = true;
2844
			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
2845
				priv->tx_path_in_lpi_mode = false;
2846
			if (status & CORE_IRQ_MTL_RX_OVERFLOW && priv->hw->dma->set_rx_tail_ptr)
A
Alexandre TORGUE 已提交
2847 2848 2849
				priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
							priv->rx_tail_addr,
							STMMAC_CHAN0);
2850
		}
2851 2852

		/* PCS link status */
2853
		if (priv->hw->pcs) {
2854 2855 2856 2857 2858
			if (priv->xstats.pcs_link)
				netif_carrier_on(dev);
			else
				netif_carrier_off(dev);
		}
2859
	}
2860

2861
	/* To handle DMA interrupts */
2862
	stmmac_dma_interrupt(priv);
2863 2864 2865 2866 2867 2868

	return IRQ_HANDLED;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
/* Polling receive - used by NETCONSOLE and other diagnostic tools
G
Giuseppe CAVALLARO 已提交
2869 2870
 * to allow network I/O with interrupts disabled.
 */
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
static void stmmac_poll_controller(struct net_device *dev)
{
	disable_irq(dev->irq);
	stmmac_interrupt(dev->irq, dev);
	enable_irq(dev->irq);
}
#endif

/**
 *  stmmac_ioctl - Entry point for the Ioctl
 *  @dev: Device pointer.
 *  @rq: An IOCTL specefic structure, that can contain a pointer to
 *  a proprietary structure used to pass information to the driver.
 *  @cmd: IOCTL command
 *  Description:
2886
 *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
2887 2888 2889
 */
static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
{
2890
	int ret = -EOPNOTSUPP;
2891 2892 2893 2894

	if (!netif_running(dev))
		return -EINVAL;

2895 2896 2897 2898
	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
2899
		if (!dev->phydev)
2900
			return -EINVAL;
2901
		ret = phy_mii_ioctl(dev->phydev, rq, cmd);
2902 2903 2904 2905 2906 2907 2908
		break;
	case SIOCSHWTSTAMP:
		ret = stmmac_hwtstamp_ioctl(dev, rq);
		break;
	default:
		break;
	}
2909

2910 2911 2912
	return ret;
}

2913
#ifdef CONFIG_DEBUG_FS
2914 2915
static struct dentry *stmmac_fs_dir;

2916
static void sysfs_display_ring(void *head, int size, int extend_desc,
G
Giuseppe CAVALLARO 已提交
2917
			       struct seq_file *seq)
2918 2919
{
	int i;
G
Giuseppe CAVALLARO 已提交
2920 2921
	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
	struct dma_desc *p = (struct dma_desc *)head;
2922

2923 2924 2925 2926 2927
	for (i = 0; i < size; i++) {
		u64 x;
		if (extend_desc) {
			x = *(u64 *) ep;
			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
G
Giuseppe CAVALLARO 已提交
2928
				   i, (unsigned int)virt_to_phys(ep),
2929 2930 2931 2932
				   le32_to_cpu(ep->basic.des0),
				   le32_to_cpu(ep->basic.des1),
				   le32_to_cpu(ep->basic.des2),
				   le32_to_cpu(ep->basic.des3));
2933 2934 2935 2936
			ep++;
		} else {
			x = *(u64 *) p;
			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
G
Giuseppe CAVALLARO 已提交
2937
				   i, (unsigned int)virt_to_phys(ep),
2938 2939
				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
2940 2941
			p++;
		}
2942 2943
		seq_printf(seq, "\n");
	}
2944
}
2945

2946 2947 2948 2949
static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
{
	struct net_device *dev = seq->private;
	struct stmmac_priv *priv = netdev_priv(dev);
2950

2951 2952
	if (priv->extend_desc) {
		seq_printf(seq, "Extended RX descriptor ring:\n");
2953
		sysfs_display_ring((void *)priv->dma_erx, DMA_RX_SIZE, 1, seq);
2954
		seq_printf(seq, "Extended TX descriptor ring:\n");
2955
		sysfs_display_ring((void *)priv->dma_etx, DMA_TX_SIZE, 1, seq);
2956 2957
	} else {
		seq_printf(seq, "RX descriptor ring:\n");
2958
		sysfs_display_ring((void *)priv->dma_rx, DMA_RX_SIZE, 0, seq);
2959
		seq_printf(seq, "TX descriptor ring:\n");
2960
		sysfs_display_ring((void *)priv->dma_tx, DMA_TX_SIZE, 0, seq);
2961 2962 2963 2964 2965 2966 2967 2968 2969 2970
	}

	return 0;
}

static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
{
	return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
}

2971 2972
/* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */

2973 2974 2975 2976 2977
static const struct file_operations stmmac_rings_status_fops = {
	.owner = THIS_MODULE,
	.open = stmmac_sysfs_ring_open,
	.read = seq_read,
	.llseek = seq_lseek,
2978
	.release = single_release,
2979 2980
};

2981 2982 2983 2984 2985
static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
{
	struct net_device *dev = seq->private;
	struct stmmac_priv *priv = netdev_priv(dev);

2986
	if (!priv->hw_cap_support) {
2987 2988 2989 2990 2991 2992 2993 2994
		seq_printf(seq, "DMA HW features not supported\n");
		return 0;
	}

	seq_printf(seq, "==============================\n");
	seq_printf(seq, "\tDMA HW features\n");
	seq_printf(seq, "==============================\n");

2995
	seq_printf(seq, "\t10/100 Mbps: %s\n",
2996
		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
2997
	seq_printf(seq, "\t1000 Mbps: %s\n",
2998
		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
2999
	seq_printf(seq, "\tHalf duplex: %s\n",
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016
		   (priv->dma_cap.half_duplex) ? "Y" : "N");
	seq_printf(seq, "\tHash Filter: %s\n",
		   (priv->dma_cap.hash_filter) ? "Y" : "N");
	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
		   (priv->dma_cap.multi_addr) ? "Y" : "N");
	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfatces): %s\n",
		   (priv->dma_cap.pcs) ? "Y" : "N");
	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
	seq_printf(seq, "\tPMT Remote wake up: %s\n",
		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
	seq_printf(seq, "\tPMT Magic Frame: %s\n",
		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
	seq_printf(seq, "\tRMON module: %s\n",
		   (priv->dma_cap.rmon) ? "Y" : "N");
	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
		   (priv->dma_cap.time_stamp) ? "Y" : "N");
3017
	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3018
		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
3019
	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3020 3021 3022 3023
		   (priv->dma_cap.eee) ? "Y" : "N");
	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
		   (priv->dma_cap.tx_coe) ? "Y" : "N");
A
Alexandre TORGUE 已提交
3024 3025 3026 3027 3028 3029 3030 3031 3032
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
			   (priv->dma_cap.rx_coe) ? "Y" : "N");
	} else {
		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
	}
3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054
	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
		   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
		   priv->dma_cap.number_rx_channel);
	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
		   priv->dma_cap.number_tx_channel);
	seq_printf(seq, "\tEnhanced descriptors: %s\n",
		   (priv->dma_cap.enh_desc) ? "Y" : "N");

	return 0;
}

static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
{
	return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
}

static const struct file_operations stmmac_dma_cap_fops = {
	.owner = THIS_MODULE,
	.open = stmmac_sysfs_dma_cap_open,
	.read = seq_read,
	.llseek = seq_lseek,
3055
	.release = single_release,
3056 3057
};

3058 3059
static int stmmac_init_fs(struct net_device *dev)
{
3060 3061 3062 3063
	struct stmmac_priv *priv = netdev_priv(dev);

	/* Create per netdev entries */
	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3064

3065
	if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3066
		netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3067 3068 3069 3070 3071

		return -ENOMEM;
	}

	/* Entry to report DMA RX/TX rings */
3072 3073 3074 3075
	priv->dbgfs_rings_status =
		debugfs_create_file("descriptors_status", S_IRUGO,
				    priv->dbgfs_dir, dev,
				    &stmmac_rings_status_fops);
3076

3077
	if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3078
		netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3079
		debugfs_remove_recursive(priv->dbgfs_dir);
3080 3081 3082 3083

		return -ENOMEM;
	}

3084
	/* Entry to report the DMA HW features */
3085 3086 3087
	priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
					    priv->dbgfs_dir,
					    dev, &stmmac_dma_cap_fops);
3088

3089
	if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3090
		netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3091
		debugfs_remove_recursive(priv->dbgfs_dir);
3092 3093 3094 3095

		return -ENOMEM;
	}

3096 3097 3098
	return 0;
}

3099
static void stmmac_exit_fs(struct net_device *dev)
3100
{
3101 3102 3103
	struct stmmac_priv *priv = netdev_priv(dev);

	debugfs_remove_recursive(priv->dbgfs_dir);
3104
}
3105
#endif /* CONFIG_DEBUG_FS */
3106

3107 3108 3109 3110 3111
static const struct net_device_ops stmmac_netdev_ops = {
	.ndo_open = stmmac_open,
	.ndo_start_xmit = stmmac_xmit,
	.ndo_stop = stmmac_release,
	.ndo_change_mtu = stmmac_change_mtu,
3112
	.ndo_fix_features = stmmac_fix_features,
3113
	.ndo_set_features = stmmac_set_features,
3114
	.ndo_set_rx_mode = stmmac_set_rx_mode,
3115 3116 3117 3118 3119 3120 3121 3122
	.ndo_tx_timeout = stmmac_tx_timeout,
	.ndo_do_ioctl = stmmac_ioctl,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = stmmac_poll_controller,
#endif
	.ndo_set_mac_address = eth_mac_addr,
};

3123 3124
/**
 *  stmmac_hw_init - Init the MAC device
3125
 *  @priv: driver private structure
3126 3127 3128 3129
 *  Description: this function is to configure the MAC device according to
 *  some platform parameters or the HW capability register. It prepares the
 *  driver to use either ring or chain modes and to setup either enhanced or
 *  normal descriptors.
3130 3131 3132 3133 3134 3135
 */
static int stmmac_hw_init(struct stmmac_priv *priv)
{
	struct mac_device_info *mac;

	/* Identify the MAC HW device */
3136 3137
	if (priv->plat->has_gmac) {
		priv->dev->priv_flags |= IFF_UNICAST_FLT;
3138 3139
		mac = dwmac1000_setup(priv->ioaddr,
				      priv->plat->multicast_filter_bins,
3140 3141
				      priv->plat->unicast_filter_entries,
				      &priv->synopsys_id);
A
Alexandre TORGUE 已提交
3142 3143 3144 3145 3146 3147
	} else if (priv->plat->has_gmac4) {
		priv->dev->priv_flags |= IFF_UNICAST_FLT;
		mac = dwmac4_setup(priv->ioaddr,
				   priv->plat->multicast_filter_bins,
				   priv->plat->unicast_filter_entries,
				   &priv->synopsys_id);
3148
	} else {
3149
		mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3150
	}
3151 3152 3153 3154 3155
	if (!mac)
		return -ENOMEM;

	priv->hw = mac;

3156
	/* To use the chained or ring mode */
A
Alexandre TORGUE 已提交
3157 3158
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		priv->hw->mode = &dwmac4_ring_mode_ops;
3159
	} else {
A
Alexandre TORGUE 已提交
3160 3161
		if (chain_mode) {
			priv->hw->mode = &chain_mode_ops;
3162
			dev_info(priv->device, "Chain mode enabled\n");
A
Alexandre TORGUE 已提交
3163 3164 3165
			priv->mode = STMMAC_CHAIN_MODE;
		} else {
			priv->hw->mode = &ring_mode_ops;
3166
			dev_info(priv->device, "Ring mode enabled\n");
A
Alexandre TORGUE 已提交
3167 3168
			priv->mode = STMMAC_RING_MODE;
		}
3169 3170
	}

3171 3172 3173
	/* Get the HW capability (new GMAC newer than 3.50a) */
	priv->hw_cap_support = stmmac_get_hw_features(priv);
	if (priv->hw_cap_support) {
3174
		dev_info(priv->device, "DMA HW capability register supported\n");
3175 3176 3177 3178 3179 3180 3181 3182

		/* We can override some gmac/dma configuration fields: e.g.
		 * enh_desc, tx_coe (e.g. that are passed through the
		 * platform) with the values from the HW capability
		 * register (if supported).
		 */
		priv->plat->enh_desc = priv->dma_cap.enh_desc;
		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3183
		priv->hw->pmt = priv->plat->pmt;
3184

3185 3186 3187 3188 3189 3190
		/* TXCOE doesn't work in thresh DMA mode */
		if (priv->plat->force_thresh_dma_mode)
			priv->plat->tx_coe = 0;
		else
			priv->plat->tx_coe = priv->dma_cap.tx_coe;

A
Alexandre TORGUE 已提交
3191 3192
		/* In case of GMAC4 rx_coe is from HW cap register. */
		priv->plat->rx_coe = priv->dma_cap.rx_coe;
3193 3194 3195 3196 3197 3198

		if (priv->dma_cap.rx_coe_type2)
			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
		else if (priv->dma_cap.rx_coe_type1)
			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;

3199 3200 3201
	} else {
		dev_info(priv->device, "No HW DMA feature register supported\n");
	}
3202

A
Alexandre TORGUE 已提交
3203 3204 3205 3206 3207
	/* To use alternate (extended), normal or GMAC4 descriptor structures */
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
		priv->hw->desc = &dwmac4_desc_ops;
	else
		stmmac_selec_desc_mode(priv);
3208

3209 3210
	if (priv->plat->rx_coe) {
		priv->hw->rx_csum = priv->plat->rx_coe;
3211
		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
A
Alexandre TORGUE 已提交
3212
		if (priv->synopsys_id < DWMAC_CORE_4_00)
3213
			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
3214
	}
3215
	if (priv->plat->tx_coe)
3216
		dev_info(priv->device, "TX Checksum insertion supported\n");
3217 3218

	if (priv->plat->pmt) {
3219
		dev_info(priv->device, "Wake-Up On Lan supported\n");
3220 3221 3222
		device_set_wakeup_capable(priv->device, 1);
	}

A
Alexandre TORGUE 已提交
3223
	if (priv->dma_cap.tsoen)
3224
		dev_info(priv->device, "TSO supported\n");
A
Alexandre TORGUE 已提交
3225

3226
	return 0;
3227 3228
}

3229
/**
3230 3231
 * stmmac_dvr_probe
 * @device: device pointer
3232
 * @plat_dat: platform data pointer
3233
 * @res: stmmac resource pointer
3234 3235
 * Description: this is the main probe function used to
 * call the alloc_etherdev, allocate the priv structure.
3236
 * Return:
3237
 * returns 0 on success, otherwise errno.
3238
 */
3239 3240 3241
int stmmac_dvr_probe(struct device *device,
		     struct plat_stmmacenet_data *plat_dat,
		     struct stmmac_resources *res)
3242 3243
{
	int ret = 0;
3244 3245
	struct net_device *ndev = NULL;
	struct stmmac_priv *priv;
3246

3247
	ndev = alloc_etherdev(sizeof(struct stmmac_priv));
3248
	if (!ndev)
3249
		return -ENOMEM;
3250 3251 3252 3253 3254 3255

	SET_NETDEV_DEV(ndev, device);

	priv = netdev_priv(ndev);
	priv->device = device;
	priv->dev = ndev;
3256

3257
	stmmac_set_ethtool_ops(ndev);
3258 3259
	priv->pause = pause;
	priv->plat = plat_dat;
3260 3261 3262 3263 3264 3265 3266 3267 3268
	priv->ioaddr = res->addr;
	priv->dev->base_addr = (unsigned long)res->addr;

	priv->dev->irq = res->irq;
	priv->wol_irq = res->wol_irq;
	priv->lpi_irq = res->lpi_irq;

	if (res->mac)
		memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
3269

3270
	dev_set_drvdata(device, priv->dev);
3271

3272 3273
	/* Verify driver arguments */
	stmmac_verify_args();
3274

3275
	/* Override with kernel parameters if supplied XXX CRS XXX
G
Giuseppe CAVALLARO 已提交
3276 3277
	 * this needs to have multiple instances
	 */
3278 3279 3280
	if ((phyaddr >= 0) && (phyaddr <= 31))
		priv->plat->phy_addr = phyaddr;

3281 3282
	priv->stmmac_clk = devm_clk_get(priv->device, STMMAC_RESOURCE_NAME);
	if (IS_ERR(priv->stmmac_clk)) {
3283 3284
		netdev_warn(priv->dev, "%s: warning: cannot get CSR clock\n",
			    __func__);
3285 3286 3287 3288 3289 3290 3291 3292 3293
		/* If failed to obtain stmmac_clk and specific clk_csr value
		 * is NOT passed from the platform, probe fail.
		 */
		if (!priv->plat->clk_csr) {
			ret = PTR_ERR(priv->stmmac_clk);
			goto error_clk_get;
		} else {
			priv->stmmac_clk = NULL;
		}
3294 3295 3296
	}
	clk_prepare_enable(priv->stmmac_clk);

3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
	priv->pclk = devm_clk_get(priv->device, "pclk");
	if (IS_ERR(priv->pclk)) {
		if (PTR_ERR(priv->pclk) == -EPROBE_DEFER) {
			ret = -EPROBE_DEFER;
			goto error_pclk_get;
		}
		priv->pclk = NULL;
	}
	clk_prepare_enable(priv->pclk);

3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319
	priv->stmmac_rst = devm_reset_control_get(priv->device,
						  STMMAC_RESOURCE_NAME);
	if (IS_ERR(priv->stmmac_rst)) {
		if (PTR_ERR(priv->stmmac_rst) == -EPROBE_DEFER) {
			ret = -EPROBE_DEFER;
			goto error_hw_init;
		}
		dev_info(priv->device, "no reset control found\n");
		priv->stmmac_rst = NULL;
	}
	if (priv->stmmac_rst)
		reset_control_deassert(priv->stmmac_rst);

3320
	/* Init MAC and get the capabilities */
3321 3322
	ret = stmmac_hw_init(priv);
	if (ret)
3323
		goto error_hw_init;
3324 3325

	ndev->netdev_ops = &stmmac_netdev_ops;
3326

3327 3328
	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
			    NETIF_F_RXCSUM;
A
Alexandre TORGUE 已提交
3329 3330 3331 3332

	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
		ndev->hw_features |= NETIF_F_TSO;
		priv->tso = true;
3333
		dev_info(priv->device, "TSO feature enabled\n");
A
Alexandre TORGUE 已提交
3334
	}
3335 3336
	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
3337 3338
#ifdef STMMAC_VLAN_TAG_USED
	/* Both mac100 and gmac support receive VLAN tag detection */
3339
	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
3340 3341 3342
#endif
	priv->msg_enable = netif_msg_init(debug, default_msg_level);

3343 3344 3345 3346 3347 3348 3349 3350 3351
	/* MTU range: 46 - hw-specific max */
	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
	if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
		ndev->max_mtu = JUMBO_LEN;
	else
		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
	if (priv->plat->maxmtu < ndev->max_mtu)
		ndev->max_mtu = priv->plat->maxmtu;

3352 3353 3354
	if (flow_ctrl)
		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */

3355 3356 3357 3358 3359 3360 3361
	/* Rx Watchdog is available in the COREs newer than the 3.40.
	 * In some case, for example on bugged HW this feature
	 * has to be disable and this can be done by passing the
	 * riwt_off field from the platform.
	 */
	if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
		priv->use_riwt = 1;
3362
		netdev_info(priv->dev, "Enable RX Mitigation via HW Watchdog Timer\n");
3363 3364
	}

3365
	netif_napi_add(ndev, &priv->napi, stmmac_poll, 64);
3366

3367 3368
	spin_lock_init(&priv->lock);

3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
	/* If a specific clk_csr value is passed from the platform
	 * this means that the CSR Clock Range selection cannot be
	 * changed at run-time and it is fixed. Viceversa the driver'll try to
	 * set the MDC clock dynamically according to the csr actual
	 * clock input.
	 */
	if (!priv->plat->clk_csr)
		stmmac_clk_csr_set(priv);
	else
		priv->clk_csr = priv->plat->clk_csr;

3380 3381
	stmmac_check_pcs_mode(priv);

3382 3383 3384
	if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI) {
3385 3386 3387
		/* MDIO bus Registration */
		ret = stmmac_mdio_register(ndev);
		if (ret < 0) {
3388 3389 3390
			netdev_err(priv->dev,
				   "%s: MDIO bus (id: %d) registration failed",
				   __func__, priv->plat->bus_id);
3391 3392
			goto error_mdio_register;
		}
3393 3394
	}

3395
	ret = register_netdev(ndev);
3396
	if (ret) {
3397 3398
		netdev_err(priv->dev, "%s: ERROR %i registering the device\n",
			   __func__, ret);
3399 3400
		goto error_netdev_register;
	}
3401 3402

	return ret;
3403

3404
error_netdev_register:
3405 3406 3407 3408
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI)
		stmmac_mdio_unregister(ndev);
3409 3410
error_mdio_register:
	netif_napi_del(&priv->napi);
3411
error_hw_init:
3412 3413
	clk_disable_unprepare(priv->pclk);
error_pclk_get:
3414 3415
	clk_disable_unprepare(priv->stmmac_clk);
error_clk_get:
3416
	free_netdev(ndev);
3417

3418
	return ret;
3419
}
3420
EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
3421 3422 3423

/**
 * stmmac_dvr_remove
3424
 * @dev: device pointer
3425
 * Description: this function resets the TX/RX processes, disables the MAC RX/TX
3426
 * changes the link status, releases the DMA descriptor rings.
3427
 */
3428
int stmmac_dvr_remove(struct device *dev)
3429
{
3430
	struct net_device *ndev = dev_get_drvdata(dev);
3431
	struct stmmac_priv *priv = netdev_priv(ndev);
3432

3433
	netdev_info(priv->dev, "%s: removing driver", __func__);
3434

3435 3436
	priv->hw->dma->stop_rx(priv->ioaddr);
	priv->hw->dma->stop_tx(priv->ioaddr);
3437

3438
	stmmac_set_mac(priv->ioaddr, false);
3439 3440
	netif_carrier_off(ndev);
	unregister_netdev(ndev);
3441 3442
	if (priv->stmmac_rst)
		reset_control_assert(priv->stmmac_rst);
3443
	clk_disable_unprepare(priv->pclk);
3444
	clk_disable_unprepare(priv->stmmac_clk);
3445 3446 3447
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI)
3448
		stmmac_mdio_unregister(ndev);
3449 3450 3451 3452
	free_netdev(ndev);

	return 0;
}
3453
EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
3454

3455 3456
/**
 * stmmac_suspend - suspend callback
3457
 * @dev: device pointer
3458 3459 3460 3461
 * Description: this is the function to suspend the device and it is called
 * by the platform driver to stop the network queue, release the resources,
 * program the PMT register (for WoL), clean and release driver resources.
 */
3462
int stmmac_suspend(struct device *dev)
3463
{
3464
	struct net_device *ndev = dev_get_drvdata(dev);
3465
	struct stmmac_priv *priv = netdev_priv(ndev);
3466
	unsigned long flags;
3467

3468
	if (!ndev || !netif_running(ndev))
3469 3470
		return 0;

3471 3472
	if (ndev->phydev)
		phy_stop(ndev->phydev);
3473

3474
	spin_lock_irqsave(&priv->lock, flags);
3475

3476 3477
	netif_device_detach(ndev);
	netif_stop_queue(ndev);
3478

3479 3480 3481 3482 3483
	napi_disable(&priv->napi);

	/* Stop TX/RX DMA */
	priv->hw->dma->stop_tx(priv->ioaddr);
	priv->hw->dma->stop_rx(priv->ioaddr);
3484

3485
	/* Enable Power down mode by programming the PMT regs */
3486
	if (device_may_wakeup(priv->device)) {
3487
		priv->hw->mac->pmt(priv->hw, priv->wolopts);
3488 3489
		priv->irq_wake = 1;
	} else {
3490
		stmmac_set_mac(priv->ioaddr, false);
3491
		pinctrl_pm_select_sleep_state(priv->device);
3492
		/* Disable clock in case of PWM is off */
3493
		clk_disable(priv->pclk);
3494
		clk_disable(priv->stmmac_clk);
3495
	}
3496
	spin_unlock_irqrestore(&priv->lock, flags);
3497 3498 3499 3500

	priv->oldlink = 0;
	priv->speed = 0;
	priv->oldduplex = -1;
3501 3502
	return 0;
}
3503
EXPORT_SYMBOL_GPL(stmmac_suspend);
3504

3505 3506
/**
 * stmmac_resume - resume callback
3507
 * @dev: device pointer
3508 3509 3510
 * Description: when resume this function is invoked to setup the DMA and CORE
 * in a usable state.
 */
3511
int stmmac_resume(struct device *dev)
3512
{
3513
	struct net_device *ndev = dev_get_drvdata(dev);
3514
	struct stmmac_priv *priv = netdev_priv(ndev);
3515
	unsigned long flags;
3516

3517
	if (!netif_running(ndev))
3518 3519 3520 3521 3522 3523
		return 0;

	/* Power Down bit, into the PM register, is cleared
	 * automatically as soon as a magic packet or a Wake-up frame
	 * is received. Anyway, it's better to manually clear
	 * this bit because it can generate problems while resuming
G
Giuseppe CAVALLARO 已提交
3524 3525
	 * from another devices (e.g. serial console).
	 */
3526
	if (device_may_wakeup(priv->device)) {
3527
		spin_lock_irqsave(&priv->lock, flags);
3528
		priv->hw->mac->pmt(priv->hw, 0);
3529
		spin_unlock_irqrestore(&priv->lock, flags);
3530
		priv->irq_wake = 0;
3531
	} else {
3532
		pinctrl_pm_select_default_state(priv->device);
3533
		/* enable the clk prevously disabled */
3534
		clk_enable(priv->stmmac_clk);
3535
		clk_enable(priv->pclk);
3536 3537 3538 3539
		/* reset the phy so that it's ready */
		if (priv->mii)
			stmmac_mdio_reset(priv->mii);
	}
3540

3541
	netif_device_attach(ndev);
3542

3543 3544
	spin_lock_irqsave(&priv->lock, flags);

3545 3546 3547 3548
	priv->cur_rx = 0;
	priv->dirty_rx = 0;
	priv->dirty_tx = 0;
	priv->cur_tx = 0;
A
Alexandre TORGUE 已提交
3549 3550 3551 3552 3553
	/* reset private mss value to force mss context settings at
	 * next tso xmit (only used for gmac4).
	 */
	priv->mss = 0;

3554 3555
	stmmac_clear_descriptors(priv);

3556
	stmmac_hw_setup(ndev, false);
3557
	stmmac_init_tx_coalesce(priv);
3558
	stmmac_set_rx_mode(ndev);
3559 3560 3561

	napi_enable(&priv->napi);

3562
	netif_start_queue(ndev);
3563

3564
	spin_unlock_irqrestore(&priv->lock, flags);
3565

3566 3567
	if (ndev->phydev)
		phy_start(ndev->phydev);
3568

3569 3570
	return 0;
}
3571
EXPORT_SYMBOL_GPL(stmmac_resume);
3572

3573 3574 3575 3576 3577 3578 3579 3580
#ifndef MODULE
static int __init stmmac_cmdline_opt(char *str)
{
	char *opt;

	if (!str || !*str)
		return -EINVAL;
	while ((opt = strsep(&str, ",")) != NULL) {
3581
		if (!strncmp(opt, "debug:", 6)) {
3582
			if (kstrtoint(opt + 6, 0, &debug))
3583 3584
				goto err;
		} else if (!strncmp(opt, "phyaddr:", 8)) {
3585
			if (kstrtoint(opt + 8, 0, &phyaddr))
3586 3587
				goto err;
		} else if (!strncmp(opt, "buf_sz:", 7)) {
3588
			if (kstrtoint(opt + 7, 0, &buf_sz))
3589 3590
				goto err;
		} else if (!strncmp(opt, "tc:", 3)) {
3591
			if (kstrtoint(opt + 3, 0, &tc))
3592 3593
				goto err;
		} else if (!strncmp(opt, "watchdog:", 9)) {
3594
			if (kstrtoint(opt + 9, 0, &watchdog))
3595 3596
				goto err;
		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
3597
			if (kstrtoint(opt + 10, 0, &flow_ctrl))
3598 3599
				goto err;
		} else if (!strncmp(opt, "pause:", 6)) {
3600
			if (kstrtoint(opt + 6, 0, &pause))
3601
				goto err;
3602
		} else if (!strncmp(opt, "eee_timer:", 10)) {
3603 3604
			if (kstrtoint(opt + 10, 0, &eee_timer))
				goto err;
3605 3606 3607
		} else if (!strncmp(opt, "chain_mode:", 11)) {
			if (kstrtoint(opt + 11, 0, &chain_mode))
				goto err;
3608
		}
3609 3610
	}
	return 0;
3611 3612 3613 3614

err:
	pr_err("%s: ERROR broken module parameter conversion", __func__);
	return -EINVAL;
3615 3616 3617
}

__setup("stmmaceth=", stmmac_cmdline_opt);
G
Giuseppe CAVALLARO 已提交
3618
#endif /* MODULE */
3619

3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648
static int __init stmmac_init(void)
{
#ifdef CONFIG_DEBUG_FS
	/* Create debugfs main directory if it doesn't exist yet */
	if (!stmmac_fs_dir) {
		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);

		if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
			pr_err("ERROR %s, debugfs create directory failed\n",
			       STMMAC_RESOURCE_NAME);

			return -ENOMEM;
		}
	}
#endif

	return 0;
}

static void __exit stmmac_exit(void)
{
#ifdef CONFIG_DEBUG_FS
	debugfs_remove_recursive(stmmac_fs_dir);
#endif
}

module_init(stmmac_init)
module_exit(stmmac_exit)

3649 3650 3651
MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
MODULE_LICENSE("GPL");