stmmac_main.c 126.9 KB
Newer Older
1 2 3 4
/*******************************************************************************
  This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
  ST Ethernet IPs are built around a Synopsys IP Core.

5
	Copyright(C) 2007-2011 STMicroelectronics Ltd
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>

  Documentation available at:
	http://www.stlinux.com
  Support available at:
	https://bugzilla.stlinux.com/
*******************************************************************************/

27
#include <linux/clk.h>
28 29 30 31 32 33 34 35 36
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/skbuff.h>
#include <linux/ethtool.h>
#include <linux/if_ether.h>
#include <linux/crc32.h>
#include <linux/mii.h>
37
#include <linux/if.h>
38 39
#include <linux/if_vlan.h>
#include <linux/dma-mapping.h>
40
#include <linux/slab.h>
41
#include <linux/prefetch.h>
42
#include <linux/pinctrl/consumer.h>
43
#ifdef CONFIG_DEBUG_FS
44 45
#include <linux/debugfs.h>
#include <linux/seq_file.h>
46
#endif /* CONFIG_DEBUG_FS */
47 48
#include <linux/net_tstamp.h>
#include "stmmac_ptp.h"
49
#include "stmmac.h"
50
#include <linux/reset.h>
51
#include <linux/of_mdio.h>
52
#include "dwmac1000.h"
53
#include "hwif.h"
54 55

#define STMMAC_ALIGN(x)	L1_CACHE_ALIGN(x)
A
Alexandre TORGUE 已提交
56
#define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
57 58

/* Module parameters */
59
#define TX_TIMEO	5000
60
static int watchdog = TX_TIMEO;
61
module_param(watchdog, int, 0644);
62
MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
63

64
static int debug = -1;
65
module_param(debug, int, 0644);
66
MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
67

68
static int phyaddr = -1;
69
module_param(phyaddr, int, 0444);
70 71
MODULE_PARM_DESC(phyaddr, "Physical device address");

72
#define STMMAC_TX_THRESH	(DMA_TX_SIZE / 4)
73
#define STMMAC_RX_THRESH	(DMA_RX_SIZE / 4)
74 75

static int flow_ctrl = FLOW_OFF;
76
module_param(flow_ctrl, int, 0644);
77 78 79
MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");

static int pause = PAUSE_TIME;
80
module_param(pause, int, 0644);
81 82 83 84
MODULE_PARM_DESC(pause, "Flow Control Pause Time");

#define TC_DEFAULT 64
static int tc = TC_DEFAULT;
85
module_param(tc, int, 0644);
86 87
MODULE_PARM_DESC(tc, "DMA threshold control value");

88 89
#define	DEFAULT_BUFSIZE	1536
static int buf_sz = DEFAULT_BUFSIZE;
90
module_param(buf_sz, int, 0644);
91 92
MODULE_PARM_DESC(buf_sz, "DMA buffer size");

93 94
#define	STMMAC_RX_COPYBREAK	256

95 96 97 98
static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);

99 100
#define STMMAC_DEFAULT_LPI_TIMER	1000
static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
101
module_param(eee_timer, int, 0644);
102
MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
G
Giuseppe CAVALLARO 已提交
103
#define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
104

105 106
/* By default the driver will use the ring mode to manage tx and rx descriptors,
 * but allow user to force to use the chain instead of the ring
107 108
 */
static unsigned int chain_mode;
109
module_param(chain_mode, int, 0444);
110 111
MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");

112 113
static irqreturn_t stmmac_interrupt(int irq, void *dev_id);

114
#ifdef CONFIG_DEBUG_FS
115
static int stmmac_init_fs(struct net_device *dev);
116
static void stmmac_exit_fs(struct net_device *dev);
117 118
#endif

119 120
#define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))

121 122
/**
 * stmmac_verify_args - verify the driver parameters.
123 124
 * Description: it checks the driver parameters and set a default in case of
 * errors.
125 126 127 128 129
 */
static void stmmac_verify_args(void)
{
	if (unlikely(watchdog < 0))
		watchdog = TX_TIMEO;
130 131
	if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
		buf_sz = DEFAULT_BUFSIZE;
132 133 134 135 136 137
	if (unlikely(flow_ctrl > 1))
		flow_ctrl = FLOW_AUTO;
	else if (likely(flow_ctrl < 0))
		flow_ctrl = FLOW_OFF;
	if (unlikely((pause < 0) || (pause > 0xffff)))
		pause = PAUSE_TIME;
138 139
	if (eee_timer < 0)
		eee_timer = STMMAC_DEFAULT_LPI_TIMER;
140 141
}

142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
/**
 * stmmac_disable_all_queues - Disable all queues
 * @priv: driver private structure
 */
static void stmmac_disable_all_queues(struct stmmac_priv *priv)
{
	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
	u32 queue;

	for (queue = 0; queue < rx_queues_cnt; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		napi_disable(&rx_q->napi);
	}
}

/**
 * stmmac_enable_all_queues - Enable all queues
 * @priv: driver private structure
 */
static void stmmac_enable_all_queues(struct stmmac_priv *priv)
{
	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
	u32 queue;

	for (queue = 0; queue < rx_queues_cnt; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		napi_enable(&rx_q->napi);
	}
}

/**
 * stmmac_stop_all_queues - Stop all queues
 * @priv: driver private structure
 */
static void stmmac_stop_all_queues(struct stmmac_priv *priv)
{
	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
	u32 queue;

	for (queue = 0; queue < tx_queues_cnt; queue++)
		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
}

/**
 * stmmac_start_all_queues - Start all queues
 * @priv: driver private structure
 */
static void stmmac_start_all_queues(struct stmmac_priv *priv)
{
	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
	u32 queue;

	for (queue = 0; queue < tx_queues_cnt; queue++)
		netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213
static void stmmac_service_event_schedule(struct stmmac_priv *priv)
{
	if (!test_bit(STMMAC_DOWN, &priv->state) &&
	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
		queue_work(priv->wq, &priv->service_task);
}

static void stmmac_global_err(struct stmmac_priv *priv)
{
	netif_carrier_off(priv->dev);
	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
	stmmac_service_event_schedule(priv);
}

214 215 216 217 218 219 220 221 222 223 224 225
/**
 * stmmac_clk_csr_set - dynamically set the MDC clock
 * @priv: driver private structure
 * Description: this is to dynamically set the MDC clock according to the csr
 * clock input.
 * Note:
 *	If a specific clk_csr value is passed from the platform
 *	this means that the CSR Clock Range selection cannot be
 *	changed at run-time and it is fixed (as reported in the driver
 *	documentation). Viceversa the driver will try to set the MDC
 *	clock dynamically according to the actual clock input.
 */
226 227 228 229
static void stmmac_clk_csr_set(struct stmmac_priv *priv)
{
	u32 clk_rate;

230
	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
231 232

	/* Platform provided default clk_csr would be assumed valid
G
Giuseppe CAVALLARO 已提交
233 234 235 236 237 238
	 * for all other cases except for the below mentioned ones.
	 * For values higher than the IEEE 802.3 specified frequency
	 * we can not estimate the proper divider as it is not known
	 * the frequency of clk_csr_i. So we do not change the default
	 * divider.
	 */
239 240 241 242 243 244 245 246 247 248 249
	if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
		if (clk_rate < CSR_F_35M)
			priv->clk_csr = STMMAC_CSR_20_35M;
		else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
			priv->clk_csr = STMMAC_CSR_35_60M;
		else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
			priv->clk_csr = STMMAC_CSR_60_100M;
		else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
			priv->clk_csr = STMMAC_CSR_100_150M;
		else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
			priv->clk_csr = STMMAC_CSR_150_250M;
250
		else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
251
			priv->clk_csr = STMMAC_CSR_250_300M;
G
Giuseppe CAVALLARO 已提交
252
	}
253 254 255 256 257 258 259 260 261 262 263

	if (priv->plat->has_sun8i) {
		if (clk_rate > 160000000)
			priv->clk_csr = 0x03;
		else if (clk_rate > 80000000)
			priv->clk_csr = 0x02;
		else if (clk_rate > 40000000)
			priv->clk_csr = 0x01;
		else
			priv->clk_csr = 0;
	}
264 265
}

266 267
static void print_pkt(unsigned char *buf, int len)
{
268 269
	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
270 271
}

272
static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
273
{
274
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
275
	u32 avail;
276

277 278
	if (tx_q->dirty_tx > tx_q->cur_tx)
		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
279
	else
280
		avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
281 282 283 284

	return avail;
}

285 286 287 288 289 290
/**
 * stmmac_rx_dirty - Get RX queue dirty
 * @priv: driver private structure
 * @queue: RX queue index
 */
static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
291
{
292
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
293
	u32 dirty;
294

295 296
	if (rx_q->dirty_rx <= rx_q->cur_rx)
		dirty = rx_q->cur_rx - rx_q->dirty_rx;
297
	else
298
		dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
299 300

	return dirty;
301 302
}

303
/**
304
 * stmmac_hw_fix_mac_speed - callback for speed selection
305
 * @priv: driver private structure
306
 * Description: on some platforms (e.g. ST), some HW system configuration
307
 * registers have to be set according to the link speed negotiated.
308 309 310
 */
static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
{
311 312
	struct net_device *ndev = priv->dev;
	struct phy_device *phydev = ndev->phydev;
313 314

	if (likely(priv->plat->fix_mac_speed))
G
Giuseppe CAVALLARO 已提交
315
		priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
316 317
}

318
/**
319
 * stmmac_enable_eee_mode - check and enter in LPI mode
320
 * @priv: driver private structure
321 322
 * Description: this function is to verify and enter in LPI mode in case of
 * EEE.
323
 */
324 325
static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
{
326 327 328 329 330 331 332 333 334 335 336
	u32 tx_cnt = priv->plat->tx_queues_to_use;
	u32 queue;

	/* check if all TX queues have the work finished */
	for (queue = 0; queue < tx_cnt; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];

		if (tx_q->dirty_tx != tx_q->cur_tx)
			return; /* still unfinished work */
	}

337
	/* Check and enter in LPI mode */
338
	if (!priv->tx_path_in_lpi_mode)
339 340
		stmmac_set_eee_mode(priv, priv->hw,
				priv->plat->en_tx_lpi_clockgating);
341 342
}

343
/**
344
 * stmmac_disable_eee_mode - disable and exit from LPI mode
345 346 347 348
 * @priv: driver private structure
 * Description: this function is to exit and disable EEE in case of
 * LPI state is true. This is called by the xmit.
 */
349 350
void stmmac_disable_eee_mode(struct stmmac_priv *priv)
{
351
	stmmac_reset_eee_mode(priv, priv->hw);
352 353 354 355 356
	del_timer_sync(&priv->eee_ctrl_timer);
	priv->tx_path_in_lpi_mode = false;
}

/**
357
 * stmmac_eee_ctrl_timer - EEE TX SW timer.
358 359
 * @arg : data hook
 * Description:
360
 *  if there is no data transfer and if we are not in LPI state,
361 362
 *  then MAC Transmitter can be moved to LPI state.
 */
363
static void stmmac_eee_ctrl_timer(struct timer_list *t)
364
{
365
	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
366 367

	stmmac_enable_eee_mode(priv);
G
Giuseppe CAVALLARO 已提交
368
	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
369 370 371
}

/**
372
 * stmmac_eee_init - init EEE
373
 * @priv: driver private structure
374
 * Description:
375 376 377
 *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 *  can also manage EEE, this function enable the LPI state and start related
 *  timer.
378 379 380
 */
bool stmmac_eee_init(struct stmmac_priv *priv)
{
381
	struct net_device *ndev = priv->dev;
382
	int interface = priv->plat->interface;
383
	unsigned long flags;
384 385
	bool ret = false;

386 387 388 389 390
	if ((interface != PHY_INTERFACE_MODE_MII) &&
	    (interface != PHY_INTERFACE_MODE_GMII) &&
	    !phy_interface_mode_is_rgmii(interface))
		goto out;

G
Giuseppe CAVALLARO 已提交
391 392 393
	/* Using PCS we cannot dial with the phy registers at this stage
	 * so we do not support extra feature like EEE.
	 */
394 395 396
	if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
	    (priv->hw->pcs == STMMAC_PCS_TBI) ||
	    (priv->hw->pcs == STMMAC_PCS_RTBI))
G
Giuseppe CAVALLARO 已提交
397 398
		goto out;

399 400
	/* MAC core supports the EEE feature. */
	if (priv->dma_cap.eee) {
401 402
		int tx_lpi_timer = priv->tx_lpi_timer;

403
		/* Check if the PHY supports EEE */
404
		if (phy_init_eee(ndev->phydev, 1)) {
405 406 407 408 409
			/* To manage at run-time if the EEE cannot be supported
			 * anymore (for example because the lp caps have been
			 * changed).
			 * In that case the driver disable own timers.
			 */
410
			spin_lock_irqsave(&priv->lock, flags);
411
			if (priv->eee_active) {
412
				netdev_dbg(priv->dev, "disable EEE\n");
413
				del_timer_sync(&priv->eee_ctrl_timer);
414 415
				stmmac_set_eee_timer(priv, priv->hw, 0,
						tx_lpi_timer);
416 417
			}
			priv->eee_active = 0;
418
			spin_unlock_irqrestore(&priv->lock, flags);
419
			goto out;
420 421
		}
		/* Activate the EEE and start timers */
422
		spin_lock_irqsave(&priv->lock, flags);
G
Giuseppe CAVALLARO 已提交
423 424
		if (!priv->eee_active) {
			priv->eee_active = 1;
425 426
			timer_setup(&priv->eee_ctrl_timer,
				    stmmac_eee_ctrl_timer, 0);
427 428
			mod_timer(&priv->eee_ctrl_timer,
				  STMMAC_LPI_T(eee_timer));
G
Giuseppe CAVALLARO 已提交
429

430 431
			stmmac_set_eee_timer(priv, priv->hw,
					STMMAC_DEFAULT_LIT_LS, tx_lpi_timer);
432 433
		}
		/* Set HW EEE according to the speed */
434
		stmmac_set_eee_pls(priv, priv->hw, ndev->phydev->link);
435 436

		ret = true;
437 438
		spin_unlock_irqrestore(&priv->lock, flags);

439
		netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
440 441 442 443 444
	}
out:
	return ret;
}

445
/* stmmac_get_tx_hwtstamp - get HW TX timestamps
446
 * @priv: driver private structure
447
 * @p : descriptor pointer
448 449 450 451 452 453
 * @skb : the socket buffer
 * Description :
 * This function will read timestamp from the descriptor & pass it to stack.
 * and also perform some sanity checks.
 */
static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
454
				   struct dma_desc *p, struct sk_buff *skb)
455 456 457 458 459 460 461
{
	struct skb_shared_hwtstamps shhwtstamp;
	u64 ns;

	if (!priv->hwts_tx_en)
		return;

G
Giuseppe CAVALLARO 已提交
462
	/* exit if skb doesn't support hw tstamp */
463
	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
464 465 466
		return;

	/* check tx tstamp status */
467
	if (stmmac_get_tx_timestamp_status(priv, p)) {
468
		/* get the valid tstamp */
469
		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
470

471 472
		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
		shhwtstamp.hwtstamp = ns_to_ktime(ns);
473

474
		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
475 476 477
		/* pass tstamp to stack */
		skb_tstamp_tx(skb, &shhwtstamp);
	}
478 479 480 481

	return;
}

482
/* stmmac_get_rx_hwtstamp - get HW RX timestamps
483
 * @priv: driver private structure
484 485
 * @p : descriptor pointer
 * @np : next descriptor pointer
486 487 488 489 490
 * @skb : the socket buffer
 * Description :
 * This function will read received packet's timestamp from the descriptor
 * and pass it to stack. It also perform some sanity checks.
 */
491 492
static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
				   struct dma_desc *np, struct sk_buff *skb)
493 494
{
	struct skb_shared_hwtstamps *shhwtstamp = NULL;
495
	struct dma_desc *desc = p;
496 497 498 499
	u64 ns;

	if (!priv->hwts_rx_en)
		return;
500 501 502
	/* For GMAC4, the valid timestamp is from CTX next desc. */
	if (priv->plat->has_gmac4)
		desc = np;
503

504
	/* Check if timestamp is available */
505 506
	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
507
		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
508 509 510 511
		shhwtstamp = skb_hwtstamps(skb);
		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
		shhwtstamp->hwtstamp = ns_to_ktime(ns);
	} else  {
512
		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
513
	}
514 515 516 517 518
}

/**
 *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 *  @dev: device pointer.
519
 *  @ifr: An IOCTL specific structure, that can contain a pointer to
520 521 522 523 524 525 526 527 528 529 530
 *  a proprietary structure used to pass information to the driver.
 *  Description:
 *  This function configures the MAC to enable/disable both outgoing(TX)
 *  and incoming(RX) packets time stamping based on user input.
 *  Return Value:
 *  0 on success and an appropriate -ve integer on failure.
 */
static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	struct hwtstamp_config config;
A
Arnd Bergmann 已提交
531
	struct timespec64 now;
532 533 534 535 536 537 538 539 540 541
	u64 temp = 0;
	u32 ptp_v2 = 0;
	u32 tstamp_all = 0;
	u32 ptp_over_ipv4_udp = 0;
	u32 ptp_over_ipv6_udp = 0;
	u32 ptp_over_ethernet = 0;
	u32 snap_type_sel = 0;
	u32 ts_master_en = 0;
	u32 ts_event_en = 0;
	u32 value = 0;
542
	u32 sec_inc;
543 544 545 546 547 548 549 550 551 552

	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
		netdev_alert(priv->dev, "No support for HW time stamping\n");
		priv->hwts_tx_en = 0;
		priv->hwts_rx_en = 0;

		return -EOPNOTSUPP;
	}

	if (copy_from_user(&config, ifr->ifr_data,
G
Giuseppe CAVALLARO 已提交
553
			   sizeof(struct hwtstamp_config)))
554 555
		return -EFAULT;

556 557
	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
		   __func__, config.flags, config.tx_type, config.rx_filter);
558 559 560 561 562

	/* reserved for future extensions */
	if (config.flags)
		return -EINVAL;

563 564
	if (config.tx_type != HWTSTAMP_TX_OFF &&
	    config.tx_type != HWTSTAMP_TX_ON)
565 566 567 568 569
		return -ERANGE;

	if (priv->adv_ts) {
		switch (config.rx_filter) {
		case HWTSTAMP_FILTER_NONE:
G
Giuseppe CAVALLARO 已提交
570
			/* time stamp no incoming packet at all */
571 572 573 574
			config.rx_filter = HWTSTAMP_FILTER_NONE;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
G
Giuseppe CAVALLARO 已提交
575
			/* PTP v1, UDP, any kind of event packet */
576 577
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
			/* take time stamp for all event messages */
578 579 580 581
			if (priv->plat->has_gmac4)
				snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
			else
				snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
582 583 584 585 586 587

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
G
Giuseppe CAVALLARO 已提交
588
			/* PTP v1, UDP, Sync packet */
589 590 591 592 593 594 595 596 597
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
598
			/* PTP v1, UDP, Delay_req packet */
599 600 601 602 603 604 605 606 607 608
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
G
Giuseppe CAVALLARO 已提交
609
			/* PTP v2, UDP, any kind of event packet */
610 611 612
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for all event messages */
613 614 615 616
			if (priv->plat->has_gmac4)
				snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
			else
				snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
617 618 619 620 621 622

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
G
Giuseppe CAVALLARO 已提交
623
			/* PTP v2, UDP, Sync packet */
624 625 626 627 628 629 630 631 632 633
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
634
			/* PTP v2, UDP, Delay_req packet */
635 636 637 638 639 640 641 642 643 644 645
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_EVENT:
G
Giuseppe CAVALLARO 已提交
646
			/* PTP v2/802.AS1 any layer, any kind of event packet */
647 648 649
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for all event messages */
650 651 652 653
			if (priv->plat->has_gmac4)
				snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
			else
				snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
654 655 656 657 658 659 660

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_SYNC:
G
Giuseppe CAVALLARO 已提交
661
			/* PTP v2/802.AS1, any layer, Sync packet */
662 663 664 665 666 667 668 669 670 671 672
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for SYNC messages only */
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
G
Giuseppe CAVALLARO 已提交
673
			/* PTP v2/802.AS1, any layer, Delay_req packet */
674 675 676 677 678 679 680 681 682 683 684
			config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
			ptp_v2 = PTP_TCR_TSVER2ENA;
			/* take time stamp for Delay_Req messages only */
			ts_master_en = PTP_TCR_TSMSTRENA;
			ts_event_en = PTP_TCR_TSEVNTENA;

			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
			ptp_over_ethernet = PTP_TCR_TSIPENA;
			break;

685
		case HWTSTAMP_FILTER_NTP_ALL:
686
		case HWTSTAMP_FILTER_ALL:
G
Giuseppe CAVALLARO 已提交
687
			/* time stamp any incoming packet */
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
			config.rx_filter = HWTSTAMP_FILTER_ALL;
			tstamp_all = PTP_TCR_TSENALL;
			break;

		default:
			return -ERANGE;
		}
	} else {
		switch (config.rx_filter) {
		case HWTSTAMP_FILTER_NONE:
			config.rx_filter = HWTSTAMP_FILTER_NONE;
			break;
		default:
			/* PTP v1, UDP, any kind of event packet */
			config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
			break;
		}
	}
	priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
707
	priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
708 709

	if (!priv->hwts_tx_en && !priv->hwts_rx_en)
710
		stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
711 712
	else {
		value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
G
Giuseppe CAVALLARO 已提交
713 714 715
			 tstamp_all | ptp_v2 | ptp_over_ethernet |
			 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
			 ts_master_en | snap_type_sel);
716
		stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
717 718

		/* program Sub Second Increment reg */
719 720 721
		stmmac_config_sub_second_increment(priv,
				priv->ptpaddr, priv->plat->clk_ptp_rate,
				priv->plat->has_gmac4, &sec_inc);
722
		temp = div_u64(1000000000ULL, sec_inc);
723 724 725 726

		/* calculate default added value:
		 * formula is :
		 * addend = (2^32)/freq_div_ratio;
727
		 * where, freq_div_ratio = 1e9ns/sec_inc
728
		 */
729
		temp = (u64)(temp << 32);
730
		priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
731
		stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
732 733

		/* initialize system time */
A
Arnd Bergmann 已提交
734 735 736
		ktime_get_real_ts64(&now);

		/* lower 32 bits of tv_sec are safe until y2106 */
737 738
		stmmac_init_systime(priv, priv->ptpaddr,
				(u32)now.tv_sec, now.tv_nsec);
739 740 741 742 743 744
	}

	return copy_to_user(ifr->ifr_data, &config,
			    sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
}

745
/**
746
 * stmmac_init_ptp - init PTP
747
 * @priv: driver private structure
748
 * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
749
 * This is done by looking at the HW cap. register.
750
 * This function also registers the ptp driver.
751
 */
752
static int stmmac_init_ptp(struct stmmac_priv *priv)
753
{
754 755 756
	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
		return -EOPNOTSUPP;

757
	priv->adv_ts = 0;
758 759 760 761 762
	/* Check if adv_ts can be enabled for dwmac 4.x core */
	if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
		priv->adv_ts = 1;
	/* Dwmac 3.x core with extend_desc can support adv_ts */
	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
763 764
		priv->adv_ts = 1;

765 766
	if (priv->dma_cap.time_stamp)
		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
767

768 769 770
	if (priv->adv_ts)
		netdev_info(priv->dev,
			    "IEEE 1588-2008 Advanced Timestamp supported\n");
771 772 773 774

	priv->hw->ptp = &stmmac_ptp;
	priv->hwts_tx_en = 0;
	priv->hwts_rx_en = 0;
775

776 777 778
	stmmac_ptp_register(priv);

	return 0;
779 780 781 782
}

static void stmmac_release_ptp(struct stmmac_priv *priv)
{
783 784
	if (priv->plat->clk_ptp_ref)
		clk_disable_unprepare(priv->plat->clk_ptp_ref);
785
	stmmac_ptp_unregister(priv);
786 787
}

788 789 790 791 792 793 794 795 796
/**
 *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 *  @priv: driver private structure
 *  Description: It is used for configuring the flow control in all queues
 */
static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
{
	u32 tx_cnt = priv->plat->tx_queues_to_use;

797 798
	stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
			priv->pause, tx_cnt);
799 800
}

801
/**
802
 * stmmac_adjust_link - adjusts the link parameters
803
 * @dev: net device structure
804 805 806 807 808
 * Description: this is the helper called by the physical abstraction layer
 * drivers to communicate the phy link status. According the speed and duplex
 * this driver can invoke registered glue-logic as well.
 * It also invoke the eee initialization because it could happen when switch
 * on different networks (that are eee capable).
809 810 811 812
 */
static void stmmac_adjust_link(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
813
	struct phy_device *phydev = dev->phydev;
814
	unsigned long flags;
815
	bool new_state = false;
816

817
	if (!phydev)
818 819 820
		return;

	spin_lock_irqsave(&priv->lock, flags);
821

822
	if (phydev->link) {
823
		u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
824 825 826 827

		/* Now we make sure that we can be in full duplex mode.
		 * If not, we operate in half-duplex mode. */
		if (phydev->duplex != priv->oldduplex) {
828
			new_state = true;
829
			if (!phydev->duplex)
830
				ctrl &= ~priv->hw->link.duplex;
831
			else
832
				ctrl |= priv->hw->link.duplex;
833 834 835 836
			priv->oldduplex = phydev->duplex;
		}
		/* Flow Control operation */
		if (phydev->pause)
837
			stmmac_mac_flow_ctrl(priv, phydev->duplex);
838 839

		if (phydev->speed != priv->speed) {
840
			new_state = true;
841
			ctrl &= ~priv->hw->link.speed_mask;
842
			switch (phydev->speed) {
843
			case SPEED_1000:
844
				ctrl |= priv->hw->link.speed1000;
845
				break;
846
			case SPEED_100:
847
				ctrl |= priv->hw->link.speed100;
848
				break;
849
			case SPEED_10:
850
				ctrl |= priv->hw->link.speed10;
851 852
				break;
			default:
853
				netif_warn(priv, link, priv->dev,
854
					   "broken speed: %d\n", phydev->speed);
855
				phydev->speed = SPEED_UNKNOWN;
856 857
				break;
			}
858 859
			if (phydev->speed != SPEED_UNKNOWN)
				stmmac_hw_fix_mac_speed(priv);
860 861 862
			priv->speed = phydev->speed;
		}

863
		writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
864 865

		if (!priv->oldlink) {
866
			new_state = true;
867
			priv->oldlink = true;
868 869
		}
	} else if (priv->oldlink) {
870
		new_state = true;
871
		priv->oldlink = false;
872 873
		priv->speed = SPEED_UNKNOWN;
		priv->oldduplex = DUPLEX_UNKNOWN;
874 875 876 877 878
	}

	if (new_state && netif_msg_link(priv))
		phy_print_status(phydev);

879 880
	spin_unlock_irqrestore(&priv->lock, flags);

881 882 883 884 885 886 887 888 889 890
	if (phydev->is_pseudo_fixed_link)
		/* Stop PHY layer to call the hook to adjust the link in case
		 * of a switch is attached to the stmmac driver.
		 */
		phydev->irq = PHY_IGNORE_INTERRUPT;
	else
		/* At this stage, init the EEE if supported.
		 * Never called in case of fixed_link.
		 */
		priv->eee_enabled = stmmac_eee_init(priv);
891 892
}

893
/**
894
 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
895 896 897 898 899
 * @priv: driver private structure
 * Description: this is to verify if the HW supports the PCS.
 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 * configured for the TBI, RTBI, or SGMII PHY interface.
 */
900 901 902 903 904
static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
{
	int interface = priv->plat->interface;

	if (priv->dma_cap.pcs) {
B
Byungho An 已提交
905 906 907 908
		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
909
			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
910
			priv->hw->pcs = STMMAC_PCS_RGMII;
B
Byungho An 已提交
911
		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
912
			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
913
			priv->hw->pcs = STMMAC_PCS_SGMII;
914 915 916 917
		}
	}
}

918 919 920 921 922 923 924 925 926 927 928 929
/**
 * stmmac_init_phy - PHY initialization
 * @dev: net device structure
 * Description: it initializes the driver's PHY state, and attaches the PHY
 * to the mac driver.
 *  Return value:
 *  0 on success
 */
static int stmmac_init_phy(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	struct phy_device *phydev;
930
	char phy_id_fmt[MII_BUS_ID_SIZE + 3];
931
	char bus_id[MII_BUS_ID_SIZE];
932
	int interface = priv->plat->interface;
933
	int max_speed = priv->plat->max_speed;
934
	priv->oldlink = false;
935 936
	priv->speed = SPEED_UNKNOWN;
	priv->oldduplex = DUPLEX_UNKNOWN;
937

938 939 940 941
	if (priv->plat->phy_node) {
		phydev = of_phy_connect(dev, priv->plat->phy_node,
					&stmmac_adjust_link, 0, interface);
	} else {
G
Giuseppe CAVALLARO 已提交
942 943
		snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
			 priv->plat->bus_id);
944 945 946

		snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
			 priv->plat->phy_addr);
947
		netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
948
			   phy_id_fmt);
949 950 951 952

		phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
				     interface);
	}
953

954
	if (IS_ERR_OR_NULL(phydev)) {
955
		netdev_err(priv->dev, "Could not attach to PHY\n");
956 957 958
		if (!phydev)
			return -ENODEV;

959 960 961
		return PTR_ERR(phydev);
	}

962
	/* Stop Advertising 1000BASE Capability if interface is not GMII */
963
	if ((interface == PHY_INTERFACE_MODE_MII) ||
964
	    (interface == PHY_INTERFACE_MODE_RMII) ||
P
Pavel Machek 已提交
965
		(max_speed < 1000 && max_speed > 0))
966 967
		phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
					 SUPPORTED_1000baseT_Full);
968

969 970 971 972 973 974 975
	/*
	 * Broken HW is sometimes missing the pull-up resistor on the
	 * MDIO line, which results in reads to non-existent devices returning
	 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
	 * device as well.
	 * Note: phydev->phy_id is the result of reading the UID PHY registers.
	 */
976
	if (!priv->plat->phy_node && phydev->phy_id == 0) {
977 978 979
		phy_disconnect(phydev);
		return -ENODEV;
	}
980

981 982 983 984 985 986 987
	/* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
	 * subsequent PHY polling, make sure we force a link transition if
	 * we have a UP/DOWN/UP transition
	 */
	if (phydev->is_pseudo_fixed_link)
		phydev->irq = PHY_POLL;

988
	phy_attached_info(phydev);
989 990 991
	return 0;
}

992
static void stmmac_display_rx_rings(struct stmmac_priv *priv)
993
{
994
	u32 rx_cnt = priv->plat->rx_queues_to_use;
995
	void *head_rx;
996
	u32 queue;
997

998 999 1000
	/* Display RX rings */
	for (queue = 0; queue < rx_cnt; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1001

1002 1003 1004 1005 1006 1007 1008 1009
		pr_info("\tRX Queue %u rings\n", queue);

		if (priv->extend_desc)
			head_rx = (void *)rx_q->dma_erx;
		else
			head_rx = (void *)rx_q->dma_rx;

		/* Display RX ring */
1010
		stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
1011
	}
1012 1013 1014 1015
}

static void stmmac_display_tx_rings(struct stmmac_priv *priv)
{
1016
	u32 tx_cnt = priv->plat->tx_queues_to_use;
1017
	void *head_tx;
1018
	u32 queue;
1019

1020 1021 1022
	/* Display TX rings */
	for (queue = 0; queue < tx_cnt; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1023

1024 1025 1026 1027 1028 1029 1030
		pr_info("\tTX Queue %d rings\n", queue);

		if (priv->extend_desc)
			head_tx = (void *)tx_q->dma_etx;
		else
			head_tx = (void *)tx_q->dma_tx;

1031
		stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
1032
	}
1033 1034
}

1035 1036 1037 1038 1039 1040 1041 1042 1043
static void stmmac_display_rings(struct stmmac_priv *priv)
{
	/* Display RX ring */
	stmmac_display_rx_rings(priv);

	/* Display TX ring */
	stmmac_display_tx_rings(priv);
}

1044 1045 1046 1047 1048 1049 1050 1051
static int stmmac_set_bfsize(int mtu, int bufsize)
{
	int ret = bufsize;

	if (mtu >= BUF_SIZE_4KiB)
		ret = BUF_SIZE_8KiB;
	else if (mtu >= BUF_SIZE_2KiB)
		ret = BUF_SIZE_4KiB;
1052
	else if (mtu > DEFAULT_BUFSIZE)
1053 1054
		ret = BUF_SIZE_2KiB;
	else
1055
		ret = DEFAULT_BUFSIZE;
1056 1057 1058 1059

	return ret;
}

1060
/**
1061
 * stmmac_clear_rx_descriptors - clear RX descriptors
1062
 * @priv: driver private structure
1063
 * @queue: RX queue index
1064
 * Description: this function is called to clear the RX descriptors
1065 1066
 * in case of both basic and extended descriptors are used.
 */
1067
static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1068
{
1069
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1070
	int i;
1071

1072
	/* Clear the RX descriptors */
1073
	for (i = 0; i < DMA_RX_SIZE; i++)
1074
		if (priv->extend_desc)
1075 1076 1077
			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
					priv->use_riwt, priv->mode,
					(i == DMA_RX_SIZE - 1));
1078
		else
1079 1080 1081
			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
					priv->use_riwt, priv->mode,
					(i == DMA_RX_SIZE - 1));
1082 1083 1084 1085 1086
}

/**
 * stmmac_clear_tx_descriptors - clear tx descriptors
 * @priv: driver private structure
1087
 * @queue: TX queue index.
1088 1089 1090
 * Description: this function is called to clear the TX descriptors
 * in case of both basic and extended descriptors are used.
 */
1091
static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1092
{
1093
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1094 1095 1096
	int i;

	/* Clear the TX descriptors */
1097
	for (i = 0; i < DMA_TX_SIZE; i++)
1098
		if (priv->extend_desc)
1099 1100
			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
					priv->mode, (i == DMA_TX_SIZE - 1));
1101
		else
1102 1103
			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
					priv->mode, (i == DMA_TX_SIZE - 1));
1104 1105
}

1106 1107 1108 1109 1110 1111 1112 1113
/**
 * stmmac_clear_descriptors - clear descriptors
 * @priv: driver private structure
 * Description: this function is called to clear the TX and RX descriptors
 * in case of both basic and extended descriptors are used.
 */
static void stmmac_clear_descriptors(struct stmmac_priv *priv)
{
1114
	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1115
	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1116 1117
	u32 queue;

1118
	/* Clear the RX descriptors */
1119 1120
	for (queue = 0; queue < rx_queue_cnt; queue++)
		stmmac_clear_rx_descriptors(priv, queue);
1121 1122

	/* Clear the TX descriptors */
1123 1124
	for (queue = 0; queue < tx_queue_cnt; queue++)
		stmmac_clear_tx_descriptors(priv, queue);
1125 1126
}

1127 1128 1129 1130 1131
/**
 * stmmac_init_rx_buffers - init the RX descriptor buffer.
 * @priv: driver private structure
 * @p: descriptor pointer
 * @i: descriptor index
1132 1133
 * @flags: gfp flag
 * @queue: RX queue index
1134 1135 1136
 * Description: this function is called to allocate a receive buffer, perform
 * the DMA mapping and init the descriptor.
 */
1137
static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1138
				  int i, gfp_t flags, u32 queue)
1139
{
1140
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1141 1142
	struct sk_buff *skb;

1143
	skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1144
	if (!skb) {
1145 1146
		netdev_err(priv->dev,
			   "%s: Rx init fails; skb is NULL\n", __func__);
1147
		return -ENOMEM;
1148
	}
1149 1150
	rx_q->rx_skbuff[i] = skb;
	rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1151 1152
						priv->dma_buf_sz,
						DMA_FROM_DEVICE);
1153
	if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1154
		netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1155 1156 1157
		dev_kfree_skb_any(skb);
		return -EINVAL;
	}
1158

A
Alexandre TORGUE 已提交
1159
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
1160
		p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
A
Alexandre TORGUE 已提交
1161
	else
1162
		p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1163

G
Giuseppe CAVALLARO 已提交
1164
	if ((priv->hw->mode->init_desc3) &&
1165
	    (priv->dma_buf_sz == BUF_SIZE_16KiB))
G
Giuseppe CAVALLARO 已提交
1166
		priv->hw->mode->init_desc3(p);
1167 1168 1169 1170

	return 0;
}

1171 1172 1173
/**
 * stmmac_free_rx_buffer - free RX dma buffers
 * @priv: private structure
1174
 * @queue: RX queue index
1175 1176
 * @i: buffer index.
 */
1177
static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1178
{
1179 1180 1181 1182
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

	if (rx_q->rx_skbuff[i]) {
		dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1183
				 priv->dma_buf_sz, DMA_FROM_DEVICE);
1184
		dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1185
	}
1186
	rx_q->rx_skbuff[i] = NULL;
1187 1188 1189
}

/**
1190 1191
 * stmmac_free_tx_buffer - free RX dma buffers
 * @priv: private structure
1192
 * @queue: RX queue index
1193 1194
 * @i: buffer index.
 */
1195
static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1196
{
1197 1198 1199 1200
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];

	if (tx_q->tx_skbuff_dma[i].buf) {
		if (tx_q->tx_skbuff_dma[i].map_as_page)
1201
			dma_unmap_page(priv->device,
1202 1203
				       tx_q->tx_skbuff_dma[i].buf,
				       tx_q->tx_skbuff_dma[i].len,
1204 1205 1206
				       DMA_TO_DEVICE);
		else
			dma_unmap_single(priv->device,
1207 1208
					 tx_q->tx_skbuff_dma[i].buf,
					 tx_q->tx_skbuff_dma[i].len,
1209 1210 1211
					 DMA_TO_DEVICE);
	}

1212 1213 1214 1215 1216
	if (tx_q->tx_skbuff[i]) {
		dev_kfree_skb_any(tx_q->tx_skbuff[i]);
		tx_q->tx_skbuff[i] = NULL;
		tx_q->tx_skbuff_dma[i].buf = 0;
		tx_q->tx_skbuff_dma[i].map_as_page = false;
1217 1218 1219 1220 1221
	}
}

/**
 * init_dma_rx_desc_rings - init the RX descriptor rings
1222
 * @dev: net device structure
1223
 * @flags: gfp flag.
1224
 * Description: this function initializes the DMA RX descriptors
1225
 * and allocates the socket buffers. It supports the chained and ring
1226
 * modes.
1227
 */
1228
static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1229 1230
{
	struct stmmac_priv *priv = netdev_priv(dev);
1231
	u32 rx_count = priv->plat->rx_queues_to_use;
1232
	unsigned int bfsize = 0;
1233
	int ret = -ENOMEM;
1234
	int queue;
1235
	int i;
1236

G
Giuseppe CAVALLARO 已提交
1237 1238
	if (priv->hw->mode->set_16kib_bfsize)
		bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1239

1240
	if (bfsize < BUF_SIZE_16KiB)
1241
		bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1242

1243 1244
	priv->dma_buf_sz = bfsize;

1245
	/* RX INITIALIZATION */
1246 1247
	netif_dbg(priv, probe, priv->dev,
		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
1248

1249 1250
	for (queue = 0; queue < rx_count; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1251

1252 1253 1254
		netif_dbg(priv, probe, priv->dev,
			  "(%s) dma_rx_phy=0x%08x\n", __func__,
			  (u32)rx_q->dma_rx_phy);
A
Alexandre TORGUE 已提交
1255

1256 1257
		for (i = 0; i < DMA_RX_SIZE; i++) {
			struct dma_desc *p;
1258

1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
			if (priv->extend_desc)
				p = &((rx_q->dma_erx + i)->basic);
			else
				p = rx_q->dma_rx + i;

			ret = stmmac_init_rx_buffers(priv, p, i, flags,
						     queue);
			if (ret)
				goto err_init_rx_buffers;

			netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
				  rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
				  (unsigned int)rx_q->rx_skbuff_dma[i]);
		}

		rx_q->cur_rx = 0;
		rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);

		stmmac_clear_rx_descriptors(priv, queue);

		/* Setup the chained descriptor addresses */
		if (priv->mode == STMMAC_CHAIN_MODE) {
			if (priv->extend_desc)
				priv->hw->mode->init(rx_q->dma_erx,
						     rx_q->dma_rx_phy,
						     DMA_RX_SIZE, 1);
			else
				priv->hw->mode->init(rx_q->dma_rx,
						     rx_q->dma_rx_phy,
						     DMA_RX_SIZE, 0);
		}
1290 1291
	}

1292 1293
	buf_sz = bfsize;

1294
	return 0;
1295

1296
err_init_rx_buffers:
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
	while (queue >= 0) {
		while (--i >= 0)
			stmmac_free_rx_buffer(priv, queue, i);

		if (queue == 0)
			break;

		i = DMA_RX_SIZE;
		queue--;
	}

1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
	return ret;
}

/**
 * init_dma_tx_desc_rings - init the TX descriptor rings
 * @dev: net device structure.
 * Description: this function initializes the DMA TX descriptors
 * and allocates the socket buffers. It supports the chained and ring
 * modes.
 */
static int init_dma_tx_desc_rings(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
1321 1322
	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
	u32 queue;
1323 1324
	int i;

1325 1326
	for (queue = 0; queue < tx_queue_cnt; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1327

1328 1329 1330
		netif_dbg(priv, probe, priv->dev,
			  "(%s) dma_tx_phy=0x%08x\n", __func__,
			 (u32)tx_q->dma_tx_phy);
A
Alexandre TORGUE 已提交
1331

1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
		/* Setup the chained descriptor addresses */
		if (priv->mode == STMMAC_CHAIN_MODE) {
			if (priv->extend_desc)
				priv->hw->mode->init(tx_q->dma_etx,
						     tx_q->dma_tx_phy,
						     DMA_TX_SIZE, 1);
			else
				priv->hw->mode->init(tx_q->dma_tx,
						     tx_q->dma_tx_phy,
						     DMA_TX_SIZE, 0);
		}
1343

1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
		for (i = 0; i < DMA_TX_SIZE; i++) {
			struct dma_desc *p;
			if (priv->extend_desc)
				p = &((tx_q->dma_etx + i)->basic);
			else
				p = tx_q->dma_tx + i;

			if (priv->synopsys_id >= DWMAC_CORE_4_00) {
				p->des0 = 0;
				p->des1 = 0;
				p->des2 = 0;
				p->des3 = 0;
			} else {
				p->des2 = 0;
			}

			tx_q->tx_skbuff_dma[i].buf = 0;
			tx_q->tx_skbuff_dma[i].map_as_page = false;
			tx_q->tx_skbuff_dma[i].len = 0;
			tx_q->tx_skbuff_dma[i].last_segment = false;
			tx_q->tx_skbuff[i] = NULL;
1365
		}
1366

1367 1368
		tx_q->dirty_tx = 0;
		tx_q->cur_tx = 0;
1369
		tx_q->mss = 0;
1370

1371 1372
		netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
	}
1373

1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	return 0;
}

/**
 * init_dma_desc_rings - init the RX/TX descriptor rings
 * @dev: net device structure
 * @flags: gfp flag.
 * Description: this function initializes the DMA RX/TX descriptors
 * and allocates the socket buffers. It supports the chained and ring
 * modes.
 */
static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	int ret;

	ret = init_dma_rx_desc_rings(dev, flags);
	if (ret)
		return ret;

	ret = init_dma_tx_desc_rings(dev);

1396
	stmmac_clear_descriptors(priv);
1397

1398 1399
	if (netif_msg_hw(priv))
		stmmac_display_rings(priv);
1400 1401

	return ret;
1402 1403
}

1404 1405 1406
/**
 * dma_free_rx_skbufs - free RX dma buffers
 * @priv: private structure
1407
 * @queue: RX queue index
1408
 */
1409
static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1410 1411 1412
{
	int i;

1413
	for (i = 0; i < DMA_RX_SIZE; i++)
1414
		stmmac_free_rx_buffer(priv, queue, i);
1415 1416
}

1417 1418 1419
/**
 * dma_free_tx_skbufs - free TX dma buffers
 * @priv: private structure
1420
 * @queue: TX queue index
1421
 */
1422
static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1423 1424 1425
{
	int i;

1426
	for (i = 0; i < DMA_TX_SIZE; i++)
1427
		stmmac_free_tx_buffer(priv, queue, i);
1428 1429
}

1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
/**
 * free_dma_rx_desc_resources - free RX dma desc resources
 * @priv: private structure
 */
static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
{
	u32 rx_count = priv->plat->rx_queues_to_use;
	u32 queue;

	/* Free RX queue resources */
	for (queue = 0; queue < rx_count; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		/* Release the DMA RX socket buffers */
		dma_free_rx_skbufs(priv, queue);

		/* Free DMA regions of consistent memory previously allocated */
		if (!priv->extend_desc)
			dma_free_coherent(priv->device,
					  DMA_RX_SIZE * sizeof(struct dma_desc),
					  rx_q->dma_rx, rx_q->dma_rx_phy);
		else
			dma_free_coherent(priv->device, DMA_RX_SIZE *
					  sizeof(struct dma_extended_desc),
					  rx_q->dma_erx, rx_q->dma_rx_phy);

		kfree(rx_q->rx_skbuff_dma);
		kfree(rx_q->rx_skbuff);
	}
}

1461 1462 1463 1464 1465 1466 1467
/**
 * free_dma_tx_desc_resources - free TX dma desc resources
 * @priv: private structure
 */
static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
{
	u32 tx_count = priv->plat->tx_queues_to_use;
1468
	u32 queue;
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491

	/* Free TX queue resources */
	for (queue = 0; queue < tx_count; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];

		/* Release the DMA TX socket buffers */
		dma_free_tx_skbufs(priv, queue);

		/* Free DMA regions of consistent memory previously allocated */
		if (!priv->extend_desc)
			dma_free_coherent(priv->device,
					  DMA_TX_SIZE * sizeof(struct dma_desc),
					  tx_q->dma_tx, tx_q->dma_tx_phy);
		else
			dma_free_coherent(priv->device, DMA_TX_SIZE *
					  sizeof(struct dma_extended_desc),
					  tx_q->dma_etx, tx_q->dma_tx_phy);

		kfree(tx_q->tx_skbuff_dma);
		kfree(tx_q->tx_skbuff);
	}
}

1492
/**
1493
 * alloc_dma_rx_desc_resources - alloc RX resources.
1494 1495
 * @priv: private structure
 * Description: according to which descriptor can be used (extend or basic)
1496 1497 1498
 * this function allocates the resources for TX and RX paths. In case of
 * reception, for example, it pre-allocated the RX socket buffer in order to
 * allow zero-copy mechanism.
1499
 */
1500
static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1501
{
1502
	u32 rx_count = priv->plat->rx_queues_to_use;
1503
	int ret = -ENOMEM;
1504
	u32 queue;
1505

1506 1507 1508
	/* RX queues buffers and DMA */
	for (queue = 0; queue < rx_count; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1509

1510 1511
		rx_q->queue_index = queue;
		rx_q->priv_data = priv;
1512

1513 1514
		rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
						    sizeof(dma_addr_t),
1515
						    GFP_KERNEL);
1516
		if (!rx_q->rx_skbuff_dma)
1517
			goto err_dma;
1518

1519 1520 1521 1522
		rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
						sizeof(struct sk_buff *),
						GFP_KERNEL);
		if (!rx_q->rx_skbuff)
1523
			goto err_dma;
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544

		if (priv->extend_desc) {
			rx_q->dma_erx = dma_zalloc_coherent(priv->device,
							    DMA_RX_SIZE *
							    sizeof(struct
							    dma_extended_desc),
							    &rx_q->dma_rx_phy,
							    GFP_KERNEL);
			if (!rx_q->dma_erx)
				goto err_dma;

		} else {
			rx_q->dma_rx = dma_zalloc_coherent(priv->device,
							   DMA_RX_SIZE *
							   sizeof(struct
							   dma_desc),
							   &rx_q->dma_rx_phy,
							   GFP_KERNEL);
			if (!rx_q->dma_rx)
				goto err_dma;
		}
1545 1546 1547 1548 1549
	}

	return 0;

err_dma:
1550 1551
	free_dma_rx_desc_resources(priv);

1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
	return ret;
}

/**
 * alloc_dma_tx_desc_resources - alloc TX resources.
 * @priv: private structure
 * Description: according to which descriptor can be used (extend or basic)
 * this function allocates the resources for TX and RX paths. In case of
 * reception, for example, it pre-allocated the RX socket buffer in order to
 * allow zero-copy mechanism.
 */
static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
{
1565
	u32 tx_count = priv->plat->tx_queues_to_use;
1566
	int ret = -ENOMEM;
1567
	u32 queue;
1568

1569 1570 1571
	/* TX queues buffers and DMA */
	for (queue = 0; queue < tx_count; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1572

1573 1574
		tx_q->queue_index = queue;
		tx_q->priv_data = priv;
1575

1576 1577
		tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
						    sizeof(*tx_q->tx_skbuff_dma),
1578
						    GFP_KERNEL);
1579
		if (!tx_q->tx_skbuff_dma)
1580
			goto err_dma;
1581 1582 1583 1584 1585

		tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
						sizeof(struct sk_buff *),
						GFP_KERNEL);
		if (!tx_q->tx_skbuff)
1586
			goto err_dma;
1587 1588 1589 1590 1591 1592 1593 1594 1595

		if (priv->extend_desc) {
			tx_q->dma_etx = dma_zalloc_coherent(priv->device,
							    DMA_TX_SIZE *
							    sizeof(struct
							    dma_extended_desc),
							    &tx_q->dma_tx_phy,
							    GFP_KERNEL);
			if (!tx_q->dma_etx)
1596
				goto err_dma;
1597 1598 1599 1600 1601 1602 1603 1604
		} else {
			tx_q->dma_tx = dma_zalloc_coherent(priv->device,
							   DMA_TX_SIZE *
							   sizeof(struct
								  dma_desc),
							   &tx_q->dma_tx_phy,
							   GFP_KERNEL);
			if (!tx_q->dma_tx)
1605
				goto err_dma;
1606
		}
1607 1608 1609 1610
	}

	return 0;

1611
err_dma:
1612 1613
	free_dma_tx_desc_resources(priv);

1614 1615 1616
	return ret;
}

1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
/**
 * alloc_dma_desc_resources - alloc TX/RX resources.
 * @priv: private structure
 * Description: according to which descriptor can be used (extend or basic)
 * this function allocates the resources for TX and RX paths. In case of
 * reception, for example, it pre-allocated the RX socket buffer in order to
 * allow zero-copy mechanism.
 */
static int alloc_dma_desc_resources(struct stmmac_priv *priv)
{
1627
	/* RX Allocation */
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
	int ret = alloc_dma_rx_desc_resources(priv);

	if (ret)
		return ret;

	ret = alloc_dma_tx_desc_resources(priv);

	return ret;
}

/**
 * free_dma_desc_resources - free dma desc resources
 * @priv: private structure
 */
static void free_dma_desc_resources(struct stmmac_priv *priv)
{
	/* Release the DMA RX socket buffers */
	free_dma_rx_desc_resources(priv);

	/* Release the DMA TX socket buffers */
	free_dma_tx_desc_resources(priv);
}

J
jpinto 已提交
1651 1652 1653 1654 1655 1656 1657
/**
 *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
 *  @priv: driver private structure
 *  Description: It is used for enabling the rx queues in the MAC
 */
static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
{
1658 1659 1660
	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	int queue;
	u8 mode;
J
jpinto 已提交
1661

1662 1663
	for (queue = 0; queue < rx_queues_count; queue++) {
		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1664
		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1665
	}
J
jpinto 已提交
1666 1667
}

1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
/**
 * stmmac_start_rx_dma - start RX DMA channel
 * @priv: driver private structure
 * @chan: RX channel index
 * Description:
 * This starts a RX DMA channel
 */
static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
{
	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1678
	stmmac_start_rx(priv, priv->ioaddr, chan);
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690
}

/**
 * stmmac_start_tx_dma - start TX DMA channel
 * @priv: driver private structure
 * @chan: TX channel index
 * Description:
 * This starts a TX DMA channel
 */
static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
{
	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1691
	stmmac_start_tx(priv, priv->ioaddr, chan);
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
}

/**
 * stmmac_stop_rx_dma - stop RX DMA channel
 * @priv: driver private structure
 * @chan: RX channel index
 * Description:
 * This stops a RX DMA channel
 */
static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
{
	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1704
	stmmac_stop_rx(priv, priv->ioaddr, chan);
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
}

/**
 * stmmac_stop_tx_dma - stop TX DMA channel
 * @priv: driver private structure
 * @chan: TX channel index
 * Description:
 * This stops a TX DMA channel
 */
static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
{
	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1717
	stmmac_stop_tx(priv, priv->ioaddr, chan);
1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757
}

/**
 * stmmac_start_all_dma - start all RX and TX DMA channels
 * @priv: driver private structure
 * Description:
 * This starts all the RX and TX DMA channels
 */
static void stmmac_start_all_dma(struct stmmac_priv *priv)
{
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	u32 chan = 0;

	for (chan = 0; chan < rx_channels_count; chan++)
		stmmac_start_rx_dma(priv, chan);

	for (chan = 0; chan < tx_channels_count; chan++)
		stmmac_start_tx_dma(priv, chan);
}

/**
 * stmmac_stop_all_dma - stop all RX and TX DMA channels
 * @priv: driver private structure
 * Description:
 * This stops the RX and TX DMA channels
 */
static void stmmac_stop_all_dma(struct stmmac_priv *priv)
{
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	u32 chan = 0;

	for (chan = 0; chan < rx_channels_count; chan++)
		stmmac_stop_rx_dma(priv, chan);

	for (chan = 0; chan < tx_channels_count; chan++)
		stmmac_stop_tx_dma(priv, chan);
}

1758 1759
/**
 *  stmmac_dma_operation_mode - HW DMA operation mode
1760
 *  @priv: driver private structure
1761 1762
 *  Description: it is used for configuring the DMA operation mode register in
 *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1763 1764 1765
 */
static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
{
1766 1767
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1768
	int rxfifosz = priv->plat->rx_fifo_size;
1769
	int txfifosz = priv->plat->tx_fifo_size;
1770 1771 1772
	u32 txmode = 0;
	u32 rxmode = 0;
	u32 chan = 0;
1773
	u8 qmode = 0;
1774

1775 1776
	if (rxfifosz == 0)
		rxfifosz = priv->dma_cap.rx_fifo_size;
1777 1778 1779 1780 1781 1782
	if (txfifosz == 0)
		txfifosz = priv->dma_cap.tx_fifo_size;

	/* Adjust for real per queue fifo size */
	rxfifosz /= rx_channels_count;
	txfifosz /= tx_channels_count;
1783

1784 1785 1786 1787
	if (priv->plat->force_thresh_dma_mode) {
		txmode = tc;
		rxmode = tc;
	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1788 1789 1790
		/*
		 * In case of GMAC, SF mode can be enabled
		 * to perform the TX COE in HW. This depends on:
1791 1792 1793 1794
		 * 1) TX COE if actually supported
		 * 2) There is no bugged Jumbo frame support
		 *    that needs to not insert csum in the TDES.
		 */
1795 1796
		txmode = SF_DMA_MODE;
		rxmode = SF_DMA_MODE;
1797
		priv->xstats.threshold = SF_DMA_MODE;
1798 1799 1800 1801 1802 1803 1804
	} else {
		txmode = tc;
		rxmode = SF_DMA_MODE;
	}

	/* configure all channels */
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1805 1806 1807
		for (chan = 0; chan < rx_channels_count; chan++) {
			qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;

1808 1809
			stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
					rxfifosz, qmode);
1810 1811 1812 1813
		}

		for (chan = 0; chan < tx_channels_count; chan++) {
			qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1814

1815 1816
			stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
					txfifosz, qmode);
1817
		}
1818
	} else {
1819
		stmmac_dma_mode(priv, priv->ioaddr, txmode, rxmode, rxfifosz);
1820
	}
1821 1822 1823
}

/**
1824
 * stmmac_tx_clean - to manage the transmission completion
1825
 * @priv: driver private structure
1826
 * @queue: TX queue index
1827
 * Description: it reclaims the transmit resources after transmission completes.
1828
 */
1829
static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1830
{
1831
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
B
Beniamino Galvani 已提交
1832
	unsigned int bytes_compl = 0, pkts_compl = 0;
1833
	unsigned int entry;
1834

1835
	netif_tx_lock(priv->dev);
1836

1837 1838
	priv->xstats.tx_clean++;

1839
	entry = tx_q->dirty_tx;
1840 1841
	while (entry != tx_q->cur_tx) {
		struct sk_buff *skb = tx_q->tx_skbuff[entry];
1842
		struct dma_desc *p;
1843
		int status;
1844 1845

		if (priv->extend_desc)
1846
			p = (struct dma_desc *)(tx_q->dma_etx + entry);
1847
		else
1848
			p = tx_q->dma_tx + entry;
1849

1850 1851
		status = stmmac_tx_status(priv, &priv->dev->stats,
				&priv->xstats, p, priv->ioaddr);
1852 1853 1854 1855
		/* Check if the descriptor is owned by the DMA */
		if (unlikely(status & tx_dma_own))
			break;

1856 1857 1858 1859 1860
		/* Make sure descriptor fields are read after reading
		 * the own bit.
		 */
		dma_rmb();

1861 1862 1863 1864 1865 1866
		/* Just consider the last segment and ...*/
		if (likely(!(status & tx_not_ls))) {
			/* ... verify the status error condition */
			if (unlikely(status & tx_err)) {
				priv->dev->stats.tx_errors++;
			} else {
1867 1868
				priv->dev->stats.tx_packets++;
				priv->xstats.tx_pkt_n++;
1869
			}
1870
			stmmac_get_tx_hwtstamp(priv, p, skb);
1871 1872
		}

1873 1874
		if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
			if (tx_q->tx_skbuff_dma[entry].map_as_page)
G
Giuseppe CAVALLARO 已提交
1875
				dma_unmap_page(priv->device,
1876 1877
					       tx_q->tx_skbuff_dma[entry].buf,
					       tx_q->tx_skbuff_dma[entry].len,
G
Giuseppe CAVALLARO 已提交
1878 1879 1880
					       DMA_TO_DEVICE);
			else
				dma_unmap_single(priv->device,
1881 1882
						 tx_q->tx_skbuff_dma[entry].buf,
						 tx_q->tx_skbuff_dma[entry].len,
G
Giuseppe CAVALLARO 已提交
1883
						 DMA_TO_DEVICE);
1884 1885 1886
			tx_q->tx_skbuff_dma[entry].buf = 0;
			tx_q->tx_skbuff_dma[entry].len = 0;
			tx_q->tx_skbuff_dma[entry].map_as_page = false;
1887
		}
A
Alexandre TORGUE 已提交
1888 1889

		if (priv->hw->mode->clean_desc3)
1890
			priv->hw->mode->clean_desc3(tx_q, p);
A
Alexandre TORGUE 已提交
1891

1892 1893
		tx_q->tx_skbuff_dma[entry].last_segment = false;
		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1894 1895

		if (likely(skb != NULL)) {
B
Beniamino Galvani 已提交
1896 1897
			pkts_compl++;
			bytes_compl += skb->len;
1898
			dev_consume_skb_any(skb);
1899
			tx_q->tx_skbuff[entry] = NULL;
1900 1901
		}

1902
		stmmac_release_tx_desc(priv, p, priv->mode);
1903

1904
		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1905
	}
1906
	tx_q->dirty_tx = entry;
B
Beniamino Galvani 已提交
1907

1908 1909 1910 1911 1912 1913
	netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
				  pkts_compl, bytes_compl);

	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
								queue))) &&
	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
B
Beniamino Galvani 已提交
1914

1915 1916
		netif_dbg(priv, tx_done, priv->dev,
			  "%s: restart transmit\n", __func__);
1917
		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1918
	}
1919 1920 1921

	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
		stmmac_enable_eee_mode(priv);
G
Giuseppe CAVALLARO 已提交
1922
		mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1923
	}
1924
	netif_tx_unlock(priv->dev);
1925 1926 1927
}

/**
1928
 * stmmac_tx_err - to manage the tx error
1929
 * @priv: driver private structure
1930
 * @chan: channel index
1931
 * Description: it cleans the descriptors and restarts the transmission
1932
 * in case of transmission errors.
1933
 */
1934
static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1935
{
1936
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1937
	int i;
1938

1939
	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1940

1941
	stmmac_stop_tx_dma(priv, chan);
1942
	dma_free_tx_skbufs(priv, chan);
1943
	for (i = 0; i < DMA_TX_SIZE; i++)
1944
		if (priv->extend_desc)
1945 1946
			stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
					priv->mode, (i == DMA_TX_SIZE - 1));
1947
		else
1948 1949
			stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
					priv->mode, (i == DMA_TX_SIZE - 1));
1950 1951
	tx_q->dirty_tx = 0;
	tx_q->cur_tx = 0;
1952
	tx_q->mss = 0;
1953
	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1954
	stmmac_start_tx_dma(priv, chan);
1955 1956

	priv->dev->stats.tx_errors++;
1957
	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1958 1959
}

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
/**
 *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
 *  @priv: driver private structure
 *  @txmode: TX operating mode
 *  @rxmode: RX operating mode
 *  @chan: channel index
 *  Description: it is used for configuring of the DMA operation mode in
 *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
 *  mode.
 */
static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
					  u32 rxmode, u32 chan)
{
1973 1974
	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1975 1976
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
1977
	int rxfifosz = priv->plat->rx_fifo_size;
1978
	int txfifosz = priv->plat->tx_fifo_size;
1979 1980 1981

	if (rxfifosz == 0)
		rxfifosz = priv->dma_cap.rx_fifo_size;
1982 1983 1984 1985 1986 1987
	if (txfifosz == 0)
		txfifosz = priv->dma_cap.tx_fifo_size;

	/* Adjust for real per queue fifo size */
	rxfifosz /= rx_channels_count;
	txfifosz /= tx_channels_count;
1988 1989

	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1990 1991 1992 1993
		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz,
				rxqmode);
		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz,
				txqmode);
1994
	} else {
1995
		stmmac_dma_mode(priv, priv->ioaddr, txmode, rxmode, rxfifosz);
1996 1997 1998
	}
}

1999 2000
static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
{
2001
	int ret = false;
2002 2003 2004 2005

	/* Safety features are only available in cores >= 5.10 */
	if (priv->synopsys_id < DWMAC_CORE_5_10)
		return ret;
2006 2007 2008
	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
	if (ret && (ret != -EINVAL)) {
2009
		stmmac_global_err(priv);
2010 2011 2012 2013
		return true;
	}

	return false;
2014 2015
}

2016
/**
2017
 * stmmac_dma_interrupt - DMA ISR
2018 2019
 * @priv: driver private structure
 * Description: this is the DMA ISR. It is called by the main ISR.
2020 2021
 * It calls the dwmac dma routine and schedule poll method in case of some
 * work can be done.
2022
 */
2023 2024
static void stmmac_dma_interrupt(struct stmmac_priv *priv)
{
2025
	u32 tx_channel_count = priv->plat->tx_queues_to_use;
2026 2027 2028
	u32 rx_channel_count = priv->plat->rx_queues_to_use;
	u32 channels_to_check = tx_channel_count > rx_channel_count ?
				tx_channel_count : rx_channel_count;
2029
	u32 chan;
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039
	bool poll_scheduled = false;
	int status[channels_to_check];

	/* Each DMA channel can be used for rx and tx simultaneously, yet
	 * napi_struct is embedded in struct stmmac_rx_queue rather than in a
	 * stmmac_channel struct.
	 * Because of this, stmmac_poll currently checks (and possibly wakes)
	 * all tx queues rather than just a single tx queue.
	 */
	for (chan = 0; chan < channels_to_check; chan++)
2040 2041
		status[chan] = stmmac_dma_interrupt_status(priv, priv->ioaddr,
				&priv->xstats, chan);
2042

2043 2044 2045
	for (chan = 0; chan < rx_channel_count; chan++) {
		if (likely(status[chan] & handle_rx)) {
			struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2046 2047

			if (likely(napi_schedule_prep(&rx_q->napi))) {
2048
				stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2049
				__napi_schedule(&rx_q->napi);
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
				poll_scheduled = true;
			}
		}
	}

	/* If we scheduled poll, we already know that tx queues will be checked.
	 * If we didn't schedule poll, see if any DMA channel (used by tx) has a
	 * completed transmission, if so, call stmmac_poll (once).
	 */
	if (!poll_scheduled) {
		for (chan = 0; chan < tx_channel_count; chan++) {
			if (status[chan] & handle_tx) {
				/* It doesn't matter what rx queue we choose
				 * here. We use 0 since it always exists.
				 */
				struct stmmac_rx_queue *rx_q =
					&priv->rx_queue[0];

				if (likely(napi_schedule_prep(&rx_q->napi))) {
2069 2070
					stmmac_disable_dma_irq(priv,
							priv->ioaddr, chan);
2071 2072 2073
					__napi_schedule(&rx_q->napi);
				}
				break;
2074
			}
2075
		}
2076
	}
2077

2078 2079
	for (chan = 0; chan < tx_channel_count; chan++) {
		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
			/* Try to bump up the dma threshold on this failure */
			if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
			    (tc <= 256)) {
				tc += 64;
				if (priv->plat->force_thresh_dma_mode)
					stmmac_set_dma_operation_mode(priv,
								      tc,
								      tc,
								      chan);
				else
					stmmac_set_dma_operation_mode(priv,
								    tc,
								    SF_DMA_MODE,
								    chan);
				priv->xstats.threshold = tc;
			}
2096
		} else if (unlikely(status[chan] == tx_hard_error)) {
2097
			stmmac_tx_err(priv, chan);
2098
		}
2099
	}
2100 2101
}

2102 2103 2104 2105 2106
/**
 * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
 * @priv: driver private structure
 * Description: this masks the MMC irq, in fact, the counters are managed in SW.
 */
2107 2108 2109
static void stmmac_mmc_setup(struct stmmac_priv *priv)
{
	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2110
			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2111

2112 2113
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
A
Alexandre TORGUE 已提交
2114
		priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
2115 2116
	} else {
		priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
A
Alexandre TORGUE 已提交
2117
		priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
2118
	}
2119 2120

	dwmac_mmc_intr_all_mask(priv->mmcaddr);
G
Giuseppe CAVALLARO 已提交
2121 2122

	if (priv->dma_cap.rmon) {
2123
		dwmac_mmc_ctrl(priv->mmcaddr, mode);
G
Giuseppe CAVALLARO 已提交
2124 2125
		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
	} else
2126
		netdev_info(priv->dev, "No MAC Management Counters available\n");
2127 2128
}

2129
/**
2130
 * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
2131 2132
 * @priv: driver private structure
 * Description: select the Enhanced/Alternate or Normal descriptors.
2133 2134
 * In case of Enhanced/Alternate, it checks if the extended descriptors are
 * supported by the HW capability register.
2135
 */
2136 2137 2138
static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
{
	if (priv->plat->enh_desc) {
2139
		dev_info(priv->device, "Enhanced/Alternate descriptors\n");
2140 2141 2142

		/* GMAC older than 3.50 has no extended descriptors */
		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
2143
			dev_info(priv->device, "Enabled extended descriptors\n");
2144 2145
			priv->extend_desc = 1;
		} else
2146
			dev_warn(priv->device, "Extended descriptors not supported\n");
2147

2148 2149
		priv->hw->desc = &enh_desc_ops;
	} else {
2150
		dev_info(priv->device, "Normal descriptors\n");
2151 2152 2153 2154 2155
		priv->hw->desc = &ndesc_ops;
	}
}

/**
2156
 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2157
 * @priv: driver private structure
2158 2159 2160 2161 2162
 * Description:
 *  new GMAC chip generations have a new register to indicate the
 *  presence of the optional feature/functions.
 *  This can be also used to override the value passed through the
 *  platform and necessary for old MAC10/100 and GMAC chips.
2163 2164 2165
 */
static int stmmac_get_hw_features(struct stmmac_priv *priv)
{
2166
	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2167 2168
}

2169
/**
2170
 * stmmac_check_ether_addr - check if the MAC addr is valid
2171 2172 2173 2174 2175
 * @priv: driver private structure
 * Description:
 * it is to verify if the MAC address is valid, in case of failures it
 * generates a random MAC address
 */
2176 2177 2178
static void stmmac_check_ether_addr(struct stmmac_priv *priv)
{
	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2179
		stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
G
Giuseppe CAVALLARO 已提交
2180
		if (!is_valid_ether_addr(priv->dev->dev_addr))
2181
			eth_hw_addr_random(priv->dev);
2182 2183
		netdev_info(priv->dev, "device MAC address %pM\n",
			    priv->dev->dev_addr);
2184 2185 2186
	}
}

2187
/**
2188
 * stmmac_init_dma_engine - DMA init.
2189 2190 2191 2192 2193 2194
 * @priv: driver private structure
 * Description:
 * It inits the DMA invoking the specific MAC/GMAC callback.
 * Some DMA parameters can be passed from the platform;
 * in case of these are not passed a default is kept for the MAC or GMAC.
 */
2195 2196
static int stmmac_init_dma_engine(struct stmmac_priv *priv)
{
2197 2198
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2199
	struct stmmac_rx_queue *rx_q;
2200
	struct stmmac_tx_queue *tx_q;
2201 2202 2203
	u32 dummy_dma_rx_phy = 0;
	u32 dummy_dma_tx_phy = 0;
	u32 chan = 0;
2204
	int atds = 0;
2205
	int ret = 0;
2206

2207 2208
	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
		dev_err(priv->device, "Invalid DMA configuration\n");
2209
		return -EINVAL;
2210 2211
	}

2212 2213 2214
	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
		atds = 1;

2215
	ret = stmmac_reset(priv, priv->ioaddr);
2216 2217 2218 2219 2220
	if (ret) {
		dev_err(priv->device, "Failed to reset the dma\n");
		return ret;
	}

A
Alexandre TORGUE 已提交
2221
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2222
		/* DMA Configuration */
2223 2224
		stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg,
				dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2225 2226 2227

		/* DMA RX Channel Configuration */
		for (chan = 0; chan < rx_channels_count; chan++) {
2228 2229
			rx_q = &priv->rx_queue[chan];

2230 2231 2232
			stmmac_init_rx_chan(priv, priv->ioaddr,
					priv->plat->dma_cfg, rx_q->dma_rx_phy,
					chan);
2233

2234
			rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2235
				    (DMA_RX_SIZE * sizeof(struct dma_desc));
2236 2237
			stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
					rx_q->rx_tail_addr, chan);
2238 2239 2240 2241
		}

		/* DMA TX Channel Configuration */
		for (chan = 0; chan < tx_channels_count; chan++) {
2242 2243
			tx_q = &priv->tx_queue[chan];

2244 2245
			stmmac_init_chan(priv, priv->ioaddr,
					priv->plat->dma_cfg, chan);
2246

2247 2248 2249
			stmmac_init_tx_chan(priv, priv->ioaddr,
					priv->plat->dma_cfg, tx_q->dma_tx_phy,
					chan);
2250

2251
			tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2252
				    (DMA_TX_SIZE * sizeof(struct dma_desc));
2253 2254
			stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
					tx_q->tx_tail_addr, chan);
2255 2256
		}
	} else {
2257
		rx_q = &priv->rx_queue[chan];
2258
		tx_q = &priv->tx_queue[chan];
2259 2260
		stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg,
				tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
A
Alexandre TORGUE 已提交
2261 2262
	}

2263 2264
	if (priv->plat->axi)
		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2265

2266
	return ret;
2267 2268
}

2269
/**
2270
 * stmmac_tx_timer - mitigation sw timer for tx.
2271 2272 2273 2274
 * @data: data pointer
 * Description:
 * This is the timer handler to directly invoke the stmmac_tx_clean.
 */
2275
static void stmmac_tx_timer(struct timer_list *t)
2276
{
2277
	struct stmmac_priv *priv = from_timer(priv, t, txtimer);
2278 2279
	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	u32 queue;
2280

2281 2282 2283
	/* let's scan all the tx queues */
	for (queue = 0; queue < tx_queues_count; queue++)
		stmmac_tx_clean(priv, queue);
2284 2285 2286
}

/**
2287
 * stmmac_init_tx_coalesce - init tx mitigation options.
2288
 * @priv: driver private structure
2289 2290 2291 2292 2293 2294 2295 2296 2297
 * Description:
 * This inits the transmit coalesce parameters: i.e. timer rate,
 * timer handler and default threshold used for enabling the
 * interrupt on completion bit.
 */
static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
{
	priv->tx_coal_frames = STMMAC_TX_FRAMES;
	priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2298
	timer_setup(&priv->txtimer, stmmac_tx_timer, 0);
2299 2300 2301 2302
	priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
	add_timer(&priv->txtimer);
}

2303 2304 2305 2306 2307 2308 2309
static void stmmac_set_rings_length(struct stmmac_priv *priv)
{
	u32 rx_channels_count = priv->plat->rx_queues_to_use;
	u32 tx_channels_count = priv->plat->tx_queues_to_use;
	u32 chan;

	/* set TX ring length */
2310 2311 2312
	for (chan = 0; chan < tx_channels_count; chan++)
		stmmac_set_tx_ring_len(priv, priv->ioaddr,
				(DMA_TX_SIZE - 1), chan);
2313 2314

	/* set RX ring length */
2315 2316 2317
	for (chan = 0; chan < rx_channels_count; chan++)
		stmmac_set_rx_ring_len(priv, priv->ioaddr,
				(DMA_RX_SIZE - 1), chan);
2318 2319
}

2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332
/**
 *  stmmac_set_tx_queue_weight - Set TX queue weight
 *  @priv: driver private structure
 *  Description: It is used for setting TX queues weight
 */
static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
{
	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	u32 weight;
	u32 queue;

	for (queue = 0; queue < tx_queues_count; queue++) {
		weight = priv->plat->tx_queues_cfg[queue].weight;
2333
		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2334 2335 2336
	}
}

2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347
/**
 *  stmmac_configure_cbs - Configure CBS in TX queue
 *  @priv: driver private structure
 *  Description: It is used for configuring CBS in AVB TX queues
 */
static void stmmac_configure_cbs(struct stmmac_priv *priv)
{
	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	u32 mode_to_use;
	u32 queue;

J
Joao Pinto 已提交
2348 2349
	/* queue 0 is reserved for legacy traffic */
	for (queue = 1; queue < tx_queues_count; queue++) {
2350 2351 2352 2353
		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
		if (mode_to_use == MTL_QUEUE_DCB)
			continue;

2354
		stmmac_config_cbs(priv, priv->hw,
2355 2356 2357 2358 2359 2360 2361 2362
				priv->plat->tx_queues_cfg[queue].send_slope,
				priv->plat->tx_queues_cfg[queue].idle_slope,
				priv->plat->tx_queues_cfg[queue].high_credit,
				priv->plat->tx_queues_cfg[queue].low_credit,
				queue);
	}
}

2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
/**
 *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
 *  @priv: driver private structure
 *  Description: It is used for mapping RX queues to RX dma channels
 */
static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
{
	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	u32 queue;
	u32 chan;

	for (queue = 0; queue < rx_queues_count; queue++) {
		chan = priv->plat->rx_queues_cfg[queue].chan;
2376
		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2377 2378 2379
	}
}

2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395
/**
 *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
 *  @priv: driver private structure
 *  Description: It is used for configuring the RX Queue Priority
 */
static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
{
	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	u32 queue;
	u32 prio;

	for (queue = 0; queue < rx_queues_count; queue++) {
		if (!priv->plat->rx_queues_cfg[queue].use_prio)
			continue;

		prio = priv->plat->rx_queues_cfg[queue].prio;
2396
		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
	}
}

/**
 *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
 *  @priv: driver private structure
 *  Description: It is used for configuring the TX Queue Priority
 */
static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
{
	u32 tx_queues_count = priv->plat->tx_queues_to_use;
	u32 queue;
	u32 prio;

	for (queue = 0; queue < tx_queues_count; queue++) {
		if (!priv->plat->tx_queues_cfg[queue].use_prio)
			continue;

		prio = priv->plat->tx_queues_cfg[queue].prio;
2416
		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2417 2418 2419
	}
}

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436
/**
 *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
 *  @priv: driver private structure
 *  Description: It is used for configuring the RX queue routing
 */
static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
{
	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	u32 queue;
	u8 packet;

	for (queue = 0; queue < rx_queues_count; queue++) {
		/* no specific packet type routing specified for the queue */
		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
			continue;

		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2437
		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2438 2439 2440
	}
}

2441 2442 2443 2444 2445 2446 2447 2448 2449 2450
/**
 *  stmmac_mtl_configuration - Configure MTL
 *  @priv: driver private structure
 *  Description: It is used for configurring MTL
 */
static void stmmac_mtl_configuration(struct stmmac_priv *priv)
{
	u32 rx_queues_count = priv->plat->rx_queues_to_use;
	u32 tx_queues_count = priv->plat->tx_queues_to_use;

2451
	if (tx_queues_count > 1)
2452 2453
		stmmac_set_tx_queue_weight(priv);

2454
	/* Configure MTL RX algorithms */
2455 2456 2457
	if (rx_queues_count > 1)
		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
				priv->plat->rx_sched_algorithm);
2458 2459

	/* Configure MTL TX algorithms */
2460 2461 2462
	if (tx_queues_count > 1)
		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
				priv->plat->tx_sched_algorithm);
2463

2464
	/* Configure CBS in AVB TX queues */
2465
	if (tx_queues_count > 1)
2466 2467
		stmmac_configure_cbs(priv);

2468
	/* Map RX MTL to DMA channels */
2469
	stmmac_rx_queue_dma_chan_map(priv);
2470

2471
	/* Enable MAC RX Queues */
2472
	stmmac_mac_enable_rx_queues(priv);
2473

2474
	/* Set RX priorities */
2475
	if (rx_queues_count > 1)
2476 2477 2478
		stmmac_mac_config_rx_queues_prio(priv);

	/* Set TX priorities */
2479
	if (tx_queues_count > 1)
2480
		stmmac_mac_config_tx_queues_prio(priv);
2481 2482

	/* Set RX routing */
2483
	if (rx_queues_count > 1)
2484
		stmmac_mac_config_rx_queues_routing(priv);
2485 2486
}

2487 2488
static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
{
2489
	if (priv->dma_cap.asp) {
2490
		netdev_info(priv->dev, "Enabling Safety Features\n");
2491
		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2492 2493 2494 2495 2496
	} else {
		netdev_info(priv->dev, "No Safety Features support found\n");
	}
}

2497
/**
2498
 * stmmac_hw_setup - setup mac in a usable state.
2499 2500
 *  @dev : pointer to the device structure.
 *  Description:
2501 2502 2503 2504
 *  this is the main function to setup the HW in a usable state because the
 *  dma engine is reset, the core registers are configured (e.g. AXI,
 *  Checksum features, timers). The DMA is ready to start receiving and
 *  transmitting.
2505 2506 2507 2508
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
2509
static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2510 2511
{
	struct stmmac_priv *priv = netdev_priv(dev);
2512
	u32 rx_cnt = priv->plat->rx_queues_to_use;
2513 2514
	u32 tx_cnt = priv->plat->tx_queues_to_use;
	u32 chan;
2515 2516 2517 2518 2519
	int ret;

	/* DMA initialization and SW reset */
	ret = stmmac_init_dma_engine(priv);
	if (ret < 0) {
2520 2521
		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
			   __func__);
2522 2523 2524 2525
		return ret;
	}

	/* Copy the MAC addr into the HW  */
2526
	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2527

2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
	/* PS and related bits will be programmed according to the speed */
	if (priv->hw->pcs) {
		int speed = priv->plat->mac_port_sel_speed;

		if ((speed == SPEED_10) || (speed == SPEED_100) ||
		    (speed == SPEED_1000)) {
			priv->hw->ps = speed;
		} else {
			dev_warn(priv->device, "invalid port speed\n");
			priv->hw->ps = 0;
		}
	}

2541
	/* Initialize the MAC Core */
2542
	stmmac_core_init(priv, priv->hw, dev);
2543

2544 2545 2546
	/* Initialize MTL*/
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
		stmmac_mtl_configuration(priv);
J
jpinto 已提交
2547

2548 2549 2550 2551
	/* Initialize Safety Features */
	if (priv->synopsys_id >= DWMAC_CORE_5_10)
		stmmac_safety_feat_configuration(priv);

2552
	ret = stmmac_rx_ipc(priv, priv->hw);
2553
	if (!ret) {
2554
		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2555
		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2556
		priv->hw->rx_csum = 0;
2557 2558
	}

2559
	/* Enable the MAC Rx/Tx */
2560
	stmmac_mac_set(priv, priv->ioaddr, true);
2561

2562 2563 2564
	/* Set the HW DMA mode and the COE */
	stmmac_dma_operation_mode(priv);

2565 2566
	stmmac_mmc_setup(priv);

2567
	if (init_ptp) {
2568 2569 2570 2571
		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
		if (ret < 0)
			netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);

2572
		ret = stmmac_init_ptp(priv);
2573 2574 2575 2576
		if (ret == -EOPNOTSUPP)
			netdev_warn(priv->dev, "PTP not supported by HW\n");
		else if (ret)
			netdev_warn(priv->dev, "PTP init failed\n");
2577
	}
2578

2579
#ifdef CONFIG_DEBUG_FS
2580 2581
	ret = stmmac_init_fs(dev);
	if (ret < 0)
2582 2583
		netdev_warn(priv->dev, "%s: failed debugFS registration\n",
			    __func__);
2584 2585
#endif
	/* Start the ball rolling... */
2586
	stmmac_start_all_dma(priv);
2587 2588 2589

	priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;

2590 2591 2592 2593
	if (priv->use_riwt) {
		ret = stmmac_rx_watchdog(priv, priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
		if (!ret)
			priv->rx_riwt = MAX_DMA_RIWT;
2594 2595
	}

2596 2597
	if (priv->hw->pcs)
		stmmac_pcs_ctrl_ane(priv, priv->hw, 1, priv->hw->ps, 0);
2598

2599 2600 2601
	/* set TX and RX rings length */
	stmmac_set_rings_length(priv);

A
Alexandre TORGUE 已提交
2602
	/* Enable TSO */
2603 2604
	if (priv->tso) {
		for (chan = 0; chan < tx_cnt; chan++)
2605
			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2606
	}
A
Alexandre TORGUE 已提交
2607

2608 2609 2610
	return 0;
}

2611 2612 2613 2614 2615 2616 2617
static void stmmac_hw_teardown(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);

	clk_disable_unprepare(priv->plat->clk_ptp_ref);
}

2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631
/**
 *  stmmac_open - open entry point of the driver
 *  @dev : pointer to the device structure.
 *  Description:
 *  This function is the open entry point of the driver.
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
static int stmmac_open(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
	int ret;

2632 2633
	stmmac_check_ether_addr(priv);

2634 2635 2636
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI) {
2637 2638
		ret = stmmac_init_phy(dev);
		if (ret) {
2639 2640 2641
			netdev_err(priv->dev,
				   "%s: Cannot attach to PHY (error: %d)\n",
				   __func__, ret);
2642
			return ret;
2643
		}
2644
	}
2645

2646 2647 2648 2649
	/* Extra statistics */
	memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
	priv->xstats.threshold = tc;

2650
	priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2651
	priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2652

2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666
	ret = alloc_dma_desc_resources(priv);
	if (ret < 0) {
		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
			   __func__);
		goto dma_desc_error;
	}

	ret = init_dma_desc_rings(dev, GFP_KERNEL);
	if (ret < 0) {
		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
			   __func__);
		goto init_error;
	}

2667
	ret = stmmac_hw_setup(dev, true);
2668
	if (ret < 0) {
2669
		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2670
		goto init_error;
2671 2672
	}

2673 2674
	stmmac_init_tx_coalesce(priv);

2675 2676
	if (dev->phydev)
		phy_start(dev->phydev);
2677

2678 2679
	/* Request the IRQ lines */
	ret = request_irq(dev->irq, stmmac_interrupt,
G
Giuseppe CAVALLARO 已提交
2680
			  IRQF_SHARED, dev->name, dev);
2681
	if (unlikely(ret < 0)) {
2682 2683 2684
		netdev_err(priv->dev,
			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
			   __func__, dev->irq, ret);
2685
		goto irq_error;
2686 2687
	}

2688 2689 2690 2691 2692
	/* Request the Wake IRQ in case of another line is used for WoL */
	if (priv->wol_irq != dev->irq) {
		ret = request_irq(priv->wol_irq, stmmac_interrupt,
				  IRQF_SHARED, dev->name, dev);
		if (unlikely(ret < 0)) {
2693 2694 2695
			netdev_err(priv->dev,
				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
				   __func__, priv->wol_irq, ret);
2696
			goto wolirq_error;
2697 2698 2699
		}
	}

2700
	/* Request the IRQ lines */
2701
	if (priv->lpi_irq > 0) {
2702 2703 2704
		ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
				  dev->name, dev);
		if (unlikely(ret < 0)) {
2705 2706 2707
			netdev_err(priv->dev,
				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
				   __func__, priv->lpi_irq, ret);
2708
			goto lpiirq_error;
2709 2710 2711
		}
	}

2712 2713
	stmmac_enable_all_queues(priv);
	stmmac_start_all_queues(priv);
2714

2715
	return 0;
2716

2717
lpiirq_error:
2718 2719
	if (priv->wol_irq != dev->irq)
		free_irq(priv->wol_irq, dev);
2720
wolirq_error:
2721
	free_irq(dev->irq, dev);
2722 2723 2724
irq_error:
	if (dev->phydev)
		phy_stop(dev->phydev);
2725

2726
	del_timer_sync(&priv->txtimer);
2727
	stmmac_hw_teardown(dev);
2728 2729
init_error:
	free_dma_desc_resources(priv);
2730
dma_desc_error:
2731 2732
	if (dev->phydev)
		phy_disconnect(dev->phydev);
2733

2734
	return ret;
2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
}

/**
 *  stmmac_release - close entry point of the driver
 *  @dev : device pointer.
 *  Description:
 *  This is the stop entry point of the driver.
 */
static int stmmac_release(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);

2747 2748 2749
	if (priv->eee_enabled)
		del_timer_sync(&priv->eee_ctrl_timer);

2750
	/* Stop and disconnect the PHY */
2751 2752 2753
	if (dev->phydev) {
		phy_stop(dev->phydev);
		phy_disconnect(dev->phydev);
2754 2755
	}

2756
	stmmac_stop_all_queues(priv);
2757

2758
	stmmac_disable_all_queues(priv);
2759

2760 2761
	del_timer_sync(&priv->txtimer);

2762 2763
	/* Free the IRQ lines */
	free_irq(dev->irq, dev);
2764 2765
	if (priv->wol_irq != dev->irq)
		free_irq(priv->wol_irq, dev);
2766
	if (priv->lpi_irq > 0)
2767
		free_irq(priv->lpi_irq, dev);
2768 2769

	/* Stop TX/RX DMA and clear the descriptors */
2770
	stmmac_stop_all_dma(priv);
2771 2772 2773 2774

	/* Release and free the Rx/Tx resources */
	free_dma_desc_resources(priv);

2775
	/* Disable the MAC Rx/Tx */
2776
	stmmac_mac_set(priv, priv->ioaddr, false);
2777 2778 2779

	netif_carrier_off(dev);

2780
#ifdef CONFIG_DEBUG_FS
2781
	stmmac_exit_fs(dev);
2782 2783
#endif

2784 2785
	stmmac_release_ptp(priv);

2786 2787 2788
	return 0;
}

A
Alexandre TORGUE 已提交
2789 2790 2791 2792 2793 2794
/**
 *  stmmac_tso_allocator - close entry point of the driver
 *  @priv: driver private structure
 *  @des: buffer start address
 *  @total_len: total length to fill in descriptors
 *  @last_segmant: condition for the last descriptor
2795
 *  @queue: TX queue index
A
Alexandre TORGUE 已提交
2796 2797 2798 2799 2800
 *  Description:
 *  This function fills descriptor and request new descriptors according to
 *  buffer length to fill
 */
static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2801
				 int total_len, bool last_segment, u32 queue)
A
Alexandre TORGUE 已提交
2802
{
2803
	struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
A
Alexandre TORGUE 已提交
2804
	struct dma_desc *desc;
2805
	u32 buff_size;
2806
	int tmp_len;
A
Alexandre TORGUE 已提交
2807 2808 2809 2810

	tmp_len = total_len;

	while (tmp_len > 0) {
2811
		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2812
		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2813
		desc = tx_q->dma_tx + tx_q->cur_tx;
A
Alexandre TORGUE 已提交
2814

2815
		desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
A
Alexandre TORGUE 已提交
2816 2817 2818
		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
			    TSO_MAX_BUFF_SIZE : tmp_len;

2819 2820 2821 2822
		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
				0, 1,
				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
				0, 0);
A
Alexandre TORGUE 已提交
2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856

		tmp_len -= TSO_MAX_BUFF_SIZE;
	}
}

/**
 *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
 *  @skb : the socket buffer
 *  @dev : device pointer
 *  Description: this is the transmit function that is called on TSO frames
 *  (support available on GMAC4 and newer chips).
 *  Diagram below show the ring programming in case of TSO frames:
 *
 *  First Descriptor
 *   --------
 *   | DES0 |---> buffer1 = L2/L3/L4 header
 *   | DES1 |---> TCP Payload (can continue on next descr...)
 *   | DES2 |---> buffer 1 and 2 len
 *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
 *   --------
 *	|
 *     ...
 *	|
 *   --------
 *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
 *   | DES1 | --|
 *   | DES2 | --> buffer 1 and 2 len
 *   | DES3 |
 *   --------
 *
 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
 */
static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
{
2857
	struct dma_desc *desc, *first, *mss_desc = NULL;
A
Alexandre TORGUE 已提交
2858 2859
	struct stmmac_priv *priv = netdev_priv(dev);
	int nfrags = skb_shinfo(skb)->nr_frags;
2860
	u32 queue = skb_get_queue_mapping(skb);
A
Alexandre TORGUE 已提交
2861
	unsigned int first_entry, des;
2862 2863 2864
	struct stmmac_tx_queue *tx_q;
	int tmp_pay_len = 0;
	u32 pay_len, mss;
A
Alexandre TORGUE 已提交
2865 2866 2867
	u8 proto_hdr_len;
	int i;

2868 2869
	tx_q = &priv->tx_queue[queue];

A
Alexandre TORGUE 已提交
2870 2871 2872 2873
	/* Compute header lengths */
	proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);

	/* Desc availability based on threshold should be enough safe */
2874
	if (unlikely(stmmac_tx_avail(priv, queue) <
A
Alexandre TORGUE 已提交
2875
		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2876 2877 2878
		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
								queue));
A
Alexandre TORGUE 已提交
2879
			/* This is a hard error, log it. */
2880 2881 2882
			netdev_err(priv->dev,
				   "%s: Tx Ring full when queue awake\n",
				   __func__);
A
Alexandre TORGUE 已提交
2883 2884 2885 2886 2887 2888 2889 2890 2891
		}
		return NETDEV_TX_BUSY;
	}

	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */

	mss = skb_shinfo(skb)->gso_size;

	/* set new MSS value if needed */
2892
	if (mss != tx_q->mss) {
2893
		mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2894
		stmmac_set_mss(priv, mss_desc, mss);
2895
		tx_q->mss = mss;
2896
		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2897
		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
A
Alexandre TORGUE 已提交
2898 2899 2900 2901 2902 2903 2904 2905 2906
	}

	if (netif_msg_tx_queued(priv)) {
		pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
			__func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
			skb->data_len);
	}

2907
	first_entry = tx_q->cur_tx;
2908
	WARN_ON(tx_q->tx_skbuff[first_entry]);
A
Alexandre TORGUE 已提交
2909

2910
	desc = tx_q->dma_tx + first_entry;
A
Alexandre TORGUE 已提交
2911 2912 2913 2914 2915 2916 2917 2918
	first = desc;

	/* first descriptor: fill Headers on Buf1 */
	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
			     DMA_TO_DEVICE);
	if (dma_mapping_error(priv->device, des))
		goto dma_map_err;

2919 2920
	tx_q->tx_skbuff_dma[first_entry].buf = des;
	tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
A
Alexandre TORGUE 已提交
2921

2922
	first->des0 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
2923 2924 2925

	/* Fill start of payload in buff2 of first descriptor */
	if (pay_len)
2926
		first->des1 = cpu_to_le32(des + proto_hdr_len);
A
Alexandre TORGUE 已提交
2927 2928 2929 2930

	/* If needed take extra descriptors to fill the remaining payload */
	tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;

2931
	stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
A
Alexandre TORGUE 已提交
2932 2933 2934 2935 2936 2937 2938 2939

	/* Prepare fragments */
	for (i = 0; i < nfrags; i++) {
		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

		des = skb_frag_dma_map(priv->device, frag, 0,
				       skb_frag_size(frag),
				       DMA_TO_DEVICE);
2940 2941
		if (dma_mapping_error(priv->device, des))
			goto dma_map_err;
A
Alexandre TORGUE 已提交
2942 2943

		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2944
				     (i == nfrags - 1), queue);
A
Alexandre TORGUE 已提交
2945

2946 2947 2948
		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
A
Alexandre TORGUE 已提交
2949 2950
	}

2951
	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
A
Alexandre TORGUE 已提交
2952

2953 2954 2955 2956 2957 2958 2959 2960
	/* Only the last descriptor gets to point to the skb. */
	tx_q->tx_skbuff[tx_q->cur_tx] = skb;

	/* We've used all descriptors we need for this skb, however,
	 * advance cur_tx so that it references a fresh descriptor.
	 * ndo_start_xmit will fill this descriptor the next time it's
	 * called and stmmac_tx_clean may clean up to this descriptor.
	 */
2961
	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
A
Alexandre TORGUE 已提交
2962

2963
	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2964 2965
		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
			  __func__);
2966
		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
A
Alexandre TORGUE 已提交
2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979
	}

	dev->stats.tx_bytes += skb->len;
	priv->xstats.tx_tso_frames++;
	priv->xstats.tx_tso_nfrags += nfrags;

	/* Manage tx mitigation */
	priv->tx_count_frames += nfrags + 1;
	if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
		mod_timer(&priv->txtimer,
			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
	} else {
		priv->tx_count_frames = 0;
2980
		stmmac_set_tx_ic(priv, desc);
A
Alexandre TORGUE 已提交
2981 2982 2983
		priv->xstats.tx_set_ic_bit++;
	}

2984
	skb_tx_timestamp(skb);
A
Alexandre TORGUE 已提交
2985 2986 2987 2988 2989

	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
		     priv->hwts_tx_en)) {
		/* declare that device is doing timestamping */
		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2990
		stmmac_enable_tx_timestamp(priv, first);
A
Alexandre TORGUE 已提交
2991 2992 2993
	}

	/* Complete the first descriptor before granting the DMA */
2994
	stmmac_prepare_tso_tx_desc(priv, first, 1,
A
Alexandre TORGUE 已提交
2995 2996
			proto_hdr_len,
			pay_len,
2997
			1, tx_q->tx_skbuff_dma[first_entry].last_segment,
A
Alexandre TORGUE 已提交
2998 2999 3000
			tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));

	/* If context desc is used to change MSS */
3001 3002 3003 3004 3005 3006 3007
	if (mss_desc) {
		/* Make sure that first descriptor has been completely
		 * written, including its own bit. This is because MSS is
		 * actually before first descriptor, so we need to make
		 * sure that MSS's own bit is the last thing written.
		 */
		dma_wmb();
3008
		stmmac_set_tx_owner(priv, mss_desc);
3009
	}
A
Alexandre TORGUE 已提交
3010 3011 3012 3013 3014

	/* The own bit must be the latest setting done when prepare the
	 * descriptor and then barrier is needed to make sure that
	 * all is coherent before granting the DMA engine.
	 */
3015
	wmb();
A
Alexandre TORGUE 已提交
3016 3017 3018

	if (netif_msg_pktdata(priv)) {
		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
3019 3020
			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
			tx_q->cur_tx, first, nfrags);
A
Alexandre TORGUE 已提交
3021

3022
		stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0);
A
Alexandre TORGUE 已提交
3023 3024 3025 3026 3027

		pr_info(">>> frame to be transmitted: ");
		print_pkt(skb->data, skb_headlen(skb));
	}

3028
	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
A
Alexandre TORGUE 已提交
3029

3030
	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
A
Alexandre TORGUE 已提交
3031 3032 3033 3034 3035 3036 3037 3038 3039 3040

	return NETDEV_TX_OK;

dma_map_err:
	dev_err(priv->device, "Tx dma map failed\n");
	dev_kfree_skb(skb);
	priv->dev->stats.tx_dropped++;
	return NETDEV_TX_OK;
}

3041
/**
3042
 *  stmmac_xmit - Tx entry point of the driver
3043 3044
 *  @skb : the socket buffer
 *  @dev : device pointer
3045 3046 3047
 *  Description : this is the tx entry point of the driver.
 *  It programs the chain or the ring and supports oversized frames
 *  and SG feature.
3048 3049 3050 3051
 */
static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);
3052
	unsigned int nopaged_len = skb_headlen(skb);
3053
	int i, csum_insertion = 0, is_jumbo = 0;
3054
	u32 queue = skb_get_queue_mapping(skb);
3055
	int nfrags = skb_shinfo(skb)->nr_frags;
3056 3057
	int entry;
	unsigned int first_entry;
3058
	struct dma_desc *desc, *first;
3059
	struct stmmac_tx_queue *tx_q;
3060
	unsigned int enh_desc;
A
Alexandre TORGUE 已提交
3061 3062
	unsigned int des;

3063 3064
	tx_q = &priv->tx_queue[queue];

A
Alexandre TORGUE 已提交
3065 3066
	/* Manage oversized TCP frames for GMAC4 device */
	if (skb_is_gso(skb) && priv->tso) {
N
Niklas Cassel 已提交
3067
		if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
A
Alexandre TORGUE 已提交
3068 3069
			return stmmac_tso_xmit(skb, dev);
	}
3070

3071
	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3072 3073 3074
		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
								queue));
3075
			/* This is a hard error, log it. */
3076 3077 3078
			netdev_err(priv->dev,
				   "%s: Tx Ring full when queue awake\n",
				   __func__);
3079 3080 3081 3082
		}
		return NETDEV_TX_BUSY;
	}

3083 3084 3085
	if (priv->tx_path_in_lpi_mode)
		stmmac_disable_eee_mode(priv);

3086
	entry = tx_q->cur_tx;
3087
	first_entry = entry;
3088
	WARN_ON(tx_q->tx_skbuff[first_entry]);
3089

3090
	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3091

3092
	if (likely(priv->extend_desc))
3093
		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3094
	else
3095
		desc = tx_q->dma_tx + entry;
3096

3097 3098
	first = desc;

3099
	enh_desc = priv->plat->enh_desc;
3100
	/* To program the descriptors according to the size of the frame */
G
Giuseppe CAVALLARO 已提交
3101 3102 3103
	if (enh_desc)
		is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);

A
Alexandre TORGUE 已提交
3104 3105
	if (unlikely(is_jumbo) && likely(priv->synopsys_id <
					 DWMAC_CORE_4_00)) {
3106
		entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
G
Giuseppe CAVALLARO 已提交
3107 3108
		if (unlikely(entry < 0))
			goto dma_map_err;
G
Giuseppe CAVALLARO 已提交
3109
	}
3110 3111

	for (i = 0; i < nfrags; i++) {
E
Eric Dumazet 已提交
3112 3113
		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
		int len = skb_frag_size(frag);
3114
		bool last_segment = (i == (nfrags - 1));
3115

3116
		entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3117
		WARN_ON(tx_q->tx_skbuff[entry]);
3118

3119
		if (likely(priv->extend_desc))
3120
			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3121
		else
3122
			desc = tx_q->dma_tx + entry;
3123

A
Alexandre TORGUE 已提交
3124 3125 3126
		des = skb_frag_dma_map(priv->device, frag, 0, len,
				       DMA_TO_DEVICE);
		if (dma_mapping_error(priv->device, des))
G
Giuseppe CAVALLARO 已提交
3127 3128
			goto dma_map_err; /* should reuse desc w/o issues */

3129
		tx_q->tx_skbuff_dma[entry].buf = des;
3130 3131 3132 3133
		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
			desc->des0 = cpu_to_le32(des);
		else
			desc->des2 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
3134

3135 3136 3137
		tx_q->tx_skbuff_dma[entry].map_as_page = true;
		tx_q->tx_skbuff_dma[entry].len = len;
		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3138 3139

		/* Prepare the descriptor and set the own bit too */
3140 3141
		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
				priv->mode, 1, last_segment, skb->len);
3142 3143
	}

3144 3145
	/* Only the last descriptor gets to point to the skb. */
	tx_q->tx_skbuff[entry] = skb;
3146

3147 3148 3149 3150 3151 3152
	/* We've used all descriptors we need for this skb, however,
	 * advance cur_tx so that it references a fresh descriptor.
	 * ndo_start_xmit will fill this descriptor the next time it's
	 * called and stmmac_tx_clean may clean up to this descriptor.
	 */
	entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3153
	tx_q->cur_tx = entry;
3154 3155

	if (netif_msg_pktdata(priv)) {
3156 3157
		void *tx_head;

3158 3159
		netdev_dbg(priv->dev,
			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3160
			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3161
			   entry, first, nfrags);
3162

3163
		if (priv->extend_desc)
3164
			tx_head = (void *)tx_q->dma_etx;
3165
		else
3166
			tx_head = (void *)tx_q->dma_tx;
3167

3168
		stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false);
3169

3170
		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3171 3172
		print_pkt(skb->data, skb->len);
	}
3173

3174
	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3175 3176
		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
			  __func__);
3177
		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3178 3179 3180 3181
	}

	dev->stats.tx_bytes += skb->len;

3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192
	/* According to the coalesce parameter the IC bit for the latest
	 * segment is reset and the timer re-started to clean the tx status.
	 * This approach takes care about the fragments: desc is the first
	 * element in case of no SG.
	 */
	priv->tx_count_frames += nfrags + 1;
	if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
		mod_timer(&priv->txtimer,
			  STMMAC_COAL_TIMER(priv->tx_coal_timer));
	} else {
		priv->tx_count_frames = 0;
3193
		stmmac_set_tx_ic(priv, desc);
3194
		priv->xstats.tx_set_ic_bit++;
3195 3196
	}

3197
	skb_tx_timestamp(skb);
3198

3199 3200 3201 3202 3203 3204 3205
	/* Ready to fill the first descriptor and set the OWN bit w/o any
	 * problems because all the descriptors are actually ready to be
	 * passed to the DMA engine.
	 */
	if (likely(!is_jumbo)) {
		bool last_segment = (nfrags == 0);

A
Alexandre TORGUE 已提交
3206 3207 3208
		des = dma_map_single(priv->device, skb->data,
				     nopaged_len, DMA_TO_DEVICE);
		if (dma_mapping_error(priv->device, des))
3209 3210
			goto dma_map_err;

3211
		tx_q->tx_skbuff_dma[first_entry].buf = des;
3212 3213 3214 3215
		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
			first->des0 = cpu_to_le32(des);
		else
			first->des2 = cpu_to_le32(des);
A
Alexandre TORGUE 已提交
3216

3217 3218
		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3219 3220 3221 3222 3223

		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
			     priv->hwts_tx_en)) {
			/* declare that device is doing timestamping */
			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3224
			stmmac_enable_tx_timestamp(priv, first);
3225 3226 3227
		}

		/* Prepare the first descriptor setting the OWN bit too */
3228 3229 3230
		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
				csum_insertion, priv->mode, 1, last_segment,
				skb->len);
3231 3232 3233 3234 3235

		/* The own bit must be the latest setting done when prepare the
		 * descriptor and then barrier is needed to make sure that
		 * all is coherent before granting the DMA engine.
		 */
3236
		wmb();
3237 3238
	}

3239
	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
A
Alexandre TORGUE 已提交
3240 3241

	if (priv->synopsys_id < DWMAC_CORE_4_00)
3242
		stmmac_enable_dma_transmission(priv, priv->ioaddr);
A
Alexandre TORGUE 已提交
3243
	else
3244 3245
		stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr,
				queue);
3246

G
Giuseppe CAVALLARO 已提交
3247
	return NETDEV_TX_OK;
3248

G
Giuseppe CAVALLARO 已提交
3249
dma_map_err:
3250
	netdev_err(priv->dev, "Tx DMA map failed\n");
G
Giuseppe CAVALLARO 已提交
3251 3252
	dev_kfree_skb(skb);
	priv->dev->stats.tx_dropped++;
3253 3254 3255
	return NETDEV_TX_OK;
}

3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272
static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
{
	struct ethhdr *ehdr;
	u16 vlanid;

	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
	    NETIF_F_HW_VLAN_CTAG_RX &&
	    !__vlan_get_tag(skb, &vlanid)) {
		/* pop the vlan tag */
		ehdr = (struct ethhdr *)skb->data;
		memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
		skb_pull(skb, VLAN_HLEN);
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
	}
}


3273
static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3274
{
3275
	if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3276 3277 3278 3279 3280
		return 0;

	return 1;
}

3281
/**
3282
 * stmmac_rx_refill - refill used skb preallocated buffers
3283
 * @priv: driver private structure
3284
 * @queue: RX queue index
3285 3286 3287
 * Description : this is to reallocate the skb for the reception process
 * that is based on zero-copy.
 */
3288
static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3289
{
3290 3291 3292 3293
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	int dirty = stmmac_rx_dirty(priv, queue);
	unsigned int entry = rx_q->dirty_rx;

3294 3295
	int bfsize = priv->dma_buf_sz;

3296
	while (dirty-- > 0) {
3297 3298 3299
		struct dma_desc *p;

		if (priv->extend_desc)
3300
			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3301
		else
3302
			p = rx_q->dma_rx + entry;
3303

3304
		if (likely(!rx_q->rx_skbuff[entry])) {
3305 3306
			struct sk_buff *skb;

E
Eric Dumazet 已提交
3307
			skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3308 3309
			if (unlikely(!skb)) {
				/* so for a while no zero-copy! */
3310
				rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3311 3312 3313 3314
				if (unlikely(net_ratelimit()))
					dev_err(priv->device,
						"fail to alloc skb entry %d\n",
						entry);
3315
				break;
3316
			}
3317

3318 3319
			rx_q->rx_skbuff[entry] = skb;
			rx_q->rx_skbuff_dma[entry] =
3320 3321
			    dma_map_single(priv->device, skb->data, bfsize,
					   DMA_FROM_DEVICE);
G
Giuseppe CAVALLARO 已提交
3322
			if (dma_mapping_error(priv->device,
3323
					      rx_q->rx_skbuff_dma[entry])) {
3324
				netdev_err(priv->dev, "Rx DMA map failed\n");
G
Giuseppe CAVALLARO 已提交
3325 3326 3327
				dev_kfree_skb(skb);
				break;
			}
3328

A
Alexandre TORGUE 已提交
3329
			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3330
				p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
A
Alexandre TORGUE 已提交
3331 3332
				p->des1 = 0;
			} else {
3333
				p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
A
Alexandre TORGUE 已提交
3334 3335
			}
			if (priv->hw->mode->refill_desc3)
3336
				priv->hw->mode->refill_desc3(rx_q, p);
3337

3338 3339
			if (rx_q->rx_zeroc_thresh > 0)
				rx_q->rx_zeroc_thresh--;
3340

3341 3342
			netif_dbg(priv, rx_status, priv->dev,
				  "refill entry #%d\n", entry);
3343
		}
P
Pavel Machek 已提交
3344
		dma_wmb();
A
Alexandre TORGUE 已提交
3345 3346

		if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3347
			stmmac_init_rx_desc(priv, p, priv->use_riwt, 0, 0);
A
Alexandre TORGUE 已提交
3348
		else
3349
			stmmac_set_rx_owner(priv, p);
A
Alexandre TORGUE 已提交
3350

P
Pavel Machek 已提交
3351
		dma_wmb();
3352 3353

		entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3354
	}
3355
	rx_q->dirty_rx = entry;
3356 3357
}

3358
/**
3359
 * stmmac_rx - manage the receive process
3360
 * @priv: driver private structure
3361 3362
 * @limit: napi bugget
 * @queue: RX queue index.
3363 3364 3365
 * Description :  this the function called by the napi poll method.
 * It gets all the frames inside the ring.
 */
3366
static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3367
{
3368 3369 3370
	struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
	unsigned int entry = rx_q->cur_rx;
	int coe = priv->hw->rx_csum;
3371 3372 3373
	unsigned int next_entry;
	unsigned int count = 0;

3374
	if (netif_msg_rx_status(priv)) {
3375 3376
		void *rx_head;

3377
		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3378
		if (priv->extend_desc)
3379
			rx_head = (void *)rx_q->dma_erx;
3380
		else
3381
			rx_head = (void *)rx_q->dma_rx;
3382

3383
		stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
3384
	}
3385
	while (count < limit) {
3386
		int status;
3387
		struct dma_desc *p;
3388
		struct dma_desc *np;
3389

3390
		if (priv->extend_desc)
3391
			p = (struct dma_desc *)(rx_q->dma_erx + entry);
3392
		else
3393
			p = rx_q->dma_rx + entry;
3394

3395
		/* read the status of the incoming frame */
3396 3397
		status = stmmac_rx_status(priv, &priv->dev->stats,
				&priv->xstats, p);
3398 3399
		/* check if managed by the DMA otherwise go ahead */
		if (unlikely(status & dma_own))
3400 3401 3402 3403
			break;

		count++;

3404 3405
		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
		next_entry = rx_q->cur_rx;
3406

3407
		if (priv->extend_desc)
3408
			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3409
		else
3410
			np = rx_q->dma_rx + next_entry;
3411 3412

		prefetch(np);
3413

3414 3415 3416
		if (priv->extend_desc)
			stmmac_rx_extended_status(priv, &priv->dev->stats,
					&priv->xstats, rx_q->dma_erx + entry);
3417
		if (unlikely(status == discard_frame)) {
3418
			priv->dev->stats.rx_errors++;
3419
			if (priv->hwts_rx_en && !priv->extend_desc) {
3420
				/* DESC2 & DESC3 will be overwritten by device
3421 3422 3423 3424
				 * with timestamp value, hence reinitialize
				 * them in stmmac_rx_refill() function so that
				 * device can reuse it.
				 */
3425
				dev_kfree_skb_any(rx_q->rx_skbuff[entry]);
3426
				rx_q->rx_skbuff[entry] = NULL;
3427
				dma_unmap_single(priv->device,
3428
						 rx_q->rx_skbuff_dma[entry],
G
Giuseppe CAVALLARO 已提交
3429 3430
						 priv->dma_buf_sz,
						 DMA_FROM_DEVICE);
3431 3432
			}
		} else {
3433
			struct sk_buff *skb;
3434
			int frame_len;
A
Alexandre TORGUE 已提交
3435 3436 3437
			unsigned int des;

			if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3438
				des = le32_to_cpu(p->des0);
A
Alexandre TORGUE 已提交
3439
			else
3440
				des = le32_to_cpu(p->des2);
3441

3442
			frame_len = stmmac_get_rx_frame_len(priv, p, coe);
G
Giuseppe CAVALLARO 已提交
3443

3444
			/*  If frame length is greater than skb buffer size
A
Alexandre TORGUE 已提交
3445 3446 3447
			 *  (preallocated during init) then the packet is
			 *  ignored
			 */
3448
			if (frame_len > priv->dma_buf_sz) {
3449 3450 3451
				netdev_err(priv->dev,
					   "len %d larger than size (%d)\n",
					   frame_len, priv->dma_buf_sz);
3452 3453 3454 3455
				priv->dev->stats.rx_length_errors++;
				break;
			}

3456
			/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
G
Giuseppe CAVALLARO 已提交
3457 3458
			 * Type frames (LLC/LLC-SNAP)
			 */
3459 3460
			if (unlikely(status != llc_snap))
				frame_len -= ETH_FCS_LEN;
3461

3462
			if (netif_msg_rx_status(priv)) {
3463 3464
				netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
					   p, entry, des);
3465 3466
				netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
					   frame_len, status);
3467
			}
3468

A
Alexandre TORGUE 已提交
3469 3470 3471 3472 3473 3474
			/* The zero-copy is always used for all the sizes
			 * in case of GMAC4 because it needs
			 * to refill the used descriptors, always.
			 */
			if (unlikely(!priv->plat->has_gmac4 &&
				     ((frame_len < priv->rx_copybreak) ||
3475
				     stmmac_rx_threshold_count(rx_q)))) {
3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486
				skb = netdev_alloc_skb_ip_align(priv->dev,
								frame_len);
				if (unlikely(!skb)) {
					if (net_ratelimit())
						dev_warn(priv->device,
							 "packet dropped\n");
					priv->dev->stats.rx_dropped++;
					break;
				}

				dma_sync_single_for_cpu(priv->device,
3487
							rx_q->rx_skbuff_dma
3488 3489 3490
							[entry], frame_len,
							DMA_FROM_DEVICE);
				skb_copy_to_linear_data(skb,
3491
							rx_q->
3492 3493 3494 3495 3496
							rx_skbuff[entry]->data,
							frame_len);

				skb_put(skb, frame_len);
				dma_sync_single_for_device(priv->device,
3497
							   rx_q->rx_skbuff_dma
3498 3499 3500
							   [entry], frame_len,
							   DMA_FROM_DEVICE);
			} else {
3501
				skb = rx_q->rx_skbuff[entry];
3502
				if (unlikely(!skb)) {
3503 3504 3505
					netdev_err(priv->dev,
						   "%s: Inconsistent Rx chain\n",
						   priv->dev->name);
3506 3507 3508 3509
					priv->dev->stats.rx_dropped++;
					break;
				}
				prefetch(skb->data - NET_IP_ALIGN);
3510 3511
				rx_q->rx_skbuff[entry] = NULL;
				rx_q->rx_zeroc_thresh++;
3512 3513 3514

				skb_put(skb, frame_len);
				dma_unmap_single(priv->device,
3515
						 rx_q->rx_skbuff_dma[entry],
3516 3517
						 priv->dma_buf_sz,
						 DMA_FROM_DEVICE);
3518 3519 3520
			}

			if (netif_msg_pktdata(priv)) {
3521 3522
				netdev_dbg(priv->dev, "frame received (%dbytes)",
					   frame_len);
3523 3524
				print_pkt(skb->data, frame_len);
			}
3525

3526 3527
			stmmac_get_rx_hwtstamp(priv, p, np, skb);

3528 3529
			stmmac_rx_vlan(priv->dev, skb);

3530 3531
			skb->protocol = eth_type_trans(skb, priv->dev);

G
Giuseppe CAVALLARO 已提交
3532
			if (unlikely(!coe))
3533
				skb_checksum_none_assert(skb);
3534
			else
3535
				skb->ip_summed = CHECKSUM_UNNECESSARY;
3536

3537
			napi_gro_receive(&rx_q->napi, skb);
3538 3539 3540 3541 3542 3543 3544

			priv->dev->stats.rx_packets++;
			priv->dev->stats.rx_bytes += frame_len;
		}
		entry = next_entry;
	}

3545
	stmmac_rx_refill(priv, queue);
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557

	priv->xstats.rx_pkt_n += count;

	return count;
}

/**
 *  stmmac_poll - stmmac poll method (NAPI)
 *  @napi : pointer to the napi structure.
 *  @budget : maximum number of packets that the current CPU can receive from
 *	      all interfaces.
 *  Description :
3558
 *  To look at the incoming frames and clear the tx resources.
3559 3560 3561
 */
static int stmmac_poll(struct napi_struct *napi, int budget)
{
3562 3563 3564
	struct stmmac_rx_queue *rx_q =
		container_of(napi, struct stmmac_rx_queue, napi);
	struct stmmac_priv *priv = rx_q->priv_data;
3565
	u32 tx_count = priv->plat->tx_queues_to_use;
3566
	u32 chan = rx_q->queue_index;
3567
	int work_done = 0;
3568
	u32 queue;
3569

3570
	priv->xstats.napi_poll++;
3571 3572 3573 3574 3575

	/* check all the queues */
	for (queue = 0; queue < tx_count; queue++)
		stmmac_tx_clean(priv, queue);

3576
	work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3577
	if (work_done < budget) {
3578
		napi_complete_done(napi, work_done);
3579
		stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3580 3581 3582 3583 3584 3585 3586 3587
	}
	return work_done;
}

/**
 *  stmmac_tx_timeout
 *  @dev : Pointer to net device structure
 *  Description: this function is called when a packet transmission fails to
3588
 *   complete within a reasonable time. The driver will mark the error in the
3589 3590 3591 3592 3593 3594 3595
 *   netdev structure and arrange for the device to be reset to a sane state
 *   in order to transmit a new packet.
 */
static void stmmac_tx_timeout(struct net_device *dev)
{
	struct stmmac_priv *priv = netdev_priv(dev);

3596
	stmmac_global_err(priv);
3597 3598 3599
}

/**
3600
 *  stmmac_set_rx_mode - entry point for multicast addressing
3601 3602 3603 3604 3605 3606 3607
 *  @dev : pointer to the device structure
 *  Description:
 *  This function is a driver entry point which gets called by the kernel
 *  whenever multicast addresses must be enabled/disabled.
 *  Return value:
 *  void.
 */
3608
static void stmmac_set_rx_mode(struct net_device *dev)
3609 3610 3611
{
	struct stmmac_priv *priv = netdev_priv(dev);

3612
	stmmac_set_filter(priv, priv->hw, dev);
3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627
}

/**
 *  stmmac_change_mtu - entry point to change MTU size for the device.
 *  @dev : device pointer.
 *  @new_mtu : the new MTU size for the device.
 *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
 *  to drive packet transmission. Ethernet has an MTU of 1500 octets
 *  (ETH_DATA_LEN). This value can be changed with ifconfig.
 *  Return value:
 *  0 on success and an appropriate (-)ve integer as defined in errno.h
 *  file on failure.
 */
static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
{
3628 3629
	struct stmmac_priv *priv = netdev_priv(dev);

3630
	if (netif_running(dev)) {
3631
		netdev_err(priv->dev, "must be stopped to change its MTU\n");
3632 3633 3634
		return -EBUSY;
	}

3635
	dev->mtu = new_mtu;
A
Alexandre TORGUE 已提交
3636

3637 3638 3639 3640 3641
	netdev_update_features(dev);

	return 0;
}

3642
static netdev_features_t stmmac_fix_features(struct net_device *dev,
G
Giuseppe CAVALLARO 已提交
3643
					     netdev_features_t features)
3644 3645 3646
{
	struct stmmac_priv *priv = netdev_priv(dev);

3647
	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3648
		features &= ~NETIF_F_RXCSUM;
3649

3650
	if (!priv->plat->tx_coe)
3651
		features &= ~NETIF_F_CSUM_MASK;
3652

3653 3654 3655
	/* Some GMAC devices have a bugged Jumbo frame support that
	 * needs to have the Tx COE disabled for oversized frames
	 * (due to limited buffer sizes). In this case we disable
3656
	 * the TX csum insertion in the TDES and not use SF.
G
Giuseppe CAVALLARO 已提交
3657
	 */
3658
	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3659
		features &= ~NETIF_F_CSUM_MASK;
3660

A
Alexandre TORGUE 已提交
3661 3662 3663 3664 3665 3666 3667 3668
	/* Disable tso if asked by ethtool */
	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
		if (features & NETIF_F_TSO)
			priv->tso = true;
		else
			priv->tso = false;
	}

3669
	return features;
3670 3671
}

3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684
static int stmmac_set_features(struct net_device *netdev,
			       netdev_features_t features)
{
	struct stmmac_priv *priv = netdev_priv(netdev);

	/* Keep the COE Type in case of csum is supporting */
	if (features & NETIF_F_RXCSUM)
		priv->hw->rx_csum = priv->plat->rx_coe;
	else
		priv->hw->rx_csum = 0;
	/* No check needed because rx_coe has been set before and it will be
	 * fixed in case of issue.
	 */
3685
	stmmac_rx_ipc(priv, priv->hw);
3686 3687 3688 3689

	return 0;
}

3690 3691 3692 3693 3694
/**
 *  stmmac_interrupt - main ISR
 *  @irq: interrupt number.
 *  @dev_id: to pass the net device pointer.
 *  Description: this is the main driver interrupt service routine.
3695 3696 3697 3698 3699
 *  It can call:
 *  o DMA service routine (to manage incoming frame reception and transmission
 *    status)
 *  o Core interrupts to manage: remote wake-up, management counter, LPI
 *    interrupts.
3700
 */
3701 3702 3703 3704
static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
{
	struct net_device *dev = (struct net_device *)dev_id;
	struct stmmac_priv *priv = netdev_priv(dev);
3705 3706 3707 3708 3709 3710
	u32 rx_cnt = priv->plat->rx_queues_to_use;
	u32 tx_cnt = priv->plat->tx_queues_to_use;
	u32 queues_count;
	u32 queue;

	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3711

3712 3713 3714
	if (priv->irq_wake)
		pm_wakeup_event(priv->device, 0);

3715
	if (unlikely(!dev)) {
3716
		netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3717 3718 3719
		return IRQ_NONE;
	}

3720 3721 3722
	/* Check if adapter is up */
	if (test_bit(STMMAC_DOWN, &priv->state))
		return IRQ_HANDLED;
3723 3724 3725
	/* Check if a fatal error happened */
	if (stmmac_safety_feat_interrupt(priv))
		return IRQ_HANDLED;
3726

3727
	/* To handle GMAC own interrupts */
A
Alexandre TORGUE 已提交
3728
	if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3729
		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
3730

3731 3732
		if (unlikely(status)) {
			/* For LPI we need to save the tx status */
3733
			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3734
				priv->tx_path_in_lpi_mode = true;
3735
			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3736
				priv->tx_path_in_lpi_mode = false;
3737 3738 3739 3740
		}

		if (priv->synopsys_id >= DWMAC_CORE_4_00) {
			for (queue = 0; queue < queues_count; queue++) {
3741 3742 3743
				struct stmmac_rx_queue *rx_q =
				&priv->rx_queue[queue];

3744 3745
				status |= stmmac_host_mtl_irq_status(priv,
						priv->hw, queue);
3746

3747 3748 3749 3750 3751
				if (status & CORE_IRQ_MTL_RX_OVERFLOW)
					stmmac_set_rx_tail_ptr(priv,
							priv->ioaddr,
							rx_q->rx_tail_addr,
							queue);
3752
			}
3753
		}
3754 3755

		/* PCS link status */
3756
		if (priv->hw->pcs) {
3757 3758 3759 3760 3761
			if (priv->xstats.pcs_link)
				netif_carrier_on(dev);
			else
				netif_carrier_off(dev);
		}
3762
	}
3763

3764
	/* To handle DMA interrupts */
3765
	stmmac_dma_interrupt(priv);
3766 3767 3768 3769 3770 3771

	return IRQ_HANDLED;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
/* Polling receive - used by NETCONSOLE and other diagnostic tools
G
Giuseppe CAVALLARO 已提交
3772 3773
 * to allow network I/O with interrupts disabled.
 */
3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
static void stmmac_poll_controller(struct net_device *dev)
{
	disable_irq(dev->irq);
	stmmac_interrupt(dev->irq, dev);
	enable_irq(dev->irq);
}
#endif

/**
 *  stmmac_ioctl - Entry point for the Ioctl
 *  @dev: Device pointer.
 *  @rq: An IOCTL specefic structure, that can contain a pointer to
 *  a proprietary structure used to pass information to the driver.
 *  @cmd: IOCTL command
 *  Description:
3789
 *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3790 3791 3792
 */
static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
{
3793
	int ret = -EOPNOTSUPP;
3794 3795 3796 3797

	if (!netif_running(dev))
		return -EINVAL;

3798 3799 3800 3801
	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
3802
		if (!dev->phydev)
3803
			return -EINVAL;
3804
		ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3805 3806 3807 3808 3809 3810 3811
		break;
	case SIOCSHWTSTAMP:
		ret = stmmac_hwtstamp_ioctl(dev, rq);
		break;
	default:
		break;
	}
3812

3813 3814 3815
	return ret;
}

3816 3817 3818 3819 3820 3821 3822 3823 3824
static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
{
	struct stmmac_priv *priv = netdev_priv(ndev);
	int ret = 0;

	ret = eth_mac_addr(ndev, addr);
	if (ret)
		return ret;

3825
	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
3826 3827 3828 3829

	return ret;
}

3830
#ifdef CONFIG_DEBUG_FS
3831 3832
static struct dentry *stmmac_fs_dir;

3833
static void sysfs_display_ring(void *head, int size, int extend_desc,
G
Giuseppe CAVALLARO 已提交
3834
			       struct seq_file *seq)
3835 3836
{
	int i;
G
Giuseppe CAVALLARO 已提交
3837 3838
	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
	struct dma_desc *p = (struct dma_desc *)head;
3839

3840 3841 3842
	for (i = 0; i < size; i++) {
		if (extend_desc) {
			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
G
Giuseppe CAVALLARO 已提交
3843
				   i, (unsigned int)virt_to_phys(ep),
3844 3845 3846 3847
				   le32_to_cpu(ep->basic.des0),
				   le32_to_cpu(ep->basic.des1),
				   le32_to_cpu(ep->basic.des2),
				   le32_to_cpu(ep->basic.des3));
3848 3849 3850
			ep++;
		} else {
			seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3851
				   i, (unsigned int)virt_to_phys(p),
3852 3853
				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3854 3855
			p++;
		}
3856 3857
		seq_printf(seq, "\n");
	}
3858
}
3859

3860 3861 3862 3863
static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
{
	struct net_device *dev = seq->private;
	struct stmmac_priv *priv = netdev_priv(dev);
3864
	u32 rx_count = priv->plat->rx_queues_to_use;
3865
	u32 tx_count = priv->plat->tx_queues_to_use;
3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882
	u32 queue;

	for (queue = 0; queue < rx_count; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		seq_printf(seq, "RX Queue %d:\n", queue);

		if (priv->extend_desc) {
			seq_printf(seq, "Extended descriptor ring:\n");
			sysfs_display_ring((void *)rx_q->dma_erx,
					   DMA_RX_SIZE, 1, seq);
		} else {
			seq_printf(seq, "Descriptor ring:\n");
			sysfs_display_ring((void *)rx_q->dma_rx,
					   DMA_RX_SIZE, 0, seq);
		}
	}
3883

3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897
	for (queue = 0; queue < tx_count; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];

		seq_printf(seq, "TX Queue %d:\n", queue);

		if (priv->extend_desc) {
			seq_printf(seq, "Extended descriptor ring:\n");
			sysfs_display_ring((void *)tx_q->dma_etx,
					   DMA_TX_SIZE, 1, seq);
		} else {
			seq_printf(seq, "Descriptor ring:\n");
			sysfs_display_ring((void *)tx_q->dma_tx,
					   DMA_TX_SIZE, 0, seq);
		}
3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
	}

	return 0;
}

static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
{
	return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
}

3908 3909
/* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */

3910 3911 3912 3913 3914
static const struct file_operations stmmac_rings_status_fops = {
	.owner = THIS_MODULE,
	.open = stmmac_sysfs_ring_open,
	.read = seq_read,
	.llseek = seq_lseek,
3915
	.release = single_release,
3916 3917
};

3918 3919 3920 3921 3922
static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
{
	struct net_device *dev = seq->private;
	struct stmmac_priv *priv = netdev_priv(dev);

3923
	if (!priv->hw_cap_support) {
3924 3925 3926 3927 3928 3929 3930 3931
		seq_printf(seq, "DMA HW features not supported\n");
		return 0;
	}

	seq_printf(seq, "==============================\n");
	seq_printf(seq, "\tDMA HW features\n");
	seq_printf(seq, "==============================\n");

3932
	seq_printf(seq, "\t10/100 Mbps: %s\n",
3933
		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3934
	seq_printf(seq, "\t1000 Mbps: %s\n",
3935
		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
3936
	seq_printf(seq, "\tHalf duplex: %s\n",
3937 3938 3939 3940 3941
		   (priv->dma_cap.half_duplex) ? "Y" : "N");
	seq_printf(seq, "\tHash Filter: %s\n",
		   (priv->dma_cap.hash_filter) ? "Y" : "N");
	seq_printf(seq, "\tMultiple MAC address registers: %s\n",
		   (priv->dma_cap.multi_addr) ? "Y" : "N");
3942
	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953
		   (priv->dma_cap.pcs) ? "Y" : "N");
	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
	seq_printf(seq, "\tPMT Remote wake up: %s\n",
		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
	seq_printf(seq, "\tPMT Magic Frame: %s\n",
		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
	seq_printf(seq, "\tRMON module: %s\n",
		   (priv->dma_cap.rmon) ? "Y" : "N");
	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
		   (priv->dma_cap.time_stamp) ? "Y" : "N");
3954
	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3955
		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
3956
	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3957 3958 3959 3960
		   (priv->dma_cap.eee) ? "Y" : "N");
	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
		   (priv->dma_cap.tx_coe) ? "Y" : "N");
A
Alexandre TORGUE 已提交
3961 3962 3963 3964 3965 3966 3967 3968 3969
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
			   (priv->dma_cap.rx_coe) ? "Y" : "N");
	} else {
		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
	}
3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991
	seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
		   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
		   priv->dma_cap.number_rx_channel);
	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
		   priv->dma_cap.number_tx_channel);
	seq_printf(seq, "\tEnhanced descriptors: %s\n",
		   (priv->dma_cap.enh_desc) ? "Y" : "N");

	return 0;
}

static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
{
	return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
}

static const struct file_operations stmmac_dma_cap_fops = {
	.owner = THIS_MODULE,
	.open = stmmac_sysfs_dma_cap_open,
	.read = seq_read,
	.llseek = seq_lseek,
3992
	.release = single_release,
3993 3994
};

3995 3996
static int stmmac_init_fs(struct net_device *dev)
{
3997 3998 3999 4000
	struct stmmac_priv *priv = netdev_priv(dev);

	/* Create per netdev entries */
	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
4001

4002
	if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
4003
		netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
4004 4005 4006 4007 4008

		return -ENOMEM;
	}

	/* Entry to report DMA RX/TX rings */
4009
	priv->dbgfs_rings_status =
4010
		debugfs_create_file("descriptors_status", 0444,
4011 4012
				    priv->dbgfs_dir, dev,
				    &stmmac_rings_status_fops);
4013

4014
	if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
4015
		netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
4016
		debugfs_remove_recursive(priv->dbgfs_dir);
4017 4018 4019 4020

		return -ENOMEM;
	}

4021
	/* Entry to report the DMA HW features */
4022 4023 4024
	priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", 0444,
						  priv->dbgfs_dir,
						  dev, &stmmac_dma_cap_fops);
4025

4026
	if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
4027
		netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
4028
		debugfs_remove_recursive(priv->dbgfs_dir);
4029 4030 4031 4032

		return -ENOMEM;
	}

4033 4034 4035
	return 0;
}

4036
static void stmmac_exit_fs(struct net_device *dev)
4037
{
4038 4039 4040
	struct stmmac_priv *priv = netdev_priv(dev);

	debugfs_remove_recursive(priv->dbgfs_dir);
4041
}
4042
#endif /* CONFIG_DEBUG_FS */
4043

4044 4045 4046 4047 4048
static const struct net_device_ops stmmac_netdev_ops = {
	.ndo_open = stmmac_open,
	.ndo_start_xmit = stmmac_xmit,
	.ndo_stop = stmmac_release,
	.ndo_change_mtu = stmmac_change_mtu,
4049
	.ndo_fix_features = stmmac_fix_features,
4050
	.ndo_set_features = stmmac_set_features,
4051
	.ndo_set_rx_mode = stmmac_set_rx_mode,
4052 4053 4054 4055 4056
	.ndo_tx_timeout = stmmac_tx_timeout,
	.ndo_do_ioctl = stmmac_ioctl,
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = stmmac_poll_controller,
#endif
4057
	.ndo_set_mac_address = stmmac_set_mac_address,
4058 4059
};

4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090
static void stmmac_reset_subtask(struct stmmac_priv *priv)
{
	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
		return;
	if (test_bit(STMMAC_DOWN, &priv->state))
		return;

	netdev_err(priv->dev, "Reset adapter.\n");

	rtnl_lock();
	netif_trans_update(priv->dev);
	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
		usleep_range(1000, 2000);

	set_bit(STMMAC_DOWN, &priv->state);
	dev_close(priv->dev);
	dev_open(priv->dev);
	clear_bit(STMMAC_DOWN, &priv->state);
	clear_bit(STMMAC_RESETING, &priv->state);
	rtnl_unlock();
}

static void stmmac_service_task(struct work_struct *work)
{
	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
			service_task);

	stmmac_reset_subtask(priv);
	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
}

4091 4092
/**
 *  stmmac_hw_init - Init the MAC device
4093
 *  @priv: driver private structure
4094 4095 4096 4097
 *  Description: this function is to configure the MAC device according to
 *  some platform parameters or the HW capability register. It prepares the
 *  driver to use either ring or chain modes and to setup either enhanced or
 *  normal descriptors.
4098 4099 4100 4101 4102 4103
 */
static int stmmac_hw_init(struct stmmac_priv *priv)
{
	struct mac_device_info *mac;

	/* Identify the MAC HW device */
4104 4105 4106
	if (priv->plat->setup) {
		mac = priv->plat->setup(priv);
	} else if (priv->plat->has_gmac) {
4107
		priv->dev->priv_flags |= IFF_UNICAST_FLT;
4108 4109
		mac = dwmac1000_setup(priv->ioaddr,
				      priv->plat->multicast_filter_bins,
4110 4111
				      priv->plat->unicast_filter_entries,
				      &priv->synopsys_id);
A
Alexandre TORGUE 已提交
4112 4113 4114 4115 4116 4117
	} else if (priv->plat->has_gmac4) {
		priv->dev->priv_flags |= IFF_UNICAST_FLT;
		mac = dwmac4_setup(priv->ioaddr,
				   priv->plat->multicast_filter_bins,
				   priv->plat->unicast_filter_entries,
				   &priv->synopsys_id);
4118
	} else {
4119
		mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
4120
	}
4121 4122 4123 4124 4125
	if (!mac)
		return -ENOMEM;

	priv->hw = mac;

4126 4127 4128 4129
	/* dwmac-sun8i only work in chain mode */
	if (priv->plat->has_sun8i)
		chain_mode = 1;

4130
	/* To use the chained or ring mode */
A
Alexandre TORGUE 已提交
4131 4132
	if (priv->synopsys_id >= DWMAC_CORE_4_00) {
		priv->hw->mode = &dwmac4_ring_mode_ops;
4133
	} else {
A
Alexandre TORGUE 已提交
4134 4135
		if (chain_mode) {
			priv->hw->mode = &chain_mode_ops;
4136
			dev_info(priv->device, "Chain mode enabled\n");
A
Alexandre TORGUE 已提交
4137 4138 4139
			priv->mode = STMMAC_CHAIN_MODE;
		} else {
			priv->hw->mode = &ring_mode_ops;
4140
			dev_info(priv->device, "Ring mode enabled\n");
A
Alexandre TORGUE 已提交
4141 4142
			priv->mode = STMMAC_RING_MODE;
		}
4143 4144
	}

4145 4146 4147
	/* Get the HW capability (new GMAC newer than 3.50a) */
	priv->hw_cap_support = stmmac_get_hw_features(priv);
	if (priv->hw_cap_support) {
4148
		dev_info(priv->device, "DMA HW capability register supported\n");
4149 4150 4151 4152 4153 4154 4155 4156

		/* We can override some gmac/dma configuration fields: e.g.
		 * enh_desc, tx_coe (e.g. that are passed through the
		 * platform) with the values from the HW capability
		 * register (if supported).
		 */
		priv->plat->enh_desc = priv->dma_cap.enh_desc;
		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4157
		priv->hw->pmt = priv->plat->pmt;
4158

4159 4160 4161 4162 4163 4164
		/* TXCOE doesn't work in thresh DMA mode */
		if (priv->plat->force_thresh_dma_mode)
			priv->plat->tx_coe = 0;
		else
			priv->plat->tx_coe = priv->dma_cap.tx_coe;

A
Alexandre TORGUE 已提交
4165 4166
		/* In case of GMAC4 rx_coe is from HW cap register. */
		priv->plat->rx_coe = priv->dma_cap.rx_coe;
4167 4168 4169 4170 4171 4172

		if (priv->dma_cap.rx_coe_type2)
			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
		else if (priv->dma_cap.rx_coe_type1)
			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;

4173 4174 4175
	} else {
		dev_info(priv->device, "No HW DMA feature register supported\n");
	}
4176

A
Alexandre TORGUE 已提交
4177 4178 4179 4180 4181
	/* To use alternate (extended), normal or GMAC4 descriptor structures */
	if (priv->synopsys_id >= DWMAC_CORE_4_00)
		priv->hw->desc = &dwmac4_desc_ops;
	else
		stmmac_selec_desc_mode(priv);
4182

4183 4184
	if (priv->plat->rx_coe) {
		priv->hw->rx_csum = priv->plat->rx_coe;
4185
		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
A
Alexandre TORGUE 已提交
4186
		if (priv->synopsys_id < DWMAC_CORE_4_00)
4187
			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4188
	}
4189
	if (priv->plat->tx_coe)
4190
		dev_info(priv->device, "TX Checksum insertion supported\n");
4191 4192

	if (priv->plat->pmt) {
4193
		dev_info(priv->device, "Wake-Up On Lan supported\n");
4194 4195 4196
		device_set_wakeup_capable(priv->device, 1);
	}

A
Alexandre TORGUE 已提交
4197
	if (priv->dma_cap.tsoen)
4198
		dev_info(priv->device, "TSO supported\n");
A
Alexandre TORGUE 已提交
4199

4200
	return 0;
4201 4202
}

4203
/**
4204 4205
 * stmmac_dvr_probe
 * @device: device pointer
4206
 * @plat_dat: platform data pointer
4207
 * @res: stmmac resource pointer
4208 4209
 * Description: this is the main probe function used to
 * call the alloc_etherdev, allocate the priv structure.
4210
 * Return:
4211
 * returns 0 on success, otherwise errno.
4212
 */
4213 4214 4215
int stmmac_dvr_probe(struct device *device,
		     struct plat_stmmacenet_data *plat_dat,
		     struct stmmac_resources *res)
4216
{
4217 4218
	struct net_device *ndev = NULL;
	struct stmmac_priv *priv;
4219 4220
	int ret = 0;
	u32 queue;
4221

4222 4223 4224
	ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
				  MTL_MAX_TX_QUEUES,
				  MTL_MAX_RX_QUEUES);
4225
	if (!ndev)
4226
		return -ENOMEM;
4227 4228 4229 4230 4231 4232

	SET_NETDEV_DEV(ndev, device);

	priv = netdev_priv(ndev);
	priv->device = device;
	priv->dev = ndev;
4233

4234
	stmmac_set_ethtool_ops(ndev);
4235 4236
	priv->pause = pause;
	priv->plat = plat_dat;
4237 4238 4239 4240 4241 4242 4243 4244 4245
	priv->ioaddr = res->addr;
	priv->dev->base_addr = (unsigned long)res->addr;

	priv->dev->irq = res->irq;
	priv->wol_irq = res->wol_irq;
	priv->lpi_irq = res->lpi_irq;

	if (res->mac)
		memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4246

4247
	dev_set_drvdata(device, priv->dev);
4248

4249 4250
	/* Verify driver arguments */
	stmmac_verify_args();
4251

4252 4253 4254 4255 4256 4257 4258 4259 4260
	/* Allocate workqueue */
	priv->wq = create_singlethread_workqueue("stmmac_wq");
	if (!priv->wq) {
		dev_err(priv->device, "failed to create workqueue\n");
		goto error_wq;
	}

	INIT_WORK(&priv->service_task, stmmac_service_task);

4261
	/* Override with kernel parameters if supplied XXX CRS XXX
G
Giuseppe CAVALLARO 已提交
4262 4263
	 * this needs to have multiple instances
	 */
4264 4265 4266
	if ((phyaddr >= 0) && (phyaddr <= 31))
		priv->plat->phy_addr = phyaddr;

4267 4268
	if (priv->plat->stmmac_rst) {
		ret = reset_control_assert(priv->plat->stmmac_rst);
4269
		reset_control_deassert(priv->plat->stmmac_rst);
4270 4271 4272 4273 4274 4275
		/* Some reset controllers have only reset callback instead of
		 * assert + deassert callbacks pair.
		 */
		if (ret == -ENOTSUPP)
			reset_control_reset(priv->plat->stmmac_rst);
	}
4276

4277
	/* Init MAC and get the capabilities */
4278 4279
	ret = stmmac_hw_init(priv);
	if (ret)
4280
		goto error_hw_init;
4281

4282
	/* Configure real RX and TX queues */
4283 4284
	netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
	netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4285

4286
	ndev->netdev_ops = &stmmac_netdev_ops;
4287

4288 4289
	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
			    NETIF_F_RXCSUM;
A
Alexandre TORGUE 已提交
4290 4291

	if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
N
Niklas Cassel 已提交
4292
		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
A
Alexandre TORGUE 已提交
4293
		priv->tso = true;
4294
		dev_info(priv->device, "TSO feature enabled\n");
A
Alexandre TORGUE 已提交
4295
	}
4296 4297
	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4298 4299
#ifdef STMMAC_VLAN_TAG_USED
	/* Both mac100 and gmac support receive VLAN tag detection */
4300
	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4301 4302 4303
#endif
	priv->msg_enable = netif_msg_init(debug, default_msg_level);

4304 4305 4306 4307 4308 4309
	/* MTU range: 46 - hw-specific max */
	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
	if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
		ndev->max_mtu = JUMBO_LEN;
	else
		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4310 4311 4312 4313 4314
	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
	 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
	 */
	if ((priv->plat->maxmtu < ndev->max_mtu) &&
	    (priv->plat->maxmtu >= ndev->min_mtu))
4315
		ndev->max_mtu = priv->plat->maxmtu;
4316
	else if (priv->plat->maxmtu < ndev->min_mtu)
4317 4318 4319
		dev_warn(priv->device,
			 "%s: warning: maxmtu having invalid value (%d)\n",
			 __func__, priv->plat->maxmtu);
4320

4321 4322 4323
	if (flow_ctrl)
		priv->flow_ctrl = FLOW_AUTO;	/* RX/TX pause on */

4324 4325 4326 4327 4328 4329 4330
	/* Rx Watchdog is available in the COREs newer than the 3.40.
	 * In some case, for example on bugged HW this feature
	 * has to be disable and this can be done by passing the
	 * riwt_off field from the platform.
	 */
	if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
		priv->use_riwt = 1;
4331 4332
		dev_info(priv->device,
			 "Enable RX Mitigation via HW Watchdog Timer\n");
4333 4334
	}

4335 4336 4337 4338 4339 4340
	for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
			       (8 * priv->plat->rx_queues_to_use));
	}
4341

4342 4343
	spin_lock_init(&priv->lock);

4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354
	/* If a specific clk_csr value is passed from the platform
	 * this means that the CSR Clock Range selection cannot be
	 * changed at run-time and it is fixed. Viceversa the driver'll try to
	 * set the MDC clock dynamically according to the csr actual
	 * clock input.
	 */
	if (!priv->plat->clk_csr)
		stmmac_clk_csr_set(priv);
	else
		priv->clk_csr = priv->plat->clk_csr;

4355 4356
	stmmac_check_pcs_mode(priv);

4357 4358 4359
	if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI) {
4360 4361 4362
		/* MDIO bus Registration */
		ret = stmmac_mdio_register(ndev);
		if (ret < 0) {
4363 4364 4365
			dev_err(priv->device,
				"%s: MDIO bus (id: %d) registration failed",
				__func__, priv->plat->bus_id);
4366 4367
			goto error_mdio_register;
		}
4368 4369
	}

4370
	ret = register_netdev(ndev);
4371
	if (ret) {
4372 4373
		dev_err(priv->device, "%s: ERROR %i registering the device\n",
			__func__, ret);
4374 4375
		goto error_netdev_register;
	}
4376 4377

	return ret;
4378

4379
error_netdev_register:
4380 4381 4382 4383
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI)
		stmmac_mdio_unregister(ndev);
4384
error_mdio_register:
4385 4386 4387 4388 4389
	for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		netif_napi_del(&rx_q->napi);
	}
4390
error_hw_init:
4391 4392
	destroy_workqueue(priv->wq);
error_wq:
4393
	free_netdev(ndev);
4394

4395
	return ret;
4396
}
4397
EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4398 4399 4400

/**
 * stmmac_dvr_remove
4401
 * @dev: device pointer
4402
 * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4403
 * changes the link status, releases the DMA descriptor rings.
4404
 */
4405
int stmmac_dvr_remove(struct device *dev)
4406
{
4407
	struct net_device *ndev = dev_get_drvdata(dev);
4408
	struct stmmac_priv *priv = netdev_priv(ndev);
4409

4410
	netdev_info(priv->dev, "%s: removing driver", __func__);
4411

4412
	stmmac_stop_all_dma(priv);
4413

4414
	stmmac_mac_set(priv, priv->ioaddr, false);
4415 4416
	netif_carrier_off(ndev);
	unregister_netdev(ndev);
4417 4418 4419 4420
	if (priv->plat->stmmac_rst)
		reset_control_assert(priv->plat->stmmac_rst);
	clk_disable_unprepare(priv->plat->pclk);
	clk_disable_unprepare(priv->plat->stmmac_clk);
4421 4422 4423
	if (priv->hw->pcs != STMMAC_PCS_RGMII &&
	    priv->hw->pcs != STMMAC_PCS_TBI &&
	    priv->hw->pcs != STMMAC_PCS_RTBI)
4424
		stmmac_mdio_unregister(ndev);
4425
	destroy_workqueue(priv->wq);
4426 4427 4428 4429
	free_netdev(ndev);

	return 0;
}
4430
EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4431

4432 4433
/**
 * stmmac_suspend - suspend callback
4434
 * @dev: device pointer
4435 4436 4437 4438
 * Description: this is the function to suspend the device and it is called
 * by the platform driver to stop the network queue, release the resources,
 * program the PMT register (for WoL), clean and release driver resources.
 */
4439
int stmmac_suspend(struct device *dev)
4440
{
4441
	struct net_device *ndev = dev_get_drvdata(dev);
4442
	struct stmmac_priv *priv = netdev_priv(ndev);
4443
	unsigned long flags;
4444

4445
	if (!ndev || !netif_running(ndev))
4446 4447
		return 0;

4448 4449
	if (ndev->phydev)
		phy_stop(ndev->phydev);
4450

4451
	spin_lock_irqsave(&priv->lock, flags);
4452

4453
	netif_device_detach(ndev);
4454
	stmmac_stop_all_queues(priv);
4455

4456
	stmmac_disable_all_queues(priv);
4457 4458

	/* Stop TX/RX DMA */
4459
	stmmac_stop_all_dma(priv);
4460

4461
	/* Enable Power down mode by programming the PMT regs */
4462
	if (device_may_wakeup(priv->device)) {
4463
		stmmac_pmt(priv, priv->hw, priv->wolopts);
4464 4465
		priv->irq_wake = 1;
	} else {
4466
		stmmac_mac_set(priv, priv->ioaddr, false);
4467
		pinctrl_pm_select_sleep_state(priv->device);
4468
		/* Disable clock in case of PWM is off */
4469 4470
		clk_disable(priv->plat->pclk);
		clk_disable(priv->plat->stmmac_clk);
4471
	}
4472
	spin_unlock_irqrestore(&priv->lock, flags);
4473

4474
	priv->oldlink = false;
4475 4476
	priv->speed = SPEED_UNKNOWN;
	priv->oldduplex = DUPLEX_UNKNOWN;
4477 4478
	return 0;
}
4479
EXPORT_SYMBOL_GPL(stmmac_suspend);
4480

4481 4482 4483 4484 4485 4486 4487
/**
 * stmmac_reset_queues_param - reset queue parameters
 * @dev: device pointer
 */
static void stmmac_reset_queues_param(struct stmmac_priv *priv)
{
	u32 rx_cnt = priv->plat->rx_queues_to_use;
4488
	u32 tx_cnt = priv->plat->tx_queues_to_use;
4489 4490 4491 4492 4493 4494 4495 4496 4497
	u32 queue;

	for (queue = 0; queue < rx_cnt; queue++) {
		struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];

		rx_q->cur_rx = 0;
		rx_q->dirty_rx = 0;
	}

4498 4499 4500 4501 4502
	for (queue = 0; queue < tx_cnt; queue++) {
		struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];

		tx_q->cur_tx = 0;
		tx_q->dirty_tx = 0;
4503
		tx_q->mss = 0;
4504
	}
4505 4506
}

4507 4508
/**
 * stmmac_resume - resume callback
4509
 * @dev: device pointer
4510 4511 4512
 * Description: when resume this function is invoked to setup the DMA and CORE
 * in a usable state.
 */
4513
int stmmac_resume(struct device *dev)
4514
{
4515
	struct net_device *ndev = dev_get_drvdata(dev);
4516
	struct stmmac_priv *priv = netdev_priv(ndev);
4517
	unsigned long flags;
4518

4519
	if (!netif_running(ndev))
4520 4521 4522 4523 4524 4525
		return 0;

	/* Power Down bit, into the PM register, is cleared
	 * automatically as soon as a magic packet or a Wake-up frame
	 * is received. Anyway, it's better to manually clear
	 * this bit because it can generate problems while resuming
G
Giuseppe CAVALLARO 已提交
4526 4527
	 * from another devices (e.g. serial console).
	 */
4528
	if (device_may_wakeup(priv->device)) {
4529
		spin_lock_irqsave(&priv->lock, flags);
4530
		stmmac_pmt(priv, priv->hw, 0);
4531
		spin_unlock_irqrestore(&priv->lock, flags);
4532
		priv->irq_wake = 0;
4533
	} else {
4534
		pinctrl_pm_select_default_state(priv->device);
4535
		/* enable the clk previously disabled */
4536 4537
		clk_enable(priv->plat->stmmac_clk);
		clk_enable(priv->plat->pclk);
4538 4539 4540 4541
		/* reset the phy so that it's ready */
		if (priv->mii)
			stmmac_mdio_reset(priv->mii);
	}
4542

4543
	netif_device_attach(ndev);
4544

4545 4546
	spin_lock_irqsave(&priv->lock, flags);

4547 4548
	stmmac_reset_queues_param(priv);

4549 4550
	stmmac_clear_descriptors(priv);

4551
	stmmac_hw_setup(ndev, false);
4552
	stmmac_init_tx_coalesce(priv);
4553
	stmmac_set_rx_mode(ndev);
4554

4555
	stmmac_enable_all_queues(priv);
4556

4557
	stmmac_start_all_queues(priv);
4558

4559
	spin_unlock_irqrestore(&priv->lock, flags);
4560

4561 4562
	if (ndev->phydev)
		phy_start(ndev->phydev);
4563

4564 4565
	return 0;
}
4566
EXPORT_SYMBOL_GPL(stmmac_resume);
4567

4568 4569 4570 4571 4572 4573 4574 4575
#ifndef MODULE
static int __init stmmac_cmdline_opt(char *str)
{
	char *opt;

	if (!str || !*str)
		return -EINVAL;
	while ((opt = strsep(&str, ",")) != NULL) {
4576
		if (!strncmp(opt, "debug:", 6)) {
4577
			if (kstrtoint(opt + 6, 0, &debug))
4578 4579
				goto err;
		} else if (!strncmp(opt, "phyaddr:", 8)) {
4580
			if (kstrtoint(opt + 8, 0, &phyaddr))
4581 4582
				goto err;
		} else if (!strncmp(opt, "buf_sz:", 7)) {
4583
			if (kstrtoint(opt + 7, 0, &buf_sz))
4584 4585
				goto err;
		} else if (!strncmp(opt, "tc:", 3)) {
4586
			if (kstrtoint(opt + 3, 0, &tc))
4587 4588
				goto err;
		} else if (!strncmp(opt, "watchdog:", 9)) {
4589
			if (kstrtoint(opt + 9, 0, &watchdog))
4590 4591
				goto err;
		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
4592
			if (kstrtoint(opt + 10, 0, &flow_ctrl))
4593 4594
				goto err;
		} else if (!strncmp(opt, "pause:", 6)) {
4595
			if (kstrtoint(opt + 6, 0, &pause))
4596
				goto err;
4597
		} else if (!strncmp(opt, "eee_timer:", 10)) {
4598 4599
			if (kstrtoint(opt + 10, 0, &eee_timer))
				goto err;
4600 4601 4602
		} else if (!strncmp(opt, "chain_mode:", 11)) {
			if (kstrtoint(opt + 11, 0, &chain_mode))
				goto err;
4603
		}
4604 4605
	}
	return 0;
4606 4607 4608 4609

err:
	pr_err("%s: ERROR broken module parameter conversion", __func__);
	return -EINVAL;
4610 4611 4612
}

__setup("stmmaceth=", stmmac_cmdline_opt);
G
Giuseppe CAVALLARO 已提交
4613
#endif /* MODULE */
4614

4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643
static int __init stmmac_init(void)
{
#ifdef CONFIG_DEBUG_FS
	/* Create debugfs main directory if it doesn't exist yet */
	if (!stmmac_fs_dir) {
		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);

		if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
			pr_err("ERROR %s, debugfs create directory failed\n",
			       STMMAC_RESOURCE_NAME);

			return -ENOMEM;
		}
	}
#endif

	return 0;
}

static void __exit stmmac_exit(void)
{
#ifdef CONFIG_DEBUG_FS
	debugfs_remove_recursive(stmmac_fs_dir);
#endif
}

module_init(stmmac_init)
module_exit(stmmac_exit)

4644 4645 4646
MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
MODULE_LICENSE("GPL");