davinci_cpdma.c 22.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Texas Instruments CPDMA Driver
 *
 * Copyright (C) 2010 Texas Instruments
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation version 2.
 *
 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 * kind, whether express or implied; without even the implied warranty
 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */
#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/device.h>
18
#include <linux/module.h>
19 20 21 22
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
23
#include <linux/delay.h>
24
#include <linux/genalloc.h>
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
#include "davinci_cpdma.h"

/* DMA Registers */
#define CPDMA_TXIDVER		0x00
#define CPDMA_TXCONTROL		0x04
#define CPDMA_TXTEARDOWN	0x08
#define CPDMA_RXIDVER		0x10
#define CPDMA_RXCONTROL		0x14
#define CPDMA_SOFTRESET		0x1c
#define CPDMA_RXTEARDOWN	0x18
#define CPDMA_TXINTSTATRAW	0x80
#define CPDMA_TXINTSTATMASKED	0x84
#define CPDMA_TXINTMASKSET	0x88
#define CPDMA_TXINTMASKCLEAR	0x8c
#define CPDMA_MACINVECTOR	0x90
#define CPDMA_MACEOIVECTOR	0x94
#define CPDMA_RXINTSTATRAW	0xa0
#define CPDMA_RXINTSTATMASKED	0xa4
#define CPDMA_RXINTMASKSET	0xa8
#define CPDMA_RXINTMASKCLEAR	0xac
#define CPDMA_DMAINTSTATRAW	0xb0
#define CPDMA_DMAINTSTATMASKED	0xb4
#define CPDMA_DMAINTMASKSET	0xb8
#define CPDMA_DMAINTMASKCLEAR	0xbc
#define CPDMA_DMAINT_HOSTERR	BIT(1)

/* the following exist only if has_ext_regs is set */
#define CPDMA_DMACONTROL	0x20
#define CPDMA_DMASTATUS		0x24
#define CPDMA_RXBUFFOFS		0x28
#define CPDMA_EM_CONTROL	0x2c

/* Descriptor mode bits */
#define CPDMA_DESC_SOP		BIT(31)
#define CPDMA_DESC_EOP		BIT(30)
#define CPDMA_DESC_OWNER	BIT(29)
#define CPDMA_DESC_EOQ		BIT(28)
#define CPDMA_DESC_TD_COMPLETE	BIT(27)
#define CPDMA_DESC_PASS_CRC	BIT(26)
64 65 66
#define CPDMA_DESC_TO_PORT_EN	BIT(20)
#define CPDMA_TO_PORT_SHIFT	16
#define CPDMA_DESC_PORT_MASK	(BIT(18) | BIT(17) | BIT(16))
67
#define CPDMA_DESC_CRC_LEN	4
68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83

#define CPDMA_TEARDOWN_VALUE	0xfffffffc

struct cpdma_desc {
	/* hardware fields */
	u32			hw_next;
	u32			hw_buffer;
	u32			hw_len;
	u32			hw_mode;
	/* software fields */
	void			*sw_token;
	u32			sw_buffer;
	u32			sw_len;
};

struct cpdma_desc_pool {
84
	phys_addr_t		phys;
85
	dma_addr_t		hw_addr;
86 87 88
	void __iomem		*iomap;		/* ioremap map */
	void			*cpumap;	/* dma_alloc map */
	int			desc_size, mem_size;
89
	int			num_desc;
90
	struct device		*dev;
91
	struct gen_pool		*gen_pool;
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
};

enum cpdma_state {
	CPDMA_STATE_IDLE,
	CPDMA_STATE_ACTIVE,
	CPDMA_STATE_TEARDOWN,
};

struct cpdma_ctlr {
	enum cpdma_state	state;
	struct cpdma_params	params;
	struct device		*dev;
	struct cpdma_desc_pool	*pool;
	spinlock_t		lock;
	struct cpdma_chan	*channels[2 * CPDMA_MAX_CHANNELS];
};

struct cpdma_chan {
110 111
	struct cpdma_desc __iomem	*head, *tail;
	void __iomem			*hdp, *cp, *rxfree;
112 113 114 115 116
	enum cpdma_state		state;
	struct cpdma_ctlr		*ctlr;
	int				chan_num;
	spinlock_t			lock;
	int				count;
117
	u32				desc_num;
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
	u32				mask;
	cpdma_handler_fn		handler;
	enum dma_data_direction		dir;
	struct cpdma_chan_stats		stats;
	/* offsets into dmaregs */
	int	int_set, int_clear, td;
};

/* The following make access to common cpdma_ctlr params more readable */
#define dmaregs		params.dmaregs
#define num_chan	params.num_chan

/* various accessors */
#define dma_reg_read(ctlr, ofs)		__raw_readl((ctlr)->dmaregs + (ofs))
#define chan_read(chan, fld)		__raw_readl((chan)->fld)
#define desc_read(desc, fld)		__raw_readl(&(desc)->fld)
#define dma_reg_write(ctlr, ofs, v)	__raw_writel(v, (ctlr)->dmaregs + (ofs))
#define chan_write(chan, fld, v)	__raw_writel(v, (chan)->fld)
#define desc_write(desc, fld, v)	__raw_writel((u32)(v), &(desc)->fld)

138 139 140 141 142 143 144 145
#define cpdma_desc_to_port(chan, mode, directed)			\
	do {								\
		if (!is_rx_chan(chan) && ((directed == 1) ||		\
					  (directed == 2)))		\
			mode |= (CPDMA_DESC_TO_PORT_EN |		\
				 (directed << CPDMA_TO_PORT_SHIFT));	\
	} while (0)

146 147 148 149 150
static void cpdma_desc_pool_destroy(struct cpdma_desc_pool *pool)
{
	if (!pool)
		return;

151 152 153 154
	WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool),
	     "cpdma_desc_pool size %d != avail %d",
	     gen_pool_size(pool->gen_pool),
	     gen_pool_avail(pool->gen_pool));
155 156 157 158 159 160 161
	if (pool->cpumap)
		dma_free_coherent(pool->dev, pool->mem_size, pool->cpumap,
				  pool->phys);
	else
		iounmap(pool->iomap);
}

162 163 164 165 166 167 168
/*
 * Utility constructs for a cpdma descriptor pool.  Some devices (e.g. davinci
 * emac) have dedicated on-chip memory for these descriptors.  Some other
 * devices (e.g. cpsw switches) use plain old memory.  Descriptor pools
 * abstract out these details
 */
static struct cpdma_desc_pool *
169
cpdma_desc_pool_create(struct device *dev, u32 phys, dma_addr_t hw_addr,
170
				int size, int align)
171 172
{
	struct cpdma_desc_pool *pool;
173
	int ret;
174

175
	pool = devm_kzalloc(dev, sizeof(*pool), GFP_KERNEL);
176
	if (!pool)
177
		goto gen_pool_create_fail;
178 179 180 181 182 183

	pool->dev	= dev;
	pool->mem_size	= size;
	pool->desc_size	= ALIGN(sizeof(struct cpdma_desc), align);
	pool->num_desc	= size / pool->desc_size;

184 185 186 187 188 189 190
	pool->gen_pool = devm_gen_pool_create(dev, ilog2(pool->desc_size), -1,
					      "cpdma");
	if (IS_ERR(pool->gen_pool)) {
		dev_err(dev, "pool create failed %ld\n",
			PTR_ERR(pool->gen_pool));
		goto gen_pool_create_fail;
	}
191 192 193

	if (phys) {
		pool->phys  = phys;
194
		pool->iomap = ioremap(phys, size); /* should be memremap? */
195
		pool->hw_addr = hw_addr;
196
	} else {
197
		pool->cpumap = dma_alloc_coherent(dev, size, &pool->hw_addr,
198
						  GFP_KERNEL);
199 200
		pool->iomap = (void __iomem __force *)pool->cpumap;
		pool->phys = pool->hw_addr; /* assumes no IOMMU, don't use this value */
201 202
	}

203 204
	if (!pool->iomap)
		goto gen_pool_create_fail;
205

206 207 208 209 210
	ret = gen_pool_add_virt(pool->gen_pool, (unsigned long)pool->iomap,
				pool->phys, pool->mem_size, -1);
	if (ret < 0) {
		dev_err(dev, "pool add failed %d\n", ret);
		goto gen_pool_add_virt_fail;
211
	}
212 213 214 215 216 217 218

	return pool;

gen_pool_add_virt_fail:
	cpdma_desc_pool_destroy(pool);
gen_pool_create_fail:
	return NULL;
219 220 221 222 223 224 225
}

static inline dma_addr_t desc_phys(struct cpdma_desc_pool *pool,
		  struct cpdma_desc __iomem *desc)
{
	if (!desc)
		return 0;
226
	return pool->hw_addr + (__force long)desc - (__force long)pool->iomap;
227 228 229 230 231
}

static inline struct cpdma_desc __iomem *
desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma)
{
232
	return dma ? pool->iomap + dma - pool->hw_addr : NULL;
233 234 235
}

static struct cpdma_desc __iomem *
236
cpdma_desc_alloc(struct cpdma_desc_pool *pool)
237
{
238 239
	return (struct cpdma_desc __iomem *)
		gen_pool_alloc(pool->gen_pool, pool->desc_size);
240 241 242 243 244
}

static void cpdma_desc_free(struct cpdma_desc_pool *pool,
			    struct cpdma_desc __iomem *desc, int num_desc)
{
245
	gen_pool_free(pool->gen_pool, (unsigned long)desc, pool->desc_size);
246 247 248 249 250 251
}

struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params)
{
	struct cpdma_ctlr *ctlr;

252
	ctlr = devm_kzalloc(params->dev, sizeof(*ctlr), GFP_KERNEL);
253 254 255 256 257 258 259 260 261 262
	if (!ctlr)
		return NULL;

	ctlr->state = CPDMA_STATE_IDLE;
	ctlr->params = *params;
	ctlr->dev = params->dev;
	spin_lock_init(&ctlr->lock);

	ctlr->pool = cpdma_desc_pool_create(ctlr->dev,
					    ctlr->params.desc_mem_phys,
263
					    ctlr->params.desc_hw_addr,
264 265
					    ctlr->params.desc_mem_size,
					    ctlr->params.desc_align);
266
	if (!ctlr->pool)
267 268 269 270 271 272
		return NULL;

	if (WARN_ON(ctlr->num_chan > CPDMA_MAX_CHANNELS))
		ctlr->num_chan = CPDMA_MAX_CHANNELS;
	return ctlr;
}
273
EXPORT_SYMBOL_GPL(cpdma_ctlr_create);
274 275 276 277 278 279 280 281 282 283 284 285 286

int cpdma_ctlr_start(struct cpdma_ctlr *ctlr)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&ctlr->lock, flags);
	if (ctlr->state != CPDMA_STATE_IDLE) {
		spin_unlock_irqrestore(&ctlr->lock, flags);
		return -EBUSY;
	}

	if (ctlr->params.has_soft_reset) {
287
		unsigned timeout = 10 * 100;
288 289

		dma_reg_write(ctlr, CPDMA_SOFTRESET, 1);
290
		while (timeout) {
291 292
			if (dma_reg_read(ctlr, CPDMA_SOFTRESET) == 0)
				break;
293 294
			udelay(10);
			timeout--;
295
		}
296
		WARN_ON(!timeout);
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	}

	for (i = 0; i < ctlr->num_chan; i++) {
		__raw_writel(0, ctlr->params.txhdp + 4 * i);
		__raw_writel(0, ctlr->params.rxhdp + 4 * i);
		__raw_writel(0, ctlr->params.txcp + 4 * i);
		__raw_writel(0, ctlr->params.rxcp + 4 * i);
	}

	dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff);
	dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff);

	dma_reg_write(ctlr, CPDMA_TXCONTROL, 1);
	dma_reg_write(ctlr, CPDMA_RXCONTROL, 1);

	ctlr->state = CPDMA_STATE_ACTIVE;

	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) {
		if (ctlr->channels[i])
			cpdma_chan_start(ctlr->channels[i]);
	}
	spin_unlock_irqrestore(&ctlr->lock, flags);
	return 0;
}
321
EXPORT_SYMBOL_GPL(cpdma_ctlr_start);
322 323 324 325 326 327 328

int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&ctlr->lock, flags);
329
	if (ctlr->state == CPDMA_STATE_TEARDOWN) {
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
		spin_unlock_irqrestore(&ctlr->lock, flags);
		return -EINVAL;
	}

	ctlr->state = CPDMA_STATE_TEARDOWN;

	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) {
		if (ctlr->channels[i])
			cpdma_chan_stop(ctlr->channels[i]);
	}

	dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff);
	dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff);

	dma_reg_write(ctlr, CPDMA_TXCONTROL, 0);
	dma_reg_write(ctlr, CPDMA_RXCONTROL, 0);

	ctlr->state = CPDMA_STATE_IDLE;

	spin_unlock_irqrestore(&ctlr->lock, flags);
	return 0;
}
352
EXPORT_SYMBOL_GPL(cpdma_ctlr_stop);
353 354 355 356 357 358 359 360 361 362 363

int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr)
{
	int ret = 0, i;

	if (!ctlr)
		return -EINVAL;

	if (ctlr->state != CPDMA_STATE_IDLE)
		cpdma_ctlr_stop(ctlr);

364 365
	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++)
		cpdma_chan_destroy(ctlr->channels[i]);
366 367 368 369

	cpdma_desc_pool_destroy(ctlr->pool);
	return ret;
}
370
EXPORT_SYMBOL_GPL(cpdma_ctlr_destroy);
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393

int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable)
{
	unsigned long flags;
	int i, reg;

	spin_lock_irqsave(&ctlr->lock, flags);
	if (ctlr->state != CPDMA_STATE_ACTIVE) {
		spin_unlock_irqrestore(&ctlr->lock, flags);
		return -EINVAL;
	}

	reg = enable ? CPDMA_DMAINTMASKSET : CPDMA_DMAINTMASKCLEAR;
	dma_reg_write(ctlr, reg, CPDMA_DMAINT_HOSTERR);

	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) {
		if (ctlr->channels[i])
			cpdma_chan_int_ctrl(ctlr->channels[i], enable);
	}

	spin_unlock_irqrestore(&ctlr->lock, flags);
	return 0;
}
394
EXPORT_SYMBOL_GPL(cpdma_ctlr_int_ctrl);
395

396
void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value)
397
{
398
	dma_reg_write(ctlr, CPDMA_MACEOIVECTOR, value);
399
}
400
EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi);
401 402 403 404 405

struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
				     cpdma_handler_fn handler)
{
	struct cpdma_chan *chan;
406
	int offset = (chan_num % CPDMA_MAX_CHANNELS) * 4;
407 408 409 410 411
	unsigned long flags;

	if (__chan_linear(chan_num) >= ctlr->num_chan)
		return NULL;

412
	chan = devm_kzalloc(ctlr->dev, sizeof(*chan), GFP_KERNEL);
413
	if (!chan)
414
		return ERR_PTR(-ENOMEM);
415 416

	spin_lock_irqsave(&ctlr->lock, flags);
417 418 419 420 421
	if (ctlr->channels[chan_num]) {
		spin_unlock_irqrestore(&ctlr->lock, flags);
		devm_kfree(ctlr->dev, chan);
		return ERR_PTR(-EBUSY);
	}
422 423 424 425 426

	chan->ctlr	= ctlr;
	chan->state	= CPDMA_STATE_IDLE;
	chan->chan_num	= chan_num;
	chan->handler	= handler;
427
	chan->desc_num = ctlr->pool->num_desc / 2;
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452

	if (is_rx_chan(chan)) {
		chan->hdp	= ctlr->params.rxhdp + offset;
		chan->cp	= ctlr->params.rxcp + offset;
		chan->rxfree	= ctlr->params.rxfree + offset;
		chan->int_set	= CPDMA_RXINTMASKSET;
		chan->int_clear	= CPDMA_RXINTMASKCLEAR;
		chan->td	= CPDMA_RXTEARDOWN;
		chan->dir	= DMA_FROM_DEVICE;
	} else {
		chan->hdp	= ctlr->params.txhdp + offset;
		chan->cp	= ctlr->params.txcp + offset;
		chan->int_set	= CPDMA_TXINTMASKSET;
		chan->int_clear	= CPDMA_TXINTMASKCLEAR;
		chan->td	= CPDMA_TXTEARDOWN;
		chan->dir	= DMA_TO_DEVICE;
	}
	chan->mask = BIT(chan_linear(chan));

	spin_lock_init(&chan->lock);

	ctlr->channels[chan_num] = chan;
	spin_unlock_irqrestore(&ctlr->lock, flags);
	return chan;
}
453
EXPORT_SYMBOL_GPL(cpdma_chan_create);
454

455 456 457 458 459 460
int cpdma_chan_get_rx_buf_num(struct cpdma_ctlr *ctlr)
{
	return ctlr->pool->num_desc / 2;
}
EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num);

461 462
int cpdma_chan_destroy(struct cpdma_chan *chan)
{
463
	struct cpdma_ctlr *ctlr;
464 465 466 467
	unsigned long flags;

	if (!chan)
		return -EINVAL;
468
	ctlr = chan->ctlr;
469 470 471 472 473 474 475 476

	spin_lock_irqsave(&ctlr->lock, flags);
	if (chan->state != CPDMA_STATE_IDLE)
		cpdma_chan_stop(chan);
	ctlr->channels[chan->chan_num] = NULL;
	spin_unlock_irqrestore(&ctlr->lock, flags);
	return 0;
}
477
EXPORT_SYMBOL_GPL(cpdma_chan_destroy);
478 479 480 481 482 483 484 485 486 487 488 489

int cpdma_chan_get_stats(struct cpdma_chan *chan,
			 struct cpdma_chan_stats *stats)
{
	unsigned long flags;
	if (!chan)
		return -EINVAL;
	spin_lock_irqsave(&chan->lock, flags);
	memcpy(stats, &chan->stats, sizeof(*stats));
	spin_unlock_irqrestore(&chan->lock, flags);
	return 0;
}
490
EXPORT_SYMBOL_GPL(cpdma_chan_get_stats);
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528

static void __cpdma_chan_submit(struct cpdma_chan *chan,
				struct cpdma_desc __iomem *desc)
{
	struct cpdma_ctlr		*ctlr = chan->ctlr;
	struct cpdma_desc __iomem	*prev = chan->tail;
	struct cpdma_desc_pool		*pool = ctlr->pool;
	dma_addr_t			desc_dma;
	u32				mode;

	desc_dma = desc_phys(pool, desc);

	/* simple case - idle channel */
	if (!chan->head) {
		chan->stats.head_enqueue++;
		chan->head = desc;
		chan->tail = desc;
		if (chan->state == CPDMA_STATE_ACTIVE)
			chan_write(chan, hdp, desc_dma);
		return;
	}

	/* first chain the descriptor at the tail of the list */
	desc_write(prev, hw_next, desc_dma);
	chan->tail = desc;
	chan->stats.tail_enqueue++;

	/* next check if EOQ has been triggered already */
	mode = desc_read(prev, hw_mode);
	if (((mode & (CPDMA_DESC_EOQ | CPDMA_DESC_OWNER)) == CPDMA_DESC_EOQ) &&
	    (chan->state == CPDMA_STATE_ACTIVE)) {
		desc_write(prev, hw_mode, mode & ~CPDMA_DESC_EOQ);
		chan_write(chan, hdp, desc_dma);
		chan->stats.misqueued++;
	}
}

int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,
529
		      int len, int directed)
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
{
	struct cpdma_ctlr		*ctlr = chan->ctlr;
	struct cpdma_desc __iomem	*desc;
	dma_addr_t			buffer;
	unsigned long			flags;
	u32				mode;
	int				ret = 0;

	spin_lock_irqsave(&chan->lock, flags);

	if (chan->state == CPDMA_STATE_TEARDOWN) {
		ret = -EINVAL;
		goto unlock_ret;
	}

545 546 547 548 549 550 551
	if (chan->count >= chan->desc_num)	{
		chan->stats.desc_alloc_fail++;
		ret = -ENOMEM;
		goto unlock_ret;
	}

	desc = cpdma_desc_alloc(ctlr->pool);
552 553 554 555 556 557 558 559 560 561 562 563
	if (!desc) {
		chan->stats.desc_alloc_fail++;
		ret = -ENOMEM;
		goto unlock_ret;
	}

	if (len < ctlr->params.min_packet_size) {
		len = ctlr->params.min_packet_size;
		chan->stats.runt_transmit_buff++;
	}

	buffer = dma_map_single(ctlr->dev, data, len, chan->dir);
564 565 566 567 568 569 570
	ret = dma_mapping_error(ctlr->dev, buffer);
	if (ret) {
		cpdma_desc_free(ctlr->pool, desc, 1);
		ret = -EINVAL;
		goto unlock_ret;
	}

571
	mode = CPDMA_DESC_OWNER | CPDMA_DESC_SOP | CPDMA_DESC_EOP;
572
	cpdma_desc_to_port(chan, mode, directed);
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592

	desc_write(desc, hw_next,   0);
	desc_write(desc, hw_buffer, buffer);
	desc_write(desc, hw_len,    len);
	desc_write(desc, hw_mode,   mode | len);
	desc_write(desc, sw_token,  token);
	desc_write(desc, sw_buffer, buffer);
	desc_write(desc, sw_len,    len);

	__cpdma_chan_submit(chan, desc);

	if (chan->state == CPDMA_STATE_ACTIVE && chan->rxfree)
		chan_write(chan, rxfree, 1);

	chan->count++;

unlock_ret:
	spin_unlock_irqrestore(&chan->lock, flags);
	return ret;
}
593
EXPORT_SYMBOL_GPL(cpdma_chan_submit);
594

595 596 597 598
bool cpdma_check_free_tx_desc(struct cpdma_chan *chan)
{
	struct cpdma_ctlr	*ctlr = chan->ctlr;
	struct cpdma_desc_pool	*pool = ctlr->pool;
599 600
	bool			free_tx_desc;
	unsigned long		flags;
601

602 603 604 605 606
	spin_lock_irqsave(&chan->lock, flags);
	free_tx_desc = (chan->count < chan->desc_num) &&
			 gen_pool_avail(pool->gen_pool);
	spin_unlock_irqrestore(&chan->lock, flags);
	return free_tx_desc;
607 608 609
}
EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc);

610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
static void __cpdma_chan_free(struct cpdma_chan *chan,
			      struct cpdma_desc __iomem *desc,
			      int outlen, int status)
{
	struct cpdma_ctlr		*ctlr = chan->ctlr;
	struct cpdma_desc_pool		*pool = ctlr->pool;
	dma_addr_t			buff_dma;
	int				origlen;
	void				*token;

	token      = (void *)desc_read(desc, sw_token);
	buff_dma   = desc_read(desc, sw_buffer);
	origlen    = desc_read(desc, sw_len);

	dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir);
	cpdma_desc_free(pool, desc, 1);
	(*chan->handler)(token, outlen, status);
}

static int __cpdma_chan_process(struct cpdma_chan *chan)
{
	struct cpdma_ctlr		*ctlr = chan->ctlr;
	struct cpdma_desc __iomem	*desc;
	int				status, outlen;
634
	int				cb_status = 0;
635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
	struct cpdma_desc_pool		*pool = ctlr->pool;
	dma_addr_t			desc_dma;
	unsigned long			flags;

	spin_lock_irqsave(&chan->lock, flags);

	desc = chan->head;
	if (!desc) {
		chan->stats.empty_dequeue++;
		status = -ENOENT;
		goto unlock_ret;
	}
	desc_dma = desc_phys(pool, desc);

	status	= __raw_readl(&desc->hw_mode);
	outlen	= status & 0x7ff;
	if (status & CPDMA_DESC_OWNER) {
		chan->stats.busy_dequeue++;
		status = -EBUSY;
		goto unlock_ret;
	}
656 657 658 659

	if (status & CPDMA_DESC_PASS_CRC)
		outlen -= CPDMA_DESC_CRC_LEN;

660 661
	status	= status & (CPDMA_DESC_EOQ | CPDMA_DESC_TD_COMPLETE |
			    CPDMA_DESC_PORT_MASK);
662 663 664 665 666 667 668 669 670 671 672 673

	chan->head = desc_from_phys(pool, desc_read(desc, hw_next));
	chan_write(chan, cp, desc_dma);
	chan->count--;
	chan->stats.good_dequeue++;

	if (status & CPDMA_DESC_EOQ) {
		chan->stats.requeue++;
		chan_write(chan, hdp, desc_phys(pool, chan->head));
	}

	spin_unlock_irqrestore(&chan->lock, flags);
674 675 676 677
	if (unlikely(status & CPDMA_DESC_TD_COMPLETE))
		cb_status = -ENOSYS;
	else
		cb_status = status;
678

679
	__cpdma_chan_free(chan, desc, outlen, cb_status);
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
	return status;

unlock_ret:
	spin_unlock_irqrestore(&chan->lock, flags);
	return status;
}

int cpdma_chan_process(struct cpdma_chan *chan, int quota)
{
	int used = 0, ret = 0;

	if (chan->state != CPDMA_STATE_ACTIVE)
		return -EINVAL;

	while (used < quota) {
		ret = __cpdma_chan_process(chan);
		if (ret < 0)
			break;
		used++;
	}
	return used;
}
702
EXPORT_SYMBOL_GPL(cpdma_chan_process);
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729

int cpdma_chan_start(struct cpdma_chan *chan)
{
	struct cpdma_ctlr	*ctlr = chan->ctlr;
	struct cpdma_desc_pool	*pool = ctlr->pool;
	unsigned long		flags;

	spin_lock_irqsave(&chan->lock, flags);
	if (chan->state != CPDMA_STATE_IDLE) {
		spin_unlock_irqrestore(&chan->lock, flags);
		return -EBUSY;
	}
	if (ctlr->state != CPDMA_STATE_ACTIVE) {
		spin_unlock_irqrestore(&chan->lock, flags);
		return -EINVAL;
	}
	dma_reg_write(ctlr, chan->int_set, chan->mask);
	chan->state = CPDMA_STATE_ACTIVE;
	if (chan->head) {
		chan_write(chan, hdp, desc_phys(pool, chan->head));
		if (chan->rxfree)
			chan_write(chan, rxfree, chan->count);
	}

	spin_unlock_irqrestore(&chan->lock, flags);
	return 0;
}
730
EXPORT_SYMBOL_GPL(cpdma_chan_start);
731 732 733 734 735 736 737

int cpdma_chan_stop(struct cpdma_chan *chan)
{
	struct cpdma_ctlr	*ctlr = chan->ctlr;
	struct cpdma_desc_pool	*pool = ctlr->pool;
	unsigned long		flags;
	int			ret;
738
	unsigned		timeout;
739 740

	spin_lock_irqsave(&chan->lock, flags);
741
	if (chan->state == CPDMA_STATE_TEARDOWN) {
742 743 744 745 746 747 748 749
		spin_unlock_irqrestore(&chan->lock, flags);
		return -EINVAL;
	}

	chan->state = CPDMA_STATE_TEARDOWN;
	dma_reg_write(ctlr, chan->int_clear, chan->mask);

	/* trigger teardown */
750
	dma_reg_write(ctlr, chan->td, chan_linear(chan));
751 752

	/* wait for teardown complete */
753 754
	timeout = 100 * 100; /* 100 ms */
	while (timeout) {
755 756 757
		u32 cp = chan_read(chan, cp);
		if ((cp & CPDMA_TEARDOWN_VALUE) == CPDMA_TEARDOWN_VALUE)
			break;
758 759
		udelay(10);
		timeout--;
760
	}
761
	WARN_ON(!timeout);
762 763 764
	chan_write(chan, cp, CPDMA_TEARDOWN_VALUE);

	/* handle completed packets */
765
	spin_unlock_irqrestore(&chan->lock, flags);
766 767 768 769 770
	do {
		ret = __cpdma_chan_process(chan);
		if (ret < 0)
			break;
	} while ((ret & CPDMA_DESC_TD_COMPLETE) == 0);
771
	spin_lock_irqsave(&chan->lock, flags);
772 773 774 775 776 777 778 779

	/* remaining packets haven't been tx/rx'ed, clean them up */
	while (chan->head) {
		struct cpdma_desc __iomem *desc = chan->head;
		dma_addr_t next_dma;

		next_dma = desc_read(desc, hw_next);
		chan->head = desc_from_phys(pool, next_dma);
780
		chan->count--;
781 782 783 784 785 786 787 788 789 790 791 792
		chan->stats.teardown_dequeue++;

		/* issue callback without locks held */
		spin_unlock_irqrestore(&chan->lock, flags);
		__cpdma_chan_free(chan, desc, 0, -ENOSYS);
		spin_lock_irqsave(&chan->lock, flags);
	}

	chan->state = CPDMA_STATE_IDLE;
	spin_unlock_irqrestore(&chan->lock, flags);
	return 0;
}
793
EXPORT_SYMBOL_GPL(cpdma_chan_stop);
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable)
{
	unsigned long flags;

	spin_lock_irqsave(&chan->lock, flags);
	if (chan->state != CPDMA_STATE_ACTIVE) {
		spin_unlock_irqrestore(&chan->lock, flags);
		return -EINVAL;
	}

	dma_reg_write(chan->ctlr, enable ? chan->int_set : chan->int_clear,
		      chan->mask);
	spin_unlock_irqrestore(&chan->lock, flags);

	return 0;
}

struct cpdma_control_info {
	u32		reg;
	u32		shift, mask;
	int		access;
#define ACCESS_RO	BIT(0)
#define ACCESS_WO	BIT(1)
#define ACCESS_RW	(ACCESS_RO | ACCESS_WO)
};

821
static struct cpdma_control_info controls[] = {
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
	[CPDMA_CMD_IDLE]	  = {CPDMA_DMACONTROL,	3,  1,      ACCESS_WO},
	[CPDMA_COPY_ERROR_FRAMES] = {CPDMA_DMACONTROL,	4,  1,      ACCESS_RW},
	[CPDMA_RX_OFF_LEN_UPDATE] = {CPDMA_DMACONTROL,	2,  1,      ACCESS_RW},
	[CPDMA_RX_OWNERSHIP_FLIP] = {CPDMA_DMACONTROL,	1,  1,      ACCESS_RW},
	[CPDMA_TX_PRIO_FIXED]	  = {CPDMA_DMACONTROL,	0,  1,      ACCESS_RW},
	[CPDMA_STAT_IDLE]	  = {CPDMA_DMASTATUS,	31, 1,      ACCESS_RO},
	[CPDMA_STAT_TX_ERR_CODE]  = {CPDMA_DMASTATUS,	20, 0xf,    ACCESS_RW},
	[CPDMA_STAT_TX_ERR_CHAN]  = {CPDMA_DMASTATUS,	16, 0x7,    ACCESS_RW},
	[CPDMA_STAT_RX_ERR_CODE]  = {CPDMA_DMASTATUS,	12, 0xf,    ACCESS_RW},
	[CPDMA_STAT_RX_ERR_CHAN]  = {CPDMA_DMASTATUS,	8,  0x7,    ACCESS_RW},
	[CPDMA_RX_BUFFER_OFFSET]  = {CPDMA_RXBUFFOFS,	0,  0xffff, ACCESS_RW},
};

int cpdma_control_get(struct cpdma_ctlr *ctlr, int control)
{
	unsigned long flags;
	struct cpdma_control_info *info = &controls[control];
	int ret;

	spin_lock_irqsave(&ctlr->lock, flags);

	ret = -ENOTSUPP;
	if (!ctlr->params.has_ext_regs)
		goto unlock_ret;

	ret = -EINVAL;
	if (ctlr->state != CPDMA_STATE_ACTIVE)
		goto unlock_ret;

	ret = -ENOENT;
	if (control < 0 || control >= ARRAY_SIZE(controls))
		goto unlock_ret;

	ret = -EPERM;
	if ((info->access & ACCESS_RO) != ACCESS_RO)
		goto unlock_ret;

	ret = (dma_reg_read(ctlr, info->reg) >> info->shift) & info->mask;

unlock_ret:
	spin_unlock_irqrestore(&ctlr->lock, flags);
	return ret;
}

int cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value)
{
	unsigned long flags;
	struct cpdma_control_info *info = &controls[control];
	int ret;
	u32 val;

	spin_lock_irqsave(&ctlr->lock, flags);

	ret = -ENOTSUPP;
	if (!ctlr->params.has_ext_regs)
		goto unlock_ret;

	ret = -EINVAL;
	if (ctlr->state != CPDMA_STATE_ACTIVE)
		goto unlock_ret;

	ret = -ENOENT;
	if (control < 0 || control >= ARRAY_SIZE(controls))
		goto unlock_ret;

	ret = -EPERM;
	if ((info->access & ACCESS_WO) != ACCESS_WO)
		goto unlock_ret;

	val  = dma_reg_read(ctlr, info->reg);
	val &= ~(info->mask << info->shift);
	val |= (value & info->mask) << info->shift;
	dma_reg_write(ctlr, info->reg, val);
	ret = 0;

unlock_ret:
	spin_unlock_irqrestore(&ctlr->lock, flags);
	return ret;
}
901
EXPORT_SYMBOL_GPL(cpdma_control_set);
902 903

MODULE_LICENSE("GPL");