fsldma.c 36.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Freescale MPC85xx, MPC83xx DMA Engine support
 *
 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
 *
 * Author:
 *   Zhang Wei <wei.zhang@freescale.com>, Jul 2007
 *   Ebony Zhu <ebony.zhu@freescale.com>, May 2007
 *
 * Description:
 *   DMA engine driver for Freescale MPC8540 DMA controller, which is
 *   also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
 *   The support for MPC8349 DMA contorller is also added.
 *
15 16 17 18 19
 * This driver instructs the DMA controller to issue the PCI Read Multiple
 * command for PCI read operations, instead of using the default PCI Read Line
 * command. Please be aware that this setting may result in read pre-fetching
 * on some platforms.
 *
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/of_platform.h>

I
Ira Snyder 已提交
37
#include <asm/fsldma.h>
38 39
#include "fsldma.h"

I
Ira Snyder 已提交
40
static void dma_init(struct fsldma_chan *chan)
41 42
{
	/* Reset the channel */
I
Ira Snyder 已提交
43
	DMA_OUT(chan, &chan->regs->mr, 0, 32);
44

I
Ira Snyder 已提交
45
	switch (chan->feature & FSL_DMA_IP_MASK) {
46 47 48 49 50 51
	case FSL_DMA_IP_85XX:
		/* Set the channel to below modes:
		 * EIE - Error interrupt enable
		 * EOSIE - End of segments interrupt enable (basic mode)
		 * EOLNIE - End of links interrupt enable
		 */
I
Ira Snyder 已提交
52
		DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EIE
53 54 55 56 57
				| FSL_DMA_MR_EOLNIE | FSL_DMA_MR_EOSIE, 32);
		break;
	case FSL_DMA_IP_83XX:
		/* Set the channel to below modes:
		 * EOTIE - End-of-transfer interrupt enable
58
		 * PRC_RM - PCI read multiple
59
		 */
I
Ira Snyder 已提交
60
		DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE
61
				| FSL_DMA_MR_PRC_RM, 32);
62 63 64 65
		break;
	}
}

I
Ira Snyder 已提交
66
static void set_sr(struct fsldma_chan *chan, u32 val)
67
{
I
Ira Snyder 已提交
68
	DMA_OUT(chan, &chan->regs->sr, val, 32);
69 70
}

I
Ira Snyder 已提交
71
static u32 get_sr(struct fsldma_chan *chan)
72
{
I
Ira Snyder 已提交
73
	return DMA_IN(chan, &chan->regs->sr, 32);
74 75
}

I
Ira Snyder 已提交
76
static void set_desc_cnt(struct fsldma_chan *chan,
77 78
				struct fsl_dma_ld_hw *hw, u32 count)
{
I
Ira Snyder 已提交
79
	hw->count = CPU_TO_DMA(chan, count, 32);
80 81
}

I
Ira Snyder 已提交
82
static void set_desc_src(struct fsldma_chan *chan,
83 84 85 86
				struct fsl_dma_ld_hw *hw, dma_addr_t src)
{
	u64 snoop_bits;

I
Ira Snyder 已提交
87
	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
88
		? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
I
Ira Snyder 已提交
89
	hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);
90 91
}

I
Ira Snyder 已提交
92
static void set_desc_dst(struct fsldma_chan *chan,
93
				struct fsl_dma_ld_hw *hw, dma_addr_t dst)
94 95 96
{
	u64 snoop_bits;

I
Ira Snyder 已提交
97
	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
98
		? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
I
Ira Snyder 已提交
99
	hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);
100 101
}

I
Ira Snyder 已提交
102
static void set_desc_next(struct fsldma_chan *chan,
103 104 105 106
				struct fsl_dma_ld_hw *hw, dma_addr_t next)
{
	u64 snoop_bits;

I
Ira Snyder 已提交
107
	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
108
		? FSL_DMA_SNEN : 0;
I
Ira Snyder 已提交
109
	hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);
110 111
}

I
Ira Snyder 已提交
112
static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
113
{
I
Ira Snyder 已提交
114
	DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
115 116
}

I
Ira Snyder 已提交
117
static dma_addr_t get_cdar(struct fsldma_chan *chan)
118
{
I
Ira Snyder 已提交
119
	return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
120 121
}

I
Ira Snyder 已提交
122
static dma_addr_t get_ndar(struct fsldma_chan *chan)
123
{
I
Ira Snyder 已提交
124
	return DMA_IN(chan, &chan->regs->ndar, 64);
125 126
}

I
Ira Snyder 已提交
127
static u32 get_bcr(struct fsldma_chan *chan)
128
{
I
Ira Snyder 已提交
129
	return DMA_IN(chan, &chan->regs->bcr, 32);
130 131
}

I
Ira Snyder 已提交
132
static int dma_is_idle(struct fsldma_chan *chan)
133
{
I
Ira Snyder 已提交
134
	u32 sr = get_sr(chan);
135 136 137
	return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
}

I
Ira Snyder 已提交
138
static void dma_start(struct fsldma_chan *chan)
139
{
I
Ira Snyder 已提交
140 141
	u32 mode;

I
Ira Snyder 已提交
142
	mode = DMA_IN(chan, &chan->regs->mr, 32);
I
Ira Snyder 已提交
143

I
Ira Snyder 已提交
144 145 146
	if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
		if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
			DMA_OUT(chan, &chan->regs->bcr, 0, 32);
I
Ira Snyder 已提交
147 148 149 150
			mode |= FSL_DMA_MR_EMP_EN;
		} else {
			mode &= ~FSL_DMA_MR_EMP_EN;
		}
151
	}
152

I
Ira Snyder 已提交
153
	if (chan->feature & FSL_DMA_CHAN_START_EXT)
I
Ira Snyder 已提交
154
		mode |= FSL_DMA_MR_EMS_EN;
155
	else
I
Ira Snyder 已提交
156
		mode |= FSL_DMA_MR_CS;
157

I
Ira Snyder 已提交
158
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
159 160
}

I
Ira Snyder 已提交
161
static void dma_halt(struct fsldma_chan *chan)
162
{
I
Ira Snyder 已提交
163
	u32 mode;
164 165
	int i;

I
Ira Snyder 已提交
166
	mode = DMA_IN(chan, &chan->regs->mr, 32);
I
Ira Snyder 已提交
167
	mode |= FSL_DMA_MR_CA;
I
Ira Snyder 已提交
168
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
I
Ira Snyder 已提交
169 170

	mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA);
I
Ira Snyder 已提交
171
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
172

173
	for (i = 0; i < 100; i++) {
I
Ira Snyder 已提交
174
		if (dma_is_idle(chan))
I
Ira Snyder 已提交
175 176
			return;

177
		udelay(10);
178
	}
I
Ira Snyder 已提交
179

I
Ira Snyder 已提交
180
	if (!dma_is_idle(chan))
I
Ira Snyder 已提交
181
		dev_err(chan->dev, "DMA halt timeout!\n");
182 183
}

I
Ira Snyder 已提交
184
static void set_ld_eol(struct fsldma_chan *chan,
185 186
			struct fsl_desc_sw *desc)
{
187 188
	u64 snoop_bits;

I
Ira Snyder 已提交
189
	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
190 191
		? FSL_DMA_SNEN : 0;

I
Ira Snyder 已提交
192 193
	desc->hw.next_ln_addr = CPU_TO_DMA(chan,
		DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
194
			| snoop_bits, 64);
195 196 197 198
}

/**
 * fsl_chan_set_src_loop_size - Set source address hold transfer size
I
Ira Snyder 已提交
199
 * @chan : Freescale DMA channel
200 201 202 203 204 205 206 207
 * @size     : Address loop size, 0 for disable loop
 *
 * The set source address hold transfer size. The source
 * address hold or loop transfer size is when the DMA transfer
 * data from source address (SA), if the loop size is 4, the DMA will
 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
 * SA + 1 ... and so on.
 */
I
Ira Snyder 已提交
208
static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)
209
{
I
Ira Snyder 已提交
210 211
	u32 mode;

I
Ira Snyder 已提交
212
	mode = DMA_IN(chan, &chan->regs->mr, 32);
I
Ira Snyder 已提交
213

214 215
	switch (size) {
	case 0:
I
Ira Snyder 已提交
216
		mode &= ~FSL_DMA_MR_SAHE;
217 218 219 220 221
		break;
	case 1:
	case 2:
	case 4:
	case 8:
I
Ira Snyder 已提交
222
		mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14);
223 224
		break;
	}
I
Ira Snyder 已提交
225

I
Ira Snyder 已提交
226
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
227 228 229
}

/**
230
 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
I
Ira Snyder 已提交
231
 * @chan : Freescale DMA channel
232 233 234 235 236 237 238 239
 * @size     : Address loop size, 0 for disable loop
 *
 * The set destination address hold transfer size. The destination
 * address hold or loop transfer size is when the DMA transfer
 * data to destination address (TA), if the loop size is 4, the DMA will
 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
 * TA + 1 ... and so on.
 */
I
Ira Snyder 已提交
240
static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)
241
{
I
Ira Snyder 已提交
242 243
	u32 mode;

I
Ira Snyder 已提交
244
	mode = DMA_IN(chan, &chan->regs->mr, 32);
I
Ira Snyder 已提交
245

246 247
	switch (size) {
	case 0:
I
Ira Snyder 已提交
248
		mode &= ~FSL_DMA_MR_DAHE;
249 250 251 252 253
		break;
	case 1:
	case 2:
	case 4:
	case 8:
I
Ira Snyder 已提交
254
		mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16);
255 256
		break;
	}
I
Ira Snyder 已提交
257

I
Ira Snyder 已提交
258
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
259 260 261
}

/**
262
 * fsl_chan_set_request_count - Set DMA Request Count for external control
I
Ira Snyder 已提交
263
 * @chan : Freescale DMA channel
264 265 266 267 268 269
 * @size     : Number of bytes to transfer in a single request
 *
 * The Freescale DMA channel can be controlled by the external signal DREQ#.
 * The DMA request count is how many bytes are allowed to transfer before
 * pausing the channel, after which a new assertion of DREQ# resumes channel
 * operation.
270
 *
271
 * A size of 0 disables external pause control. The maximum size is 1024.
272
 */
I
Ira Snyder 已提交
273
static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)
274
{
I
Ira Snyder 已提交
275 276
	u32 mode;

277
	BUG_ON(size > 1024);
I
Ira Snyder 已提交
278

I
Ira Snyder 已提交
279
	mode = DMA_IN(chan, &chan->regs->mr, 32);
I
Ira Snyder 已提交
280 281
	mode |= (__ilog2(size) << 24) & 0x0f000000;

I
Ira Snyder 已提交
282
	DMA_OUT(chan, &chan->regs->mr, mode, 32);
283
}
284

285 286
/**
 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
I
Ira Snyder 已提交
287
 * @chan : Freescale DMA channel
288 289 290 291 292 293
 * @enable   : 0 is disabled, 1 is enabled.
 *
 * The Freescale DMA channel can be controlled by the external signal DREQ#.
 * The DMA Request Count feature should be used in addition to this feature
 * to set the number of bytes to transfer before pausing the channel.
 */
I
Ira Snyder 已提交
294
static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)
295 296
{
	if (enable)
I
Ira Snyder 已提交
297
		chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
298
	else
I
Ira Snyder 已提交
299
		chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
300 301 302 303
}

/**
 * fsl_chan_toggle_ext_start - Toggle channel external start status
I
Ira Snyder 已提交
304
 * @chan : Freescale DMA channel
305 306 307 308 309 310 311
 * @enable   : 0 is disabled, 1 is enabled.
 *
 * If enable the external start, the channel can be started by an
 * external DMA start pin. So the dma_start() does not start the
 * transfer immediately. The DMA channel will wait for the
 * control pin asserted.
 */
I
Ira Snyder 已提交
312
static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
313 314
{
	if (enable)
I
Ira Snyder 已提交
315
		chan->feature |= FSL_DMA_CHAN_START_EXT;
316
	else
I
Ira Snyder 已提交
317
		chan->feature &= ~FSL_DMA_CHAN_START_EXT;
318 319
}

I
Ira Snyder 已提交
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
static void append_ld_queue(struct fsldma_chan *chan,
			    struct fsl_desc_sw *desc)
{
	struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);

	if (list_empty(&chan->ld_pending))
		goto out_splice;

	/*
	 * Add the hardware descriptor to the chain of hardware descriptors
	 * that already exists in memory.
	 *
	 * This will un-set the EOL bit of the existing transaction, and the
	 * last link in this transaction will become the EOL descriptor.
	 */
	set_desc_next(chan, &tail->hw, desc->async_tx.phys);

	/*
	 * Add the software descriptor and all children to the list
	 * of pending transactions
	 */
out_splice:
	list_splice_tail_init(&desc->tx_list, &chan->ld_pending);
}

345 346
static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
{
I
Ira Snyder 已提交
347
	struct fsldma_chan *chan = to_fsl_chan(tx->chan);
348 349
	struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
	struct fsl_desc_sw *child;
350 351 352
	unsigned long flags;
	dma_cookie_t cookie;

I
Ira Snyder 已提交
353
	spin_lock_irqsave(&chan->desc_lock, flags);
354

I
Ira Snyder 已提交
355 356 357 358
	/*
	 * assign cookies to all of the software descriptors
	 * that make up this transaction
	 */
I
Ira Snyder 已提交
359
	cookie = chan->common.cookie;
360
	list_for_each_entry(child, &desc->tx_list, node) {
361 362 363 364
		cookie++;
		if (cookie < 0)
			cookie = 1;

S
Steven J. Magnani 已提交
365
		child->async_tx.cookie = cookie;
366 367
	}

I
Ira Snyder 已提交
368
	chan->common.cookie = cookie;
I
Ira Snyder 已提交
369 370

	/* put this transaction onto the tail of the pending queue */
I
Ira Snyder 已提交
371
	append_ld_queue(chan, desc);
372

I
Ira Snyder 已提交
373
	spin_unlock_irqrestore(&chan->desc_lock, flags);
374 375 376 377 378 379

	return cookie;
}

/**
 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
I
Ira Snyder 已提交
380
 * @chan : Freescale DMA channel
381 382 383 384
 *
 * Return - The descriptor allocated. NULL for failed.
 */
static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
I
Ira Snyder 已提交
385
					struct fsldma_chan *chan)
386
{
I
Ira Snyder 已提交
387
	struct fsl_desc_sw *desc;
388
	dma_addr_t pdesc;
I
Ira Snyder 已提交
389 390 391 392 393

	desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
	if (!desc) {
		dev_dbg(chan->dev, "out of memory for link desc\n");
		return NULL;
394 395
	}

I
Ira Snyder 已提交
396 397 398 399 400 401 402
	memset(desc, 0, sizeof(*desc));
	INIT_LIST_HEAD(&desc->tx_list);
	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
	desc->async_tx.tx_submit = fsl_dma_tx_submit;
	desc->async_tx.phys = pdesc;

	return desc;
403 404 405 406 407
}


/**
 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
I
Ira Snyder 已提交
408
 * @chan : Freescale DMA channel
409 410 411 412 413
 *
 * This function will create a dma pool for descriptor allocation.
 *
 * Return - The number of descriptors allocated.
 */
I
Ira Snyder 已提交
414
static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
415
{
I
Ira Snyder 已提交
416
	struct fsldma_chan *chan = to_fsl_chan(dchan);
417 418

	/* Has this channel already been allocated? */
I
Ira Snyder 已提交
419
	if (chan->desc_pool)
420
		return 1;
421

I
Ira Snyder 已提交
422 423
	/*
	 * We need the descriptor to be aligned to 32bytes
424 425
	 * for meeting FSL DMA specification requirement.
	 */
I
Ira Snyder 已提交
426
	chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool",
I
Ira Snyder 已提交
427 428 429
					  chan->dev,
					  sizeof(struct fsl_desc_sw),
					  __alignof__(struct fsl_desc_sw), 0);
I
Ira Snyder 已提交
430
	if (!chan->desc_pool) {
I
Ira Snyder 已提交
431 432 433
		dev_err(chan->dev, "unable to allocate channel %d "
				   "descriptor pool\n", chan->id);
		return -ENOMEM;
434 435
	}

I
Ira Snyder 已提交
436
	/* there is at least one descriptor free to be allocated */
437 438 439
	return 1;
}

I
Ira Snyder 已提交
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
/**
 * fsldma_free_desc_list - Free all descriptors in a queue
 * @chan: Freescae DMA channel
 * @list: the list to free
 *
 * LOCKING: must hold chan->desc_lock
 */
static void fsldma_free_desc_list(struct fsldma_chan *chan,
				  struct list_head *list)
{
	struct fsl_desc_sw *desc, *_desc;

	list_for_each_entry_safe(desc, _desc, list, node) {
		list_del(&desc->node);
		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
	}
}

static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
					  struct list_head *list)
{
	struct fsl_desc_sw *desc, *_desc;

	list_for_each_entry_safe_reverse(desc, _desc, list, node) {
		list_del(&desc->node);
		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
	}
}

469 470
/**
 * fsl_dma_free_chan_resources - Free all resources of the channel.
I
Ira Snyder 已提交
471
 * @chan : Freescale DMA channel
472
 */
I
Ira Snyder 已提交
473
static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
474
{
I
Ira Snyder 已提交
475
	struct fsldma_chan *chan = to_fsl_chan(dchan);
476 477
	unsigned long flags;

I
Ira Snyder 已提交
478 479
	dev_dbg(chan->dev, "Free all channel resources.\n");
	spin_lock_irqsave(&chan->desc_lock, flags);
I
Ira Snyder 已提交
480 481
	fsldma_free_desc_list(chan, &chan->ld_pending);
	fsldma_free_desc_list(chan, &chan->ld_running);
I
Ira Snyder 已提交
482
	spin_unlock_irqrestore(&chan->desc_lock, flags);
483

I
Ira Snyder 已提交
484
	dma_pool_destroy(chan->desc_pool);
I
Ira Snyder 已提交
485
	chan->desc_pool = NULL;
486 487
}

488
static struct dma_async_tx_descriptor *
I
Ira Snyder 已提交
489
fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)
490
{
I
Ira Snyder 已提交
491
	struct fsldma_chan *chan;
492 493
	struct fsl_desc_sw *new;

I
Ira Snyder 已提交
494
	if (!dchan)
495 496
		return NULL;

I
Ira Snyder 已提交
497
	chan = to_fsl_chan(dchan);
498

I
Ira Snyder 已提交
499
	new = fsl_dma_alloc_descriptor(chan);
500
	if (!new) {
I
Ira Snyder 已提交
501
		dev_err(chan->dev, "No free memory for link descriptor\n");
502 503 504 505
		return NULL;
	}

	new->async_tx.cookie = -EBUSY;
506
	new->async_tx.flags = flags;
507

508
	/* Insert the link descriptor to the LD ring */
509
	list_add_tail(&new->node, &new->tx_list);
510

511
	/* Set End-of-link to the last link descriptor of new list*/
I
Ira Snyder 已提交
512
	set_ld_eol(chan, new);
513 514 515 516

	return &new->async_tx;
}

517
static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
I
Ira Snyder 已提交
518
	struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src,
519 520
	size_t len, unsigned long flags)
{
I
Ira Snyder 已提交
521
	struct fsldma_chan *chan;
522 523 524
	struct fsl_desc_sw *first = NULL, *prev = NULL, *new;
	size_t copy;

I
Ira Snyder 已提交
525
	if (!dchan)
526 527 528 529 530
		return NULL;

	if (!len)
		return NULL;

I
Ira Snyder 已提交
531
	chan = to_fsl_chan(dchan);
532 533 534 535

	do {

		/* Allocate the link descriptor from DMA pool */
I
Ira Snyder 已提交
536
		new = fsl_dma_alloc_descriptor(chan);
537
		if (!new) {
I
Ira Snyder 已提交
538
			dev_err(chan->dev,
539
					"No free memory for link descriptor\n");
540
			goto fail;
541 542
		}
#ifdef FSL_DMA_LD_DEBUG
I
Ira Snyder 已提交
543
		dev_dbg(chan->dev, "new link desc alloc %p\n", new);
544 545
#endif

546
		copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
547

I
Ira Snyder 已提交
548 549 550
		set_desc_cnt(chan, &new->hw, copy);
		set_desc_src(chan, &new->hw, dma_src);
		set_desc_dst(chan, &new->hw, dma_dst);
551 552 553 554

		if (!first)
			first = new;
		else
I
Ira Snyder 已提交
555
			set_desc_next(chan, &prev->hw, new->async_tx.phys);
556 557

		new->async_tx.cookie = 0;
558
		async_tx_ack(&new->async_tx);
559 560 561 562

		prev = new;
		len -= copy;
		dma_src += copy;
563
		dma_dst += copy;
564 565

		/* Insert the link descriptor to the LD ring */
566
		list_add_tail(&new->node, &first->tx_list);
567 568
	} while (len);

569
	new->async_tx.flags = flags; /* client is in control of this ack */
570 571 572
	new->async_tx.cookie = -EBUSY;

	/* Set End-of-link to the last link descriptor of new list*/
I
Ira Snyder 已提交
573
	set_ld_eol(chan, new);
574

575 576 577 578 579 580
	return &first->async_tx;

fail:
	if (!first)
		return NULL;

I
Ira Snyder 已提交
581
	fsldma_free_desc_list_reverse(chan, &first->tx_list);
582
	return NULL;
583 584
}

I
Ira Snyder 已提交
585 586 587 588 589 590 591 592 593 594 595 596 597
/**
 * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
 * @chan: DMA channel
 * @sgl: scatterlist to transfer to/from
 * @sg_len: number of entries in @scatterlist
 * @direction: DMA direction
 * @flags: DMAEngine flags
 *
 * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
 * DMA_SLAVE API, this gets the device-specific information from the
 * chan->private variable.
 */
static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
I
Ira Snyder 已提交
598
	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
I
Ira Snyder 已提交
599 600
	enum dma_data_direction direction, unsigned long flags)
{
I
Ira Snyder 已提交
601
	struct fsldma_chan *chan;
I
Ira Snyder 已提交
602 603 604 605 606 607 608 609 610 611 612
	struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
	struct fsl_dma_slave *slave;
	size_t copy;

	int i;
	struct scatterlist *sg;
	size_t sg_used;
	size_t hw_used;
	struct fsl_dma_hw_addr *hw;
	dma_addr_t dma_dst, dma_src;

I
Ira Snyder 已提交
613
	if (!dchan)
I
Ira Snyder 已提交
614 615
		return NULL;

I
Ira Snyder 已提交
616
	if (!dchan->private)
I
Ira Snyder 已提交
617 618
		return NULL;

I
Ira Snyder 已提交
619 620
	chan = to_fsl_chan(dchan);
	slave = dchan->private;
I
Ira Snyder 已提交
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668

	if (list_empty(&slave->addresses))
		return NULL;

	hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
	hw_used = 0;

	/*
	 * Build the hardware transaction to copy from the scatterlist to
	 * the hardware, or from the hardware to the scatterlist
	 *
	 * If you are copying from the hardware to the scatterlist and it
	 * takes two hardware entries to fill an entire page, then both
	 * hardware entries will be coalesced into the same page
	 *
	 * If you are copying from the scatterlist to the hardware and a
	 * single page can fill two hardware entries, then the data will
	 * be read out of the page into the first hardware entry, and so on
	 */
	for_each_sg(sgl, sg, sg_len, i) {
		sg_used = 0;

		/* Loop until the entire scatterlist entry is used */
		while (sg_used < sg_dma_len(sg)) {

			/*
			 * If we've used up the current hardware address/length
			 * pair, we need to load a new one
			 *
			 * This is done in a while loop so that descriptors with
			 * length == 0 will be skipped
			 */
			while (hw_used >= hw->length) {

				/*
				 * If the current hardware entry is the last
				 * entry in the list, we're finished
				 */
				if (list_is_last(&hw->entry, &slave->addresses))
					goto finished;

				/* Get the next hardware address/length pair */
				hw = list_entry(hw->entry.next,
						struct fsl_dma_hw_addr, entry);
				hw_used = 0;
			}

			/* Allocate the link descriptor from DMA pool */
I
Ira Snyder 已提交
669
			new = fsl_dma_alloc_descriptor(chan);
I
Ira Snyder 已提交
670
			if (!new) {
I
Ira Snyder 已提交
671
				dev_err(chan->dev, "No free memory for "
I
Ira Snyder 已提交
672 673 674 675
						       "link descriptor\n");
				goto fail;
			}
#ifdef FSL_DMA_LD_DEBUG
I
Ira Snyder 已提交
676
			dev_dbg(chan->dev, "new link desc alloc %p\n", new);
I
Ira Snyder 已提交
677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
#endif

			/*
			 * Calculate the maximum number of bytes to transfer,
			 * making sure it is less than the DMA controller limit
			 */
			copy = min_t(size_t, sg_dma_len(sg) - sg_used,
					     hw->length - hw_used);
			copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);

			/*
			 * DMA_FROM_DEVICE
			 * from the hardware to the scatterlist
			 *
			 * DMA_TO_DEVICE
			 * from the scatterlist to the hardware
			 */
			if (direction == DMA_FROM_DEVICE) {
				dma_src = hw->address + hw_used;
				dma_dst = sg_dma_address(sg) + sg_used;
			} else {
				dma_src = sg_dma_address(sg) + sg_used;
				dma_dst = hw->address + hw_used;
			}

			/* Fill in the descriptor */
I
Ira Snyder 已提交
703 704 705
			set_desc_cnt(chan, &new->hw, copy);
			set_desc_src(chan, &new->hw, dma_src);
			set_desc_dst(chan, &new->hw, dma_dst);
I
Ira Snyder 已提交
706 707 708 709 710 711 712 713

			/*
			 * If this is not the first descriptor, chain the
			 * current descriptor after the previous descriptor
			 */
			if (!first) {
				first = new;
			} else {
I
Ira Snyder 已提交
714
				set_desc_next(chan, &prev->hw,
I
Ira Snyder 已提交
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
					      new->async_tx.phys);
			}

			new->async_tx.cookie = 0;
			async_tx_ack(&new->async_tx);

			prev = new;
			sg_used += copy;
			hw_used += copy;

			/* Insert the link descriptor into the LD ring */
			list_add_tail(&new->node, &first->tx_list);
		}
	}

finished:

	/* All of the hardware address/length pairs had length == 0 */
	if (!first || !new)
		return NULL;

	new->async_tx.flags = flags;
	new->async_tx.cookie = -EBUSY;

	/* Set End-of-link to the last link descriptor of new list */
I
Ira Snyder 已提交
740
	set_ld_eol(chan, new);
I
Ira Snyder 已提交
741 742

	/* Enable extra controller features */
I
Ira Snyder 已提交
743 744
	if (chan->set_src_loop_size)
		chan->set_src_loop_size(chan, slave->src_loop_size);
I
Ira Snyder 已提交
745

I
Ira Snyder 已提交
746 747
	if (chan->set_dst_loop_size)
		chan->set_dst_loop_size(chan, slave->dst_loop_size);
I
Ira Snyder 已提交
748

I
Ira Snyder 已提交
749 750
	if (chan->toggle_ext_start)
		chan->toggle_ext_start(chan, slave->external_start);
I
Ira Snyder 已提交
751

I
Ira Snyder 已提交
752 753
	if (chan->toggle_ext_pause)
		chan->toggle_ext_pause(chan, slave->external_pause);
I
Ira Snyder 已提交
754

I
Ira Snyder 已提交
755 756
	if (chan->set_request_count)
		chan->set_request_count(chan, slave->request_count);
I
Ira Snyder 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772

	return &first->async_tx;

fail:
	/* If first was not set, then we failed to allocate the very first
	 * descriptor, and we're done */
	if (!first)
		return NULL;

	/*
	 * First is set, so all of the descriptors we allocated have been added
	 * to first->tx_list, INCLUDING "first" itself. Therefore we
	 * must traverse the list backwards freeing each descriptor in turn
	 *
	 * We're re-using variables for the loop, oh well
	 */
I
Ira Snyder 已提交
773
	fsldma_free_desc_list_reverse(chan, &first->tx_list);
I
Ira Snyder 已提交
774 775 776
	return NULL;
}

777 778
static int fsl_dma_device_control(struct dma_chan *dchan,
				  enum dma_ctrl_cmd cmd)
I
Ira Snyder 已提交
779
{
I
Ira Snyder 已提交
780
	struct fsldma_chan *chan;
I
Ira Snyder 已提交
781 782
	unsigned long flags;

783 784 785 786
	/* Only supports DMA_TERMINATE_ALL */
	if (cmd != DMA_TERMINATE_ALL)
		return -ENXIO;

I
Ira Snyder 已提交
787
	if (!dchan)
788
		return -EINVAL;
I
Ira Snyder 已提交
789

I
Ira Snyder 已提交
790
	chan = to_fsl_chan(dchan);
I
Ira Snyder 已提交
791 792

	/* Halt the DMA engine */
I
Ira Snyder 已提交
793
	dma_halt(chan);
I
Ira Snyder 已提交
794

I
Ira Snyder 已提交
795
	spin_lock_irqsave(&chan->desc_lock, flags);
I
Ira Snyder 已提交
796 797

	/* Remove and free all of the descriptors in the LD queue */
I
Ira Snyder 已提交
798 799
	fsldma_free_desc_list(chan, &chan->ld_pending);
	fsldma_free_desc_list(chan, &chan->ld_running);
I
Ira Snyder 已提交
800

I
Ira Snyder 已提交
801
	spin_unlock_irqrestore(&chan->desc_lock, flags);
802 803

	return 0;
I
Ira Snyder 已提交
804 805
}

806 807
/**
 * fsl_dma_update_completed_cookie - Update the completed cookie.
I
Ira Snyder 已提交
808
 * @chan : Freescale DMA channel
I
Ira Snyder 已提交
809 810
 *
 * CONTEXT: hardirq
811
 */
I
Ira Snyder 已提交
812
static void fsl_dma_update_completed_cookie(struct fsldma_chan *chan)
813
{
I
Ira Snyder 已提交
814 815 816
	struct fsl_desc_sw *desc;
	unsigned long flags;
	dma_cookie_t cookie;
817

I
Ira Snyder 已提交
818
	spin_lock_irqsave(&chan->desc_lock, flags);
819

I
Ira Snyder 已提交
820 821 822
	if (list_empty(&chan->ld_running)) {
		dev_dbg(chan->dev, "no running descriptors\n");
		goto out_unlock;
823
	}
I
Ira Snyder 已提交
824 825 826 827 828

	/* Get the last descriptor, update the cookie to that */
	desc = to_fsl_desc(chan->ld_running.prev);
	if (dma_is_idle(chan))
		cookie = desc->async_tx.cookie;
S
Steven J. Magnani 已提交
829
	else {
I
Ira Snyder 已提交
830
		cookie = desc->async_tx.cookie - 1;
S
Steven J. Magnani 已提交
831 832 833
		if (unlikely(cookie < DMA_MIN_COOKIE))
			cookie = DMA_MAX_COOKIE;
	}
I
Ira Snyder 已提交
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853

	chan->completed_cookie = cookie;

out_unlock:
	spin_unlock_irqrestore(&chan->desc_lock, flags);
}

/**
 * fsldma_desc_status - Check the status of a descriptor
 * @chan: Freescale DMA channel
 * @desc: DMA SW descriptor
 *
 * This function will return the status of the given descriptor
 */
static enum dma_status fsldma_desc_status(struct fsldma_chan *chan,
					  struct fsl_desc_sw *desc)
{
	return dma_async_is_complete(desc->async_tx.cookie,
				     chan->completed_cookie,
				     chan->common.cookie);
854 855 856 857
}

/**
 * fsl_chan_ld_cleanup - Clean up link descriptors
I
Ira Snyder 已提交
858
 * @chan : Freescale DMA channel
859 860 861
 *
 * This function clean up the ld_queue of DMA channel.
 */
I
Ira Snyder 已提交
862
static void fsl_chan_ld_cleanup(struct fsldma_chan *chan)
863 864 865 866
{
	struct fsl_desc_sw *desc, *_desc;
	unsigned long flags;

I
Ira Snyder 已提交
867
	spin_lock_irqsave(&chan->desc_lock, flags);
868

I
Ira Snyder 已提交
869 870
	dev_dbg(chan->dev, "chan completed_cookie = %d\n", chan->completed_cookie);
	list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {
871 872 873
		dma_async_tx_callback callback;
		void *callback_param;

I
Ira Snyder 已提交
874
		if (fsldma_desc_status(chan, desc) == DMA_IN_PROGRESS)
875 876
			break;

I
Ira Snyder 已提交
877
		/* Remove from the list of running transactions */
878 879 880
		list_del(&desc->node);

		/* Run the link descriptor callback function */
I
Ira Snyder 已提交
881 882
		callback = desc->async_tx.callback;
		callback_param = desc->async_tx.callback_param;
883
		if (callback) {
I
Ira Snyder 已提交
884
			spin_unlock_irqrestore(&chan->desc_lock, flags);
I
Ira Snyder 已提交
885
			dev_dbg(chan->dev, "LD %p callback\n", desc);
886
			callback(callback_param);
I
Ira Snyder 已提交
887
			spin_lock_irqsave(&chan->desc_lock, flags);
888
		}
I
Ira Snyder 已提交
889 890 891 892

		/* Run any dependencies, then free the descriptor */
		dma_run_dependencies(&desc->async_tx);
		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
893
	}
I
Ira Snyder 已提交
894

I
Ira Snyder 已提交
895
	spin_unlock_irqrestore(&chan->desc_lock, flags);
896 897 898
}

/**
I
Ira Snyder 已提交
899
 * fsl_chan_xfer_ld_queue - transfer any pending transactions
I
Ira Snyder 已提交
900
 * @chan : Freescale DMA channel
I
Ira Snyder 已提交
901 902 903 904 905
 *
 * This will make sure that any pending transactions will be run.
 * If the DMA controller is idle, it will be started. Otherwise,
 * the DMA controller's interrupt handler will start any pending
 * transactions when it becomes idle.
906
 */
I
Ira Snyder 已提交
907
static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
908
{
I
Ira Snyder 已提交
909
	struct fsl_desc_sw *desc;
910 911
	unsigned long flags;

I
Ira Snyder 已提交
912
	spin_lock_irqsave(&chan->desc_lock, flags);
913

I
Ira Snyder 已提交
914 915 916 917 918 919
	/*
	 * If the list of pending descriptors is empty, then we
	 * don't need to do any work at all
	 */
	if (list_empty(&chan->ld_pending)) {
		dev_dbg(chan->dev, "no pending LDs\n");
920
		goto out_unlock;
I
Ira Snyder 已提交
921
	}
922

I
Ira Snyder 已提交
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937
	/*
	 * The DMA controller is not idle, which means the interrupt
	 * handler will start any queued transactions when it runs
	 * at the end of the current transaction
	 */
	if (!dma_is_idle(chan)) {
		dev_dbg(chan->dev, "DMA controller still busy\n");
		goto out_unlock;
	}

	/*
	 * TODO:
	 * make sure the dma_halt() function really un-wedges the
	 * controller as much as possible
	 */
I
Ira Snyder 已提交
938
	dma_halt(chan);
939

I
Ira Snyder 已提交
940 941 942
	/*
	 * If there are some link descriptors which have not been
	 * transferred, we need to start the controller
943 944
	 */

I
Ira Snyder 已提交
945 946 947 948 949 950 951 952 953 954 955 956 957
	/*
	 * Move all elements from the queue of pending transactions
	 * onto the list of running transactions
	 */
	desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
	list_splice_tail_init(&chan->ld_pending, &chan->ld_running);

	/*
	 * Program the descriptor's address into the DMA controller,
	 * then start the DMA transaction
	 */
	set_cdar(chan, desc->async_tx.phys);
	dma_start(chan);
958 959

out_unlock:
I
Ira Snyder 已提交
960
	spin_unlock_irqrestore(&chan->desc_lock, flags);
961 962 963 964
}

/**
 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
I
Ira Snyder 已提交
965
 * @chan : Freescale DMA channel
966
 */
I
Ira Snyder 已提交
967
static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
968
{
I
Ira Snyder 已提交
969 970
	struct fsldma_chan *chan = to_fsl_chan(dchan);
	fsl_chan_xfer_ld_queue(chan);
971 972 973 974
}

/**
 * fsl_dma_is_complete - Determine the DMA status
I
Ira Snyder 已提交
975
 * @chan : Freescale DMA channel
976
 */
I
Ira Snyder 已提交
977
static enum dma_status fsl_dma_is_complete(struct dma_chan *dchan,
978 979 980 981
					dma_cookie_t cookie,
					dma_cookie_t *done,
					dma_cookie_t *used)
{
I
Ira Snyder 已提交
982
	struct fsldma_chan *chan = to_fsl_chan(dchan);
983 984 985
	dma_cookie_t last_used;
	dma_cookie_t last_complete;

I
Ira Snyder 已提交
986
	fsl_chan_ld_cleanup(chan);
987

I
Ira Snyder 已提交
988 989
	last_used = dchan->cookie;
	last_complete = chan->completed_cookie;
990 991 992 993 994 995 996 997 998 999

	if (done)
		*done = last_complete;

	if (used)
		*used = last_used;

	return dma_async_is_complete(cookie, last_complete, last_used);
}

1000 1001 1002 1003
/*----------------------------------------------------------------------------*/
/* Interrupt Handling                                                         */
/*----------------------------------------------------------------------------*/

1004
static irqreturn_t fsldma_chan_irq(int irq, void *data)
1005
{
I
Ira Snyder 已提交
1006
	struct fsldma_chan *chan = data;
1007 1008
	int update_cookie = 0;
	int xfer_ld_q = 0;
I
Ira Snyder 已提交
1009
	u32 stat;
1010

I
Ira Snyder 已提交
1011
	/* save and clear the status register */
I
Ira Snyder 已提交
1012
	stat = get_sr(chan);
I
Ira Snyder 已提交
1013 1014
	set_sr(chan, stat);
	dev_dbg(chan->dev, "irq: channel %d, stat = 0x%x\n", chan->id, stat);
1015 1016 1017 1018 1019 1020

	stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
	if (!stat)
		return IRQ_NONE;

	if (stat & FSL_DMA_SR_TE)
I
Ira Snyder 已提交
1021
		dev_err(chan->dev, "Transfer Error!\n");
1022

I
Ira Snyder 已提交
1023 1024
	/*
	 * Programming Error
1025 1026 1027 1028
	 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
	 * triger a PE interrupt.
	 */
	if (stat & FSL_DMA_SR_PE) {
I
Ira Snyder 已提交
1029
		dev_dbg(chan->dev, "irq: Programming Error INT\n");
I
Ira Snyder 已提交
1030
		if (get_bcr(chan) == 0) {
1031 1032 1033 1034
			/* BCR register is 0, this is a DMA_INTERRUPT async_tx.
			 * Now, update the completed cookie, and continue the
			 * next uncompleted transfer.
			 */
1035 1036
			update_cookie = 1;
			xfer_ld_q = 1;
1037 1038 1039 1040
		}
		stat &= ~FSL_DMA_SR_PE;
	}

I
Ira Snyder 已提交
1041 1042
	/*
	 * If the link descriptor segment transfer finishes,
1043 1044 1045
	 * we will recycle the used descriptor.
	 */
	if (stat & FSL_DMA_SR_EOSI) {
I
Ira Snyder 已提交
1046 1047
		dev_dbg(chan->dev, "irq: End-of-segments INT\n");
		dev_dbg(chan->dev, "irq: clndar 0x%llx, nlndar 0x%llx\n",
I
Ira Snyder 已提交
1048 1049
			(unsigned long long)get_cdar(chan),
			(unsigned long long)get_ndar(chan));
1050
		stat &= ~FSL_DMA_SR_EOSI;
1051 1052 1053
		update_cookie = 1;
	}

I
Ira Snyder 已提交
1054 1055
	/*
	 * For MPC8349, EOCDI event need to update cookie
1056 1057 1058
	 * and start the next transfer if it exist.
	 */
	if (stat & FSL_DMA_SR_EOCDI) {
I
Ira Snyder 已提交
1059
		dev_dbg(chan->dev, "irq: End-of-Chain link INT\n");
1060 1061 1062
		stat &= ~FSL_DMA_SR_EOCDI;
		update_cookie = 1;
		xfer_ld_q = 1;
1063 1064
	}

I
Ira Snyder 已提交
1065 1066
	/*
	 * If it current transfer is the end-of-transfer,
1067 1068 1069
	 * we should clear the Channel Start bit for
	 * prepare next transfer.
	 */
1070
	if (stat & FSL_DMA_SR_EOLNI) {
I
Ira Snyder 已提交
1071
		dev_dbg(chan->dev, "irq: End-of-link INT\n");
1072
		stat &= ~FSL_DMA_SR_EOLNI;
1073
		xfer_ld_q = 1;
1074 1075
	}

1076
	if (update_cookie)
I
Ira Snyder 已提交
1077
		fsl_dma_update_completed_cookie(chan);
1078
	if (xfer_ld_q)
I
Ira Snyder 已提交
1079
		fsl_chan_xfer_ld_queue(chan);
1080
	if (stat)
I
Ira Snyder 已提交
1081
		dev_dbg(chan->dev, "irq: unhandled sr 0x%02x\n", stat);
1082

I
Ira Snyder 已提交
1083
	dev_dbg(chan->dev, "irq: Exit\n");
I
Ira Snyder 已提交
1084
	tasklet_schedule(&chan->tasklet);
1085 1086 1087
	return IRQ_HANDLED;
}

1088 1089
static void dma_do_tasklet(unsigned long data)
{
I
Ira Snyder 已提交
1090 1091
	struct fsldma_chan *chan = (struct fsldma_chan *)data;
	fsl_chan_ld_cleanup(chan);
1092 1093 1094
}

static irqreturn_t fsldma_ctrl_irq(int irq, void *data)
1095
{
1096
	struct fsldma_device *fdev = data;
1097 1098 1099 1100
	struct fsldma_chan *chan;
	unsigned int handled = 0;
	u32 gsr, mask;
	int i;
1101

1102
	gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)
1103 1104 1105
						   : in_le32(fdev->regs);
	mask = 0xff000000;
	dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);
1106

1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
		chan = fdev->chan[i];
		if (!chan)
			continue;

		if (gsr & mask) {
			dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);
			fsldma_chan_irq(irq, chan);
			handled++;
		}

		gsr &= ~mask;
		mask >>= 8;
	}

	return IRQ_RETVAL(handled);
1123 1124
}

1125
static void fsldma_free_irqs(struct fsldma_device *fdev)
1126
{
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
	struct fsldma_chan *chan;
	int i;

	if (fdev->irq != NO_IRQ) {
		dev_dbg(fdev->dev, "free per-controller IRQ\n");
		free_irq(fdev->irq, fdev);
		return;
	}

	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
		chan = fdev->chan[i];
		if (chan && chan->irq != NO_IRQ) {
			dev_dbg(fdev->dev, "free channel %d IRQ\n", chan->id);
			free_irq(chan->irq, chan);
		}
	}
}

static int fsldma_request_irqs(struct fsldma_device *fdev)
{
	struct fsldma_chan *chan;
	int ret;
	int i;

	/* if we have a per-controller IRQ, use that */
	if (fdev->irq != NO_IRQ) {
		dev_dbg(fdev->dev, "request per-controller IRQ\n");
		ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
				  "fsldma-controller", fdev);
		return ret;
	}

	/* no per-controller IRQ, use the per-channel IRQs */
	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
		chan = fdev->chan[i];
		if (!chan)
			continue;

		if (chan->irq == NO_IRQ) {
			dev_err(fdev->dev, "no interrupts property defined for "
					   "DMA channel %d. Please fix your "
					   "device tree\n", chan->id);
			ret = -ENODEV;
			goto out_unwind;
		}

		dev_dbg(fdev->dev, "request channel %d IRQ\n", chan->id);
		ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
				  "fsldma-chan", chan);
		if (ret) {
			dev_err(fdev->dev, "unable to request IRQ for DMA "
					   "channel %d\n", chan->id);
			goto out_unwind;
		}
	}

	return 0;

out_unwind:
	for (/* none */; i >= 0; i--) {
		chan = fdev->chan[i];
		if (!chan)
			continue;

		if (chan->irq == NO_IRQ)
			continue;

		free_irq(chan->irq, chan);
	}

	return ret;
1198 1199
}

1200 1201 1202 1203 1204
/*----------------------------------------------------------------------------*/
/* OpenFirmware Subsystem                                                     */
/*----------------------------------------------------------------------------*/

static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,
1205
	struct device_node *node, u32 feature, const char *compatible)
1206
{
I
Ira Snyder 已提交
1207
	struct fsldma_chan *chan;
1208
	struct resource res;
1209 1210 1211
	int err;

	/* alloc channel */
I
Ira Snyder 已提交
1212 1213
	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	if (!chan) {
1214 1215 1216 1217 1218 1219
		dev_err(fdev->dev, "no free memory for DMA channels!\n");
		err = -ENOMEM;
		goto out_return;
	}

	/* ioremap registers for use */
I
Ira Snyder 已提交
1220 1221
	chan->regs = of_iomap(node, 0);
	if (!chan->regs) {
1222 1223
		dev_err(fdev->dev, "unable to ioremap registers\n");
		err = -ENOMEM;
I
Ira Snyder 已提交
1224
		goto out_free_chan;
1225 1226
	}

1227
	err = of_address_to_resource(node, 0, &res);
1228
	if (err) {
1229 1230
		dev_err(fdev->dev, "unable to find 'reg' property\n");
		goto out_iounmap_regs;
1231 1232
	}

I
Ira Snyder 已提交
1233
	chan->feature = feature;
1234
	if (!fdev->feature)
I
Ira Snyder 已提交
1235
		fdev->feature = chan->feature;
1236

1237 1238 1239
	/*
	 * If the DMA device's feature is different than the feature
	 * of its channels, report the bug
1240
	 */
I
Ira Snyder 已提交
1241
	WARN_ON(fdev->feature != chan->feature);
1242

I
Ira Snyder 已提交
1243 1244 1245
	chan->dev = fdev->dev;
	chan->id = ((res.start - 0x100) & 0xfff) >> 7;
	if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {
1246
		dev_err(fdev->dev, "too many channels for device\n");
1247
		err = -EINVAL;
1248
		goto out_iounmap_regs;
1249 1250
	}

I
Ira Snyder 已提交
1251 1252
	fdev->chan[chan->id] = chan;
	tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
1253 1254

	/* Initialize the channel */
I
Ira Snyder 已提交
1255
	dma_init(chan);
1256 1257

	/* Clear cdar registers */
I
Ira Snyder 已提交
1258
	set_cdar(chan, 0);
1259

I
Ira Snyder 已提交
1260
	switch (chan->feature & FSL_DMA_IP_MASK) {
1261
	case FSL_DMA_IP_85XX:
I
Ira Snyder 已提交
1262
		chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
1263
	case FSL_DMA_IP_83XX:
I
Ira Snyder 已提交
1264 1265 1266 1267
		chan->toggle_ext_start = fsl_chan_toggle_ext_start;
		chan->set_src_loop_size = fsl_chan_set_src_loop_size;
		chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;
		chan->set_request_count = fsl_chan_set_request_count;
1268 1269
	}

I
Ira Snyder 已提交
1270
	spin_lock_init(&chan->desc_lock);
I
Ira Snyder 已提交
1271 1272
	INIT_LIST_HEAD(&chan->ld_pending);
	INIT_LIST_HEAD(&chan->ld_running);
1273

I
Ira Snyder 已提交
1274
	chan->common.device = &fdev->common;
1275

1276
	/* find the IRQ line, if it exists in the device tree */
I
Ira Snyder 已提交
1277
	chan->irq = irq_of_parse_and_map(node, 0);
1278

1279
	/* Add the channel to DMA device channel list */
I
Ira Snyder 已提交
1280
	list_add_tail(&chan->common.device_node, &fdev->common.channels);
1281 1282
	fdev->common.chancnt++;

I
Ira Snyder 已提交
1283 1284
	dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
		 chan->irq != NO_IRQ ? chan->irq : fdev->irq);
1285 1286

	return 0;
1287

1288
out_iounmap_regs:
I
Ira Snyder 已提交
1289 1290 1291
	iounmap(chan->regs);
out_free_chan:
	kfree(chan);
1292
out_return:
1293 1294 1295
	return err;
}

I
Ira Snyder 已提交
1296
static void fsl_dma_chan_remove(struct fsldma_chan *chan)
1297
{
I
Ira Snyder 已提交
1298 1299 1300 1301
	irq_dispose_mapping(chan->irq);
	list_del(&chan->common.device_node);
	iounmap(chan->regs);
	kfree(chan);
1302 1303
}

1304
static int __devinit fsldma_of_probe(struct of_device *op,
1305 1306
			const struct of_device_id *match)
{
1307
	struct fsldma_device *fdev;
1308
	struct device_node *child;
1309
	int err;
1310

1311
	fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);
1312
	if (!fdev) {
1313 1314 1315
		dev_err(&op->dev, "No enough memory for 'priv'\n");
		err = -ENOMEM;
		goto out_return;
1316
	}
1317 1318

	fdev->dev = &op->dev;
1319 1320
	INIT_LIST_HEAD(&fdev->common.channels);

1321 1322 1323 1324 1325 1326
	/* ioremap the registers for use */
	fdev->regs = of_iomap(op->node, 0);
	if (!fdev->regs) {
		dev_err(&op->dev, "unable to ioremap registers\n");
		err = -ENOMEM;
		goto out_free_fdev;
1327 1328
	}

1329 1330 1331
	/* map the channel IRQ if it exists, but don't hookup the handler yet */
	fdev->irq = irq_of_parse_and_map(op->node, 0);

1332 1333
	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
I
Ira Snyder 已提交
1334
	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
1335 1336
	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
1337
	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
1338 1339 1340
	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
	fdev->common.device_is_tx_complete = fsl_dma_is_complete;
	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
I
Ira Snyder 已提交
1341
	fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
1342
	fdev->common.device_control = fsl_dma_device_control;
1343
	fdev->common.dev = &op->dev;
1344

1345
	dev_set_drvdata(&op->dev, fdev);
1346

1347 1348 1349
	/*
	 * We cannot use of_platform_bus_probe() because there is no
	 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1350 1351
	 * channel object.
	 */
1352 1353
	for_each_child_of_node(op->node, child) {
		if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {
1354 1355 1356
			fsl_dma_chan_probe(fdev, child,
				FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN,
				"fsl,eloplus-dma-channel");
1357 1358 1359
		}

		if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {
1360 1361 1362
			fsl_dma_chan_probe(fdev, child,
				FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN,
				"fsl,elo-dma-channel");
1363
		}
1364
	}
1365

1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
	/*
	 * Hookup the IRQ handler(s)
	 *
	 * If we have a per-controller interrupt, we prefer that to the
	 * per-channel interrupts to reduce the number of shared interrupt
	 * handlers on the same IRQ line
	 */
	err = fsldma_request_irqs(fdev);
	if (err) {
		dev_err(fdev->dev, "unable to request IRQs\n");
		goto out_free_fdev;
	}

1379 1380 1381
	dma_async_device_register(&fdev->common);
	return 0;

1382
out_free_fdev:
1383
	irq_dispose_mapping(fdev->irq);
1384
	kfree(fdev);
1385
out_return:
1386 1387 1388
	return err;
}

1389
static int fsldma_of_remove(struct of_device *op)
1390
{
1391
	struct fsldma_device *fdev;
1392 1393
	unsigned int i;

1394
	fdev = dev_get_drvdata(&op->dev);
1395 1396
	dma_async_device_unregister(&fdev->common);

1397 1398
	fsldma_free_irqs(fdev);

1399
	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1400 1401
		if (fdev->chan[i])
			fsl_dma_chan_remove(fdev->chan[i]);
1402
	}
1403

1404 1405
	iounmap(fdev->regs);
	dev_set_drvdata(&op->dev, NULL);
1406 1407 1408 1409 1410
	kfree(fdev);

	return 0;
}

1411
static const struct of_device_id fsldma_of_ids[] = {
1412 1413
	{ .compatible = "fsl,eloplus-dma", },
	{ .compatible = "fsl,elo-dma", },
1414 1415 1416
	{}
};

1417 1418 1419 1420 1421
static struct of_platform_driver fsldma_of_driver = {
	.name		= "fsl-elo-dma",
	.match_table	= fsldma_of_ids,
	.probe		= fsldma_of_probe,
	.remove		= fsldma_of_remove,
1422 1423
};

1424 1425 1426 1427 1428
/*----------------------------------------------------------------------------*/
/* Module Init / Exit                                                         */
/*----------------------------------------------------------------------------*/

static __init int fsldma_init(void)
1429
{
1430 1431 1432 1433
	int ret;

	pr_info("Freescale Elo / Elo Plus DMA driver\n");

1434
	ret = of_register_platform_driver(&fsldma_of_driver);
1435 1436 1437 1438 1439 1440
	if (ret)
		pr_err("fsldma: failed to register platform driver\n");

	return ret;
}

1441
static void __exit fsldma_exit(void)
1442
{
1443
	of_unregister_platform_driver(&fsldma_of_driver);
1444 1445
}

1446 1447
subsys_initcall(fsldma_init);
module_exit(fsldma_exit);
1448 1449 1450

MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");
MODULE_LICENSE("GPL");