amba-pl08x.c 54.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Copyright (c) 2006 ARM Ltd.
 * Copyright (c) 2010 ST-Ericsson SA
 *
 * Author: Peter Pearse <peter.pearse@arm.com>
 * Author: Linus Walleij <linus.walleij@stericsson.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59
 * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 *
22 23
 * The full GNU General Public License is in this distribution in the file
 * called COPYING.
24 25
 *
 * Documentation: ARM DDI 0196G == PL080
26
 * Documentation: ARM DDI 0218E == PL081
27
 *
28 29
 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
 * channel.
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
 *
 * The PL080 has 8 channels available for simultaneous use, and the PL081
 * has only two channels. So on these DMA controllers the number of channels
 * and the number of incoming DMA signals are two totally different things.
 * It is usually not possible to theoretically handle all physical signals,
 * so a multiplexing scheme with possible denial of use is necessary.
 *
 * The PL080 has a dual bus master, PL081 has a single master.
 *
 * Memory to peripheral transfer may be visualized as
 *	Get data from memory to DMAC
 *	Until no data left
 *		On burst request from peripheral
 *			Destination burst from DMAC to peripheral
 *			Clear burst request
 *	Raise terminal count interrupt
 *
 * For peripherals with a FIFO:
 * Source      burst size == half the depth of the peripheral FIFO
 * Destination burst size == the depth of the peripheral FIFO
 *
 * (Bursts are irrelevant for mem to mem transfers - there are no burst
 * signals, the DMA controller will simply facilitate its AHB master.)
 *
 * ASSUMES default (little) endianness for DMA transfers
 *
56 57 58 59 60 61 62 63 64 65 66 67 68
 * The PL08x has two flow control settings:
 *  - DMAC flow control: the transfer size defines the number of transfers
 *    which occur for the current LLI entry, and the DMAC raises TC at the
 *    end of every LLI entry.  Observed behaviour shows the DMAC listening
 *    to both the BREQ and SREQ signals (contrary to documented),
 *    transferring data if either is active.  The LBREQ and LSREQ signals
 *    are ignored.
 *
 *  - Peripheral flow control: the transfer size is ignored (and should be
 *    zero).  The data is transferred from the current LLI entry, until
 *    after the final transfer signalled by LBREQ or LSREQ.  The DMAC
 *    will then move to the next LLI entry.
 *
69 70 71
 * Global TODO:
 * - Break out common code from arch/arm/mach-s3c64xx and share
 */
72
#include <linux/amba/bus.h>
73 74
#include <linux/amba/pl08x.h>
#include <linux/debugfs.h>
75 76 77 78
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/dmapool.h>
V
Vinod Koul 已提交
79
#include <linux/dma-mapping.h>
80 81 82
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/module.h>
83
#include <linux/pm_runtime.h>
84
#include <linux/seq_file.h>
85
#include <linux/slab.h>
86 87
#include <asm/hardware/pl080.h>

88
#include "dmaengine.h"
89
#include "virt-dma.h"
90

91 92
#define DRIVER_NAME	"pl08xdmac"

93
static struct amba_driver pl08x_amba_driver;
94
struct pl08x_driver_data;
95

96
/**
97
 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
98
 * @channels: the number of channels available in this variant
99
 * @dualmaster: whether this version supports dual AHB masters or not.
100 101 102
 * @nomadik: whether the channels have Nomadik security extension bits
 *	that need to be checked for permission before use and some registers are
 *	missing
103 104 105 106
 */
struct vendor_data {
	u8 channels;
	bool dualmaster;
107
	bool nomadik;
108 109 110 111
};

/*
 * PL08X private data structures
112
 * An LLI struct - see PL08x TRM.  Note that next uses bit[0] as a bus bit,
113 114
 * start & end do not - their bus bit info is in cctl.  Also note that these
 * are fixed 32-bit quantities.
115
 */
116
struct pl08x_lli {
117 118
	u32 src;
	u32 dst;
119
	u32 lli;
120 121 122
	u32 cctl;
};

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
/**
 * struct pl08x_bus_data - information of source or destination
 * busses for a transfer
 * @addr: current address
 * @maxwidth: the maximum width of a transfer on this bus
 * @buswidth: the width of this bus in bytes: 1, 2 or 4
 */
struct pl08x_bus_data {
	dma_addr_t addr;
	u8 maxwidth;
	u8 buswidth;
};

/**
 * struct pl08x_phy_chan - holder for the physical channels
 * @id: physical index to this channel
 * @lock: a lock to use when altering an instance of this struct
 * @serving: the virtual channel currently being served by this physical
 * channel
142 143
 * @locked: channel unavailable for the system, e.g. dedicated to secure
 * world
144 145 146 147 148 149
 */
struct pl08x_phy_chan {
	unsigned int id;
	void __iomem *base;
	spinlock_t lock;
	struct pl08x_dma_chan *serving;
150
	bool locked;
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
};

/**
 * struct pl08x_sg - structure containing data per sg
 * @src_addr: src address of sg
 * @dst_addr: dst address of sg
 * @len: transfer len in bytes
 * @node: node for txd's dsg_list
 */
struct pl08x_sg {
	dma_addr_t src_addr;
	dma_addr_t dst_addr;
	size_t len;
	struct list_head node;
};

/**
 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
169
 * @vd: virtual DMA descriptor
170 171 172 173 174
 * @dsg_list: list of children sg's
 * @llis_bus: DMA memory address (physical) start for the LLIs
 * @llis_va: virtual memory address start for the LLIs
 * @cctl: control reg values for current txd
 * @ccfg: config reg values for current txd
175 176
 * @done: this marks completed descriptors, which should not have their
 *   mux released.
177 178
 */
struct pl08x_txd {
179
	struct virt_dma_desc vd;
180 181 182 183 184 185 186 187 188 189
	struct list_head dsg_list;
	dma_addr_t llis_bus;
	struct pl08x_lli *llis_va;
	/* Default cctl value for LLIs */
	u32 cctl;
	/*
	 * Settings to be put into the physical channel when we
	 * trigger this txd.  Other registers are in llis_va[0].
	 */
	u32 ccfg;
190
	bool done;
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
};

/**
 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
 * states
 * @PL08X_CHAN_IDLE: the channel is idle
 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
 * channel and is running a transfer on it
 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
 * channel, but the transfer is currently paused
 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
 * channel to become available (only pertains to memcpy channels)
 */
enum pl08x_dma_chan_state {
	PL08X_CHAN_IDLE,
	PL08X_CHAN_RUNNING,
	PL08X_CHAN_PAUSED,
	PL08X_CHAN_WAITING,
};

/**
 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
213
 * @vc: wrappped virtual channel
214 215 216 217 218 219 220 221 222
 * @phychan: the physical channel utilized by this channel, if there is one
 * @name: name of channel
 * @cd: channel platform data
 * @runtime_addr: address for RX/TX according to the runtime config
 * @at: active transaction on this channel
 * @lock: a lock for this channel data
 * @host: a pointer to the host (internal use)
 * @state: whether the channel is idle, paused, running etc
 * @slave: whether this channel is a device (slave) or for memcpy
223
 * @signal: the physical DMA request signal which this channel is using
224
 * @mux_use: count of descriptors using this DMA request signal setting
225 226
 */
struct pl08x_dma_chan {
227
	struct virt_dma_chan vc;
228
	struct pl08x_phy_chan *phychan;
229
	const char *name;
230
	const struct pl08x_channel_data *cd;
231
	struct dma_slave_config cfg;
232 233 234 235
	struct pl08x_txd *at;
	struct pl08x_driver_data *host;
	enum pl08x_dma_chan_state state;
	bool slave;
236
	int signal;
237
	unsigned mux_use;
238 239
};

240 241 242 243 244 245 246 247 248 249
/**
 * struct pl08x_driver_data - the local state holder for the PL08x
 * @slave: slave engine for this instance
 * @memcpy: memcpy engine for this instance
 * @base: virtual memory base (remapped) for the PL08x
 * @adev: the corresponding AMBA (PrimeCell) bus entry
 * @vd: vendor data for this PL08x variant
 * @pd: platform data passed in from the platform/machine
 * @phy_chans: array of data for the physical channels
 * @pool: a pool for the LLI descriptors
250 251
 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
 * fetches
252
 * @mem_buses: set to indicate memory transfers on AHB2.
253 254 255 256 257 258 259
 * @lock: a spinlock for this struct
 */
struct pl08x_driver_data {
	struct dma_device slave;
	struct dma_device memcpy;
	void __iomem *base;
	struct amba_device *adev;
260
	const struct vendor_data *vd;
261 262 263
	struct pl08x_platform_data *pd;
	struct pl08x_phy_chan *phy_chans;
	struct dma_pool *pool;
264 265
	u8 lli_buses;
	u8 mem_buses;
266 267 268 269 270 271 272 273 274
};

/*
 * PL08X specific defines
 */

/* Size (bytes) of each LLI buffer allocated for one transfer */
# define PL08X_LLI_TSFR_SIZE	0x2000

275
/* Maximum times we call dma_pool_alloc on this pool without freeing */
276
#define MAX_NUM_TSFR_LLIS	(PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli))
277 278 279 280
#define PL08X_ALIGN		8

static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
{
281
	return container_of(chan, struct pl08x_dma_chan, vc.chan);
282 283
}

284 285
static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
{
286
	return container_of(tx, struct pl08x_txd, vd.tx);
287 288
}

289 290 291 292 293 294 295 296
/*
 * Mux handling.
 *
 * This gives us the DMA request input to the PL08x primecell which the
 * peripheral described by the channel data will be routed to, possibly
 * via a board/SoC specific external MUX.  One important point to note
 * here is that this does not depend on the physical channel.
 */
297
static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
298 299 300 301
{
	const struct pl08x_platform_data *pd = plchan->host->pd;
	int ret;

302
	if (plchan->mux_use++ == 0 && pd->get_signal) {
303
		ret = pd->get_signal(plchan->cd);
304 305
		if (ret < 0) {
			plchan->mux_use = 0;
306
			return ret;
307
		}
308

309
		plchan->signal = ret;
310 311 312 313 314 315 316 317
	}
	return 0;
}

static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
{
	const struct pl08x_platform_data *pd = plchan->host->pd;

318 319 320 321 322 323 324
	if (plchan->signal >= 0) {
		WARN_ON(plchan->mux_use == 0);

		if (--plchan->mux_use == 0 && pd->put_signal) {
			pd->put_signal(plchan->cd, plchan->signal);
			plchan->signal = -1;
		}
325 326 327
	}
}

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
/*
 * Physical channel handling
 */

/* Whether a certain channel is busy or not */
static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
{
	unsigned int val;

	val = readl(ch->base + PL080_CH_CONFIG);
	return val & PL080_CONFIG_ACTIVE;
}

/*
 * Set the initial DMA register values i.e. those for the first LLI
343
 * The next LLI pointer and the configuration interrupt bit have
344 345
 * been set when the LLIs were constructed.  Poke them into the hardware
 * and start the transfer.
346
 */
347
static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
348
{
349
	struct pl08x_driver_data *pl08x = plchan->host;
350
	struct pl08x_phy_chan *phychan = plchan->phychan;
351 352
	struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
353
	struct pl08x_lli *lli;
354
	u32 val;
355

356
	list_del(&txd->vd.node);
357

358
	plchan->at = txd;
359

360 361 362
	/* Wait for channel inactive */
	while (pl08x_phy_channel_busy(phychan))
		cpu_relax();
363

364 365
	lli = &txd->llis_va[0];

366 367
	dev_vdbg(&pl08x->adev->dev,
		"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
368 369
		"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
		phychan->id, lli->src, lli->dst, lli->lli, lli->cctl,
370
		txd->ccfg);
371 372 373 374 375

	writel(lli->src, phychan->base + PL080_CH_SRC_ADDR);
	writel(lli->dst, phychan->base + PL080_CH_DST_ADDR);
	writel(lli->lli, phychan->base + PL080_CH_LLI);
	writel(lli->cctl, phychan->base + PL080_CH_CONTROL);
376
	writel(txd->ccfg, phychan->base + PL080_CH_CONFIG);
377 378 379 380

	/* Enable the DMA channel */
	/* Do not access config register until channel shows as disabled */
	while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
381
		cpu_relax();
382

383 384
	/* Do not access config register until channel shows as inactive */
	val = readl(phychan->base + PL080_CH_CONFIG);
385
	while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
386
		val = readl(phychan->base + PL080_CH_CONFIG);
387

388
	writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG);
389 390 391
}

/*
392
 * Pause the channel by setting the HALT bit.
393
 *
394 395 396
 * For M->P transfers, pause the DMAC first and then stop the peripheral -
 * the FIFO can only drain if the peripheral is still requesting data.
 * (note: this can still timeout if the DMAC FIFO never drains of data.)
397
 *
398 399
 * For P->M transfers, disable the peripheral first to stop it filling
 * the DMAC FIFO, and then pause the DMAC.
400 401 402 403
 */
static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
{
	u32 val;
404
	int timeout;
405 406 407 408 409 410 411

	/* Set the HALT bit and wait for the FIFO to drain */
	val = readl(ch->base + PL080_CH_CONFIG);
	val |= PL080_CONFIG_HALT;
	writel(val, ch->base + PL080_CH_CONFIG);

	/* Wait for channel inactive */
412 413 414 415 416 417 418
	for (timeout = 1000; timeout; timeout--) {
		if (!pl08x_phy_channel_busy(ch))
			break;
		udelay(1);
	}
	if (pl08x_phy_channel_busy(ch))
		pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
419 420 421 422 423 424 425 426 427 428 429 430
}

static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
{
	u32 val;

	/* Clear the HALT bit */
	val = readl(ch->base + PL080_CH_CONFIG);
	val &= ~PL080_CONFIG_HALT;
	writel(val, ch->base + PL080_CH_CONFIG);
}

431 432 433 434 435 436 437 438
/*
 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
 * clears any pending interrupt status.  This should not be used for
 * an on-going transfer, but as a method of shutting down a channel
 * (eg, when it's no longer used) or terminating a transfer.
 */
static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
	struct pl08x_phy_chan *ch)
439
{
440
	u32 val = readl(ch->base + PL080_CH_CONFIG);
441

442 443
	val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
	         PL080_CONFIG_TC_IRQ_MASK);
444 445

	writel(val, ch->base + PL080_CH_CONFIG);
446 447 448

	writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
	writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
}

static inline u32 get_bytes_in_cctl(u32 cctl)
{
	/* The source width defines the number of bytes */
	u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;

	switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
	case PL080_WIDTH_8BIT:
		break;
	case PL080_WIDTH_16BIT:
		bytes *= 2;
		break;
	case PL080_WIDTH_32BIT:
		bytes *= 4;
		break;
	}
	return bytes;
}

/* The channel should be paused when calling this */
static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
{
	struct pl08x_phy_chan *ch;
	struct pl08x_txd *txd;
474
	size_t bytes = 0;
475 476 477 478 479

	ch = plchan->phychan;
	txd = plchan->at;

	/*
480 481
	 * Follow the LLIs to get the number of remaining
	 * bytes in the currently active transaction.
482 483
	 */
	if (ch && txd) {
484
		u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
485

486
		/* First get the remaining bytes in the active transfer */
487 488 489
		bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));

		if (clli) {
490 491 492 493 494 495
			struct pl08x_lli *llis_va = txd->llis_va;
			dma_addr_t llis_bus = txd->llis_bus;
			int index;

			BUG_ON(clli < llis_bus || clli >= llis_bus +
				sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS);
496

497 498 499 500 501 502 503 504
			/*
			 * Locate the next LLI - as this is an array,
			 * it's simple maths to find.
			 */
			index = (clli - llis_bus) / sizeof(struct pl08x_lli);

			for (; index < MAX_NUM_TSFR_LLIS; index++) {
				bytes += get_bytes_in_cctl(llis_va[index].cctl);
505 506

				/*
507
				 * A LLI pointer of 0 terminates the LLI list
508
				 */
509 510
				if (!llis_va[index].lli)
					break;
511 512 513 514 515 516 517 518 519
			}
		}
	}

	return bytes;
}

/*
 * Allocate a physical channel for a virtual channel
520 521 522 523
 *
 * Try to locate a physical channel to be used for this transfer. If all
 * are taken return NULL and the requester will have to cope by using
 * some fallback PIO mode or retrying later.
524 525 526 527 528 529 530 531 532 533 534 535 536 537
 */
static struct pl08x_phy_chan *
pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
		      struct pl08x_dma_chan *virt_chan)
{
	struct pl08x_phy_chan *ch = NULL;
	unsigned long flags;
	int i;

	for (i = 0; i < pl08x->vd->channels; i++) {
		ch = &pl08x->phy_chans[i];

		spin_lock_irqsave(&ch->lock, flags);

538
		if (!ch->locked && !ch->serving) {
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
			ch->serving = virt_chan;
			spin_unlock_irqrestore(&ch->lock, flags);
			break;
		}

		spin_unlock_irqrestore(&ch->lock, flags);
	}

	if (i == pl08x->vd->channels) {
		/* No physical channel available, cope with it */
		return NULL;
	}

	return ch;
}

555
/* Mark the physical channel as free.  Note, this write is atomic. */
556 557 558
static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
					 struct pl08x_phy_chan *ch)
{
559 560
	ch->serving = NULL;
}
561

562 563 564 565 566 567 568 569 570
/*
 * Try to allocate a physical channel.  When successful, assign it to
 * this virtual channel, and initiate the next descriptor.  The
 * virtual channel lock must be held at this point.
 */
static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
{
	struct pl08x_driver_data *pl08x = plchan->host;
	struct pl08x_phy_chan *ch;
571

572 573 574 575 576 577
	ch = pl08x_get_phy_channel(pl08x, plchan);
	if (!ch) {
		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
		plchan->state = PL08X_CHAN_WAITING;
		return;
	}
578

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
		ch->id, plchan->name);

	plchan->phychan = ch;
	plchan->state = PL08X_CHAN_RUNNING;
	pl08x_start_next_txd(plchan);
}

static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
	struct pl08x_dma_chan *plchan)
{
	struct pl08x_driver_data *pl08x = plchan->host;

	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
		ch->id, plchan->name);

	/*
	 * We do this without taking the lock; we're really only concerned
	 * about whether this pointer is NULL or not, and we're guaranteed
	 * that this will only be called when it _already_ is non-NULL.
	 */
	ch->serving = plchan;
	plchan->phychan = ch;
	plchan->state = PL08X_CHAN_RUNNING;
	pl08x_start_next_txd(plchan);
}

/*
 * Free a physical DMA channel, potentially reallocating it to another
 * virtual channel if we have any pending.
 */
static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
{
	struct pl08x_driver_data *pl08x = plchan->host;
	struct pl08x_dma_chan *p, *next;

 retry:
	next = NULL;

	/* Find a waiting virtual channel for the next transfer. */
619
	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
620 621 622 623 624 625
		if (p->state == PL08X_CHAN_WAITING) {
			next = p;
			break;
		}

	if (!next) {
626
		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
			if (p->state == PL08X_CHAN_WAITING) {
				next = p;
				break;
			}
	}

	/* Ensure that the physical channel is stopped */
	pl08x_terminate_phy_chan(pl08x, plchan->phychan);

	if (next) {
		bool success;

		/*
		 * Eww.  We know this isn't going to deadlock
		 * but lockdep probably doesn't.
		 */
643
		spin_lock(&next->vc.lock);
644 645 646 647
		/* Re-check the state now that we have the lock */
		success = next->state == PL08X_CHAN_WAITING;
		if (success)
			pl08x_phy_reassign_start(plchan->phychan, next);
648
		spin_unlock(&next->vc.lock);
649 650 651 652 653 654 655 656 657 658 659

		/* If the state changed, try to find another channel */
		if (!success)
			goto retry;
	} else {
		/* No more jobs, so free up the physical channel */
		pl08x_put_phy_channel(pl08x, plchan->phychan);
	}

	plchan->phychan = NULL;
	plchan->state = PL08X_CHAN_IDLE;
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
}

/*
 * LLI handling
 */

static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
{
	switch (coded) {
	case PL080_WIDTH_8BIT:
		return 1;
	case PL080_WIDTH_16BIT:
		return 2;
	case PL080_WIDTH_32BIT:
		return 4;
	default:
		break;
	}
	BUG();
	return 0;
}

static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
683
				  size_t tsize)
684 685 686
{
	u32 retbits = cctl;

687
	/* Remove all src, dst and transfer size bits */
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
	retbits &= ~PL080_CONTROL_DWIDTH_MASK;
	retbits &= ~PL080_CONTROL_SWIDTH_MASK;
	retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;

	/* Then set the bits according to the parameters */
	switch (srcwidth) {
	case 1:
		retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT;
		break;
	case 2:
		retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT;
		break;
	case 4:
		retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT;
		break;
	default:
		BUG();
		break;
	}

	switch (dstwidth) {
	case 1:
		retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
		break;
	case 2:
		retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
		break;
	case 4:
		retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
		break;
	default:
		BUG();
		break;
	}

	retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
	return retbits;
}

727 728 729 730 731
struct pl08x_lli_build_data {
	struct pl08x_txd *txd;
	struct pl08x_bus_data srcbus;
	struct pl08x_bus_data dstbus;
	size_t remainder;
732
	u32 lli_bus;
733 734
};

735
/*
736 737 738 739 740 741
 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
 * victim in case src & dest are not similarly aligned. i.e. If after aligning
 * masters address with width requirements of transfer (by sending few byte by
 * byte data), slave is still not aligned, then its width will be reduced to
 * BYTE.
 * - prefers the destination bus if both available
742
 * - prefers bus with fixed address (i.e. peripheral)
743
 */
744 745
static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
	struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
746 747
{
	if (!(cctl & PL080_CONTROL_DST_INCR)) {
748 749
		*mbus = &bd->dstbus;
		*sbus = &bd->srcbus;
750 751 752
	} else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
		*mbus = &bd->srcbus;
		*sbus = &bd->dstbus;
753
	} else {
754
		if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
755 756
			*mbus = &bd->dstbus;
			*sbus = &bd->srcbus;
757
		} else {
758 759
			*mbus = &bd->srcbus;
			*sbus = &bd->dstbus;
760 761 762 763 764
		}
	}
}

/*
765
 * Fills in one LLI for a certain transfer descriptor and advance the counter
766
 */
767 768
static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd,
	int num_llis, int len, u32 cctl)
769
{
770 771
	struct pl08x_lli *llis_va = bd->txd->llis_va;
	dma_addr_t llis_bus = bd->txd->llis_bus;
772 773 774

	BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);

775
	llis_va[num_llis].cctl = cctl;
776 777
	llis_va[num_llis].src = bd->srcbus.addr;
	llis_va[num_llis].dst = bd->dstbus.addr;
778 779
	llis_va[num_llis].lli = llis_bus + (num_llis + 1) *
		sizeof(struct pl08x_lli);
780
	llis_va[num_llis].lli |= bd->lli_bus;
781 782

	if (cctl & PL080_CONTROL_SRC_INCR)
783
		bd->srcbus.addr += len;
784
	if (cctl & PL080_CONTROL_DST_INCR)
785
		bd->dstbus.addr += len;
786

787
	BUG_ON(bd->remainder < len);
788

789
	bd->remainder -= len;
790 791
}

792 793
static inline void prep_byte_width_lli(struct pl08x_lli_build_data *bd,
		u32 *cctl, u32 len, int num_llis, size_t *total_bytes)
794
{
795 796 797
	*cctl = pl08x_cctl_bits(*cctl, 1, 1, len);
	pl08x_fill_lli_for_desc(bd, num_llis, len, *cctl);
	(*total_bytes) += len;
798 799 800 801 802 803 804 805 806 807 808
}

/*
 * This fills in the table of LLIs for the transfer descriptor
 * Note that we assume we never have to change the burst sizes
 * Return 0 for error
 */
static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
			      struct pl08x_txd *txd)
{
	struct pl08x_bus_data *mbus, *sbus;
809
	struct pl08x_lli_build_data bd;
810
	int num_llis = 0;
811
	u32 cctl, early_bytes = 0;
812
	size_t max_bytes_per_lli, total_bytes;
813
	struct pl08x_lli *llis_va;
814
	struct pl08x_sg *dsg;
815

816
	txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
817 818 819 820 821
	if (!txd->llis_va) {
		dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
		return 0;
	}

822
	bd.txd = txd;
823
	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
824
	cctl = txd->cctl;
825

826
	/* Find maximum width of the source bus */
827
	bd.srcbus.maxwidth =
828 829 830 831
		pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
				       PL080_CONTROL_SWIDTH_SHIFT);

	/* Find maximum width of the destination bus */
832
	bd.dstbus.maxwidth =
833 834 835
		pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
				       PL080_CONTROL_DWIDTH_SHIFT);

836 837 838
	list_for_each_entry(dsg, &txd->dsg_list, node) {
		total_bytes = 0;
		cctl = txd->cctl;
839

840 841 842 843 844
		bd.srcbus.addr = dsg->src_addr;
		bd.dstbus.addr = dsg->dst_addr;
		bd.remainder = dsg->len;
		bd.srcbus.buswidth = bd.srcbus.maxwidth;
		bd.dstbus.buswidth = bd.dstbus.maxwidth;
845

846
		pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
847

848 849 850 851 852 853 854 855 856
		dev_vdbg(&pl08x->adev->dev, "src=0x%08x%s/%u dst=0x%08x%s/%u len=%zu\n",
			bd.srcbus.addr, cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
			bd.srcbus.buswidth,
			bd.dstbus.addr, cctl & PL080_CONTROL_DST_INCR ? "+" : "",
			bd.dstbus.buswidth,
			bd.remainder);
		dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
			mbus == &bd.srcbus ? "src" : "dst",
			sbus == &bd.srcbus ? "src" : "dst");
857

858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
		/*
		 * Zero length is only allowed if all these requirements are
		 * met:
		 * - flow controller is peripheral.
		 * - src.addr is aligned to src.width
		 * - dst.addr is aligned to dst.width
		 *
		 * sg_len == 1 should be true, as there can be two cases here:
		 *
		 * - Memory addresses are contiguous and are not scattered.
		 *   Here, Only one sg will be passed by user driver, with
		 *   memory address and zero length. We pass this to controller
		 *   and after the transfer it will receive the last burst
		 *   request from peripheral and so transfer finishes.
		 *
		 * - Memory addresses are scattered and are not contiguous.
		 *   Here, Obviously as DMA controller doesn't know when a lli's
		 *   transfer gets over, it can't load next lli. So in this
		 *   case, there has to be an assumption that only one lli is
		 *   supported. Thus, we can't have scattered addresses.
		 */
		if (!bd.remainder) {
			u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
				PL080_CONFIG_FLOW_CONTROL_SHIFT;
			if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
883
					(fc <= PL080_FLOW_SRC2DST_SRC))) {
884 885 886 887
				dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
					__func__);
				return 0;
			}
888

889
			if ((bd.srcbus.addr % bd.srcbus.buswidth) ||
890
					(bd.dstbus.addr % bd.dstbus.buswidth)) {
891 892 893 894 895 896
				dev_err(&pl08x->adev->dev,
					"%s src & dst address must be aligned to src"
					" & dst width if peripheral is flow controller",
					__func__);
				return 0;
			}
897

898 899 900 901 902
			cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
					bd.dstbus.buswidth, 0);
			pl08x_fill_lli_for_desc(&bd, num_llis++, 0, cctl);
			break;
		}
903 904

		/*
905 906 907
		 * Send byte by byte for following cases
		 * - Less than a bus width available
		 * - until master bus is aligned
908
		 */
909 910 911 912 913 914 915 916
		if (bd.remainder < mbus->buswidth)
			early_bytes = bd.remainder;
		else if ((mbus->addr) % (mbus->buswidth)) {
			early_bytes = mbus->buswidth - (mbus->addr) %
				(mbus->buswidth);
			if ((bd.remainder - early_bytes) < mbus->buswidth)
				early_bytes = bd.remainder;
		}
917

918 919 920 921 922 923
		if (early_bytes) {
			dev_vdbg(&pl08x->adev->dev,
				"%s byte width LLIs (remain 0x%08x)\n",
				__func__, bd.remainder);
			prep_byte_width_lli(&bd, &cctl, early_bytes, num_llis++,
				&total_bytes);
924 925
		}

926 927 928 929 930 931 932 933 934
		if (bd.remainder) {
			/*
			 * Master now aligned
			 * - if slave is not then we must set its width down
			 */
			if (sbus->addr % sbus->buswidth) {
				dev_dbg(&pl08x->adev->dev,
					"%s set down bus width to one byte\n",
					__func__);
935

936 937
				sbus->buswidth = 1;
			}
938 939

			/*
940 941
			 * Bytes transferred = tsize * src width, not
			 * MIN(buswidths)
942
			 */
943 944 945 946 947
			max_bytes_per_lli = bd.srcbus.buswidth *
				PL080_CONTROL_TRANSFER_SIZE_MASK;
			dev_vdbg(&pl08x->adev->dev,
				"%s max bytes per lli = %zu\n",
				__func__, max_bytes_per_lli);
948 949

			/*
950 951
			 * Make largest possible LLIs until less than one bus
			 * width left
952
			 */
953 954
			while (bd.remainder > (mbus->buswidth - 1)) {
				size_t lli_len, tsize, width;
955

956 957 958 959 960
				/*
				 * If enough left try to send max possible,
				 * otherwise try to send the remainder
				 */
				lli_len = min(bd.remainder, max_bytes_per_lli);
961

962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
				/*
				 * Check against maximum bus alignment:
				 * Calculate actual transfer size in relation to
				 * bus width an get a maximum remainder of the
				 * highest bus width - 1
				 */
				width = max(mbus->buswidth, sbus->buswidth);
				lli_len = (lli_len / width) * width;
				tsize = lli_len / bd.srcbus.buswidth;

				dev_vdbg(&pl08x->adev->dev,
					"%s fill lli with single lli chunk of "
					"size 0x%08zx (remainder 0x%08zx)\n",
					__func__, lli_len, bd.remainder);

				cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
978
					bd.dstbus.buswidth, tsize);
979 980 981 982
				pl08x_fill_lli_for_desc(&bd, num_llis++,
						lli_len, cctl);
				total_bytes += lli_len;
			}
983

984 985 986 987 988 989 990 991 992 993
			/*
			 * Send any odd bytes
			 */
			if (bd.remainder) {
				dev_vdbg(&pl08x->adev->dev,
					"%s align with boundary, send odd bytes (remain %zu)\n",
					__func__, bd.remainder);
				prep_byte_width_lli(&bd, &cctl, bd.remainder,
						num_llis++, &total_bytes);
			}
994
		}
995

996 997 998 999 1000 1001
		if (total_bytes != dsg->len) {
			dev_err(&pl08x->adev->dev,
				"%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
				__func__, total_bytes, dsg->len);
			return 0;
		}
1002

1003 1004 1005 1006 1007 1008
		if (num_llis >= MAX_NUM_TSFR_LLIS) {
			dev_err(&pl08x->adev->dev,
				"%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
				__func__, (u32) MAX_NUM_TSFR_LLIS);
			return 0;
		}
1009
	}
1010 1011

	llis_va = txd->llis_va;
1012
	/* The final LLI terminates the LLI. */
1013
	llis_va[num_llis - 1].lli = 0;
1014
	/* The final LLI element shall also fire an interrupt. */
1015
	llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
1016 1017 1018 1019 1020

#ifdef VERBOSE_DEBUG
	{
		int i;

1021 1022 1023
		dev_vdbg(&pl08x->adev->dev,
			 "%-3s %-9s  %-10s %-10s %-10s %s\n",
			 "lli", "", "csrc", "cdst", "clli", "cctl");
1024 1025
		for (i = 0; i < num_llis; i++) {
			dev_vdbg(&pl08x->adev->dev,
1026 1027 1028
				 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
				 i, &llis_va[i], llis_va[i].src,
				 llis_va[i].dst, llis_va[i].lli, llis_va[i].cctl
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
				);
		}
	}
#endif

	return num_llis;
}

static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
			   struct pl08x_txd *txd)
{
1040 1041
	struct pl08x_sg *dsg, *_dsg;

1042 1043
	if (txd->llis_va)
		dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
1044

1045 1046 1047 1048 1049
	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
		list_del(&dsg->node);
		kfree(dsg);
	}

1050 1051 1052
	kfree(txd);
}

1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
static void pl08x_unmap_buffers(struct pl08x_txd *txd)
{
	struct device *dev = txd->vd.tx.chan->device->dev;
	struct pl08x_sg *dsg;

	if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
		if (txd->vd.tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
			list_for_each_entry(dsg, &txd->dsg_list, node)
				dma_unmap_single(dev, dsg->src_addr, dsg->len,
						DMA_TO_DEVICE);
		else {
			list_for_each_entry(dsg, &txd->dsg_list, node)
				dma_unmap_page(dev, dsg->src_addr, dsg->len,
						DMA_TO_DEVICE);
		}
	}
	if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
		if (txd->vd.tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
			list_for_each_entry(dsg, &txd->dsg_list, node)
				dma_unmap_single(dev, dsg->dst_addr, dsg->len,
						DMA_FROM_DEVICE);
		else
			list_for_each_entry(dsg, &txd->dsg_list, node)
				dma_unmap_page(dev, dsg->dst_addr, dsg->len,
						DMA_FROM_DEVICE);
	}
}

static void pl08x_desc_free(struct virt_dma_desc *vd)
{
	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
	struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);

	if (!plchan->slave)
		pl08x_unmap_buffers(txd);

	if (!txd->done)
		pl08x_release_mux(plchan);

	pl08x_free_txd(plchan->host, txd);
}

1095 1096 1097
static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
				struct pl08x_dma_chan *plchan)
{
1098
	LIST_HEAD(head);
1099

1100
	vchan_get_all_descriptors(&plchan->vc, &head);
1101
	vchan_dma_desc_free_list(&plchan->vc, &head);
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
}

/*
 * The DMA ENGINE API
 */
static int pl08x_alloc_chan_resources(struct dma_chan *chan)
{
	return 0;
}

static void pl08x_free_chan_resources(struct dma_chan *chan)
{
1114 1115
	/* Ensure all queued descriptors are freed */
	vchan_free_chan_resources(to_virt_chan(chan));
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
}

static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
		struct dma_chan *chan, unsigned long flags)
{
	struct dma_async_tx_descriptor *retval = NULL;

	return retval;
}

/*
1127 1128 1129
 * Code accessing dma_async_is_complete() in a tight loop may give problems.
 * If slaves are relying on interrupts to signal completion this function
 * must not be called with interrupts disabled.
1130
 */
1131 1132
static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
		dma_cookie_t cookie, struct dma_tx_state *txstate)
1133 1134
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1135 1136
	struct virt_dma_desc *vd;
	unsigned long flags;
1137
	enum dma_status ret;
1138
	size_t bytes = 0;
1139

1140 1141
	ret = dma_cookie_status(chan, cookie, txstate);
	if (ret == DMA_SUCCESS)
1142 1143
		return ret;

1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
	/*
	 * There's no point calculating the residue if there's
	 * no txstate to store the value.
	 */
	if (!txstate) {
		if (plchan->state == PL08X_CHAN_PAUSED)
			ret = DMA_PAUSED;
		return ret;
	}

	spin_lock_irqsave(&plchan->vc.lock, flags);
	ret = dma_cookie_status(chan, cookie, txstate);
	if (ret != DMA_SUCCESS) {
		vd = vchan_find_desc(&plchan->vc, cookie);
		if (vd) {
			/* On the issued list, so hasn't been processed yet */
			struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
			struct pl08x_sg *dsg;

			list_for_each_entry(dsg, &txd->dsg_list, node)
				bytes += dsg->len;
		} else {
			bytes = pl08x_getbytes_chan(plchan);
		}
	}
	spin_unlock_irqrestore(&plchan->vc.lock, flags);

1171 1172
	/*
	 * This cookie not complete yet
1173
	 * Get number of bytes left in the active transactions and queue
1174
	 */
1175
	dma_set_residue(txstate, bytes);
1176

1177 1178
	if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
		ret = DMA_PAUSED;
1179 1180

	/* Whether waiting or running, we're in progress */
1181
	return ret;
1182 1183 1184 1185
}

/* PrimeCell DMA extension */
struct burst_table {
1186
	u32 burstwords;
1187 1188 1189 1190 1191 1192
	u32 reg;
};

static const struct burst_table burst_sizes[] = {
	{
		.burstwords = 256,
1193
		.reg = PL080_BSIZE_256,
1194 1195 1196
	},
	{
		.burstwords = 128,
1197
		.reg = PL080_BSIZE_128,
1198 1199 1200
	},
	{
		.burstwords = 64,
1201
		.reg = PL080_BSIZE_64,
1202 1203 1204
	},
	{
		.burstwords = 32,
1205
		.reg = PL080_BSIZE_32,
1206 1207 1208
	},
	{
		.burstwords = 16,
1209
		.reg = PL080_BSIZE_16,
1210 1211 1212
	},
	{
		.burstwords = 8,
1213
		.reg = PL080_BSIZE_8,
1214 1215 1216
	},
	{
		.burstwords = 4,
1217
		.reg = PL080_BSIZE_4,
1218 1219
	},
	{
1220 1221
		.burstwords = 0,
		.reg = PL080_BSIZE_1,
1222 1223 1224
	},
};

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
/*
 * Given the source and destination available bus masks, select which
 * will be routed to each port.  We try to have source and destination
 * on separate ports, but always respect the allowable settings.
 */
static u32 pl08x_select_bus(u8 src, u8 dst)
{
	u32 cctl = 0;

	if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
		cctl |= PL080_CONTROL_DST_AHB2;
	if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
		cctl |= PL080_CONTROL_SRC_AHB2;

	return cctl;
}

1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
static u32 pl08x_cctl(u32 cctl)
{
	cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
		  PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
		  PL080_CONTROL_PROT_MASK);

	/* Access the cell in privileged mode, non-bufferable, non-cacheable */
	return cctl | PL080_CONTROL_PROT_SYS;
}

1252 1253 1254 1255 1256 1257 1258 1259 1260
static u32 pl08x_width(enum dma_slave_buswidth width)
{
	switch (width) {
	case DMA_SLAVE_BUSWIDTH_1_BYTE:
		return PL080_WIDTH_8BIT;
	case DMA_SLAVE_BUSWIDTH_2_BYTES:
		return PL080_WIDTH_16BIT;
	case DMA_SLAVE_BUSWIDTH_4_BYTES:
		return PL080_WIDTH_32BIT;
1261 1262
	default:
		return ~0;
1263 1264 1265
	}
}

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
static u32 pl08x_burst(u32 maxburst)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
		if (burst_sizes[i].burstwords <= maxburst)
			break;

	return burst_sizes[i].reg;
}

1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
	enum dma_slave_buswidth addr_width, u32 maxburst)
{
	u32 width, burst, cctl = 0;

	width = pl08x_width(addr_width);
	if (width == ~0)
		return ~0;

	cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
	cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;

	/*
	 * If this channel will only request single transfers, set this
	 * down to ONE element.  Also select one element if no maxburst
	 * is specified.
	 */
	if (plchan->cd->single)
		maxburst = 1;

	burst = pl08x_burst(maxburst);
	cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
	cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;

	return pl08x_cctl(cctl);
}

1304 1305
static int dma_set_runtime_config(struct dma_chan *chan,
				  struct dma_slave_config *config)
1306 1307
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1308 1309 1310

	if (!plchan->slave)
		return -EINVAL;
1311

1312 1313 1314
	/* Reject definitely invalid configurations */
	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1315
		return -EINVAL;
1316

1317 1318
	plchan->cfg = *config;

1319
	return 0;
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
}

/*
 * Slave transactions callback to the slave device to allow
 * synchronization of slave DMA signals with the DMAC enable
 */
static void pl08x_issue_pending(struct dma_chan *chan)
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
	unsigned long flags;

1331
	spin_lock_irqsave(&plchan->vc.lock, flags);
1332
	if (vchan_issue_pending(&plchan->vc)) {
1333 1334
		if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
			pl08x_phy_alloc_and_start(plchan);
1335
	}
1336
	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1337 1338
}

1339
static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
1340
{
1341
	struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
1342 1343

	if (txd) {
1344
		INIT_LIST_HEAD(&txd->dsg_list);
1345 1346 1347 1348

		/* Always enable error and terminal interrupts */
		txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
			    PL080_CONFIG_TC_IRQ_MASK;
1349 1350 1351 1352
	}
	return txd;
}

1353 1354 1355 1356 1357 1358 1359 1360 1361 1362
/*
 * Initialize a descriptor to be used by memcpy submit
 */
static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
		size_t len, unsigned long flags)
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
	struct pl08x_driver_data *pl08x = plchan->host;
	struct pl08x_txd *txd;
1363
	struct pl08x_sg *dsg;
1364 1365
	int ret;

1366
	txd = pl08x_get_txd(plchan);
1367 1368 1369 1370 1371 1372
	if (!txd) {
		dev_err(&pl08x->adev->dev,
			"%s no memory for descriptor\n", __func__);
		return NULL;
	}

1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
	if (!dsg) {
		pl08x_free_txd(pl08x, txd);
		dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",
				__func__);
		return NULL;
	}
	list_add_tail(&dsg->node, &txd->dsg_list);

	dsg->src_addr = src;
	dsg->dst_addr = dest;
	dsg->len = len;
1385 1386

	/* Set platform data for m2m */
1387
	txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1388
	txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy &
1389
			~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
1390

1391
	/* Both to be incremented or the code will break */
1392
	txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1393 1394

	if (pl08x->vd->dualmaster)
1395 1396
		txd->cctl |= pl08x_select_bus(pl08x->mem_buses,
					      pl08x->mem_buses);
1397

1398 1399 1400
	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
	if (!ret) {
		pl08x_free_txd(pl08x, txd);
1401
		return NULL;
1402
	}
1403

1404
	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1405 1406
}

1407
static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1408
		struct dma_chan *chan, struct scatterlist *sgl,
1409
		unsigned int sg_len, enum dma_transfer_direction direction,
1410
		unsigned long flags, void *context)
1411 1412 1413 1414
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
	struct pl08x_driver_data *pl08x = plchan->host;
	struct pl08x_txd *txd;
1415 1416
	struct pl08x_sg *dsg;
	struct scatterlist *sg;
1417
	enum dma_slave_buswidth addr_width;
1418
	dma_addr_t slave_addr;
1419
	int ret, tmp;
1420
	u8 src_buses, dst_buses;
1421
	u32 maxburst, cctl;
1422 1423

	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1424
			__func__, sg_dma_len(sgl), plchan->name);
1425

1426
	txd = pl08x_get_txd(plchan);
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
	if (!txd) {
		dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
		return NULL;
	}

	/*
	 * Set up addresses, the PrimeCell configured address
	 * will take precedence since this may configure the
	 * channel target address dynamically at runtime.
	 */
1437
	if (direction == DMA_MEM_TO_DEV) {
1438
		cctl = PL080_CONTROL_SRC_INCR;
1439
		slave_addr = plchan->cfg.dst_addr;
1440 1441
		addr_width = plchan->cfg.dst_addr_width;
		maxburst = plchan->cfg.dst_maxburst;
1442 1443
		src_buses = pl08x->mem_buses;
		dst_buses = plchan->cd->periph_buses;
1444
	} else if (direction == DMA_DEV_TO_MEM) {
1445
		cctl = PL080_CONTROL_DST_INCR;
1446
		slave_addr = plchan->cfg.src_addr;
1447 1448
		addr_width = plchan->cfg.src_addr_width;
		maxburst = plchan->cfg.src_maxburst;
1449 1450
		src_buses = plchan->cd->periph_buses;
		dst_buses = pl08x->mem_buses;
1451
	} else {
1452
		pl08x_free_txd(pl08x, txd);
1453 1454 1455 1456 1457
		dev_err(&pl08x->adev->dev,
			"%s direction unsupported\n", __func__);
		return NULL;
	}

1458
	cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
1459 1460 1461 1462 1463 1464 1465
	if (cctl == ~0) {
		pl08x_free_txd(pl08x, txd);
		dev_err(&pl08x->adev->dev,
			"DMA slave configuration botched?\n");
		return NULL;
	}

1466 1467
	txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses);

1468
	if (plchan->cfg.device_fc)
1469
		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
1470 1471
			PL080_FLOW_PER2MEM_PER;
	else
1472
		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
1473 1474 1475 1476
			PL080_FLOW_PER2MEM;

	txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;

1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
	ret = pl08x_request_mux(plchan);
	if (ret < 0) {
		pl08x_free_txd(pl08x, txd);
		dev_dbg(&pl08x->adev->dev,
			"unable to mux for transfer on %s due to platform restrictions\n",
			plchan->name);
		return NULL;
	}

	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
		 plchan->signal, plchan->name);

	/* Assign the flow control signal to this channel */
	if (direction == DMA_MEM_TO_DEV)
		txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
	else
		txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;

1495 1496 1497
	for_each_sg(sgl, sg, sg_len, tmp) {
		dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
		if (!dsg) {
1498
			pl08x_release_mux(plchan);
1499 1500 1501 1502 1503 1504 1505 1506
			pl08x_free_txd(pl08x, txd);
			dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
					__func__);
			return NULL;
		}
		list_add_tail(&dsg->node, &txd->dsg_list);

		dsg->len = sg_dma_len(sg);
1507
		if (direction == DMA_MEM_TO_DEV) {
1508
			dsg->src_addr = sg_dma_address(sg);
1509 1510 1511
			dsg->dst_addr = slave_addr;
		} else {
			dsg->src_addr = slave_addr;
1512
			dsg->dst_addr = sg_dma_address(sg);
1513 1514 1515
		}
	}

1516 1517 1518 1519
	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
	if (!ret) {
		pl08x_release_mux(plchan);
		pl08x_free_txd(pl08x, txd);
1520
		return NULL;
1521
	}
1522

1523
	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
}

static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
			 unsigned long arg)
{
	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
	struct pl08x_driver_data *pl08x = plchan->host;
	unsigned long flags;
	int ret = 0;

	/* Controls applicable to inactive channels */
	if (cmd == DMA_SLAVE_CONFIG) {
1536 1537
		return dma_set_runtime_config(chan,
					      (struct dma_slave_config *)arg);
1538 1539 1540 1541 1542 1543
	}

	/*
	 * Anything succeeds on channels with no physical allocation and
	 * no queued transfers.
	 */
1544
	spin_lock_irqsave(&plchan->vc.lock, flags);
1545
	if (!plchan->phychan && !plchan->at) {
1546
		spin_unlock_irqrestore(&plchan->vc.lock, flags);
1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558
		return 0;
	}

	switch (cmd) {
	case DMA_TERMINATE_ALL:
		plchan->state = PL08X_CHAN_IDLE;

		if (plchan->phychan) {
			/*
			 * Mark physical channel as free and free any slave
			 * signal
			 */
1559
			pl08x_phy_free(plchan);
1560 1561 1562
		}
		/* Dequeue jobs and free LLIs */
		if (plchan->at) {
1563
			pl08x_desc_free(&plchan->at->vd);
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
			plchan->at = NULL;
		}
		/* Dequeue jobs not yet fired as well */
		pl08x_free_txd_list(pl08x, plchan);
		break;
	case DMA_PAUSE:
		pl08x_pause_phy_chan(plchan->phychan);
		plchan->state = PL08X_CHAN_PAUSED;
		break;
	case DMA_RESUME:
		pl08x_resume_phy_chan(plchan->phychan);
		plchan->state = PL08X_CHAN_RUNNING;
		break;
	default:
		/* Unknown command */
		ret = -ENXIO;
		break;
	}

1583
	spin_unlock_irqrestore(&plchan->vc.lock, flags);
1584 1585 1586 1587 1588 1589

	return ret;
}

bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
{
1590
	struct pl08x_dma_chan *plchan;
1591 1592
	char *name = chan_id;

1593 1594 1595 1596 1597 1598
	/* Reject channels for devices not bound to this driver */
	if (chan->device->dev->driver != &pl08x_amba_driver.drv)
		return false;

	plchan = to_pl08x_chan(chan);

1599 1600 1601 1602 1603 1604 1605 1606 1607
	/* Check that the channel is not taken! */
	if (!strcmp(plchan->name, name))
		return true;

	return false;
}

/*
 * Just check that the device is there and active
1608 1609 1610
 * TODO: turn this bit on/off depending on the number of physical channels
 * actually used, if it is zero... well shut it off. That will save some
 * power. Cut the clock at the same time.
1611 1612 1613
 */
static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
{
1614 1615 1616
	/* The Nomadik variant does not have the config register */
	if (pl08x->vd->nomadik)
		return;
1617
	writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
1618 1619 1620 1621 1622
}

static irqreturn_t pl08x_irq(int irq, void *dev)
{
	struct pl08x_driver_data *pl08x = dev;
1623 1624 1625 1626 1627 1628 1629 1630
	u32 mask = 0, err, tc, i;

	/* check & clear - ERR & TC interrupts */
	err = readl(pl08x->base + PL080_ERR_STATUS);
	if (err) {
		dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
			__func__, err);
		writel(err, pl08x->base + PL080_ERR_CLEAR);
1631
	}
1632
	tc = readl(pl08x->base + PL080_TC_STATUS);
1633 1634 1635 1636 1637 1638
	if (tc)
		writel(tc, pl08x->base + PL080_TC_CLEAR);

	if (!err && !tc)
		return IRQ_NONE;

1639
	for (i = 0; i < pl08x->vd->channels; i++) {
1640
		if (((1 << i) & err) || ((1 << i) & tc)) {
1641 1642 1643
			/* Locate physical channel */
			struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
			struct pl08x_dma_chan *plchan = phychan->serving;
1644
			struct pl08x_txd *tx;
1645

1646 1647 1648 1649 1650 1651 1652
			if (!plchan) {
				dev_err(&pl08x->adev->dev,
					"%s Error TC interrupt on unused channel: 0x%08x\n",
					__func__, i);
				continue;
			}

1653
			spin_lock(&plchan->vc.lock);
1654 1655 1656
			tx = plchan->at;
			if (tx) {
				plchan->at = NULL;
1657 1658 1659 1660 1661
				/*
				 * This descriptor is done, release its mux
				 * reservation.
				 */
				pl08x_release_mux(plchan);
1662 1663
				tx->done = true;
				vchan_cookie_complete(&tx->vd);
1664

1665 1666 1667 1668
				/*
				 * And start the next descriptor (if any),
				 * otherwise free this channel.
				 */
1669
				if (vchan_next_desc(&plchan->vc))
1670
					pl08x_start_next_txd(plchan);
1671 1672
				else
					pl08x_phy_free(plchan);
1673
			}
1674
			spin_unlock(&plchan->vc.lock);
1675

1676 1677 1678 1679 1680 1681 1682
			mask |= (1 << i);
		}
	}

	return mask ? IRQ_HANDLED : IRQ_NONE;
}

1683 1684 1685 1686
static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
{
	chan->slave = true;
	chan->name = chan->cd->bus_id;
1687 1688
	chan->cfg.src_addr = chan->cd->addr;
	chan->cfg.dst_addr = chan->cd->addr;
1689 1690
}

1691 1692 1693 1694 1695
/*
 * Initialise the DMAC memcpy/slave channels.
 * Make a local wrapper to hold required data
 */
static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
1696
		struct dma_device *dmadev, unsigned int channels, bool slave)
1697 1698 1699 1700 1701
{
	struct pl08x_dma_chan *chan;
	int i;

	INIT_LIST_HEAD(&dmadev->channels);
1702

1703 1704 1705 1706 1707 1708
	/*
	 * Register as many many memcpy as we have physical channels,
	 * we won't always be able to use all but the code will have
	 * to cope with that situation.
	 */
	for (i = 0; i < channels; i++) {
1709
		chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1710 1711 1712 1713 1714 1715 1716 1717
		if (!chan) {
			dev_err(&pl08x->adev->dev,
				"%s no memory for channel\n", __func__);
			return -ENOMEM;
		}

		chan->host = pl08x;
		chan->state = PL08X_CHAN_IDLE;
1718
		chan->signal = -1;
1719 1720 1721

		if (slave) {
			chan->cd = &pl08x->pd->slave_channels[i];
1722
			pl08x_dma_slave_init(chan);
1723 1724 1725 1726 1727 1728 1729 1730
		} else {
			chan->cd = &pl08x->pd->memcpy_channel;
			chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
			if (!chan->name) {
				kfree(chan);
				return -ENOMEM;
			}
		}
1731
		dev_dbg(&pl08x->adev->dev,
1732 1733 1734
			 "initialize virtual channel \"%s\"\n",
			 chan->name);

1735
		chan->vc.desc_free = pl08x_desc_free;
1736
		vchan_init(&chan->vc, dmadev);
1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
	}
	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
		 i, slave ? "slave" : "memcpy");
	return i;
}

static void pl08x_free_virtual_channels(struct dma_device *dmadev)
{
	struct pl08x_dma_chan *chan = NULL;
	struct pl08x_dma_chan *next;

	list_for_each_entry_safe(chan,
1749 1750
				 next, &dmadev->channels, vc.chan.device_node) {
		list_del(&chan->vc.chan.device_node);
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791
		kfree(chan);
	}
}

#ifdef CONFIG_DEBUG_FS
static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
{
	switch (state) {
	case PL08X_CHAN_IDLE:
		return "idle";
	case PL08X_CHAN_RUNNING:
		return "running";
	case PL08X_CHAN_PAUSED:
		return "paused";
	case PL08X_CHAN_WAITING:
		return "waiting";
	default:
		break;
	}
	return "UNKNOWN STATE";
}

static int pl08x_debugfs_show(struct seq_file *s, void *data)
{
	struct pl08x_driver_data *pl08x = s->private;
	struct pl08x_dma_chan *chan;
	struct pl08x_phy_chan *ch;
	unsigned long flags;
	int i;

	seq_printf(s, "PL08x physical channels:\n");
	seq_printf(s, "CHANNEL:\tUSER:\n");
	seq_printf(s, "--------\t-----\n");
	for (i = 0; i < pl08x->vd->channels; i++) {
		struct pl08x_dma_chan *virt_chan;

		ch = &pl08x->phy_chans[i];

		spin_lock_irqsave(&ch->lock, flags);
		virt_chan = ch->serving;

1792 1793 1794 1795
		seq_printf(s, "%d\t\t%s%s\n",
			   ch->id,
			   virt_chan ? virt_chan->name : "(none)",
			   ch->locked ? " LOCKED" : "");
1796 1797 1798 1799 1800 1801 1802

		spin_unlock_irqrestore(&ch->lock, flags);
	}

	seq_printf(s, "\nPL08x virtual memcpy channels:\n");
	seq_printf(s, "CHANNEL:\tSTATE:\n");
	seq_printf(s, "--------\t------\n");
1803
	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
1804
		seq_printf(s, "%s\t\t%s\n", chan->name,
1805 1806 1807 1808 1809 1810
			   pl08x_state_str(chan->state));
	}

	seq_printf(s, "\nPL08x virtual slave channels:\n");
	seq_printf(s, "CHANNEL:\tSTATE:\n");
	seq_printf(s, "--------\t------\n");
1811
	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
1812
		seq_printf(s, "%s\t\t%s\n", chan->name,
1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
			   pl08x_state_str(chan->state));
	}

	return 0;
}

static int pl08x_debugfs_open(struct inode *inode, struct file *file)
{
	return single_open(file, pl08x_debugfs_show, inode->i_private);
}

static const struct file_operations pl08x_debugfs_operations = {
	.open		= pl08x_debugfs_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
{
	/* Expose a simple debugfs interface to view all clocks */
1834 1835 1836
	(void) debugfs_create_file(dev_name(&pl08x->adev->dev),
			S_IFREG | S_IRUGO, NULL, pl08x,
			&pl08x_debugfs_operations);
1837 1838 1839 1840 1841 1842 1843 1844
}

#else
static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
{
}
#endif

1845
static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
1846 1847
{
	struct pl08x_driver_data *pl08x;
1848
	const struct vendor_data *vd = id->data;
1849 1850 1851 1852 1853 1854 1855 1856
	int ret = 0;
	int i;

	ret = amba_request_regions(adev, NULL);
	if (ret)
		return ret;

	/* Create the driver state holder */
1857
	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
	if (!pl08x) {
		ret = -ENOMEM;
		goto out_no_pl08x;
	}

	/* Initialize memcpy engine */
	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
	pl08x->memcpy.dev = &adev->dev;
	pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources;
	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
	pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
	pl08x->memcpy.device_control = pl08x_control;

	/* Initialize slave engine */
	dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
	pl08x->slave.dev = &adev->dev;
	pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources;
	pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;
	pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
	pl08x->slave.device_tx_status = pl08x_dma_tx_status;
	pl08x->slave.device_issue_pending = pl08x_issue_pending;
	pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
	pl08x->slave.device_control = pl08x_control;

	/* Get the platform data */
	pl08x->pd = dev_get_platdata(&adev->dev);
	if (!pl08x->pd) {
		dev_err(&adev->dev, "no platform data supplied\n");
1889
		ret = -EINVAL;
1890 1891 1892 1893 1894 1895 1896
		goto out_no_platdata;
	}

	/* Assign useful pointers to the driver state */
	pl08x->adev = adev;
	pl08x->vd = vd;

1897 1898 1899 1900 1901 1902 1903 1904
	/* By default, AHB1 only.  If dualmaster, from platform */
	pl08x->lli_buses = PL08X_AHB1;
	pl08x->mem_buses = PL08X_AHB1;
	if (pl08x->vd->dualmaster) {
		pl08x->lli_buses = pl08x->pd->lli_buses;
		pl08x->mem_buses = pl08x->pd->mem_buses;
	}

1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921
	/* A DMA memory pool for LLIs, align on 1-byte boundary */
	pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
			PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0);
	if (!pl08x->pool) {
		ret = -ENOMEM;
		goto out_no_lli_pool;
	}

	pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
	if (!pl08x->base) {
		ret = -ENOMEM;
		goto out_no_ioremap;
	}

	/* Turn on the PL08x */
	pl08x_ensure_on(pl08x);

1922
	/* Attach the interrupt handler */
1923 1924 1925 1926
	writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
	writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);

	ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED,
1927
			  DRIVER_NAME, pl08x);
1928 1929 1930 1931 1932 1933 1934
	if (ret) {
		dev_err(&adev->dev, "%s failed to request interrupt %d\n",
			__func__, adev->irq[0]);
		goto out_no_irq;
	}

	/* Initialize physical channels */
1935
	pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
1936 1937 1938 1939 1940
			GFP_KERNEL);
	if (!pl08x->phy_chans) {
		dev_err(&adev->dev, "%s failed to allocate "
			"physical channel holders\n",
			__func__);
1941
		ret = -ENOMEM;
1942 1943 1944 1945 1946 1947 1948 1949 1950
		goto out_no_phychans;
	}

	for (i = 0; i < vd->channels; i++) {
		struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];

		ch->id = i;
		ch->base = pl08x->base + PL080_Cx_BASE(i);
		spin_lock_init(&ch->lock);
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966

		/*
		 * Nomadik variants can have channels that are locked
		 * down for the secure world only. Lock up these channels
		 * by perpetually serving a dummy virtual channel.
		 */
		if (vd->nomadik) {
			u32 val;

			val = readl(ch->base + PL080_CH_CONFIG);
			if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
				dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
				ch->locked = true;
			}
		}

1967 1968
		dev_dbg(&adev->dev, "physical channel %d is %s\n",
			i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
	}

	/* Register as many memcpy channels as there are physical channels */
	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
					      pl08x->vd->channels, false);
	if (ret <= 0) {
		dev_warn(&pl08x->adev->dev,
			 "%s failed to enumerate memcpy channels - %d\n",
			 __func__, ret);
		goto out_no_memcpy;
	}
	pl08x->memcpy.chancnt = ret;

	/* Register slave channels */
	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
1984
			pl08x->pd->num_slave_channels, true);
1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
	if (ret <= 0) {
		dev_warn(&pl08x->adev->dev,
			"%s failed to enumerate slave channels - %d\n",
				__func__, ret);
		goto out_no_slave;
	}
	pl08x->slave.chancnt = ret;

	ret = dma_async_device_register(&pl08x->memcpy);
	if (ret) {
		dev_warn(&pl08x->adev->dev,
			"%s failed to register memcpy as an async device - %d\n",
			__func__, ret);
		goto out_no_memcpy_reg;
	}

	ret = dma_async_device_register(&pl08x->slave);
	if (ret) {
		dev_warn(&pl08x->adev->dev,
			"%s failed to register slave as an async device - %d\n",
			__func__, ret);
		goto out_no_slave_reg;
	}

	amba_set_drvdata(adev, pl08x);
	init_pl08x_debugfs(pl08x);
2011 2012 2013
	dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n",
		 amba_part(adev), amba_rev(adev),
		 (unsigned long long)adev->res.start, adev->irq[0]);
2014

2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044
	return 0;

out_no_slave_reg:
	dma_async_device_unregister(&pl08x->memcpy);
out_no_memcpy_reg:
	pl08x_free_virtual_channels(&pl08x->slave);
out_no_slave:
	pl08x_free_virtual_channels(&pl08x->memcpy);
out_no_memcpy:
	kfree(pl08x->phy_chans);
out_no_phychans:
	free_irq(adev->irq[0], pl08x);
out_no_irq:
	iounmap(pl08x->base);
out_no_ioremap:
	dma_pool_destroy(pl08x->pool);
out_no_lli_pool:
out_no_platdata:
	kfree(pl08x);
out_no_pl08x:
	amba_release_regions(adev);
	return ret;
}

/* PL080 has 8 channels and the PL080 have just 2 */
static struct vendor_data vendor_pl080 = {
	.channels = 8,
	.dualmaster = true,
};

2045 2046 2047 2048 2049 2050
static struct vendor_data vendor_nomadik = {
	.channels = 8,
	.dualmaster = true,
	.nomadik = true,
};

2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
static struct vendor_data vendor_pl081 = {
	.channels = 2,
	.dualmaster = false,
};

static struct amba_id pl08x_ids[] = {
	/* PL080 */
	{
		.id	= 0x00041080,
		.mask	= 0x000fffff,
		.data	= &vendor_pl080,
	},
	/* PL081 */
	{
		.id	= 0x00041081,
		.mask	= 0x000fffff,
		.data	= &vendor_pl081,
	},
	/* Nomadik 8815 PL080 variant */
	{
2071
		.id	= 0x00280080,
2072
		.mask	= 0x00ffffff,
2073
		.data	= &vendor_nomadik,
2074 2075 2076 2077
	},
	{ 0, 0 },
};

2078 2079
MODULE_DEVICE_TABLE(amba, pl08x_ids);

2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
static struct amba_driver pl08x_amba_driver = {
	.drv.name	= DRIVER_NAME,
	.id_table	= pl08x_ids,
	.probe		= pl08x_probe,
};

static int __init pl08x_init(void)
{
	int retval;
	retval = amba_driver_register(&pl08x_amba_driver);
	if (retval)
		printk(KERN_WARNING DRIVER_NAME
2092
		       "failed to register as an AMBA device (%d)\n",
2093 2094 2095 2096
		       retval);
	return retval;
}
subsys_initcall(pl08x_init);