dmaengine.c 33.7 KB
Newer Older
C
Chris Leech 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * The full GNU General Public License is included in this distribution in the
 * file called COPYING.
 */

/*
 * This code implements the DMA subsystem. It provides a HW-neutral interface
 * for other kernel code to use asynchronous memory copy capabilities,
 * if present, and allows different HW DMA drivers to register as providing
 * this capability.
 *
 * Due to the fact we are accelerating what is already a relatively fast
 * operation, the code goes to great lengths to avoid additional overhead,
 * such as locking.
 *
 * LOCKING:
 *
30 31
 * The subsystem keeps a global list of dma_device structs it is protected by a
 * mutex, dma_list_mutex.
C
Chris Leech 已提交
32
 *
33 34 35 36 37
 * A subsystem can get access to a channel by calling dmaengine_get() followed
 * by dma_find_channel(), or if it has need for an exclusive channel it can call
 * dma_request_channel().  Once a channel is allocated a reference is taken
 * against its corresponding driver to disable removal.
 *
C
Chris Leech 已提交
38 39 40
 * Each device has a channels list, which runs unlocked but is never modified
 * once the device is registered, it's just setup by the driver.
 *
41
 * See Documentation/driver-api/dmaengine for more details
C
Chris Leech 已提交
42 43
 */

44 45
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

46
#include <linux/platform_device.h>
47
#include <linux/dma-mapping.h>
C
Chris Leech 已提交
48 49
#include <linux/init.h>
#include <linux/module.h>
50
#include <linux/mm.h>
C
Chris Leech 已提交
51 52 53 54 55 56 57
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/mutex.h>
58
#include <linux/jiffies.h>
59
#include <linux/rculist.h>
60
#include <linux/idr.h>
61
#include <linux/slab.h>
62 63
#include <linux/acpi.h>
#include <linux/acpi_dma.h>
64
#include <linux/of_dma.h>
65
#include <linux/mempool.h>
C
Chris Leech 已提交
66 67

static DEFINE_MUTEX(dma_list_mutex);
68
static DEFINE_IDA(dma_ida);
C
Chris Leech 已提交
69
static LIST_HEAD(dma_device_list);
70
static long dmaengine_ref_count;
C
Chris Leech 已提交
71 72 73

/* --- sysfs implementation --- */

74 75 76 77 78 79 80 81 82 83 84 85 86 87
/**
 * dev_to_dma_chan - convert a device pointer to the its sysfs container object
 * @dev - device node
 *
 * Must be called under dma_list_mutex
 */
static struct dma_chan *dev_to_dma_chan(struct device *dev)
{
	struct dma_chan_dev *chan_dev;

	chan_dev = container_of(dev, typeof(*chan_dev), device);
	return chan_dev->chan;
}

88 89
static ssize_t memcpy_count_show(struct device *dev,
				 struct device_attribute *attr, char *buf)
C
Chris Leech 已提交
90
{
91
	struct dma_chan *chan;
C
Chris Leech 已提交
92 93
	unsigned long count = 0;
	int i;
94
	int err;
C
Chris Leech 已提交
95

96 97 98 99 100 101 102 103 104
	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan) {
		for_each_possible_cpu(i)
			count += per_cpu_ptr(chan->local, i)->memcpy_count;
		err = sprintf(buf, "%lu\n", count);
	} else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);
C
Chris Leech 已提交
105

106
	return err;
C
Chris Leech 已提交
107
}
108
static DEVICE_ATTR_RO(memcpy_count);
C
Chris Leech 已提交
109

110 111
static ssize_t bytes_transferred_show(struct device *dev,
				      struct device_attribute *attr, char *buf)
C
Chris Leech 已提交
112
{
113
	struct dma_chan *chan;
C
Chris Leech 已提交
114 115
	unsigned long count = 0;
	int i;
116
	int err;
C
Chris Leech 已提交
117

118 119 120 121 122 123 124 125 126
	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan) {
		for_each_possible_cpu(i)
			count += per_cpu_ptr(chan->local, i)->bytes_transferred;
		err = sprintf(buf, "%lu\n", count);
	} else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);
C
Chris Leech 已提交
127

128
	return err;
C
Chris Leech 已提交
129
}
130
static DEVICE_ATTR_RO(bytes_transferred);
C
Chris Leech 已提交
131

132 133
static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
			   char *buf)
C
Chris Leech 已提交
134
{
135 136
	struct dma_chan *chan;
	int err;
C
Chris Leech 已提交
137

138 139 140 141 142 143 144 145 146
	mutex_lock(&dma_list_mutex);
	chan = dev_to_dma_chan(dev);
	if (chan)
		err = sprintf(buf, "%d\n", chan->client_count);
	else
		err = -ENODEV;
	mutex_unlock(&dma_list_mutex);

	return err;
C
Chris Leech 已提交
147
}
148
static DEVICE_ATTR_RO(in_use);
C
Chris Leech 已提交
149

150 151 152 153 154
static struct attribute *dma_dev_attrs[] = {
	&dev_attr_memcpy_count.attr,
	&dev_attr_bytes_transferred.attr,
	&dev_attr_in_use.attr,
	NULL,
C
Chris Leech 已提交
155
};
156
ATTRIBUTE_GROUPS(dma_dev);
C
Chris Leech 已提交
157

158 159 160 161 162
static void chan_dev_release(struct device *dev)
{
	struct dma_chan_dev *chan_dev;

	chan_dev = container_of(dev, typeof(*chan_dev), device);
163 164
	if (atomic_dec_and_test(chan_dev->idr_ref)) {
		mutex_lock(&dma_list_mutex);
165
		ida_remove(&dma_ida, chan_dev->dev_id);
166 167 168
		mutex_unlock(&dma_list_mutex);
		kfree(chan_dev->idr_ref);
	}
169 170 171
	kfree(chan_dev);
}

C
Chris Leech 已提交
172
static struct class dma_devclass = {
173
	.name		= "dma",
174
	.dev_groups	= dma_dev_groups,
175
	.dev_release	= chan_dev_release,
C
Chris Leech 已提交
176 177 178 179
};

/* --- client and device registration --- */

180 181
#define dma_device_satisfies_mask(device, mask) \
	__dma_device_satisfies_mask((device), &(mask))
182
static int
183 184
__dma_device_satisfies_mask(struct dma_device *device,
			    const dma_cap_mask_t *want)
185 186 187
{
	dma_cap_mask_t has;

188
	bitmap_and(has.bits, want->bits, device->cap_mask.bits,
189 190 191 192
		DMA_TX_TYPE_END);
	return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}

193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
static struct module *dma_chan_to_owner(struct dma_chan *chan)
{
	return chan->device->dev->driver->owner;
}

/**
 * balance_ref_count - catch up the channel reference count
 * @chan - channel to balance ->client_count versus dmaengine_ref_count
 *
 * balance_ref_count must be called under dma_list_mutex
 */
static void balance_ref_count(struct dma_chan *chan)
{
	struct module *owner = dma_chan_to_owner(chan);

	while (chan->client_count < dmaengine_ref_count) {
		__module_get(owner);
		chan->client_count++;
	}
}

/**
 * dma_chan_get - try to grab a dma channel's parent driver module
 * @chan - channel to grab
 *
 * Must be called under dma_list_mutex
 */
static int dma_chan_get(struct dma_chan *chan)
{
	struct module *owner = dma_chan_to_owner(chan);
M
Maxime Ripard 已提交
223
	int ret;
224

M
Maxime Ripard 已提交
225
	/* The channel is already in use, update client count */
226 227
	if (chan->client_count) {
		__module_get(owner);
M
Maxime Ripard 已提交
228 229
		goto out;
	}
230

M
Maxime Ripard 已提交
231 232
	if (!try_module_get(owner))
		return -ENODEV;
233 234

	/* allocate upon first client reference */
235 236 237 238 239
	if (chan->device->device_alloc_chan_resources) {
		ret = chan->device->device_alloc_chan_resources(chan);
		if (ret < 0)
			goto err_out;
	}
240

M
Maxime Ripard 已提交
241 242 243 244 245 246 247 248 249 250
	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
		balance_ref_count(chan);

out:
	chan->client_count++;
	return 0;

err_out:
	module_put(owner);
	return ret;
251 252 253 254 255 256 257 258 259 260
}

/**
 * dma_chan_put - drop a reference to a dma channel's parent driver module
 * @chan - channel to release
 *
 * Must be called under dma_list_mutex
 */
static void dma_chan_put(struct dma_chan *chan)
{
261
	/* This channel is not in use, bail out */
262
	if (!chan->client_count)
263 264
		return;

265 266
	chan->client_count--;
	module_put(dma_chan_to_owner(chan));
267 268

	/* This channel is not in use anymore, free it */
269 270 271
	if (!chan->client_count && chan->device->device_free_chan_resources) {
		/* Make sure all operations have completed */
		dmaengine_synchronize(chan);
272
		chan->device->device_free_chan_resources(chan);
273
	}
274 275 276 277 278 279 280

	/* If the channel is used via a DMA request router, free the mapping */
	if (chan->router && chan->router->route_free) {
		chan->router->route_free(chan->router->dev, chan->route_data);
		chan->router = NULL;
		chan->route_data = NULL;
	}
281 282
}

283 284 285 286 287 288 289 290 291
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
	enum dma_status status;
	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);

	dma_async_issue_pending(chan);
	do {
		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
292
			dev_err(chan->device->dev, "%s: timeout!\n", __func__);
293 294
			return DMA_ERROR;
		}
295 296 297 298
		if (status != DMA_IN_PROGRESS)
			break;
		cpu_relax();
	} while (1);
299 300 301 302 303

	return status;
}
EXPORT_SYMBOL(dma_sync_wait);

304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
/**
 * dma_cap_mask_all - enable iteration over all operation types
 */
static dma_cap_mask_t dma_cap_mask_all;

/**
 * dma_chan_tbl_ent - tracks channel allocations per core/operation
 * @chan - associated channel for this entry
 */
struct dma_chan_tbl_ent {
	struct dma_chan *chan;
};

/**
 * channel_table - percpu lookup table for memory-to-memory offload providers
 */
320
static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
321 322 323 324 325 326 327 328

static int __init dma_channel_table_init(void)
{
	enum dma_transaction_type cap;
	int err = 0;

	bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);

329 330 331
	/* 'interrupt', 'private', and 'slave' are channel capabilities,
	 * but are not associated with an operation so they do not need
	 * an entry in the channel_table
332 333
	 */
	clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
334
	clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
335 336 337 338 339 340 341 342 343 344 345
	clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);

	for_each_dma_cap_mask(cap, dma_cap_mask_all) {
		channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
		if (!channel_table[cap]) {
			err = -ENOMEM;
			break;
		}
	}

	if (err) {
346
		pr_err("initialization failure\n");
347
		for_each_dma_cap_mask(cap, dma_cap_mask_all)
348
			free_percpu(channel_table[cap]);
349 350 351 352
	}

	return err;
}
353
arch_initcall(dma_channel_table_init);
354 355 356 357 358 359 360

/**
 * dma_find_channel - find a channel to carry out the operation
 * @tx_type: transaction type
 */
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
{
361
	return this_cpu_read(channel_table[tx_type]->chan);
362 363
}
EXPORT_SYMBOL(dma_find_channel);
364

365 366 367 368 369 370 371 372 373
/**
 * dma_issue_pending_all - flush all pending operations across all channels
 */
void dma_issue_pending_all(void)
{
	struct dma_device *device;
	struct dma_chan *chan;

	rcu_read_lock();
374 375 376
	list_for_each_entry_rcu(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
377 378 379
		list_for_each_entry(chan, &device->channels, device_node)
			if (chan->client_count)
				device->device_issue_pending(chan);
380
	}
381 382 383 384
	rcu_read_unlock();
}
EXPORT_SYMBOL(dma_issue_pending_all);

385
/**
386 387 388 389 390 391 392 393 394 395
 * dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu
 */
static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
{
	int node = dev_to_node(chan->device->dev);
	return node == -1 || cpumask_test_cpu(cpu, cpumask_of_node(node));
}

/**
 * min_chan - returns the channel with min count and in the same numa-node as the cpu
396
 * @cap: capability to match
397
 * @cpu: cpu index which the channel should be close to
398
 *
399 400 401 402
 * If some channels are close to the given cpu, the one with the lowest
 * reference count is returned. Otherwise, cpu is ignored and only the
 * reference count is taken into account.
 * Must be called under dma_list_mutex.
403
 */
404
static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
405 406 407 408
{
	struct dma_device *device;
	struct dma_chan *chan;
	struct dma_chan *min = NULL;
409
	struct dma_chan *localmin = NULL;
410 411

	list_for_each_entry(device, &dma_device_list, global_node) {
412 413
		if (!dma_has_cap(cap, device->cap_mask) ||
		    dma_has_cap(DMA_PRIVATE, device->cap_mask))
414 415 416 417
			continue;
		list_for_each_entry(chan, &device->channels, device_node) {
			if (!chan->client_count)
				continue;
418
			if (!min || chan->table_count < min->table_count)
419 420
				min = chan;

421 422 423 424
			if (dma_chan_is_local(chan, cpu))
				if (!localmin ||
				    chan->table_count < localmin->table_count)
					localmin = chan;
425 426 427
		}
	}

428
	chan = localmin ? localmin : min;
429

430 431
	if (chan)
		chan->table_count++;
432

433
	return chan;
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
}

/**
 * dma_channel_rebalance - redistribute the available channels
 *
 * Optimize for cpu isolation (each cpu gets a dedicated channel for an
 * operation type) in the SMP case,  and operation isolation (avoid
 * multi-tasking channels) in the non-SMP case.  Must be called under
 * dma_list_mutex.
 */
static void dma_channel_rebalance(void)
{
	struct dma_chan *chan;
	struct dma_device *device;
	int cpu;
	int cap;

	/* undo the last distribution */
	for_each_dma_cap_mask(cap, dma_cap_mask_all)
		for_each_possible_cpu(cpu)
			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;

456 457 458
	list_for_each_entry(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
459 460
		list_for_each_entry(chan, &device->channels, device_node)
			chan->table_count = 0;
461
	}
462 463 464 465 466 467 468 469

	/* don't populate the channel_table if no clients are available */
	if (!dmaengine_ref_count)
		return;

	/* redistribute available channels */
	for_each_dma_cap_mask(cap, dma_cap_mask_all)
		for_each_online_cpu(cpu) {
470
			chan = min_chan(cap, cpu);
471 472 473 474
			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
		}
}

475 476 477 478 479 480 481 482 483 484
int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
{
	struct dma_device *device;

	if (!chan || !caps)
		return -EINVAL;

	device = chan->device;

	/* check if the channel supports slave transactions */
485 486
	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))
487 488 489 490 491 492 493 494 495 496 497 498 499
		return -ENXIO;

	/*
	 * Check whether it reports it uses the generic slave
	 * capabilities, if not, that means it doesn't support any
	 * kind of slave capabilities reporting.
	 */
	if (!device->directions)
		return -ENXIO;

	caps->src_addr_widths = device->src_addr_widths;
	caps->dst_addr_widths = device->dst_addr_widths;
	caps->directions = device->directions;
500
	caps->max_burst = device->max_burst;
501
	caps->residue_granularity = device->residue_granularity;
R
Robert Jarzmik 已提交
502
	caps->descriptor_reuse = device->descriptor_reuse;
503 504
	caps->cmd_pause = !!device->device_pause;
	caps->cmd_resume = !!device->device_resume;
505 506 507 508 509 510
	caps->cmd_terminate = !!device->device_terminate_all;

	return 0;
}
EXPORT_SYMBOL_GPL(dma_get_slave_caps);

511 512
static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
					  struct dma_device *dev,
513
					  dma_filter_fn fn, void *fn_param)
514 515 516
{
	struct dma_chan *chan;

517
	if (mask && !__dma_device_satisfies_mask(dev, mask)) {
518
		dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
519 520 521 522 523 524 525 526 527 528 529 530 531 532
		return NULL;
	}
	/* devices with multiple channels need special handling as we need to
	 * ensure that all channels are either private or public.
	 */
	if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
		list_for_each_entry(chan, &dev->channels, device_node) {
			/* some channels are already publicly allocated */
			if (chan->client_count)
				return NULL;
		}

	list_for_each_entry(chan, &dev->channels, device_node) {
		if (chan->client_count) {
533
			dev_dbg(dev->dev, "%s: %s busy\n",
534
				 __func__, dma_chan_name(chan));
535 536
			continue;
		}
537
		if (fn && !fn(chan, fn_param)) {
538
			dev_dbg(dev->dev, "%s: %s filter said false\n",
539 540 541 542
				 __func__, dma_chan_name(chan));
			continue;
		}
		return chan;
543 544
	}

545
	return NULL;
546 547
}

548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
static struct dma_chan *find_candidate(struct dma_device *device,
				       const dma_cap_mask_t *mask,
				       dma_filter_fn fn, void *fn_param)
{
	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
	int err;

	if (chan) {
		/* Found a suitable channel, try to grab, prep, and return it.
		 * We first set DMA_PRIVATE to disable balance_ref_count as this
		 * channel will not be published in the general-purpose
		 * allocator
		 */
		dma_cap_set(DMA_PRIVATE, device->cap_mask);
		device->privatecnt++;
		err = dma_chan_get(chan);

		if (err) {
			if (err == -ENODEV) {
567 568
				dev_dbg(device->dev, "%s: %s module removed\n",
					__func__, dma_chan_name(chan));
569 570
				list_del_rcu(&device->global_node);
			} else
571 572
				dev_dbg(device->dev,
					"%s: failed to get %s: (%d)\n",
573 574 575 576 577 578 579 580 581 582 583 584
					 __func__, dma_chan_name(chan), err);

			if (--device->privatecnt == 0)
				dma_cap_clear(DMA_PRIVATE, device->cap_mask);

			chan = ERR_PTR(err);
		}
	}

	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
}

585
/**
586
 * dma_get_slave_channel - try to get specific channel exclusively
587 588 589 590 591 592 593 594 595
 * @chan: target channel
 */
struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
{
	int err = -EBUSY;

	/* lock against __dma_request_channel */
	mutex_lock(&dma_list_mutex);

596
	if (chan->client_count == 0) {
597 598 599 600
		struct dma_device *device = chan->device;

		dma_cap_set(DMA_PRIVATE, device->cap_mask);
		device->privatecnt++;
601
		err = dma_chan_get(chan);
602
		if (err) {
603 604
			dev_dbg(chan->device->dev,
				"%s: failed to get %s: (%d)\n",
605
				__func__, dma_chan_name(chan), err);
606 607 608 609
			chan = NULL;
			if (--device->privatecnt == 0)
				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
		}
610
	} else
611 612 613 614 615 616 617 618 619
		chan = NULL;

	mutex_unlock(&dma_list_mutex);


	return chan;
}
EXPORT_SYMBOL_GPL(dma_get_slave_channel);

620 621 622 623 624 625 626 627 628 629 630
struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
{
	dma_cap_mask_t mask;
	struct dma_chan *chan;

	dma_cap_zero(mask);
	dma_cap_set(DMA_SLAVE, mask);

	/* lock against __dma_request_channel */
	mutex_lock(&dma_list_mutex);

631
	chan = find_candidate(device, &mask, NULL, NULL);
632 633 634

	mutex_unlock(&dma_list_mutex);

635
	return IS_ERR(chan) ? NULL : chan;
636 637 638
}
EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);

639
/**
640
 * __dma_request_channel - try to allocate an exclusive channel
641 642 643
 * @mask: capabilities that the channel must satisfy
 * @fn: optional callback to disposition available channels
 * @fn_param: opaque parameter to pass to dma_filter_fn
644 645
 *
 * Returns pointer to appropriate DMA channel on success or NULL.
646
 */
647 648
struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
				       dma_filter_fn fn, void *fn_param)
649 650 651 652 653 654 655
{
	struct dma_device *device, *_d;
	struct dma_chan *chan = NULL;

	/* Find a channel */
	mutex_lock(&dma_list_mutex);
	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
656 657 658
		chan = find_candidate(device, mask, fn, fn_param);
		if (!IS_ERR(chan))
			break;
659

660
		chan = NULL;
661 662 663
	}
	mutex_unlock(&dma_list_mutex);

664
	pr_debug("%s: %s (%s)\n",
665 666
		 __func__,
		 chan ? "success" : "fail",
667
		 chan ? dma_chan_name(chan) : NULL);
668 669 670 671 672

	return chan;
}
EXPORT_SYMBOL_GPL(__dma_request_channel);

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
						    const char *name,
						    struct device *dev)
{
	int i;

	if (!device->filter.mapcnt)
		return NULL;

	for (i = 0; i < device->filter.mapcnt; i++) {
		const struct dma_slave_map *map = &device->filter.map[i];

		if (!strcmp(map->devname, dev_name(dev)) &&
		    !strcmp(map->slave, name))
			return map;
	}

	return NULL;
}

693
/**
694
 * dma_request_chan - try to allocate an exclusive slave channel
695 696
 * @dev:	pointer to client device structure
 * @name:	slave channel name
697 698
 *
 * Returns pointer to appropriate DMA channel on success or an error pointer.
699
 */
700
struct dma_chan *dma_request_chan(struct device *dev, const char *name)
701
{
702 703 704
	struct dma_device *d, *_d;
	struct dma_chan *chan = NULL;

705 706
	/* If device-tree is present get slave info from here */
	if (dev->of_node)
707
		chan = of_dma_request_slave_channel(dev->of_node, name);
708

709
	/* If device was enumerated by ACPI get slave info from here */
710 711 712 713 714 715 716 717 718 719 720 721 722 723
	if (has_acpi_companion(dev) && !chan)
		chan = acpi_dma_request_slave_chan_by_name(dev, name);

	if (chan) {
		/* Valid channel found or requester need to be deferred */
		if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER)
			return chan;
	}

	/* Try to find the channel via the DMA filter map(s) */
	mutex_lock(&dma_list_mutex);
	list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
		dma_cap_mask_t mask;
		const struct dma_slave_map *map = dma_filter_match(d, name, dev);
724

725 726 727 728 729
		if (!map)
			continue;

		dma_cap_zero(mask);
		dma_cap_set(DMA_SLAVE, mask);
730

731 732 733 734 735 736 737
		chan = find_candidate(d, &mask, d->filter.fn, map->param);
		if (!IS_ERR(chan))
			break;
	}
	mutex_unlock(&dma_list_mutex);

	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
738
}
739
EXPORT_SYMBOL_GPL(dma_request_chan);
740 741 742 743 744 745 746 747 748 749 750

/**
 * dma_request_slave_channel - try to allocate an exclusive slave channel
 * @dev:	pointer to client device structure
 * @name:	slave channel name
 *
 * Returns pointer to appropriate DMA channel on success or NULL.
 */
struct dma_chan *dma_request_slave_channel(struct device *dev,
					   const char *name)
{
751
	struct dma_chan *ch = dma_request_chan(dev, name);
752 753
	if (IS_ERR(ch))
		return NULL;
754

755
	return ch;
756 757 758
}
EXPORT_SYMBOL_GPL(dma_request_slave_channel);

759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
/**
 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
 * @mask: capabilities that the channel must satisfy
 *
 * Returns pointer to appropriate DMA channel on success or an error pointer.
 */
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
{
	struct dma_chan *chan;

	if (!mask)
		return ERR_PTR(-ENODEV);

	chan = __dma_request_channel(mask, NULL, NULL);
	if (!chan)
		chan = ERR_PTR(-ENODEV);

	return chan;
}
EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);

780 781 782 783 784 785
void dma_release_channel(struct dma_chan *chan)
{
	mutex_lock(&dma_list_mutex);
	WARN_ONCE(chan->client_count != 1,
		  "chan reference count %d != 1\n", chan->client_count);
	dma_chan_put(chan);
786 787 788
	/* drop PRIVATE cap enabled by __dma_request_channel() */
	if (--chan->device->privatecnt == 0)
		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
789 790 791 792
	mutex_unlock(&dma_list_mutex);
}
EXPORT_SYMBOL_GPL(dma_release_channel);

793
/**
794
 * dmaengine_get - register interest in dma_channels
795
 */
796
void dmaengine_get(void)
797
{
798 799 800 801
	struct dma_device *device, *_d;
	struct dma_chan *chan;
	int err;

C
Chris Leech 已提交
802
	mutex_lock(&dma_list_mutex);
803 804 805
	dmaengine_ref_count++;

	/* try to grab channels */
806 807 808
	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
809 810 811 812
		list_for_each_entry(chan, &device->channels, device_node) {
			err = dma_chan_get(chan);
			if (err == -ENODEV) {
				/* module removed before we could use it */
813
				list_del_rcu(&device->global_node);
814 815
				break;
			} else if (err)
816 817 818
				dev_dbg(chan->device->dev,
					"%s: failed to get %s: (%d)\n",
					__func__, dma_chan_name(chan), err);
819
		}
820
	}
821

822 823 824 825 826 827
	/* if this is the first reference and there were channels
	 * waiting we need to rebalance to get those channels
	 * incorporated into the channel table
	 */
	if (dmaengine_ref_count == 1)
		dma_channel_rebalance();
C
Chris Leech 已提交
828 829
	mutex_unlock(&dma_list_mutex);
}
830
EXPORT_SYMBOL(dmaengine_get);
C
Chris Leech 已提交
831 832

/**
833
 * dmaengine_put - let dma drivers be removed when ref_count == 0
C
Chris Leech 已提交
834
 */
835
void dmaengine_put(void)
C
Chris Leech 已提交
836
{
837
	struct dma_device *device;
C
Chris Leech 已提交
838 839 840
	struct dma_chan *chan;

	mutex_lock(&dma_list_mutex);
841 842 843
	dmaengine_ref_count--;
	BUG_ON(dmaengine_ref_count < 0);
	/* drop channel references */
844 845 846
	list_for_each_entry(device, &dma_device_list, global_node) {
		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
			continue;
847 848
		list_for_each_entry(chan, &device->channels, device_node)
			dma_chan_put(chan);
849
	}
C
Chris Leech 已提交
850 851
	mutex_unlock(&dma_list_mutex);
}
852
EXPORT_SYMBOL(dmaengine_put);
C
Chris Leech 已提交
853

854 855 856 857 858 859 860 861 862 863 864
static bool device_has_all_tx_types(struct dma_device *device)
{
	/* A device that satisfies this test has channels that will never cause
	 * an async_tx channel switch event as all possible operation types can
	 * be handled.
	 */
	#ifdef CONFIG_ASYNC_TX_DMA
	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
		return false;
	#endif

865
	#if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
866 867 868 869
	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
		return false;
	#endif

870
	#if IS_ENABLED(CONFIG_ASYNC_XOR)
871 872
	if (!dma_has_cap(DMA_XOR, device->cap_mask))
		return false;
873 874

	#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
875 876
	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
		return false;
877
	#endif
878
	#endif
879

880
	#if IS_ENABLED(CONFIG_ASYNC_PQ)
881 882
	if (!dma_has_cap(DMA_PQ, device->cap_mask))
		return false;
883 884

	#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
885 886
	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
		return false;
887
	#endif
888
	#endif
889 890 891 892

	return true;
}

893 894 895 896
static int get_dma_id(struct dma_device *device)
{
	int rc;

897 898 899 900 901 902 903
	do {
		if (!ida_pre_get(&dma_ida, GFP_KERNEL))
			return -ENOMEM;
		mutex_lock(&dma_list_mutex);
		rc = ida_get_new(&dma_ida, &device->dev_id);
		mutex_unlock(&dma_list_mutex);
	} while (rc == -EAGAIN);
T
Tejun Heo 已提交
904

905
	return rc;
906 907
}

C
Chris Leech 已提交
908
/**
909
 * dma_async_device_register - registers DMA devices found
C
Chris Leech 已提交
910 911 912 913
 * @device: &dma_device
 */
int dma_async_device_register(struct dma_device *device)
{
J
Jeff Garzik 已提交
914
	int chancnt = 0, rc;
C
Chris Leech 已提交
915
	struct dma_chan* chan;
916
	atomic_t *idr_ref;
C
Chris Leech 已提交
917 918 919 920

	if (!device)
		return -ENODEV;

921
	/* validate device routines */
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
	if (!device->dev) {
		pr_err("DMAdevice must have dev\n");
		return -EIO;
	}

	if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_MEMCPY");
		return -EIO;
	}

	if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_XOR");
		return -EIO;
	}

	if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_XOR_VAL");
		return -EIO;
	}

	if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_PQ");
		return -EIO;
	}

	if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_PQ_VAL");
		return -EIO;
	}

	if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_MEMSET");
		return -EIO;
	}

	if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_INTERRUPT");
		return -EIO;
	}

	if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_CYCLIC");
		return -EIO;
	}

	if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) {
		dev_err(device->dev,
			"Device claims capability %s, but op is not defined\n",
			"DMA_INTERLEAVE");
		return -EIO;
	}


	if (!device->device_tx_status) {
		dev_err(device->dev, "Device tx_status is not defined\n");
		return -EIO;
	}


	if (!device->device_issue_pending) {
		dev_err(device->dev, "Device issue_pending is not defined\n");
		return -EIO;
	}
1001

1002
	/* note: this only matters in the
1003
	 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
1004 1005 1006 1007
	 */
	if (device_has_all_tx_types(device))
		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);

1008 1009 1010
	idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
	if (!idr_ref)
		return -ENOMEM;
1011 1012 1013
	rc = get_dma_id(device);
	if (rc != 0) {
		kfree(idr_ref);
1014
		return rc;
1015 1016 1017
	}

	atomic_set(idr_ref, 0);
C
Chris Leech 已提交
1018 1019 1020

	/* represent channels in sysfs. Probably want devs too */
	list_for_each_entry(chan, &device->channels, device_node) {
1021
		rc = -ENOMEM;
C
Chris Leech 已提交
1022 1023
		chan->local = alloc_percpu(typeof(*chan->local));
		if (chan->local == NULL)
1024
			goto err_out;
1025 1026 1027
		chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
		if (chan->dev == NULL) {
			free_percpu(chan->local);
1028 1029
			chan->local = NULL;
			goto err_out;
1030
		}
C
Chris Leech 已提交
1031 1032

		chan->chan_id = chancnt++;
1033 1034 1035
		chan->dev->device.class = &dma_devclass;
		chan->dev->device.parent = device->dev;
		chan->dev->chan = chan;
1036 1037 1038
		chan->dev->idr_ref = idr_ref;
		chan->dev->dev_id = device->dev_id;
		atomic_inc(idr_ref);
1039
		dev_set_name(&chan->dev->device, "dma%dchan%d",
1040
			     device->dev_id, chan->chan_id);
C
Chris Leech 已提交
1041

1042
		rc = device_register(&chan->dev->device);
J
Jeff Garzik 已提交
1043 1044 1045
		if (rc) {
			free_percpu(chan->local);
			chan->local = NULL;
1046 1047
			kfree(chan->dev);
			atomic_dec(idr_ref);
J
Jeff Garzik 已提交
1048 1049
			goto err_out;
		}
1050
		chan->client_count = 0;
C
Chris Leech 已提交
1051
	}
1052 1053 1054 1055 1056 1057 1058

	if (!chancnt) {
		dev_err(device->dev, "%s: device has no channels!\n", __func__);
		rc = -ENODEV;
		goto err_out;
	}

1059
	device->chancnt = chancnt;
C
Chris Leech 已提交
1060 1061

	mutex_lock(&dma_list_mutex);
1062 1063
	/* take references on public channels */
	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		list_for_each_entry(chan, &device->channels, device_node) {
			/* if clients are already waiting for channels we need
			 * to take references on their behalf
			 */
			if (dma_chan_get(chan) == -ENODEV) {
				/* note we can only get here for the first
				 * channel as the remaining channels are
				 * guaranteed to get a reference
				 */
				rc = -ENODEV;
				mutex_unlock(&dma_list_mutex);
				goto err_out;
			}
		}
1078
	list_add_tail_rcu(&device->global_node, &dma_device_list);
1079 1080
	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
		device->privatecnt++;	/* Always private */
1081
	dma_channel_rebalance();
C
Chris Leech 已提交
1082 1083 1084
	mutex_unlock(&dma_list_mutex);

	return 0;
J
Jeff Garzik 已提交
1085 1086

err_out:
1087 1088 1089
	/* if we never registered a channel just release the idr */
	if (atomic_read(idr_ref) == 0) {
		mutex_lock(&dma_list_mutex);
1090
		ida_remove(&dma_ida, device->dev_id);
1091 1092 1093 1094 1095
		mutex_unlock(&dma_list_mutex);
		kfree(idr_ref);
		return rc;
	}

J
Jeff Garzik 已提交
1096 1097 1098
	list_for_each_entry(chan, &device->channels, device_node) {
		if (chan->local == NULL)
			continue;
1099 1100 1101 1102
		mutex_lock(&dma_list_mutex);
		chan->dev->chan = NULL;
		mutex_unlock(&dma_list_mutex);
		device_unregister(&chan->dev->device);
J
Jeff Garzik 已提交
1103 1104 1105
		free_percpu(chan->local);
	}
	return rc;
C
Chris Leech 已提交
1106
}
1107
EXPORT_SYMBOL(dma_async_device_register);
C
Chris Leech 已提交
1108

1109
/**
1110
 * dma_async_device_unregister - unregister a DMA device
1111
 * @device: &dma_device
1112 1113 1114
 *
 * This routine is called by dma driver exit routines, dmaengine holds module
 * references to prevent it being called while channels are in use.
1115 1116
 */
void dma_async_device_unregister(struct dma_device *device)
C
Chris Leech 已提交
1117 1118 1119 1120
{
	struct dma_chan *chan;

	mutex_lock(&dma_list_mutex);
1121
	list_del_rcu(&device->global_node);
1122
	dma_channel_rebalance();
C
Chris Leech 已提交
1123 1124 1125
	mutex_unlock(&dma_list_mutex);

	list_for_each_entry(chan, &device->channels, device_node) {
1126 1127 1128
		WARN_ONCE(chan->client_count,
			  "%s called while %d clients hold a reference\n",
			  __func__, chan->client_count);
1129 1130 1131 1132
		mutex_lock(&dma_list_mutex);
		chan->dev->chan = NULL;
		mutex_unlock(&dma_list_mutex);
		device_unregister(&chan->dev->device);
1133
		free_percpu(chan->local);
C
Chris Leech 已提交
1134 1135
	}
}
1136
EXPORT_SYMBOL(dma_async_device_unregister);
C
Chris Leech 已提交
1137

1138 1139 1140 1141 1142 1143
struct dmaengine_unmap_pool {
	struct kmem_cache *cache;
	const char *name;
	mempool_t *pool;
	size_t size;
};
1144

1145 1146 1147
#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
static struct dmaengine_unmap_pool unmap_pool[] = {
	__UNMAP_POOL(2),
1148
	#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1149 1150 1151 1152 1153
	__UNMAP_POOL(16),
	__UNMAP_POOL(128),
	__UNMAP_POOL(256),
	#endif
};
1154

1155 1156 1157 1158 1159 1160 1161
static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
{
	int order = get_count_order(nr);

	switch (order) {
	case 0 ... 1:
		return &unmap_pool[0];
1162
#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1163 1164 1165 1166 1167 1168
	case 2 ... 4:
		return &unmap_pool[1];
	case 5 ... 7:
		return &unmap_pool[2];
	case 8:
		return &unmap_pool[3];
1169
#endif
1170 1171 1172
	default:
		BUG();
		return NULL;
1173
	}
1174
}
1175

1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
static void dmaengine_unmap(struct kref *kref)
{
	struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
	struct device *dev = unmap->dev;
	int cnt, i;

	cnt = unmap->to_cnt;
	for (i = 0; i < cnt; i++)
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_TO_DEVICE);
	cnt += unmap->from_cnt;
	for (; i < cnt; i++)
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_FROM_DEVICE);
	cnt += unmap->bidi_cnt;
1191 1192 1193
	for (; i < cnt; i++) {
		if (unmap->addr[i] == 0)
			continue;
1194 1195
		dma_unmap_page(dev, unmap->addr[i], unmap->len,
			       DMA_BIDIRECTIONAL);
1196
	}
1197
	cnt = unmap->map_cnt;
1198 1199
	mempool_free(unmap, __get_unmap_pool(cnt)->pool);
}
1200

1201 1202 1203 1204 1205 1206
void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
{
	if (unmap)
		kref_put(&unmap->kref, dmaengine_unmap);
}
EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
1207

1208 1209 1210 1211 1212 1213 1214
static void dmaengine_destroy_unmap_pool(void)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
		struct dmaengine_unmap_pool *p = &unmap_pool[i];

1215
		mempool_destroy(p->pool);
1216
		p->pool = NULL;
1217
		kmem_cache_destroy(p->cache);
1218 1219
		p->cache = NULL;
	}
1220 1221
}

1222
static int __init dmaengine_init_unmap_pool(void)
1223
{
1224
	int i;
1225

1226 1227 1228
	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
		struct dmaengine_unmap_pool *p = &unmap_pool[i];
		size_t size;
1229

1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
		size = sizeof(struct dmaengine_unmap_data) +
		       sizeof(dma_addr_t) * p->size;

		p->cache = kmem_cache_create(p->name, size, 0,
					     SLAB_HWCACHE_ALIGN, NULL);
		if (!p->cache)
			break;
		p->pool = mempool_create_slab_pool(1, p->cache);
		if (!p->pool)
			break;
1240
	}
1241

1242 1243
	if (i == ARRAY_SIZE(unmap_pool))
		return 0;
1244

1245 1246 1247
	dmaengine_destroy_unmap_pool();
	return -ENOMEM;
}
1248

1249
struct dmaengine_unmap_data *
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
{
	struct dmaengine_unmap_data *unmap;

	unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
	if (!unmap)
		return NULL;

	memset(unmap, 0, sizeof(*unmap));
	kref_init(&unmap->kref);
	unmap->dev = dev;
1261
	unmap->map_cnt = nr;
1262 1263

	return unmap;
1264
}
1265
EXPORT_SYMBOL(dmaengine_get_unmap_data);
1266 1267 1268 1269 1270

void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	struct dma_chan *chan)
{
	tx->chan = chan;
1271
	#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1272
	spin_lock_init(&tx->lock);
1273
	#endif
1274 1275 1276
}
EXPORT_SYMBOL(dma_async_tx_descriptor_init);

1277 1278 1279 1280 1281 1282
/* dma_wait_for_async_tx - spin wait for a transaction to complete
 * @tx: in-flight transaction to wait on
 */
enum dma_status
dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
{
1283
	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
1284 1285

	if (!tx)
1286
		return DMA_COMPLETE;
1287

1288 1289
	while (tx->cookie == -EBUSY) {
		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
1290 1291 1292
			dev_err(tx->chan->device->dev,
				"%s timeout waiting for descriptor submission\n",
				__func__);
1293 1294 1295 1296 1297
			return DMA_ERROR;
		}
		cpu_relax();
	}
	return dma_sync_wait(tx->chan, tx->cookie);
1298 1299 1300 1301 1302 1303 1304 1305 1306
}
EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);

/* dma_run_dependencies - helper routine for dma drivers to process
 *	(start) dependent operations on their target channel
 * @tx: transaction with dependencies
 */
void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
{
1307
	struct dma_async_tx_descriptor *dep = txd_next(tx);
1308 1309 1310 1311 1312 1313
	struct dma_async_tx_descriptor *dep_next;
	struct dma_chan *chan;

	if (!dep)
		return;

1314
	/* we'll submit tx->next now, so clear the link */
1315
	txd_clear_next(tx);
1316 1317 1318 1319 1320 1321 1322
	chan = dep->chan;

	/* keep submitting up until a channel switch is detected
	 * in that case we will be called again as a result of
	 * processing the interrupt from async_tx_channel_switch
	 */
	for (; dep; dep = dep_next) {
1323 1324 1325
		txd_lock(dep);
		txd_clear_parent(dep);
		dep_next = txd_next(dep);
1326
		if (dep_next && dep_next->chan == chan)
1327
			txd_clear_next(dep); /* ->next will be submitted */
1328 1329
		else
			dep_next = NULL; /* submit current dep and terminate */
1330
		txd_unlock(dep);
1331 1332 1333 1334 1335 1336 1337 1338

		dep->tx_submit(dep);
	}

	chan->device->device_issue_pending(chan);
}
EXPORT_SYMBOL_GPL(dma_run_dependencies);

C
Chris Leech 已提交
1339 1340
static int __init dma_bus_init(void)
{
1341 1342 1343 1344
	int err = dmaengine_init_unmap_pool();

	if (err)
		return err;
C
Chris Leech 已提交
1345 1346
	return class_register(&dma_devclass);
}
1347
arch_initcall(dma_bus_init);
C
Chris Leech 已提交
1348

1349