msm_iommu.c 19.0 KB
Newer Older
1
/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
S
Stepan Moskovchenko 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/errno.h>
#include <linux/io.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/iommu.h>
29
#include <linux/clk.h>
30
#include <linux/err.h>
31
#include <linux/of_iommu.h>
S
Stepan Moskovchenko 已提交
32 33 34 35

#include <asm/cacheflush.h>
#include <asm/sizes.h>

36 37
#include "msm_iommu_hw-8xxx.h"
#include "msm_iommu.h"
38
#include "io-pgtable.h"
S
Stepan Moskovchenko 已提交
39

40 41 42 43 44
#define MRC(reg, processor, op1, crn, crm, op2)				\
__asm__ __volatile__ (							\
"   mrc   "   #processor "," #op1 ", %0,"  #crn "," #crm "," #op2 "\n"  \
: "=r" (reg))

45 46 47
/* bitmap of the page sizes currently supported */
#define MSM_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_1M | SZ_16M)

S
Stepan Moskovchenko 已提交
48
DEFINE_SPINLOCK(msm_iommu_lock);
S
Sricharan R 已提交
49
static LIST_HEAD(qcom_iommu_devices);
50
static struct iommu_ops msm_iommu_ops;
S
Stepan Moskovchenko 已提交
51 52 53

struct msm_priv {
	struct list_head list_attached;
54
	struct iommu_domain domain;
55 56 57 58
	struct io_pgtable_cfg	cfg;
	struct io_pgtable_ops	*iop;
	struct device		*dev;
	spinlock_t		pgtlock; /* pagetable lock */
S
Stepan Moskovchenko 已提交
59 60
};

61 62 63 64 65
static struct msm_priv *to_msm_priv(struct iommu_domain *dom)
{
	return container_of(dom, struct msm_priv, domain);
}

S
Sricharan R 已提交
66
static int __enable_clocks(struct msm_iommu_dev *iommu)
67 68 69
{
	int ret;

S
Sricharan R 已提交
70
	ret = clk_enable(iommu->pclk);
71 72 73
	if (ret)
		goto fail;

S
Sricharan R 已提交
74 75
	if (iommu->clk) {
		ret = clk_enable(iommu->clk);
76
		if (ret)
S
Sricharan R 已提交
77
			clk_disable(iommu->pclk);
78 79 80 81 82
	}
fail:
	return ret;
}

S
Sricharan R 已提交
83
static void __disable_clocks(struct msm_iommu_dev *iommu)
84
{
S
Sricharan R 已提交
85 86 87
	if (iommu->clk)
		clk_disable(iommu->clk);
	clk_disable(iommu->pclk);
88 89
}

90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
static void msm_iommu_reset(void __iomem *base, int ncb)
{
	int ctx;

	SET_RPUE(base, 0);
	SET_RPUEIE(base, 0);
	SET_ESRRESTORE(base, 0);
	SET_TBE(base, 0);
	SET_CR(base, 0);
	SET_SPDMBE(base, 0);
	SET_TESTBUSCR(base, 0);
	SET_TLBRSW(base, 0);
	SET_GLOBAL_TLBIALL(base, 0);
	SET_RPU_ACR(base, 0);
	SET_TLBLKCRWE(base, 1);

	for (ctx = 0; ctx < ncb; ctx++) {
		SET_BPRCOSH(base, ctx, 0);
		SET_BPRCISH(base, ctx, 0);
		SET_BPRCNSH(base, ctx, 0);
		SET_BPSHCFG(base, ctx, 0);
		SET_BPMTCFG(base, ctx, 0);
		SET_ACTLR(base, ctx, 0);
		SET_SCTLR(base, ctx, 0);
		SET_FSRRESTORE(base, ctx, 0);
		SET_TTBR0(base, ctx, 0);
		SET_TTBR1(base, ctx, 0);
		SET_TTBCR(base, ctx, 0);
		SET_BFBCR(base, ctx, 0);
		SET_PAR(base, ctx, 0);
		SET_FAR(base, ctx, 0);
		SET_CTX_TLBIALL(base, ctx, 0);
		SET_TLBFLPTER(base, ctx, 0);
		SET_TLBSLPTER(base, ctx, 0);
		SET_TLBLKCR(base, ctx, 0);
		SET_CONTEXTIDR(base, ctx, 0);
	}
}

129
static void __flush_iotlb(void *cookie)
S
Stepan Moskovchenko 已提交
130
{
131
	struct msm_priv *priv = cookie;
S
Sricharan R 已提交
132 133
	struct msm_iommu_dev *iommu = NULL;
	struct msm_iommu_ctx_dev *master;
134
	int ret = 0;
S
Sricharan R 已提交
135

136 137 138 139
	list_for_each_entry(iommu, &priv->list_attached, dom_node) {
		ret = __enable_clocks(iommu);
		if (ret)
			goto fail;
S
Stepan Moskovchenko 已提交
140

141 142
		list_for_each_entry(master, &iommu->ctx_list, list)
			SET_CTX_TLBIALL(iommu->base, master->num, 0);
S
Stepan Moskovchenko 已提交
143

144
		__disable_clocks(iommu);
145
	}
146 147 148 149 150 151 152 153 154 155 156 157
fail:
	return;
}

static void __flush_iotlb_range(unsigned long iova, size_t size,
				size_t granule, bool leaf, void *cookie)
{
	struct msm_priv *priv = cookie;
	struct msm_iommu_dev *iommu = NULL;
	struct msm_iommu_ctx_dev *master;
	int ret = 0;
	int temp_size;
S
Stepan Moskovchenko 已提交
158

S
Sricharan R 已提交
159 160
	list_for_each_entry(iommu, &priv->list_attached, dom_node) {
		ret = __enable_clocks(iommu);
161 162 163
		if (ret)
			goto fail;

164 165 166 167 168 169 170 171 172 173
		list_for_each_entry(master, &iommu->ctx_list, list) {
			temp_size = size;
			do {
				iova &= TLBIVA_VA;
				iova |= GET_CONTEXTIDR_ASID(iommu->base,
							    master->num);
				SET_TLBIVA(iommu->base, master->num, iova);
				iova += granule;
			} while (temp_size -= granule);
		}
S
Sricharan R 已提交
174 175

		__disable_clocks(iommu);
S
Stepan Moskovchenko 已提交
176
	}
177

178
fail:
179
	return;
S
Stepan Moskovchenko 已提交
180 181
}

182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
static void __flush_iotlb_sync(void *cookie)
{
	/*
	 * Nothing is needed here, the barrier to guarantee
	 * completion of the tlb sync operation is implicitly
	 * taken care when the iommu client does a writel before
	 * kick starting the other master.
	 */
}

static const struct iommu_gather_ops msm_iommu_gather_ops = {
	.tlb_flush_all = __flush_iotlb,
	.tlb_add_flush = __flush_iotlb_range,
	.tlb_sync = __flush_iotlb_sync,
};

S
Sricharan R 已提交
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end)
{
	int idx;

	do {
		idx = find_next_zero_bit(map, end, start);
		if (idx == end)
			return -ENOSPC;
	} while (test_and_set_bit(idx, map));

	return idx;
}

static void msm_iommu_free_ctx(unsigned long *map, int idx)
{
	clear_bit(idx, map);
}

static void config_mids(struct msm_iommu_dev *iommu,
			struct msm_iommu_ctx_dev *master)
{
	int mid, ctx, i;

	for (i = 0; i < master->num_mids; i++) {
		mid = master->mids[i];
		ctx = master->num;

		SET_M2VCBR_N(iommu->base, mid, 0);
		SET_CBACR_N(iommu->base, ctx, 0);

		/* Set VMID = 0 */
		SET_VMID(iommu->base, mid, 0);

		/* Set the context number for that MID to this context */
		SET_CBNDX(iommu->base, mid, ctx);

		/* Set MID associated with this context bank to 0*/
		SET_CBVMID(iommu->base, ctx, 0);

		/* Set the ASID for TLB tagging for this context */
		SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx);

		/* Set security bit override to be Non-secure */
		SET_NSCFG(iommu->base, mid, 3);
	}
}

S
Stepan Moskovchenko 已提交
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
static void __reset_context(void __iomem *base, int ctx)
{
	SET_BPRCOSH(base, ctx, 0);
	SET_BPRCISH(base, ctx, 0);
	SET_BPRCNSH(base, ctx, 0);
	SET_BPSHCFG(base, ctx, 0);
	SET_BPMTCFG(base, ctx, 0);
	SET_ACTLR(base, ctx, 0);
	SET_SCTLR(base, ctx, 0);
	SET_FSRRESTORE(base, ctx, 0);
	SET_TTBR0(base, ctx, 0);
	SET_TTBR1(base, ctx, 0);
	SET_TTBCR(base, ctx, 0);
	SET_BFBCR(base, ctx, 0);
	SET_PAR(base, ctx, 0);
	SET_FAR(base, ctx, 0);
	SET_CTX_TLBIALL(base, ctx, 0);
	SET_TLBFLPTER(base, ctx, 0);
	SET_TLBSLPTER(base, ctx, 0);
	SET_TLBLKCR(base, ctx, 0);
}

267 268
static void __program_context(void __iomem *base, int ctx,
			      struct msm_priv *priv)
S
Stepan Moskovchenko 已提交
269 270 271
{
	__reset_context(base, ctx);

272 273 274 275
	/* Turn on TEX Remap */
	SET_TRE(base, ctx, 1);
	SET_AFE(base, ctx, 1);

S
Stepan Moskovchenko 已提交
276 277 278 279 280 281 282
	/* Set up HTW mode */
	/* TLB miss configuration: perform HTW on miss */
	SET_TLBMCFG(base, ctx, 0x3);

	/* V2P configuration: HTW for access */
	SET_V2PCFG(base, ctx, 0x3);

283 284 285 286 287 288 289
	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);
	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]);
	SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]);

	/* Set prrr and nmrr */
	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);
	SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr);
S
Stepan Moskovchenko 已提交
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314

	/* Invalidate the TLB for this context */
	SET_CTX_TLBIALL(base, ctx, 0);

	/* Set interrupt number to "secure" interrupt */
	SET_IRPTNDX(base, ctx, 0);

	/* Enable context fault interrupt */
	SET_CFEIE(base, ctx, 1);

	/* Stall access on a context fault and let the handler deal with it */
	SET_CFCFG(base, ctx, 1);

	/* Redirect all cacheable requests to L2 slave port. */
	SET_RCISH(base, ctx, 1);
	SET_RCOSH(base, ctx, 1);
	SET_RCNSH(base, ctx, 1);

	/* Turn on BFB prefetch */
	SET_BFBDFE(base, ctx, 1);

	/* Enable the MMU */
	SET_M(base, ctx, 1);
}

315
static struct iommu_domain *msm_iommu_domain_alloc(unsigned type)
S
Stepan Moskovchenko 已提交
316
{
317
	struct msm_priv *priv;
S
Stepan Moskovchenko 已提交
318

319 320 321 322
	if (type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
S
Stepan Moskovchenko 已提交
323 324 325 326
	if (!priv)
		goto fail_nomem;

	INIT_LIST_HEAD(&priv->list_attached);
327

328 329 330
	priv->domain.geometry.aperture_start = 0;
	priv->domain.geometry.aperture_end   = (1ULL << 32) - 1;
	priv->domain.geometry.force_aperture = true;
331

332
	return &priv->domain;
S
Stepan Moskovchenko 已提交
333 334 335

fail_nomem:
	kfree(priv);
336
	return NULL;
S
Stepan Moskovchenko 已提交
337 338
}

339
static void msm_iommu_domain_free(struct iommu_domain *domain)
S
Stepan Moskovchenko 已提交
340 341 342 343 344
{
	struct msm_priv *priv;
	unsigned long flags;

	spin_lock_irqsave(&msm_iommu_lock, flags);
345
	priv = to_msm_priv(domain);
346 347 348
	kfree(priv);
	spin_unlock_irqrestore(&msm_iommu_lock, flags);
}
S
Stepan Moskovchenko 已提交
349

350 351 352
static int msm_iommu_domain_config(struct msm_priv *priv)
{
	spin_lock_init(&priv->pgtlock);
S
Stepan Moskovchenko 已提交
353

354 355 356 357 358 359 360 361
	priv->cfg = (struct io_pgtable_cfg) {
		.quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP,
		.pgsize_bitmap = msm_iommu_ops.pgsize_bitmap,
		.ias = 32,
		.oas = 32,
		.tlb = &msm_iommu_gather_ops,
		.iommu_dev = priv->dev,
	};
S
Stepan Moskovchenko 已提交
362

363 364 365 366 367
	priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv);
	if (!priv->iop) {
		dev_err(priv->dev, "Failed to allocate pgtable\n");
		return -EINVAL;
	}
S
Stepan Moskovchenko 已提交
368

369 370 371
	msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap;

	return 0;
S
Stepan Moskovchenko 已提交
372 373 374 375 376 377
}

static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
	int ret = 0;
	unsigned long flags;
S
Sricharan R 已提交
378 379 380
	struct msm_iommu_dev *iommu;
	struct msm_priv *priv = to_msm_priv(domain);
	struct msm_iommu_ctx_dev *master;
S
Stepan Moskovchenko 已提交
381

382 383 384
	priv->dev = dev;
	msm_iommu_domain_config(priv);

S
Stepan Moskovchenko 已提交
385
	spin_lock_irqsave(&msm_iommu_lock, flags);
S
Sricharan R 已提交
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
	list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {
		master = list_first_entry(&iommu->ctx_list,
					  struct msm_iommu_ctx_dev,
					  list);
		if (master->of_node == dev->of_node) {
			ret = __enable_clocks(iommu);
			if (ret)
				goto fail;

			list_for_each_entry(master, &iommu->ctx_list, list) {
				if (master->num) {
					dev_err(dev, "domain already attached");
					ret = -EEXIST;
					goto fail;
				}
				master->num =
					msm_iommu_alloc_ctx(iommu->context_map,
							    0, iommu->ncb);
					if (IS_ERR_VALUE(master->num)) {
						ret = -ENODEV;
						goto fail;
					}
				config_mids(iommu, master);
				__program_context(iommu->base, master->num,
410
						  priv);
S
Sricharan R 已提交
411 412 413
			}
			__disable_clocks(iommu);
			list_add(&iommu->dom_node, &priv->list_attached);
S
Stepan Moskovchenko 已提交
414
		}
S
Sricharan R 已提交
415
	}
S
Stepan Moskovchenko 已提交
416 417 418

fail:
	spin_unlock_irqrestore(&msm_iommu_lock, flags);
S
Sricharan R 已提交
419

S
Stepan Moskovchenko 已提交
420 421 422 423 424 425
	return ret;
}

static void msm_iommu_detach_dev(struct iommu_domain *domain,
				 struct device *dev)
{
S
Sricharan R 已提交
426
	struct msm_priv *priv = to_msm_priv(domain);
S
Stepan Moskovchenko 已提交
427
	unsigned long flags;
S
Sricharan R 已提交
428 429
	struct msm_iommu_dev *iommu;
	struct msm_iommu_ctx_dev *master;
430
	int ret;
S
Stepan Moskovchenko 已提交
431

432
	free_io_pgtable_ops(priv->iop);
433

434
	spin_lock_irqsave(&msm_iommu_lock, flags);
S
Sricharan R 已提交
435 436 437 438
	list_for_each_entry(iommu, &priv->list_attached, dom_node) {
		ret = __enable_clocks(iommu);
		if (ret)
			goto fail;
S
Stepan Moskovchenko 已提交
439

S
Sricharan R 已提交
440 441 442 443 444 445
		list_for_each_entry(master, &iommu->ctx_list, list) {
			msm_iommu_free_ctx(iommu->context_map, master->num);
			__reset_context(iommu->base, master->num);
		}
		__disable_clocks(iommu);
	}
S
Stepan Moskovchenko 已提交
446 447 448 449
fail:
	spin_unlock_irqrestore(&msm_iommu_lock, flags);
}

450
static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
451
			 phys_addr_t pa, size_t len, int prot)
S
Stepan Moskovchenko 已提交
452
{
453
	struct msm_priv *priv = to_msm_priv(domain);
S
Stepan Moskovchenko 已提交
454
	unsigned long flags;
455
	int ret;
S
Stepan Moskovchenko 已提交
456

457 458 459
	spin_lock_irqsave(&priv->pgtlock, flags);
	ret = priv->iop->map(priv->iop, iova, pa, len, prot);
	spin_unlock_irqrestore(&priv->pgtlock, flags);
S
Stepan Moskovchenko 已提交
460 461 462 463

	return ret;
}

464 465
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
			      size_t len)
S
Stepan Moskovchenko 已提交
466
{
467
	struct msm_priv *priv = to_msm_priv(domain);
S
Stepan Moskovchenko 已提交
468 469
	unsigned long flags;

470 471 472
	spin_lock_irqsave(&priv->pgtlock, flags);
	len = priv->iop->unmap(priv->iop, iova, len);
	spin_unlock_irqrestore(&priv->pgtlock, flags);
S
Stepan Moskovchenko 已提交
473

474
	return len;
S
Stepan Moskovchenko 已提交
475 476 477
}

static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
478
					  dma_addr_t va)
S
Stepan Moskovchenko 已提交
479 480
{
	struct msm_priv *priv;
S
Sricharan R 已提交
481 482
	struct msm_iommu_dev *iommu;
	struct msm_iommu_ctx_dev *master;
S
Stepan Moskovchenko 已提交
483 484 485 486 487 488
	unsigned int par;
	unsigned long flags;
	phys_addr_t ret = 0;

	spin_lock_irqsave(&msm_iommu_lock, flags);

489
	priv = to_msm_priv(domain);
S
Sricharan R 已提交
490 491
	iommu = list_first_entry(&priv->list_attached,
				 struct msm_iommu_dev, dom_node);
S
Stepan Moskovchenko 已提交
492

S
Sricharan R 已提交
493 494
	if (list_empty(&iommu->ctx_list))
		goto fail;
S
Stepan Moskovchenko 已提交
495

S
Sricharan R 已提交
496 497 498 499
	master = list_first_entry(&iommu->ctx_list,
				  struct msm_iommu_ctx_dev, list);
	if (!master)
		goto fail;
S
Stepan Moskovchenko 已提交
500

S
Sricharan R 已提交
501
	ret = __enable_clocks(iommu);
502 503 504
	if (ret)
		goto fail;

S
Stepan Moskovchenko 已提交
505
	/* Invalidate context TLB */
S
Sricharan R 已提交
506 507
	SET_CTX_TLBIALL(iommu->base, master->num, 0);
	SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA);
S
Stepan Moskovchenko 已提交
508

S
Sricharan R 已提交
509
	par = GET_PAR(iommu->base, master->num);
S
Stepan Moskovchenko 已提交
510 511

	/* We are dealing with a supersection */
S
Sricharan R 已提交
512
	if (GET_NOFAULT_SS(iommu->base, master->num))
S
Stepan Moskovchenko 已提交
513 514 515 516
		ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
	else	/* Upper 20 bits from PAR, lower 12 from VA */
		ret = (par & 0xFFFFF000) | (va & 0x00000FFF);

S
Sricharan R 已提交
517
	if (GET_FAULT(iommu->base, master->num))
518 519
		ret = 0;

S
Sricharan R 已提交
520
	__disable_clocks(iommu);
S
Stepan Moskovchenko 已提交
521 522 523 524 525
fail:
	spin_unlock_irqrestore(&msm_iommu_lock, flags);
	return ret;
}

526
static bool msm_iommu_capable(enum iommu_cap cap)
S
Stepan Moskovchenko 已提交
527
{
528
	return false;
S
Stepan Moskovchenko 已提交
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
}

static void print_ctx_regs(void __iomem *base, int ctx)
{
	unsigned int fsr = GET_FSR(base, ctx);
	pr_err("FAR    = %08x    PAR    = %08x\n",
	       GET_FAR(base, ctx), GET_PAR(base, ctx));
	pr_err("FSR    = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr,
			(fsr & 0x02) ? "TF " : "",
			(fsr & 0x04) ? "AFF " : "",
			(fsr & 0x08) ? "APF " : "",
			(fsr & 0x10) ? "TLBMF " : "",
			(fsr & 0x20) ? "HTWDEEF " : "",
			(fsr & 0x40) ? "HTWSEEF " : "",
			(fsr & 0x80) ? "MHF " : "",
			(fsr & 0x10000) ? "SL " : "",
			(fsr & 0x40000000) ? "SS " : "",
			(fsr & 0x80000000) ? "MULTI " : "");

	pr_err("FSYNR0 = %08x    FSYNR1 = %08x\n",
	       GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));
	pr_err("TTBR0  = %08x    TTBR1  = %08x\n",
	       GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
	pr_err("SCTLR  = %08x    ACTLR  = %08x\n",
	       GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
}

556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
static void insert_iommu_master(struct device *dev,
				struct msm_iommu_dev **iommu,
				struct of_phandle_args *spec)
{
	struct msm_iommu_ctx_dev *master = dev->archdata.iommu;
	int sid;

	if (list_empty(&(*iommu)->ctx_list)) {
		master = kzalloc(sizeof(*master), GFP_ATOMIC);
		master->of_node = dev->of_node;
		list_add(&master->list, &(*iommu)->ctx_list);
		dev->archdata.iommu = master;
	}

	for (sid = 0; sid < master->num_mids; sid++)
		if (master->mids[sid] == spec->args[0]) {
			dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n",
				 sid);
			return;
		}

	master->mids[master->num_mids++] = spec->args[0];
}

static int qcom_iommu_of_xlate(struct device *dev,
			       struct of_phandle_args *spec)
{
	struct msm_iommu_dev *iommu;
	unsigned long flags;
	int ret = 0;

	spin_lock_irqsave(&msm_iommu_lock, flags);
	list_for_each_entry(iommu, &qcom_iommu_devices, dev_node)
		if (iommu->dev->of_node == spec->np)
			break;

	if (!iommu || iommu->dev->of_node != spec->np) {
		ret = -ENODEV;
		goto fail;
	}

	insert_iommu_master(dev, &iommu, spec);
fail:
	spin_unlock_irqrestore(&msm_iommu_lock, flags);

	return ret;
}

S
Stepan Moskovchenko 已提交
604 605
irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id)
{
S
Sricharan R 已提交
606
	struct msm_iommu_dev *iommu = dev_id;
607
	unsigned int fsr;
608
	int i, ret;
S
Stepan Moskovchenko 已提交
609 610 611

	spin_lock(&msm_iommu_lock);

S
Sricharan R 已提交
612
	if (!iommu) {
S
Stepan Moskovchenko 已提交
613 614 615 616 617
		pr_err("Invalid device ID in context interrupt handler\n");
		goto fail;
	}

	pr_err("Unexpected IOMMU page fault!\n");
S
Sricharan R 已提交
618
	pr_err("base = %08x\n", (unsigned int)iommu->base);
S
Stepan Moskovchenko 已提交
619

S
Sricharan R 已提交
620
	ret = __enable_clocks(iommu);
621 622 623
	if (ret)
		goto fail;

S
Sricharan R 已提交
624 625
	for (i = 0; i < iommu->ncb; i++) {
		fsr = GET_FSR(iommu->base, i);
S
Stepan Moskovchenko 已提交
626 627 628
		if (fsr) {
			pr_err("Fault occurred in context %d.\n", i);
			pr_err("Interesting registers:\n");
S
Sricharan R 已提交
629 630
			print_ctx_regs(iommu->base, i);
			SET_FSR(iommu->base, i, 0x4000000F);
S
Stepan Moskovchenko 已提交
631 632
		}
	}
S
Sricharan R 已提交
633
	__disable_clocks(iommu);
S
Stepan Moskovchenko 已提交
634 635 636 637 638
fail:
	spin_unlock(&msm_iommu_lock);
	return 0;
}

639
static struct iommu_ops msm_iommu_ops = {
640
	.capable = msm_iommu_capable,
641 642
	.domain_alloc = msm_iommu_domain_alloc,
	.domain_free = msm_iommu_domain_free,
S
Stepan Moskovchenko 已提交
643 644 645 646
	.attach_dev = msm_iommu_attach_dev,
	.detach_dev = msm_iommu_detach_dev,
	.map = msm_iommu_map,
	.unmap = msm_iommu_unmap,
O
Olav Haugan 已提交
647
	.map_sg = default_iommu_map_sg,
S
Stepan Moskovchenko 已提交
648
	.iova_to_phys = msm_iommu_iova_to_phys,
649
	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
650
	.of_xlate = qcom_iommu_of_xlate,
S
Stepan Moskovchenko 已提交
651 652
};

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
static int msm_iommu_probe(struct platform_device *pdev)
{
	struct resource *r;
	struct msm_iommu_dev *iommu;
	int ret, par, val;

	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
	if (!iommu)
		return -ENODEV;

	iommu->dev = &pdev->dev;
	INIT_LIST_HEAD(&iommu->ctx_list);

	iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");
	if (IS_ERR(iommu->pclk)) {
		dev_err(iommu->dev, "could not get smmu_pclk\n");
		return PTR_ERR(iommu->pclk);
	}

	ret = clk_prepare(iommu->pclk);
	if (ret) {
		dev_err(iommu->dev, "could not prepare smmu_pclk\n");
		return ret;
	}

	iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");
	if (IS_ERR(iommu->clk)) {
		dev_err(iommu->dev, "could not get iommu_clk\n");
		clk_unprepare(iommu->pclk);
		return PTR_ERR(iommu->clk);
	}

	ret = clk_prepare(iommu->clk);
	if (ret) {
		dev_err(iommu->dev, "could not prepare iommu_clk\n");
		clk_unprepare(iommu->pclk);
		return ret;
	}

	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	iommu->base = devm_ioremap_resource(iommu->dev, r);
	if (IS_ERR(iommu->base)) {
		dev_err(iommu->dev, "could not get iommu base\n");
		ret = PTR_ERR(iommu->base);
		goto fail;
	}

	iommu->irq = platform_get_irq(pdev, 0);
	if (iommu->irq < 0) {
		dev_err(iommu->dev, "could not get iommu irq\n");
		ret = -ENODEV;
		goto fail;
	}

	ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val);
	if (ret) {
		dev_err(iommu->dev, "could not get ncb\n");
		goto fail;
	}
	iommu->ncb = val;

	msm_iommu_reset(iommu->base, iommu->ncb);
	SET_M(iommu->base, 0, 1);
	SET_PAR(iommu->base, 0, 0);
	SET_V2PCFG(iommu->base, 0, 1);
	SET_V2PPR(iommu->base, 0, 0);
	par = GET_PAR(iommu->base, 0);
	SET_V2PCFG(iommu->base, 0, 0);
	SET_M(iommu->base, 0, 0);

	if (!par) {
		pr_err("Invalid PAR value detected\n");
		ret = -ENODEV;
		goto fail;
	}

	ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL,
					msm_iommu_fault_handler,
					IRQF_ONESHOT | IRQF_SHARED,
					"msm_iommu_secure_irpt_handler",
					iommu);
	if (ret) {
		pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret);
		goto fail;
	}

	list_add(&iommu->dev_node, &qcom_iommu_devices);
740
	of_iommu_set_ops(pdev->dev.of_node, &msm_iommu_ops);
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793

	pr_info("device mapped at %p, irq %d with %d ctx banks\n",
		iommu->base, iommu->irq, iommu->ncb);

	return ret;
fail:
	clk_unprepare(iommu->clk);
	clk_unprepare(iommu->pclk);
	return ret;
}

static const struct of_device_id msm_iommu_dt_match[] = {
	{ .compatible = "qcom,apq8064-iommu" },
	{}
};

static int msm_iommu_remove(struct platform_device *pdev)
{
	struct msm_iommu_dev *iommu = platform_get_drvdata(pdev);

	clk_unprepare(iommu->clk);
	clk_unprepare(iommu->pclk);
	return 0;
}

static struct platform_driver msm_iommu_driver = {
	.driver = {
		.name	= "msm_iommu",
		.of_match_table = msm_iommu_dt_match,
	},
	.probe		= msm_iommu_probe,
	.remove		= msm_iommu_remove,
};

static int __init msm_iommu_driver_init(void)
{
	int ret;

	ret = platform_driver_register(&msm_iommu_driver);
	if (ret != 0)
		pr_err("Failed to register IOMMU driver\n");

	return ret;
}

static void __exit msm_iommu_driver_exit(void)
{
	platform_driver_unregister(&msm_iommu_driver);
}

subsys_initcall(msm_iommu_driver_init);
module_exit(msm_iommu_driver_exit);

794
static int __init msm_iommu_init(void)
S
Stepan Moskovchenko 已提交
795
{
796
	bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
S
Stepan Moskovchenko 已提交
797 798 799
	return 0;
}

800 801 802 803 804 805 806
static int __init msm_iommu_of_setup(struct device_node *np)
{
	msm_iommu_init();
	return 0;
}

IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu", msm_iommu_of_setup);
S
Stepan Moskovchenko 已提交
807 808 809

MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");