coherency.c 11.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Coherency fabric (Aurora) support for Armada 370 and XP platforms.
 *
 * Copyright (C) 2012 Marvell
 *
 * Yehuda Yitschak <yehuday@marvell.com>
 * Gregory Clement <gregory.clement@free-electrons.com>
 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
 *
 * This file is licensed under the terms of the GNU General Public
 * License version 2.  This program is licensed "as is" without any
 * warranty of any kind, whether express or implied.
 *
 * The Armada 370 and Armada XP SOCs have a coherency fabric which is
 * responsible for ensuring hardware coherency between all CPUs and between
 * CPUs and I/O masters. This file initializes the coherency fabric and
 * supplies basic routines for configuring and controlling hardware coherency
 */

20 21
#define pr_fmt(fmt) "mvebu-coherency: " fmt

22 23 24 25 26
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/of_address.h>
#include <linux/io.h>
#include <linux/smp.h>
27 28
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
29 30 31
#include <linux/slab.h>
#include <linux/mbus.h>
#include <linux/clk.h>
32
#include <linux/pci.h>
33
#include <asm/smp_plat.h>
34
#include <asm/cacheflush.h>
35
#include <asm/mach/map.h>
36
#include "armada-370-xp.h"
37
#include "coherency.h"
38
#include "mvebu-soc-id.h"
39

40
unsigned long coherency_phys_base;
41
void __iomem *coherency_base;
42
static void __iomem *coherency_cpu_base;
43 44 45 46

/* Coherency fabric registers */
#define COHERENCY_FABRIC_CFG_OFFSET		   0x4

47 48
#define IO_SYNC_BARRIER_CTL_OFFSET		   0x0

49
enum {
50
	COHERENCY_FABRIC_TYPE_NONE,
51
	COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
52
	COHERENCY_FABRIC_TYPE_ARMADA_375,
53
	COHERENCY_FABRIC_TYPE_ARMADA_380,
54 55
};

56
static struct of_device_id of_coherency_table[] = {
57 58
	{.compatible = "marvell,coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
59 60
	{.compatible = "marvell,armada-375-coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
61 62
	{.compatible = "marvell,armada-380-coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
63 64 65
	{ /* end of list */ },
};

66 67 68
/* Functions defined in coherency_ll.S */
int ll_enable_coherency(void);
void ll_add_cpu_to_smp_group(void);
69

70
int set_cpu_coherent(void)
71 72
{
	if (!coherency_base) {
73
		pr_warn("Can't make current CPU cache coherent.\n");
74 75 76 77
		pr_warn("Coherency fabric is not initialized\n");
		return 1;
	}

78 79
	ll_add_cpu_to_smp_group();
	return ll_enable_coherency();
80 81
}

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
/*
 * The below code implements the I/O coherency workaround on Armada
 * 375. This workaround consists in using the two channels of the
 * first XOR engine to trigger a XOR transaction that serves as the
 * I/O coherency barrier.
 */

static void __iomem *xor_base, *xor_high_base;
static dma_addr_t coherency_wa_buf_phys[CONFIG_NR_CPUS];
static void *coherency_wa_buf[CONFIG_NR_CPUS];
static bool coherency_wa_enabled;

#define XOR_CONFIG(chan)            (0x10 + (chan * 4))
#define XOR_ACTIVATION(chan)        (0x20 + (chan * 4))
#define WINDOW_BAR_ENABLE(chan)     (0x240 + ((chan) << 2))
#define WINDOW_BASE(w)              (0x250 + ((w) << 2))
#define WINDOW_SIZE(w)              (0x270 + ((w) << 2))
#define WINDOW_REMAP_HIGH(w)        (0x290 + ((w) << 2))
#define WINDOW_OVERRIDE_CTRL(chan)  (0x2A0 + ((chan) << 2))
#define XOR_DEST_POINTER(chan)      (0x2B0 + (chan * 4))
#define XOR_BLOCK_SIZE(chan)        (0x2C0 + (chan * 4))
#define XOR_INIT_VALUE_LOW           0x2E0
#define XOR_INIT_VALUE_HIGH          0x2E4

static inline void mvebu_hwcc_armada375_sync_io_barrier_wa(void)
{
	int idx = smp_processor_id();

	/* Write '1' to the first word of the buffer */
	writel(0x1, coherency_wa_buf[idx]);

	/* Wait until the engine is idle */
	while ((readl(xor_base + XOR_ACTIVATION(idx)) >> 4) & 0x3)
		;

	dmb();

	/* Trigger channel */
	writel(0x1, xor_base + XOR_ACTIVATION(idx));

	/* Poll the data until it is cleared by the XOR transaction */
	while (readl(coherency_wa_buf[idx]))
		;
}

static void __init armada_375_coherency_init_wa(void)
{
	const struct mbus_dram_target_info *dram;
	struct device_node *xor_node;
	struct property *xor_status;
	struct clk *xor_clk;
	u32 win_enable = 0;
	int i;

	pr_warn("enabling coherency workaround for Armada 375 Z1, one XOR engine disabled\n");

	/*
	 * Since the workaround uses one XOR engine, we grab a
	 * reference to its Device Tree node first.
	 */
	xor_node = of_find_compatible_node(NULL, NULL, "marvell,orion-xor");
	BUG_ON(!xor_node);

	/*
	 * Then we mark it as disabled so that the real XOR driver
	 * will not use it.
	 */
	xor_status = kzalloc(sizeof(struct property), GFP_KERNEL);
	BUG_ON(!xor_status);

	xor_status->value = kstrdup("disabled", GFP_KERNEL);
	BUG_ON(!xor_status->value);

	xor_status->length = 8;
	xor_status->name = kstrdup("status", GFP_KERNEL);
	BUG_ON(!xor_status->name);

	of_update_property(xor_node, xor_status);

	/*
	 * And we remap the registers, get the clock, and do the
	 * initial configuration of the XOR engine.
	 */
	xor_base = of_iomap(xor_node, 0);
	xor_high_base = of_iomap(xor_node, 1);

	xor_clk = of_clk_get_by_name(xor_node, NULL);
	BUG_ON(!xor_clk);

	clk_prepare_enable(xor_clk);

	dram = mv_mbus_dram_info();

	for (i = 0; i < 8; i++) {
		writel(0, xor_base + WINDOW_BASE(i));
		writel(0, xor_base + WINDOW_SIZE(i));
		if (i < 4)
			writel(0, xor_base + WINDOW_REMAP_HIGH(i));
	}

	for (i = 0; i < dram->num_cs; i++) {
		const struct mbus_dram_window *cs = dram->cs + i;
		writel((cs->base & 0xffff0000) |
		       (cs->mbus_attr << 8) |
		       dram->mbus_dram_target_id, xor_base + WINDOW_BASE(i));
		writel((cs->size - 1) & 0xffff0000, xor_base + WINDOW_SIZE(i));

		win_enable |= (1 << i);
		win_enable |= 3 << (16 + (2 * i));
	}

	writel(win_enable, xor_base + WINDOW_BAR_ENABLE(0));
	writel(win_enable, xor_base + WINDOW_BAR_ENABLE(1));
	writel(0, xor_base + WINDOW_OVERRIDE_CTRL(0));
	writel(0, xor_base + WINDOW_OVERRIDE_CTRL(1));

	for (i = 0; i < CONFIG_NR_CPUS; i++) {
		coherency_wa_buf[i] = kzalloc(PAGE_SIZE, GFP_KERNEL);
		BUG_ON(!coherency_wa_buf[i]);

		/*
		 * We can't use the DMA mapping API, since we don't
		 * have a valid 'struct device' pointer
		 */
		coherency_wa_buf_phys[i] =
			virt_to_phys(coherency_wa_buf[i]);
		BUG_ON(!coherency_wa_buf_phys[i]);

		/*
		 * Configure the XOR engine for memset operation, with
		 * a 128 bytes block size
		 */
		writel(0x444, xor_base + XOR_CONFIG(i));
		writel(128, xor_base + XOR_BLOCK_SIZE(i));
		writel(coherency_wa_buf_phys[i],
		       xor_base + XOR_DEST_POINTER(i));
	}

	writel(0x0, xor_base + XOR_INIT_VALUE_LOW);
	writel(0x0, xor_base + XOR_INIT_VALUE_HIGH);

	coherency_wa_enabled = true;
}

226 227
static inline void mvebu_hwcc_sync_io_barrier(void)
{
228 229 230 231 232
	if (coherency_wa_enabled) {
		mvebu_hwcc_armada375_sync_io_barrier_wa();
		return;
	}

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
	writel(0x1, coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET);
	while (readl(coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET) & 0x1);
}

static dma_addr_t mvebu_hwcc_dma_map_page(struct device *dev, struct page *page,
				  unsigned long offset, size_t size,
				  enum dma_data_direction dir,
				  struct dma_attrs *attrs)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
}


static void mvebu_hwcc_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
			      size_t size, enum dma_data_direction dir,
			      struct dma_attrs *attrs)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
}

static void mvebu_hwcc_dma_sync(struct device *dev, dma_addr_t dma_handle,
			size_t size, enum dma_data_direction dir)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
}

static struct dma_map_ops mvebu_hwcc_dma_ops = {
	.alloc			= arm_dma_alloc,
	.free			= arm_dma_free,
	.mmap			= arm_dma_mmap,
	.map_page		= mvebu_hwcc_dma_map_page,
	.unmap_page		= mvebu_hwcc_dma_unmap_page,
	.get_sgtable		= arm_dma_get_sgtable,
	.map_sg			= arm_dma_map_sg,
	.unmap_sg		= arm_dma_unmap_sg,
	.sync_single_for_cpu	= mvebu_hwcc_dma_sync,
	.sync_single_for_device	= mvebu_hwcc_dma_sync,
	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
	.set_dma_mask		= arm_dma_set_mask,
};

279 280
static int mvebu_hwcc_notifier(struct notifier_block *nb,
			       unsigned long event, void *__dev)
281 282 283 284 285 286 287 288 289 290
{
	struct device *dev = __dev;

	if (event != BUS_NOTIFY_ADD_DEVICE)
		return NOTIFY_DONE;
	set_dma_ops(dev, &mvebu_hwcc_dma_ops);

	return NOTIFY_OK;
}

291 292
static struct notifier_block mvebu_hwcc_nb = {
	.notifier_call = mvebu_hwcc_notifier,
293 294
};

295 296 297 298
static struct notifier_block mvebu_hwcc_pci_nb = {
	.notifier_call = mvebu_hwcc_notifier,
};

299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
static void __init armada_370_coherency_init(struct device_node *np)
{
	struct resource res;

	of_address_to_resource(np, 0, &res);
	coherency_phys_base = res.start;
	/*
	 * Ensure secondary CPUs will see the updated value,
	 * which they read before they join the coherency
	 * fabric, and therefore before they are coherent with
	 * the boot CPU cache.
	 */
	sync_cache_w(&coherency_phys_base);
	coherency_base = of_iomap(np, 0);
	coherency_cpu_base = of_iomap(np, 1);
314
	set_cpu_coherent();
315 316
}

317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
/*
 * This ioremap hook is used on Armada 375/38x to ensure that PCIe
 * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This
 * is needed as a workaround for a deadlock issue between the PCIe
 * interface and the cache controller.
 */
static void __iomem *
armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
			      unsigned int mtype, void *caller)
{
	struct resource pcie_mem;

	mvebu_mbus_get_pcie_mem_aperture(&pcie_mem);

	if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end)
		mtype = MT_UNCACHED;

	return __arm_ioremap_caller(phys_addr, size, mtype, caller);
}

337
static void __init armada_375_380_coherency_init(struct device_node *np)
338
{
339 340
	struct device_node *cache_dn;

341
	coherency_cpu_base = of_iomap(np, 0);
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
	arch_ioremap_caller = armada_pcie_wa_ioremap_caller;

	/*
	 * Add the PL310 property "arm,io-coherent". This makes sure the
	 * outer sync operation is not used, which allows to
	 * workaround the system erratum that causes deadlocks when
	 * doing PCIe in an SMP situation on Armada 375 and Armada
	 * 38x.
	 */
	for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
		struct property *p;

		p = kzalloc(sizeof(*p), GFP_KERNEL);
		p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
		of_add_property(cache_dn, p);
	}
358 359
}

360
static int coherency_type(void)
361 362
{
	struct device_node *np;
363
	const struct of_device_id *match;
364

365
	np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
366
	if (np) {
367
		int type = (int) match->data;
368

369
		/* Armada 370/XP coherency works in both UP and SMP */
370
		if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
371
			return type;
372

373 374 375 376
		/* Armada 375 coherency works only on SMP */
		else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 && is_smp())
			return type;

377 378 379
		/* Armada 380 coherency works only on SMP */
		else if (type == COHERENCY_FABRIC_TYPE_ARMADA_380 && is_smp())
			return type;
380 381
	}

382
	return COHERENCY_FABRIC_TYPE_NONE;
383
}
384

385
int coherency_available(void)
386
{
387 388 389 390 391 392
	return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
}

int __init coherency_init(void)
{
	int type = coherency_type();
393 394 395
	struct device_node *np;

	np = of_find_matching_node(NULL, of_coherency_table);
396 397 398

	if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
		armada_370_coherency_init(np);
399 400 401
	else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
		 type == COHERENCY_FABRIC_TYPE_ARMADA_380)
		armada_375_380_coherency_init(np);
402

403 404
	of_node_put(np);

405 406 407 408 409
	return 0;
}

static int __init coherency_late_init(void)
{
410 411 412 413 414
	int type = coherency_type();

	if (type == COHERENCY_FABRIC_TYPE_NONE)
		return 0;

415 416 417 418 419 420 421
	if (type == COHERENCY_FABRIC_TYPE_ARMADA_375) {
		u32 dev, rev;

		if (mvebu_get_soc_id(&dev, &rev) == 0 &&
		    rev == ARMADA_375_Z1_REV)
			armada_375_coherency_init_wa();
	}
422 423

	bus_register_notifier(&platform_bus_type,
424
			      &mvebu_hwcc_nb);
425

426 427 428 429
	return 0;
}

postcore_initcall(coherency_late_init);
430

431
#if IS_ENABLED(CONFIG_PCI)
432 433 434 435
static int __init coherency_pci_init(void)
{
	if (coherency_available())
		bus_register_notifier(&pci_bus_type,
436
				       &mvebu_hwcc_pci_nb);
437 438 439 440
	return 0;
}

arch_initcall(coherency_pci_init);
441
#endif