aperture_64.c 13.5 KB
Newer Older
1
/*
L
Linus Torvalds 已提交
2
 * Firmware replacement code.
3
 *
L
Linus Torvalds 已提交
4
 * Work around broken BIOSes that don't set an aperture or only set the
5 6 7 8 9
 * aperture in the AGP bridge.
 * If all fails map the aperture over some low memory.  This is cheaper than
 * doing bounce buffering. The memory is lost. This is done at early boot
 * because only the bootmem allocator can allocate 32+MB.
 *
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19
 * Copyright 2002 Andi Kleen, SuSE Labs.
 */
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mmzone.h>
#include <linux/pci_ids.h>
#include <linux/pci.h>
#include <linux/bitops.h>
20
#include <linux/ioport.h>
21
#include <linux/suspend.h>
L
Linus Torvalds 已提交
22 23
#include <asm/e820.h>
#include <asm/io.h>
24
#include <asm/iommu.h>
J
Joerg Roedel 已提交
25
#include <asm/gart.h>
L
Linus Torvalds 已提交
26
#include <asm/pci-direct.h>
27
#include <asm/dma.h>
28
#include <asm/k8.h>
L
Linus Torvalds 已提交
29

30
int gart_iommu_aperture;
P
Pavel Machek 已提交
31 32
int gart_iommu_aperture_disabled __initdata;
int gart_iommu_aperture_allowed __initdata;
L
Linus Torvalds 已提交
33 34

int fallback_aper_order __initdata = 1; /* 64MB */
P
Pavel Machek 已提交
35
int fallback_aper_force __initdata;
L
Linus Torvalds 已提交
36 37 38

int fix_aperture __initdata = 1;

Y
Yinghai Lu 已提交
39 40 41 42 43 44 45 46 47 48 49 50
struct bus_dev_range {
	int bus;
	int dev_base;
	int dev_limit;
};

static struct bus_dev_range bus_dev_ranges[] __initdata = {
	{ 0x00, 0x18, 0x20},
	{ 0xff, 0x00, 0x20},
	{ 0xfe, 0x00, 0x20}
};

51 52 53 54 55 56 57 58 59 60 61 62
static struct resource gart_resource = {
	.name	= "GART",
	.flags	= IORESOURCE_MEM,
};

static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
{
	gart_resource.start = aper_base;
	gart_resource.end = aper_base + aper_size - 1;
	insert_resource(&iomem_resource, &gart_resource);
}

63 64
/* This code runs before the PCI subsystem is initialized, so just
   access the northbridge directly. */
L
Linus Torvalds 已提交
65

66
static u32 __init allocate_aperture(void)
L
Linus Torvalds 已提交
67 68
{
	u32 aper_size;
69
	void *p;
L
Linus Torvalds 已提交
70

71 72 73
	/* aper_size should <= 1G */
	if (fallback_aper_order > 5)
		fallback_aper_order = 5;
74
	aper_size = (32 * 1024 * 1024) << fallback_aper_order;
L
Linus Torvalds 已提交
75

76 77 78 79 80
	/*
	 * Aperture has to be naturally aligned. This means a 2GB aperture
	 * won't have much chance of finding a place in the lower 4GB of
	 * memory. Unfortunately we cannot move it up because that would
	 * make the IOMMU useless.
L
Linus Torvalds 已提交
81
	 */
82 83 84 85 86 87 88 89 90 91 92 93 94 95
	/*
	 * using 512M as goal, in case kexec will load kernel_big
	 * that will do the on position decompress, and  could overlap with
	 * that positon with gart that is used.
	 * sequende:
	 * kernel_small
	 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
	 * ==> kernel_small(gart area become e820_reserved)
	 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
	 * ==> kerne_big (uncompressed size will be big than 64M or 128M)
	 * so don't use 512M below as gart iommu, leave the space for kernel
	 * code for safe
	 */
	p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
L
Linus Torvalds 已提交
96
	if (!p || __pa(p)+aper_size > 0xffffffff) {
97 98 99
		printk(KERN_ERR
			"Cannot allocate aperture memory hole (%p,%uK)\n",
				p, aper_size>>10);
L
Linus Torvalds 已提交
100
		if (p)
101
			free_bootmem(__pa(p), aper_size);
L
Linus Torvalds 已提交
102 103
		return 0;
	}
104 105
	printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
			aper_size >> 10, __pa(p));
106
	insert_aperture_resource((u32)__pa(p), aper_size);
107 108
	register_nosave_region((u32)__pa(p) >> PAGE_SHIFT,
				(u32)__pa(p+aper_size) >> PAGE_SHIFT);
109 110

	return (u32)__pa(p);
L
Linus Torvalds 已提交
111 112 113
}


114
/* Find a PCI capability */
P
Pavel Machek 已提交
115
static u32 __init find_cap(int bus, int slot, int func, int cap)
116
{
L
Linus Torvalds 已提交
117
	int bytes;
118 119
	u8 pos;

Y
Yinghai Lu 已提交
120
	if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
121
						PCI_STATUS_CAP_LIST))
L
Linus Torvalds 已提交
122
		return 0;
123

Y
Yinghai Lu 已提交
124
	pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
125
	for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
L
Linus Torvalds 已提交
126
		u8 id;
127 128

		pos &= ~3;
Y
Yinghai Lu 已提交
129
		id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
L
Linus Torvalds 已提交
130 131
		if (id == 0xff)
			break;
132 133
		if (id == cap)
			return pos;
Y
Yinghai Lu 已提交
134
		pos = read_pci_config_byte(bus, slot, func,
135 136
						pos+PCI_CAP_LIST_NEXT);
	}
L
Linus Torvalds 已提交
137
	return 0;
138
}
L
Linus Torvalds 已提交
139 140

/* Read a standard AGPv3 bridge header */
P
Pavel Machek 已提交
141
static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
142
{
L
Linus Torvalds 已提交
143 144 145 146 147
	u32 apsize;
	u32 apsizereg;
	int nbits;
	u32 aper_low, aper_hi;
	u64 aper;
148
	u32 old_order;
L
Linus Torvalds 已提交
149

Y
Yinghai Lu 已提交
150 151
	printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
	apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
L
Linus Torvalds 已提交
152
	if (apsizereg == 0xffffffff) {
153
		printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
L
Linus Torvalds 已提交
154 155 156
		return 0;
	}

157 158 159
	/* old_order could be the value from NB gart setting */
	old_order = *order;

L
Linus Torvalds 已提交
160 161
	apsize = apsizereg & 0xfff;
	/* Some BIOS use weird encodings not in the AGPv3 table. */
162 163
	if (apsize & 0xff)
		apsize |= 0xf00;
L
Linus Torvalds 已提交
164 165 166 167
	nbits = hweight16(apsize);
	*order = 7 - nbits;
	if ((int)*order < 0) /* < 32MB */
		*order = 0;
168

Y
Yinghai Lu 已提交
169 170
	aper_low = read_pci_config(bus, slot, func, 0x10);
	aper_hi = read_pci_config(bus, slot, func, 0x14);
L
Linus Torvalds 已提交
171 172
	aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);

173 174 175 176
	/*
	 * On some sick chips, APSIZE is 0. It means it wants 4G
	 * so let double check that order, and lets trust AMD NB settings:
	 */
Y
Yinghai Lu 已提交
177 178 179
	printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
			aper, 32 << old_order);
	if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
180 181 182 183 184
		printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
				32 << *order, apsizereg);
		*order = old_order;
	}

185 186
	printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
			aper, 32 << *order, apsizereg);
L
Linus Torvalds 已提交
187

Y
Yinghai Lu 已提交
188
	if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
189 190 191
		return 0;
	return (u32)aper;
}
L
Linus Torvalds 已提交
192

193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Look for an AGP bridge. Windows only expects the aperture in the
 * AGP bridge and some BIOS forget to initialize the Northbridge too.
 * Work around this here.
 *
 * Do an PCI bus scan by hand because we're running before the PCI
 * subsystem.
 *
 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan
 * generically. It's probably overkill to always scan all slots because
 * the AGP bridges should be always an own bus on the HT hierarchy,
 * but do it here for future safety.
 */
P
Pavel Machek 已提交
206
static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
L
Linus Torvalds 已提交
207
{
Y
Yinghai Lu 已提交
208
	int bus, slot, func;
L
Linus Torvalds 已提交
209 210

	/* Poor man's PCI discovery */
Y
Yinghai Lu 已提交
211
	for (bus = 0; bus < 256; bus++) {
212 213
		for (slot = 0; slot < 32; slot++) {
			for (func = 0; func < 8; func++) {
L
Linus Torvalds 已提交
214 215
				u32 class, cap;
				u8 type;
Y
Yinghai Lu 已提交
216
				class = read_pci_config(bus, slot, func,
L
Linus Torvalds 已提交
217 218
							PCI_CLASS_REVISION);
				if (class == 0xffffffff)
219 220 221
					break;

				switch (class >> 16) {
L
Linus Torvalds 已提交
222 223 224
				case PCI_CLASS_BRIDGE_HOST:
				case PCI_CLASS_BRIDGE_OTHER: /* needed? */
					/* AGP bridge? */
Y
Yinghai Lu 已提交
225
					cap = find_cap(bus, slot, func,
226
							PCI_CAP_ID_AGP);
L
Linus Torvalds 已提交
227 228
					if (!cap)
						break;
229
					*valid_agp = 1;
Y
Yinghai Lu 已提交
230
					return read_agp(bus, slot, func, cap,
231 232 233
							order);
				}

L
Linus Torvalds 已提交
234
				/* No multi-function device? */
Y
Yinghai Lu 已提交
235
				type = read_pci_config_byte(bus, slot, func,
L
Linus Torvalds 已提交
236 237 238
							       PCI_HEADER_TYPE);
				if (!(type & 0x80))
					break;
239 240
			}
		}
L
Linus Torvalds 已提交
241
	}
242
	printk(KERN_INFO "No AGP bridge found\n");
243

L
Linus Torvalds 已提交
244 245 246
	return 0;
}

Y
Yinghai Lu 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
static int gart_fix_e820 __initdata = 1;

static int __init parse_gart_mem(char *p)
{
	if (!p)
		return -EINVAL;

	if (!strncmp(p, "off", 3))
		gart_fix_e820 = 0;
	else if (!strncmp(p, "on", 2))
		gart_fix_e820 = 1;

	return 0;
}
early_param("gart_fix_e820", parse_gart_mem);

void __init early_gart_iommu_check(void)
{
	/*
	 * in case it is enabled before, esp for kexec/kdump,
	 * previous kernel already enable that. memset called
	 * by allocate_aperture/__alloc_bootmem_nopanic cause restart.
	 * or second kernel have different position for GART hole. and new
	 * kernel could use hole as RAM that is still used by GART set by
	 * first kernel
	 * or BIOS forget to put that in reserved.
	 * try to update e820 to make that region as reserved.
	 */
275
	int i, fix, slot;
Y
Yinghai Lu 已提交
276 277 278
	u32 ctl;
	u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
	u64 aper_base = 0, last_aper_base = 0;
279
	int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0;
Y
Yinghai Lu 已提交
280 281 282 283

	if (!early_pci_allowed())
		return;

284
	/* This is mostly duplicate of iommu_hole_init */
Y
Yinghai Lu 已提交
285
	fix = 0;
Y
Yinghai Lu 已提交
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;

		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
			aper_enabled = ctl & AMD64_GARTEN;
			aper_order = (ctl >> 1) & 7;
			aper_size = (32 * 1024 * 1024) << aper_order;
			aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
			aper_base <<= 25;

305 306 307 308 309 310 311
			if (last_valid) {
				if ((aper_order != last_aper_order) ||
				    (aper_base != last_aper_base) ||
				    (aper_enabled != last_aper_enabled)) {
					fix = 1;
					break;
				}
Y
Yinghai Lu 已提交
312
			}
313

Y
Yinghai Lu 已提交
314 315 316
			last_aper_order = aper_order;
			last_aper_base = aper_base;
			last_aper_enabled = aper_enabled;
317
			last_valid = 1;
Y
Yinghai Lu 已提交
318 319 320 321 322 323 324 325 326 327
		}
	}

	if (!fix && !aper_enabled)
		return;

	if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL)
		fix = 1;

	if (gart_fix_e820 && !fix && aper_enabled) {
328 329
		if (e820_any_mapped(aper_base, aper_base + aper_size,
				    E820_RAM)) {
330
			/* reserve it, so we can reuse it in second kernel */
Y
Yinghai Lu 已提交
331
			printk(KERN_INFO "update e820 for GART\n");
332
			e820_add_region(aper_base, aper_size, E820_RESERVED);
Y
Yinghai Lu 已提交
333 334 335 336
			update_e820();
		}
	}

337 338 339
	if (!fix)
		return;

Y
Yinghai Lu 已提交
340
	/* different nodes have different setting, disable them all at first*/
Y
Yinghai Lu 已提交
341 342 343 344 345 346 347
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;
Y
Yinghai Lu 已提交
348

Y
Yinghai Lu 已提交
349 350 351 352 353 354 355 356
		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
			ctl &= ~AMD64_GARTEN;
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
		}
Y
Yinghai Lu 已提交
357 358 359 360
	}

}

Y
Yinghai Lu 已提交
361 362
static int __initdata printed_gart_size_msg;

363
void __init gart_iommu_hole_init(void)
364
{
Y
Yinghai Lu 已提交
365
	u32 agp_aper_base = 0, agp_aper_order = 0;
366
	u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
L
Linus Torvalds 已提交
367
	u64 aper_base, last_aper_base = 0;
Y
Yinghai Lu 已提交
368 369
	int fix, slot, valid_agp = 0;
	int i, node;
L
Linus Torvalds 已提交
370

371 372
	if (gart_iommu_aperture_disabled || !fix_aperture ||
	    !early_pci_allowed())
L
Linus Torvalds 已提交
373 374
		return;

375
	printk(KERN_INFO  "Checking aperture...\n");
L
Linus Torvalds 已提交
376

Y
Yinghai Lu 已提交
377 378 379
	if (!fallback_aper_force)
		agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);

L
Linus Torvalds 已提交
380
	fix = 0;
381
	node = 0;
Y
Yinghai Lu 已提交
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;

		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			iommu_detected = 1;
			gart_iommu_aperture = 1;

			aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7;
			aper_size = (32 * 1024 * 1024) << aper_order;
			aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
			aper_base <<= 25;

			printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
					node, aper_base, aper_size >> 20);
			node++;

			if (!aperture_valid(aper_base, aper_size, 64<<20)) {
				if (valid_agp && agp_aper_base &&
				    agp_aper_base == aper_base &&
				    agp_aper_order == aper_order) {
					/* the same between two setting from NB and agp */
Y
Yinghai Lu 已提交
411 412 413
					if (!no_iommu &&
					    max_pfn > MAX_DMA32_PFN &&
					    !printed_gart_size_msg) {
Y
Yinghai Lu 已提交
414 415 416 417 418 419 420 421
						printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
						printk(KERN_ERR "please increase GART size in your BIOS setup\n");
						printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
						printed_gart_size_msg = 1;
					}
				} else {
					fix = 1;
					goto out;
Y
Yinghai Lu 已提交
422 423
				}
			}
L
Linus Torvalds 已提交
424

Y
Yinghai Lu 已提交
425 426 427 428 429 430 431
			if ((last_aper_order && aper_order != last_aper_order) ||
			    (last_aper_base && aper_base != last_aper_base)) {
				fix = 1;
				goto out;
			}
			last_aper_order = aper_order;
			last_aper_base = aper_base;
L
Linus Torvalds 已提交
432
		}
433
	}
L
Linus Torvalds 已提交
434

Y
Yinghai Lu 已提交
435
out:
436 437 438
	if (!fix && !fallback_aper_force) {
		if (last_aper_base) {
			unsigned long n = (32 * 1024 * 1024) << last_aper_order;
439

440 441
			insert_aperture_resource((u32)last_aper_base, n);
		}
442
		return;
443
	}
L
Linus Torvalds 已提交
444

Y
Yinghai Lu 已提交
445 446 447 448
	if (!fallback_aper_force) {
		aper_alloc = agp_aper_base;
		aper_order = agp_aper_order;
	}
449 450

	if (aper_alloc) {
L
Linus Torvalds 已提交
451
		/* Got the aperture from the AGP bridge */
452 453
	} else if (swiotlb && !valid_agp) {
		/* Do nothing */
Y
Yinghai Lu 已提交
454
	} else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
L
Linus Torvalds 已提交
455 456
		   force_iommu ||
		   valid_agp ||
457
		   fallback_aper_force) {
458
		printk(KERN_INFO
459
			"Your BIOS doesn't leave a aperture memory hole\n");
460
		printk(KERN_INFO
461
			"Please enable the IOMMU option in the BIOS setup\n");
462
		printk(KERN_INFO
463 464
			"This costs you %d MB of RAM\n",
				32 << fallback_aper_order);
L
Linus Torvalds 已提交
465 466 467

		aper_order = fallback_aper_order;
		aper_alloc = allocate_aperture();
468 469 470 471 472 473 474 475 476
		if (!aper_alloc) {
			/*
			 * Could disable AGP and IOMMU here, but it's
			 * probably not worth it. But the later users
			 * cannot deal with bad apertures and turning
			 * on the aperture over memory causes very
			 * strange problems, so it's better to panic
			 * early.
			 */
L
Linus Torvalds 已提交
477 478
			panic("Not enough memory for aperture");
		}
479 480 481
	} else {
		return;
	}
L
Linus Torvalds 已提交
482 483

	/* Fix up the north bridges */
Y
Yinghai Lu 已提交
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;
		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			/* Don't enable translation yet. That is done later.
			   Assume this BIOS didn't initialise the GART so
			   just overwrite all previous bits */
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
		}
501
	}
502 503

	set_up_gart_resume(aper_order, aper_alloc);
504
}