aperture_64.c 13.4 KB
Newer Older
1
/*
L
Linus Torvalds 已提交
2
 * Firmware replacement code.
3
 *
L
Linus Torvalds 已提交
4
 * Work around broken BIOSes that don't set an aperture or only set the
5 6 7 8 9
 * aperture in the AGP bridge.
 * If all fails map the aperture over some low memory.  This is cheaper than
 * doing bounce buffering. The memory is lost. This is done at early boot
 * because only the bootmem allocator can allocate 32+MB.
 *
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19
 * Copyright 2002 Andi Kleen, SuSE Labs.
 */
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mmzone.h>
#include <linux/pci_ids.h>
#include <linux/pci.h>
#include <linux/bitops.h>
20
#include <linux/ioport.h>
21
#include <linux/suspend.h>
L
Linus Torvalds 已提交
22 23
#include <asm/e820.h>
#include <asm/io.h>
J
Joerg Roedel 已提交
24
#include <asm/gart.h>
L
Linus Torvalds 已提交
25
#include <asm/pci-direct.h>
26
#include <asm/dma.h>
27
#include <asm/k8.h>
L
Linus Torvalds 已提交
28

29
int gart_iommu_aperture;
P
Pavel Machek 已提交
30 31
int gart_iommu_aperture_disabled __initdata;
int gart_iommu_aperture_allowed __initdata;
L
Linus Torvalds 已提交
32 33

int fallback_aper_order __initdata = 1; /* 64MB */
P
Pavel Machek 已提交
34
int fallback_aper_force __initdata;
L
Linus Torvalds 已提交
35 36 37

int fix_aperture __initdata = 1;

Y
Yinghai Lu 已提交
38 39 40 41 42 43 44 45 46 47 48 49
struct bus_dev_range {
	int bus;
	int dev_base;
	int dev_limit;
};

static struct bus_dev_range bus_dev_ranges[] __initdata = {
	{ 0x00, 0x18, 0x20},
	{ 0xff, 0x00, 0x20},
	{ 0xfe, 0x00, 0x20}
};

50 51 52 53 54 55 56 57 58 59 60 61
static struct resource gart_resource = {
	.name	= "GART",
	.flags	= IORESOURCE_MEM,
};

static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
{
	gart_resource.start = aper_base;
	gart_resource.end = aper_base + aper_size - 1;
	insert_resource(&iomem_resource, &gart_resource);
}

62 63
/* This code runs before the PCI subsystem is initialized, so just
   access the northbridge directly. */
L
Linus Torvalds 已提交
64

65
static u32 __init allocate_aperture(void)
L
Linus Torvalds 已提交
66 67
{
	u32 aper_size;
68
	void *p;
L
Linus Torvalds 已提交
69

70 71 72
	/* aper_size should <= 1G */
	if (fallback_aper_order > 5)
		fallback_aper_order = 5;
73
	aper_size = (32 * 1024 * 1024) << fallback_aper_order;
L
Linus Torvalds 已提交
74

75 76 77 78 79
	/*
	 * Aperture has to be naturally aligned. This means a 2GB aperture
	 * won't have much chance of finding a place in the lower 4GB of
	 * memory. Unfortunately we cannot move it up because that would
	 * make the IOMMU useless.
L
Linus Torvalds 已提交
80
	 */
81 82 83 84 85 86 87 88 89 90 91 92 93 94
	/*
	 * using 512M as goal, in case kexec will load kernel_big
	 * that will do the on position decompress, and  could overlap with
	 * that positon with gart that is used.
	 * sequende:
	 * kernel_small
	 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
	 * ==> kernel_small(gart area become e820_reserved)
	 * ==> kexec (with kdump trigger path or previous doesn't shutdown gart)
	 * ==> kerne_big (uncompressed size will be big than 64M or 128M)
	 * so don't use 512M below as gart iommu, leave the space for kernel
	 * code for safe
	 */
	p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
L
Linus Torvalds 已提交
95
	if (!p || __pa(p)+aper_size > 0xffffffff) {
96 97 98
		printk(KERN_ERR
			"Cannot allocate aperture memory hole (%p,%uK)\n",
				p, aper_size>>10);
L
Linus Torvalds 已提交
99
		if (p)
100
			free_bootmem(__pa(p), aper_size);
L
Linus Torvalds 已提交
101 102
		return 0;
	}
103 104
	printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
			aper_size >> 10, __pa(p));
105
	insert_aperture_resource((u32)__pa(p), aper_size);
106 107
	register_nosave_region((u32)__pa(p) >> PAGE_SHIFT,
				(u32)__pa(p+aper_size) >> PAGE_SHIFT);
108 109

	return (u32)__pa(p);
L
Linus Torvalds 已提交
110 111 112
}


113
/* Find a PCI capability */
P
Pavel Machek 已提交
114
static u32 __init find_cap(int bus, int slot, int func, int cap)
115
{
L
Linus Torvalds 已提交
116
	int bytes;
117 118
	u8 pos;

Y
Yinghai Lu 已提交
119
	if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
120
						PCI_STATUS_CAP_LIST))
L
Linus Torvalds 已提交
121
		return 0;
122

Y
Yinghai Lu 已提交
123
	pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
124
	for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
L
Linus Torvalds 已提交
125
		u8 id;
126 127

		pos &= ~3;
Y
Yinghai Lu 已提交
128
		id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
L
Linus Torvalds 已提交
129 130
		if (id == 0xff)
			break;
131 132
		if (id == cap)
			return pos;
Y
Yinghai Lu 已提交
133
		pos = read_pci_config_byte(bus, slot, func,
134 135
						pos+PCI_CAP_LIST_NEXT);
	}
L
Linus Torvalds 已提交
136
	return 0;
137
}
L
Linus Torvalds 已提交
138 139

/* Read a standard AGPv3 bridge header */
P
Pavel Machek 已提交
140
static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
141
{
L
Linus Torvalds 已提交
142 143 144 145 146
	u32 apsize;
	u32 apsizereg;
	int nbits;
	u32 aper_low, aper_hi;
	u64 aper;
147
	u32 old_order;
L
Linus Torvalds 已提交
148

Y
Yinghai Lu 已提交
149 150
	printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
	apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
L
Linus Torvalds 已提交
151
	if (apsizereg == 0xffffffff) {
152
		printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
L
Linus Torvalds 已提交
153 154 155
		return 0;
	}

156 157 158
	/* old_order could be the value from NB gart setting */
	old_order = *order;

L
Linus Torvalds 已提交
159 160
	apsize = apsizereg & 0xfff;
	/* Some BIOS use weird encodings not in the AGPv3 table. */
161 162
	if (apsize & 0xff)
		apsize |= 0xf00;
L
Linus Torvalds 已提交
163 164 165 166
	nbits = hweight16(apsize);
	*order = 7 - nbits;
	if ((int)*order < 0) /* < 32MB */
		*order = 0;
167

Y
Yinghai Lu 已提交
168 169
	aper_low = read_pci_config(bus, slot, func, 0x10);
	aper_hi = read_pci_config(bus, slot, func, 0x14);
L
Linus Torvalds 已提交
170 171
	aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);

172 173 174 175
	/*
	 * On some sick chips, APSIZE is 0. It means it wants 4G
	 * so let double check that order, and lets trust AMD NB settings:
	 */
Y
Yinghai Lu 已提交
176 177 178
	printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
			aper, 32 << old_order);
	if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
179 180 181 182 183
		printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
				32 << *order, apsizereg);
		*order = old_order;
	}

184 185
	printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
			aper, 32 << *order, apsizereg);
L
Linus Torvalds 已提交
186

Y
Yinghai Lu 已提交
187
	if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
188 189 190
		return 0;
	return (u32)aper;
}
L
Linus Torvalds 已提交
191

192 193 194 195 196 197 198 199 200 201 202 203 204
/*
 * Look for an AGP bridge. Windows only expects the aperture in the
 * AGP bridge and some BIOS forget to initialize the Northbridge too.
 * Work around this here.
 *
 * Do an PCI bus scan by hand because we're running before the PCI
 * subsystem.
 *
 * All K8 AGP bridges are AGPv3 compliant, so we can do this scan
 * generically. It's probably overkill to always scan all slots because
 * the AGP bridges should be always an own bus on the HT hierarchy,
 * but do it here for future safety.
 */
P
Pavel Machek 已提交
205
static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
L
Linus Torvalds 已提交
206
{
Y
Yinghai Lu 已提交
207
	int bus, slot, func;
L
Linus Torvalds 已提交
208 209

	/* Poor man's PCI discovery */
Y
Yinghai Lu 已提交
210
	for (bus = 0; bus < 256; bus++) {
211 212
		for (slot = 0; slot < 32; slot++) {
			for (func = 0; func < 8; func++) {
L
Linus Torvalds 已提交
213 214
				u32 class, cap;
				u8 type;
Y
Yinghai Lu 已提交
215
				class = read_pci_config(bus, slot, func,
L
Linus Torvalds 已提交
216 217
							PCI_CLASS_REVISION);
				if (class == 0xffffffff)
218 219 220
					break;

				switch (class >> 16) {
L
Linus Torvalds 已提交
221 222 223
				case PCI_CLASS_BRIDGE_HOST:
				case PCI_CLASS_BRIDGE_OTHER: /* needed? */
					/* AGP bridge? */
Y
Yinghai Lu 已提交
224
					cap = find_cap(bus, slot, func,
225
							PCI_CAP_ID_AGP);
L
Linus Torvalds 已提交
226 227
					if (!cap)
						break;
228
					*valid_agp = 1;
Y
Yinghai Lu 已提交
229
					return read_agp(bus, slot, func, cap,
230 231 232
							order);
				}

L
Linus Torvalds 已提交
233
				/* No multi-function device? */
Y
Yinghai Lu 已提交
234
				type = read_pci_config_byte(bus, slot, func,
L
Linus Torvalds 已提交
235 236 237
							       PCI_HEADER_TYPE);
				if (!(type & 0x80))
					break;
238 239
			}
		}
L
Linus Torvalds 已提交
240
	}
241
	printk(KERN_INFO "No AGP bridge found\n");
242

L
Linus Torvalds 已提交
243 244 245
	return 0;
}

Y
Yinghai Lu 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
static int gart_fix_e820 __initdata = 1;

static int __init parse_gart_mem(char *p)
{
	if (!p)
		return -EINVAL;

	if (!strncmp(p, "off", 3))
		gart_fix_e820 = 0;
	else if (!strncmp(p, "on", 2))
		gart_fix_e820 = 1;

	return 0;
}
early_param("gart_fix_e820", parse_gart_mem);

void __init early_gart_iommu_check(void)
{
	/*
	 * in case it is enabled before, esp for kexec/kdump,
	 * previous kernel already enable that. memset called
	 * by allocate_aperture/__alloc_bootmem_nopanic cause restart.
	 * or second kernel have different position for GART hole. and new
	 * kernel could use hole as RAM that is still used by GART set by
	 * first kernel
	 * or BIOS forget to put that in reserved.
	 * try to update e820 to make that region as reserved.
	 */
Y
Yinghai Lu 已提交
274
	int fix, slot;
Y
Yinghai Lu 已提交
275 276 277 278
	u32 ctl;
	u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
	u64 aper_base = 0, last_aper_base = 0;
	int aper_enabled = 0, last_aper_enabled = 0;
Y
Yinghai Lu 已提交
279
	int i;
Y
Yinghai Lu 已提交
280 281 282 283 284

	if (!early_pci_allowed())
		return;

	fix = 0;
Y
Yinghai Lu 已提交
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;

		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
			aper_enabled = ctl & AMD64_GARTEN;
			aper_order = (ctl >> 1) & 7;
			aper_size = (32 * 1024 * 1024) << aper_order;
			aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
			aper_base <<= 25;

			if ((last_aper_order && aper_order != last_aper_order) ||
			    (last_aper_base && aper_base != last_aper_base) ||
			    (last_aper_enabled && aper_enabled != last_aper_enabled)) {
				fix = 1;
				goto out;
			}
			last_aper_order = aper_order;
			last_aper_base = aper_base;
			last_aper_enabled = aper_enabled;
Y
Yinghai Lu 已提交
313 314 315
		}
	}

Y
Yinghai Lu 已提交
316
out:
Y
Yinghai Lu 已提交
317 318 319 320 321 322 323
	if (!fix && !aper_enabled)
		return;

	if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL)
		fix = 1;

	if (gart_fix_e820 && !fix && aper_enabled) {
Y
Yinghai Lu 已提交
324 325
		if (!e820_all_mapped(aper_base, aper_base + aper_size,
				    E820_RESERVED)) {
326
			/* reserve it, so we can reuse it in second kernel */
Y
Yinghai Lu 已提交
327 328 329 330 331 332 333 334
			printk(KERN_INFO "update e820 for GART\n");
			add_memory_region(aper_base, aper_size, E820_RESERVED);
			update_e820();
		}
		return;
	}

	/* different nodes have different setting, disable them all at first*/
Y
Yinghai Lu 已提交
335 336 337 338 339 340 341
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;
Y
Yinghai Lu 已提交
342

Y
Yinghai Lu 已提交
343 344 345 346 347 348 349 350
		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
			ctl &= ~AMD64_GARTEN;
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
		}
Y
Yinghai Lu 已提交
351 352 353 354
	}

}

Y
Yinghai Lu 已提交
355 356
static int __initdata printed_gart_size_msg;

357
void __init gart_iommu_hole_init(void)
358
{
Y
Yinghai Lu 已提交
359
	u32 agp_aper_base = 0, agp_aper_order = 0;
360
	u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
L
Linus Torvalds 已提交
361
	u64 aper_base, last_aper_base = 0;
Y
Yinghai Lu 已提交
362 363
	int fix, slot, valid_agp = 0;
	int i, node;
L
Linus Torvalds 已提交
364

365 366
	if (gart_iommu_aperture_disabled || !fix_aperture ||
	    !early_pci_allowed())
L
Linus Torvalds 已提交
367 368
		return;

369
	printk(KERN_INFO  "Checking aperture...\n");
L
Linus Torvalds 已提交
370

Y
Yinghai Lu 已提交
371 372 373
	if (!fallback_aper_force)
		agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);

L
Linus Torvalds 已提交
374
	fix = 0;
375
	node = 0;
Y
Yinghai Lu 已提交
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;

		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			iommu_detected = 1;
			gart_iommu_aperture = 1;

			aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7;
			aper_size = (32 * 1024 * 1024) << aper_order;
			aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
			aper_base <<= 25;

			printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
					node, aper_base, aper_size >> 20);
			node++;

			if (!aperture_valid(aper_base, aper_size, 64<<20)) {
				if (valid_agp && agp_aper_base &&
				    agp_aper_base == aper_base &&
				    agp_aper_order == aper_order) {
					/* the same between two setting from NB and agp */
					if (!no_iommu && end_pfn > MAX_DMA32_PFN && !printed_gart_size_msg) {
						printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
						printk(KERN_ERR "please increase GART size in your BIOS setup\n");
						printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
						printed_gart_size_msg = 1;
					}
				} else {
					fix = 1;
					goto out;
Y
Yinghai Lu 已提交
414 415
				}
			}
L
Linus Torvalds 已提交
416

Y
Yinghai Lu 已提交
417 418 419 420 421 422 423
			if ((last_aper_order && aper_order != last_aper_order) ||
			    (last_aper_base && aper_base != last_aper_base)) {
				fix = 1;
				goto out;
			}
			last_aper_order = aper_order;
			last_aper_base = aper_base;
L
Linus Torvalds 已提交
424
		}
425
	}
L
Linus Torvalds 已提交
426

Y
Yinghai Lu 已提交
427
out:
428 429 430
	if (!fix && !fallback_aper_force) {
		if (last_aper_base) {
			unsigned long n = (32 * 1024 * 1024) << last_aper_order;
431

432 433
			insert_aperture_resource((u32)last_aper_base, n);
		}
434
		return;
435
	}
L
Linus Torvalds 已提交
436

Y
Yinghai Lu 已提交
437 438 439 440
	if (!fallback_aper_force) {
		aper_alloc = agp_aper_base;
		aper_order = agp_aper_order;
	}
441 442

	if (aper_alloc) {
L
Linus Torvalds 已提交
443
		/* Got the aperture from the AGP bridge */
444 445
	} else if (swiotlb && !valid_agp) {
		/* Do nothing */
446
	} else if ((!no_iommu && end_pfn > MAX_DMA32_PFN) ||
L
Linus Torvalds 已提交
447 448
		   force_iommu ||
		   valid_agp ||
449
		   fallback_aper_force) {
450 451 452 453 454 455 456
		printk(KERN_ERR
			"Your BIOS doesn't leave a aperture memory hole\n");
		printk(KERN_ERR
			"Please enable the IOMMU option in the BIOS setup\n");
		printk(KERN_ERR
			"This costs you %d MB of RAM\n",
				32 << fallback_aper_order);
L
Linus Torvalds 已提交
457 458 459

		aper_order = fallback_aper_order;
		aper_alloc = allocate_aperture();
460 461 462 463 464 465 466 467 468
		if (!aper_alloc) {
			/*
			 * Could disable AGP and IOMMU here, but it's
			 * probably not worth it. But the later users
			 * cannot deal with bad apertures and turning
			 * on the aperture over memory causes very
			 * strange problems, so it's better to panic
			 * early.
			 */
L
Linus Torvalds 已提交
469 470
			panic("Not enough memory for aperture");
		}
471 472 473
	} else {
		return;
	}
L
Linus Torvalds 已提交
474 475

	/* Fix up the north bridges */
Y
Yinghai Lu 已提交
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
	for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
		int bus;
		int dev_base, dev_limit;

		bus = bus_dev_ranges[i].bus;
		dev_base = bus_dev_ranges[i].dev_base;
		dev_limit = bus_dev_ranges[i].dev_limit;
		for (slot = dev_base; slot < dev_limit; slot++) {
			if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
				continue;

			/* Don't enable translation yet. That is done later.
			   Assume this BIOS didn't initialise the GART so
			   just overwrite all previous bits */
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
			write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
		}
493 494
	}
}