intel-gtt.c 41.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Intel GTT (Graphics Translation Table) routines
 *
 * Caveat: This driver implements the linux agp interface, but this is far from
 * a agp driver! GTT support ended up here for purely historical reasons: The
 * old userspace intel graphics drivers needed an interface to map memory into
 * the GTT. And the drm provides a default interface for graphic devices sitting
 * on an agp port. So it made sense to fake the GTT support as an agp port to
 * avoid having to create a new api.
 *
 * With gem this does not make much sense anymore, just needlessly complicates
 * the code. But as long as the old graphics stack is still support, it's stuck
 * here.
 *
 * /fairy-tale-mode off
 */

18 19 20 21 22 23 24 25 26 27
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/pagemap.h>
#include <linux/agp_backend.h>
#include <asm/smp.h>
#include "agp.h"
#include "intel-agp.h"
#include <linux/intel-gtt.h>
28
#include <drm/intel-gtt.h>
29

30 31 32 33 34 35 36 37
/*
 * If we have Intel graphics, we're not going to have anything other than
 * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
 * on the Intel IOMMU support (CONFIG_DMAR).
 * Only newer chipsets need to bother with this, of course.
 */
#ifdef CONFIG_DMAR
#define USE_PCI_DMA_API 1
38 39
#else
#define USE_PCI_DMA_API 0
40 41
#endif

42 43 44
/* Max amount of stolen space, anything above will be returned to Linux */
int intel_max_stolen = 32 * 1024 * 1024;

45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
static const struct aper_size_info_fixed intel_i810_sizes[] =
{
	{64, 16384, 4},
	/* The 32M mode still requires a 64k gatt */
	{32, 8192, 4}
};

#define AGP_DCACHE_MEMORY	1
#define AGP_PHYS_MEMORY		2
#define INTEL_AGP_CACHED_MEMORY 3

static struct gatt_mask intel_i810_masks[] =
{
	{.mask = I810_PTE_VALID, .type = 0},
	{.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
	{.mask = I810_PTE_VALID, .type = 0},
	{.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
	 .type = INTEL_AGP_CACHED_MEMORY}
};

65 66 67 68 69 70
#define INTEL_AGP_UNCACHED_MEMORY              0
#define INTEL_AGP_CACHED_MEMORY_LLC            1
#define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
#define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
#define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4

71 72 73 74 75
struct intel_gtt_driver {
	unsigned int gen : 8;
	unsigned int is_g33 : 1;
	unsigned int is_pineview : 1;
	unsigned int is_ironlake : 1;
76
	unsigned int dma_mask_size : 8;
77 78
	/* Chipset specific GTT setup */
	int (*setup)(void);
79 80 81
	/* This should undo anything done in ->setup() save the unmapping
	 * of the mmio register file, that's done in the generic code. */
	void (*cleanup)(void);
82 83 84 85
	void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags);
	/* Flags is a more or less chipset specific opaque value.
	 * For chipsets that need to support old ums (non-gem) code, this
	 * needs to be identical to the various supported agp memory types! */
86
	bool (*check_flags)(unsigned int flags);
87
	void (*chipset_flush)(void);
88 89
};

90
static struct _intel_private {
91
	struct intel_gtt base;
92
	const struct intel_gtt_driver *driver;
93
	struct pci_dev *pcidev;	/* device one */
94
	struct pci_dev *bridge_dev;
95
	u8 __iomem *registers;
96
	phys_addr_t gtt_bus_addr;
97
	phys_addr_t gma_bus_addr;
98
	phys_addr_t pte_bus_addr;
99 100 101 102 103 104 105 106 107
	u32 __iomem *gtt;		/* I915G */
	int num_dcache_entries;
	union {
		void __iomem *i9xx_flush_page;
		void *i8xx_flush_page;
	};
	struct page *i8xx_page;
	struct resource ifp_resource;
	int resource_valid;
108 109
	struct page *scratch_page;
	dma_addr_t scratch_page_dma;
110 111
} intel_private;

112 113 114 115 116
#define INTEL_GTT_GEN	intel_private.driver->gen
#define IS_G33		intel_private.driver->is_g33
#define IS_PINEVIEW	intel_private.driver->is_pineview
#define IS_IRONLAKE	intel_private.driver->is_ironlake

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
static void intel_agp_free_sglist(struct agp_memory *mem)
{
	struct sg_table st;

	st.sgl = mem->sg_list;
	st.orig_nents = st.nents = mem->page_count;

	sg_free_table(&st);

	mem->sg_list = NULL;
	mem->num_sg = 0;
}

static int intel_agp_map_memory(struct agp_memory *mem)
{
	struct sg_table st;
	struct scatterlist *sg;
	int i;

136 137 138
	if (mem->sg_list)
		return 0; /* already mapped (for e.g. resume */

139 140 141
	DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);

	if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
142
		goto err;
143 144 145 146 147 148 149 150

	mem->sg_list = sg = st.sgl;

	for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
		sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);

	mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
				 mem->page_count, PCI_DMA_BIDIRECTIONAL);
151 152 153
	if (unlikely(!mem->num_sg))
		goto err;

154
	return 0;
155 156 157 158

err:
	sg_free_table(&st);
	return -ENOMEM;
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
}

static void intel_agp_unmap_memory(struct agp_memory *mem)
{
	DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);

	pci_unmap_sg(intel_private.pcidev, mem->sg_list,
		     mem->page_count, PCI_DMA_BIDIRECTIONAL);
	intel_agp_free_sglist(mem);
}

static int intel_i810_fetch_size(void)
{
	u32 smram_miscc;
	struct aper_size_info_fixed *values;

175 176
	pci_read_config_dword(intel_private.bridge_dev,
			      I810_SMRAM_MISCC, &smram_miscc);
177 178 179
	values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);

	if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
180
		dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
181 182 183
		return 0;
	}
	if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
184
		agp_bridge->current_size = (void *) (values + 1);
185 186 187
		agp_bridge->aperture_size_idx = 1;
		return values[1].size;
	} else {
188
		agp_bridge->current_size = (void *) (values);
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
		agp_bridge->aperture_size_idx = 0;
		return values[0].size;
	}

	return 0;
}

static int intel_i810_configure(void)
{
	struct aper_size_info_fixed *current_size;
	u32 temp;
	int i;

	current_size = A_SIZE_FIX(agp_bridge->current_size);

	if (!intel_private.registers) {
		pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
		temp &= 0xfff80000;

		intel_private.registers = ioremap(temp, 128 * 4096);
		if (!intel_private.registers) {
			dev_err(&intel_private.pcidev->dev,
				"can't remap memory\n");
			return -ENOMEM;
		}
	}

	if ((readl(intel_private.registers+I810_DRAM_CTL)
		& I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
		/* This will need to be dynamically assigned */
		dev_info(&intel_private.pcidev->dev,
			 "detected 4MB dedicated video ram\n");
		intel_private.num_dcache_entries = 1024;
	}
	pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
	agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
	writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
	readl(intel_private.registers+I810_PGETBL_CTL);	/* PCI Posting. */

	if (agp_bridge->driver->needs_scratch_page) {
		for (i = 0; i < current_size->num_entries; i++) {
			writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
		}
		readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));	/* PCI posting. */
	}
	global_cache_flush();
	return 0;
}

static void intel_i810_cleanup(void)
{
	writel(0, intel_private.registers+I810_PGETBL_CTL);
	readl(intel_private.registers);	/* PCI Posting. */
	iounmap(intel_private.registers);
}

245
static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
{
	return;
}

/* Exists to support ARGB cursors */
static struct page *i8xx_alloc_pages(void)
{
	struct page *page;

	page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
	if (page == NULL)
		return NULL;

	if (set_pages_uc(page, 4) < 0) {
		set_pages_wb(page, 4);
		__free_pages(page, 2);
		return NULL;
	}
	get_page(page);
	atomic_inc(&agp_bridge->current_memory_agp);
	return page;
}

static void i8xx_destroy_pages(struct page *page)
{
	if (page == NULL)
		return;

	set_pages_wb(page, 4);
	put_page(page);
	__free_pages(page, 2);
	atomic_dec(&agp_bridge->current_memory_agp);
}

static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
				int type)
{
	int i, j, num_entries;
	void *temp;
	int ret = -EINVAL;
	int mask_type;

	if (mem->page_count == 0)
		goto out;

	temp = agp_bridge->current_size;
	num_entries = A_SIZE_FIX(temp)->num_entries;

	if ((pg_start + mem->page_count) > num_entries)
		goto out_err;


	for (j = pg_start; j < (pg_start + mem->page_count); j++) {
		if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
			ret = -EBUSY;
			goto out_err;
		}
	}

	if (type != mem->type)
		goto out_err;

	mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);

	switch (mask_type) {
	case AGP_DCACHE_MEMORY:
		if (!mem->is_flushed)
			global_cache_flush();
		for (i = pg_start; i < (pg_start + mem->page_count); i++) {
			writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
			       intel_private.registers+I810_PTE_BASE+(i*4));
		}
		readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
		break;
	case AGP_PHYS_MEMORY:
	case AGP_NORMAL_MEMORY:
		if (!mem->is_flushed)
			global_cache_flush();
		for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
			writel(agp_bridge->driver->mask_memory(agp_bridge,
					page_to_phys(mem->pages[i]), mask_type),
			       intel_private.registers+I810_PTE_BASE+(j*4));
		}
		readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
		break;
	default:
		goto out_err;
	}

out:
	ret = 0;
out_err:
	mem->is_flushed = true;
	return ret;
}

static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
				int type)
{
	int i;

	if (mem->page_count == 0)
		return 0;

	for (i = pg_start; i < (mem->page_count + pg_start); i++) {
		writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
	}
	readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));

	return 0;
}

/*
 * The i810/i830 requires a physical address to program its mouse
 * pointer into hardware.
 * However the Xserver still writes to it through the agp aperture.
 */
static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
{
	struct agp_memory *new;
	struct page *page;

	switch (pg_count) {
	case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
		break;
	case 4:
		/* kludge to get 4 physical pages for ARGB cursor */
		page = i8xx_alloc_pages();
		break;
	default:
		return NULL;
	}

	if (page == NULL)
		return NULL;

	new = agp_create_memory(pg_count);
	if (new == NULL)
		return NULL;

	new->pages[0] = page;
	if (pg_count == 4) {
		/* kludge to get 4 physical pages for ARGB cursor */
		new->pages[1] = new->pages[0] + 1;
		new->pages[2] = new->pages[1] + 1;
		new->pages[3] = new->pages[2] + 1;
	}
	new->page_count = pg_count;
	new->num_scratch_pages = pg_count;
	new->type = AGP_PHYS_MEMORY;
	new->physical = page_to_phys(new->pages[0]);
	return new;
}

static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
{
	struct agp_memory *new;

	if (type == AGP_DCACHE_MEMORY) {
		if (pg_count != intel_private.num_dcache_entries)
			return NULL;

		new = agp_create_memory(1);
		if (new == NULL)
			return NULL;

		new->type = AGP_DCACHE_MEMORY;
		new->page_count = pg_count;
		new->num_scratch_pages = 0;
		agp_free_page_array(new);
		return new;
	}
	if (type == AGP_PHYS_MEMORY)
		return alloc_agpphysmem_i8xx(pg_count, type);
	return NULL;
}

static void intel_i810_free_by_type(struct agp_memory *curr)
{
	agp_free_key(curr->key);
	if (curr->type == AGP_PHYS_MEMORY) {
		if (curr->page_count == 4)
			i8xx_destroy_pages(curr->pages[0]);
		else {
			agp_bridge->driver->agp_destroy_page(curr->pages[0],
							     AGP_PAGE_DESTROY_UNMAP);
			agp_bridge->driver->agp_destroy_page(curr->pages[0],
							     AGP_PAGE_DESTROY_FREE);
		}
		agp_free_page_array(curr);
	}
	kfree(curr);
}

static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
					    dma_addr_t addr, int type)
{
	/* Type checking must be done elsewhere */
	return addr | bridge->driver->masks[type].mask;
}

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
static int intel_gtt_setup_scratch_page(void)
{
	struct page *page;
	dma_addr_t dma_addr;

	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
	if (page == NULL)
		return -ENOMEM;
	get_page(page);
	set_pages_uc(page, 1);

	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
		dma_addr = pci_map_page(intel_private.pcidev, page, 0,
				    PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
		if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
			return -EINVAL;

		intel_private.scratch_page_dma = dma_addr;
	} else
		intel_private.scratch_page_dma = page_to_phys(page);

	intel_private.scratch_page = page;

	return 0;
}

473
static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
474 475 476 477 478 479 480
	{128, 32768, 5},
	/* The 64M mode still requires a 128k gatt */
	{64, 16384, 5},
	{256, 65536, 6},
	{512, 131072, 7},
};

481
static unsigned int intel_gtt_stolen_entries(void)
482 483 484 485 486
{
	u16 gmch_ctrl;
	u8 rdct;
	int local = 0;
	static const int ddt[4] = { 0, 16, 32, 64 };
487 488
	unsigned int overhead_entries, stolen_entries;
	unsigned int stolen_size = 0;
489

490 491
	pci_read_config_word(intel_private.bridge_dev,
			     I830_GMCH_CTRL, &gmch_ctrl);
492

493
	if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
494 495 496 497
		overhead_entries = 0;
	else
		overhead_entries = intel_private.base.gtt_mappable_entries
			/ 1024;
498

499
	overhead_entries += 1; /* BIOS popup */
500

501 502
	if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
	    intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
503 504
		switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
		case I830_GMCH_GMS_STOLEN_512:
505
			stolen_size = KB(512);
506 507
			break;
		case I830_GMCH_GMS_STOLEN_1024:
508
			stolen_size = MB(1);
509 510
			break;
		case I830_GMCH_GMS_STOLEN_8192:
511
			stolen_size = MB(8);
512 513 514
			break;
		case I830_GMCH_GMS_LOCAL:
			rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
515
			stolen_size = (I830_RDRAM_ND(rdct) + 1) *
516 517 518 519
					MB(ddt[I830_RDRAM_DDT(rdct)]);
			local = 1;
			break;
		default:
520
			stolen_size = 0;
521 522
			break;
		}
523
	} else if (INTEL_GTT_GEN == 6) {
524 525 526 527 528 529 530
		/*
		 * SandyBridge has new memory control reg at 0x50.w
		 */
		u16 snb_gmch_ctl;
		pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
		switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
		case SNB_GMCH_GMS_STOLEN_32M:
531
			stolen_size = MB(32);
532 533
			break;
		case SNB_GMCH_GMS_STOLEN_64M:
534
			stolen_size = MB(64);
535 536
			break;
		case SNB_GMCH_GMS_STOLEN_96M:
537
			stolen_size = MB(96);
538 539
			break;
		case SNB_GMCH_GMS_STOLEN_128M:
540
			stolen_size = MB(128);
541 542
			break;
		case SNB_GMCH_GMS_STOLEN_160M:
543
			stolen_size = MB(160);
544 545
			break;
		case SNB_GMCH_GMS_STOLEN_192M:
546
			stolen_size = MB(192);
547 548
			break;
		case SNB_GMCH_GMS_STOLEN_224M:
549
			stolen_size = MB(224);
550 551
			break;
		case SNB_GMCH_GMS_STOLEN_256M:
552
			stolen_size = MB(256);
553 554
			break;
		case SNB_GMCH_GMS_STOLEN_288M:
555
			stolen_size = MB(288);
556 557
			break;
		case SNB_GMCH_GMS_STOLEN_320M:
558
			stolen_size = MB(320);
559 560
			break;
		case SNB_GMCH_GMS_STOLEN_352M:
561
			stolen_size = MB(352);
562 563
			break;
		case SNB_GMCH_GMS_STOLEN_384M:
564
			stolen_size = MB(384);
565 566
			break;
		case SNB_GMCH_GMS_STOLEN_416M:
567
			stolen_size = MB(416);
568 569
			break;
		case SNB_GMCH_GMS_STOLEN_448M:
570
			stolen_size = MB(448);
571 572
			break;
		case SNB_GMCH_GMS_STOLEN_480M:
573
			stolen_size = MB(480);
574 575
			break;
		case SNB_GMCH_GMS_STOLEN_512M:
576
			stolen_size = MB(512);
577 578 579 580 581
			break;
		}
	} else {
		switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
		case I855_GMCH_GMS_STOLEN_1M:
582
			stolen_size = MB(1);
583 584
			break;
		case I855_GMCH_GMS_STOLEN_4M:
585
			stolen_size = MB(4);
586 587
			break;
		case I855_GMCH_GMS_STOLEN_8M:
588
			stolen_size = MB(8);
589 590
			break;
		case I855_GMCH_GMS_STOLEN_16M:
591
			stolen_size = MB(16);
592 593
			break;
		case I855_GMCH_GMS_STOLEN_32M:
594
			stolen_size = MB(32);
595 596
			break;
		case I915_GMCH_GMS_STOLEN_48M:
597
			stolen_size = MB(48);
598 599
			break;
		case I915_GMCH_GMS_STOLEN_64M:
600
			stolen_size = MB(64);
601 602
			break;
		case G33_GMCH_GMS_STOLEN_128M:
603
			stolen_size = MB(128);
604 605
			break;
		case G33_GMCH_GMS_STOLEN_256M:
606
			stolen_size = MB(256);
607 608
			break;
		case INTEL_GMCH_GMS_STOLEN_96M:
609
			stolen_size = MB(96);
610 611
			break;
		case INTEL_GMCH_GMS_STOLEN_160M:
612
			stolen_size = MB(160);
613 614
			break;
		case INTEL_GMCH_GMS_STOLEN_224M:
615
			stolen_size = MB(224);
616 617
			break;
		case INTEL_GMCH_GMS_STOLEN_352M:
618
			stolen_size = MB(352);
619 620
			break;
		default:
621
			stolen_size = 0;
622 623 624
			break;
		}
	}
625

626
	if (!local && stolen_size > intel_max_stolen) {
627
		dev_info(&intel_private.bridge_dev->dev,
628
			 "detected %dK stolen memory, trimming to %dK\n",
629 630 631
			 stolen_size / KB(1), intel_max_stolen / KB(1));
		stolen_size = intel_max_stolen;
	} else if (stolen_size > 0) {
632
		dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
633
		       stolen_size / KB(1), local ? "local" : "stolen");
634
	} else {
635
		dev_info(&intel_private.bridge_dev->dev,
636
		       "no pre-allocated video memory detected\n");
637
		stolen_size = 0;
638 639
	}

640 641 642
	stolen_entries = stolen_size/KB(4) - overhead_entries;

	return stolen_entries;
643 644
}

645 646 647 648
static unsigned int intel_gtt_total_entries(void)
{
	int size;

649
	if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) {
650 651 652 653 654
		u32 pgetbl_ctl;
		pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);

		switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
		case I965_PGETBL_SIZE_128KB:
655
			size = KB(128);
656 657
			break;
		case I965_PGETBL_SIZE_256KB:
658
			size = KB(256);
659 660
			break;
		case I965_PGETBL_SIZE_512KB:
661
			size = KB(512);
662 663
			break;
		case I965_PGETBL_SIZE_1MB:
664
			size = KB(1024);
665 666
			break;
		case I965_PGETBL_SIZE_2MB:
667
			size = KB(2048);
668 669
			break;
		case I965_PGETBL_SIZE_1_5MB:
670
			size = KB(1024 + 512);
671 672 673 674
			break;
		default:
			dev_info(&intel_private.pcidev->dev,
				 "unknown page table size, assuming 512KB\n");
675
			size = KB(512);
676
		}
677

678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
		return size/4;
	} else if (INTEL_GTT_GEN == 6) {
		u16 snb_gmch_ctl;

		pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
		switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
		default:
		case SNB_GTT_SIZE_0M:
			printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
			size = MB(0);
			break;
		case SNB_GTT_SIZE_1M:
			size = MB(1);
			break;
		case SNB_GTT_SIZE_2M:
			size = MB(2);
			break;
		}
696
		return size/4;
697 698 699 700
	} else {
		/* On previous hardware, the GTT size was just what was
		 * required to map the aperture.
		 */
701
		return intel_private.base.gtt_mappable_entries;
702 703 704
	}
}

705 706 707 708
static unsigned int intel_gtt_mappable_entries(void)
{
	unsigned int aperture_size;

709 710
	if (INTEL_GTT_GEN == 2) {
		u16 gmch_ctrl;
711

712 713
		pci_read_config_word(intel_private.bridge_dev,
				     I830_GMCH_CTRL, &gmch_ctrl);
714 715

		if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
716
			aperture_size = MB(64);
717
		else
718
			aperture_size = MB(128);
719
	} else {
720 721 722 723 724 725 726
		/* 9xx supports large sizes, just look at the length */
		aperture_size = pci_resource_len(intel_private.pcidev, 2);
	}

	return aperture_size >> PAGE_SHIFT;
}

727 728 729 730 731 732 733 734 735 736 737
static void intel_gtt_teardown_scratch_page(void)
{
	set_pages_wb(intel_private.scratch_page, 1);
	pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,
		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
	put_page(intel_private.scratch_page);
	__free_page(intel_private.scratch_page);
}

static void intel_gtt_cleanup(void)
{
738 739
	intel_private.driver->cleanup();

740 741 742 743 744 745
	iounmap(intel_private.gtt);
	iounmap(intel_private.registers);
	
	intel_gtt_teardown_scratch_page();
}

746 747
static int intel_gtt_init(void)
{
748
	u32 gtt_map_size;
749 750 751 752 753
	int ret;

	ret = intel_private.driver->setup();
	if (ret != 0)
		return ret;
754 755 756 757

	intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
	intel_private.base.gtt_total_entries = intel_gtt_total_entries();

758 759 760 761 762
	dev_info(&intel_private.bridge_dev->dev,
			"detected gtt size: %dK total, %dK mappable\n",
			intel_private.base.gtt_total_entries * 4,
			intel_private.base.gtt_mappable_entries * 4);

763 764 765 766 767
	gtt_map_size = intel_private.base.gtt_total_entries * 4;

	intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
				    gtt_map_size);
	if (!intel_private.gtt) {
768
		intel_private.driver->cleanup();
769 770 771 772 773 774
		iounmap(intel_private.registers);
		return -ENOMEM;
	}

	global_cache_flush();   /* FIXME: ? */

775 776 777
	/* we have to call this as early as possible after the MMIO base address is known */
	intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
	if (intel_private.base.gtt_stolen_entries == 0) {
778
		intel_private.driver->cleanup();
779
		iounmap(intel_private.registers);
780
		iounmap(intel_private.gtt);
781 782 783
		return -ENOMEM;
	}

784 785 786 787 788 789
	ret = intel_gtt_setup_scratch_page();
	if (ret != 0) {
		intel_gtt_cleanup();
		return ret;
	}

790 791 792
	return 0;
}

793 794
static int intel_fake_agp_fetch_size(void)
{
795
	int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
796 797 798 799 800 801 802
	unsigned int aper_size;
	int i;

	aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
		    / MB(1);

	for (i = 0; i < num_sizes; i++) {
803
		if (aper_size == intel_fake_agp_sizes[i].size) {
804 805
			agp_bridge->current_size =
				(void *) (intel_fake_agp_sizes + i);
806 807 808 809 810 811 812
			return aper_size;
		}
	}

	return 0;
}

813
static void i830_cleanup(void)
814
{
815 816 817 818
	if (intel_private.i8xx_flush_page) {
		kunmap(intel_private.i8xx_flush_page);
		intel_private.i8xx_flush_page = NULL;
	}
819 820 821 822 823 824 825 826 827 828 829

	__free_page(intel_private.i8xx_page);
	intel_private.i8xx_page = NULL;
}

static void intel_i830_setup_flush(void)
{
	/* return if we've already set the flush mechanism up */
	if (intel_private.i8xx_page)
		return;

J
Jan Beulich 已提交
830
	intel_private.i8xx_page = alloc_page(GFP_KERNEL);
831 832 833 834 835
	if (!intel_private.i8xx_page)
		return;

	intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
	if (!intel_private.i8xx_flush_page)
836
		i830_cleanup();
837 838 839 840 841 842 843 844 845 846 847 848
}

/* The chipset_flush interface needs to get data that has already been
 * flushed out of the CPU all the way out to main memory, because the GPU
 * doesn't snoop those buffers.
 *
 * The 8xx series doesn't have the same lovely interface for flushing the
 * chipset write buffers that the later chips do. According to the 865
 * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 * that buffer out, we just fill 1KB and clflush it out, on the assumption
 * that it'll push whatever was in there out.  It appears to work.
 */
849
static void i830_chipset_flush(void)
850 851 852 853 854 855 856 857 858 859 860
{
	unsigned int *pg = intel_private.i8xx_flush_page;

	memset(pg, 0, 1024);

	if (cpu_has_clflush)
		clflush_cache_range(pg, 1024);
	else if (wbinvd_on_all_cpus() != 0)
		printk(KERN_ERR "Timed out waiting for cache flush.\n");
}

861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877
static void i830_write_entry(dma_addr_t addr, unsigned int entry,
			     unsigned int flags)
{
	u32 pte_flags = I810_PTE_VALID;
	
	switch (flags) {
	case AGP_DCACHE_MEMORY:
		pte_flags |= I810_PTE_LOCAL;
		break;
	case AGP_USER_CACHED_MEMORY:
		pte_flags |= I830_PTE_SYSTEM_CACHED;
		break;
	}

	writel(addr | pte_flags, intel_private.gtt + entry);
}

878
static void intel_enable_gtt(void)
879
{
880
	u32 gma_addr;
881
	u16 gmch_ctrl;
882

883 884 885 886 887 888 889
	if (INTEL_GTT_GEN == 2)
		pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
				      &gma_addr);
	else
		pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
				      &gma_addr);

890
	intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
891

892 893 894 895
	pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl);
	gmch_ctrl |= I830_GMCH_ENABLED;
	pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);

896 897
	writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED,
	       intel_private.registers+I810_PGETBL_CTL);
898 899 900 901 902 903 904 905 906 907 908
	readl(intel_private.registers+I810_PGETBL_CTL);	/* PCI Posting. */
}

static int i830_setup(void)
{
	u32 reg_addr;

	pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
	reg_addr &= 0xfff80000;

	intel_private.registers = ioremap(reg_addr, KB(64));
909 910 911
	if (!intel_private.registers)
		return -ENOMEM;

912
	intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
913 914
	intel_private.pte_bus_addr =
		readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
915 916 917 918 919 920

	intel_i830_setup_flush();

	return 0;
}

921
static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
922 923
{
	agp_bridge->gatt_table_real = NULL;
924
	agp_bridge->gatt_table = NULL;
925
	agp_bridge->gatt_bus_addr = 0;
926 927 928 929

	return 0;
}

930
static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
931 932 933 934
{
	return 0;
}

935
static int intel_fake_agp_configure(void)
936 937 938
{
	int i;

939
	intel_enable_gtt();
940

941
	agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
942

943 944 945 946
	for (i = intel_private.base.gtt_stolen_entries;
			i < intel_private.base.gtt_total_entries; i++) {
		intel_private.driver->write_entry(intel_private.scratch_page_dma,
						  i, 0);
947
	}
948
	readl(intel_private.gtt+i-1);	/* PCI Posting. */
949 950 951 952 953 954

	global_cache_flush();

	return 0;
}

955
static bool i830_check_flags(unsigned int flags)
956
{
957 958 959 960 961 962 963 964 965 966 967
	switch (flags) {
	case 0:
	case AGP_PHYS_MEMORY:
	case AGP_USER_CACHED_MEMORY:
	case AGP_USER_MEMORY:
		return true;
	}

	return false;
}

968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
					unsigned int sg_len,
					unsigned int pg_start,
					unsigned int flags)
{
	struct scatterlist *sg;
	unsigned int len, m;
	int i, j;

	j = pg_start;

	/* sg may merge pages, but we have to separate
	 * per-page addr for GTT */
	for_each_sg(sg_list, sg, sg_len, i) {
		len = sg_dma_len(sg) >> PAGE_SHIFT;
		for (m = 0; m < len; m++) {
			dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
			intel_private.driver->write_entry(addr,
							  j, flags);
			j++;
		}
	}
	readl(intel_private.gtt+j-1);
}

993 994 995 996
static int intel_fake_agp_insert_entries(struct agp_memory *mem,
					 off_t pg_start, int type)
{
	int i, j;
997 998 999 1000 1001
	int ret = -EINVAL;

	if (mem->page_count == 0)
		goto out;

1002
	if (pg_start < intel_private.base.gtt_stolen_entries) {
1003
		dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1004 1005
			   "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
			   pg_start, intel_private.base.gtt_stolen_entries);
1006 1007 1008 1009 1010 1011

		dev_info(&intel_private.pcidev->dev,
			 "trying to insert into local/stolen memory\n");
		goto out_err;
	}

1012
	if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
1013 1014 1015 1016 1017
		goto out_err;

	if (type != mem->type)
		goto out_err;

1018
	if (!intel_private.driver->check_flags(type))
1019 1020 1021 1022 1023
		goto out_err;

	if (!mem->is_flushed)
		global_cache_flush();

1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
		ret = intel_agp_map_memory(mem);
		if (ret != 0)
			return ret;

		intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
					    pg_start, type);
	} else {
		for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
			dma_addr_t addr = page_to_phys(mem->pages[i]);
			intel_private.driver->write_entry(addr,
							  j, type);
		}
		readl(intel_private.gtt+j-1);
1038 1039 1040 1041 1042 1043 1044 1045 1046
	}

out:
	ret = 0;
out_err:
	mem->is_flushed = true;
	return ret;
}

1047 1048
static int intel_fake_agp_remove_entries(struct agp_memory *mem,
					 off_t pg_start, int type)
1049 1050 1051 1052 1053 1054
{
	int i;

	if (mem->page_count == 0)
		return 0;

1055
	if (pg_start < intel_private.base.gtt_stolen_entries) {
1056 1057 1058 1059 1060
		dev_info(&intel_private.pcidev->dev,
			 "trying to disable local/stolen memory\n");
		return -EINVAL;
	}

1061 1062 1063
	if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2)
		intel_agp_unmap_memory(mem);

1064
	for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1065 1066
		intel_private.driver->write_entry(intel_private.scratch_page_dma,
						  i, 0);
1067
	}
1068
	readl(intel_private.gtt+i-1);
1069 1070 1071 1072

	return 0;
}

1073 1074 1075 1076 1077
static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
{
	intel_private.driver->chipset_flush();
}

1078 1079
static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
						       int type)
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
{
	if (type == AGP_PHYS_MEMORY)
		return alloc_agpphysmem_i8xx(pg_count, type);
	/* always return NULL for other allocation types for now */
	return NULL;
}

static int intel_alloc_chipset_flush_resource(void)
{
	int ret;
1090
	ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1091
				     PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1092
				     pcibios_align_resource, intel_private.bridge_dev);
1093 1094 1095 1096 1097 1098 1099 1100 1101

	return ret;
}

static void intel_i915_setup_chipset_flush(void)
{
	int ret;
	u32 temp;

1102
	pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
1103 1104 1105
	if (!(temp & 0x1)) {
		intel_alloc_chipset_flush_resource();
		intel_private.resource_valid = 1;
1106
		pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
	} else {
		temp &= ~1;

		intel_private.resource_valid = 1;
		intel_private.ifp_resource.start = temp;
		intel_private.ifp_resource.end = temp + PAGE_SIZE;
		ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
		/* some BIOSes reserve this area in a pnp some don't */
		if (ret)
			intel_private.resource_valid = 0;
	}
}

static void intel_i965_g33_setup_chipset_flush(void)
{
	u32 temp_hi, temp_lo;
	int ret;

1125 1126
	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
1127 1128 1129 1130 1131 1132

	if (!(temp_lo & 0x1)) {

		intel_alloc_chipset_flush_resource();

		intel_private.resource_valid = 1;
1133
		pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
1134
			upper_32_bits(intel_private.ifp_resource.start));
1135
		pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
	} else {
		u64 l64;

		temp_lo &= ~0x1;
		l64 = ((u64)temp_hi << 32) | temp_lo;

		intel_private.resource_valid = 1;
		intel_private.ifp_resource.start = l64;
		intel_private.ifp_resource.end = l64 + PAGE_SIZE;
		ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
		/* some BIOSes reserve this area in a pnp some don't */
		if (ret)
			intel_private.resource_valid = 0;
	}
}

static void intel_i9xx_setup_flush(void)
{
	/* return if already configured */
	if (intel_private.ifp_resource.start)
		return;

1158
	if (INTEL_GTT_GEN == 6)
1159 1160 1161 1162 1163 1164 1165
		return;

	/* setup a resource for this object */
	intel_private.ifp_resource.name = "Intel Flush Page";
	intel_private.ifp_resource.flags = IORESOURCE_MEM;

	/* Setup chipset flush for 915 */
1166
	if (IS_G33 || INTEL_GTT_GEN >= 4) {
1167 1168 1169 1170 1171
		intel_i965_g33_setup_chipset_flush();
	} else {
		intel_i915_setup_chipset_flush();
	}

1172
	if (intel_private.ifp_resource.start)
1173
		intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1174 1175 1176
	if (!intel_private.i9xx_flush_page)
		dev_err(&intel_private.pcidev->dev,
			"can't ioremap flush page - no chipset flushing\n");
1177 1178
}

1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
static void i9xx_cleanup(void)
{
	if (intel_private.i9xx_flush_page)
		iounmap(intel_private.i9xx_flush_page);
	if (intel_private.resource_valid)
		release_resource(&intel_private.ifp_resource);
	intel_private.ifp_resource.start = 0;
	intel_private.resource_valid = 0;
}

1189
static void i9xx_chipset_flush(void)
1190 1191 1192 1193 1194
{
	if (intel_private.i9xx_flush_page)
		writel(1, intel_private.i9xx_flush_page);
}

1195 1196 1197 1198 1199 1200 1201 1202
static void i965_write_entry(dma_addr_t addr, unsigned int entry,
			     unsigned int flags)
{
	/* Shift high bits down */
	addr |= (addr >> 28) & 0xf0;
	writel(addr | I810_PTE_VALID, intel_private.gtt + entry);
}

1203 1204 1205 1206 1207
static bool gen6_check_flags(unsigned int flags)
{
	return true;
}

1208 1209 1210 1211 1212 1213 1214
static void gen6_write_entry(dma_addr_t addr, unsigned int entry,
			     unsigned int flags)
{
	unsigned int type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
	unsigned int gfdt = flags & AGP_USER_CACHED_MEMORY_GFDT;
	u32 pte_flags;

1215
	if (type_mask == AGP_USER_MEMORY)
1216
		pte_flags = GEN6_PTE_UNCACHED | I810_PTE_VALID;
1217
	else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC) {
1218
		pte_flags = GEN6_PTE_LLC_MLC | I810_PTE_VALID;
1219 1220 1221
		if (gfdt)
			pte_flags |= GEN6_PTE_GFDT;
	} else { /* set 'normal'/'cached' to LLC by default */
1222
		pte_flags = GEN6_PTE_LLC | I810_PTE_VALID;
1223 1224 1225 1226 1227 1228 1229 1230 1231
		if (gfdt)
			pte_flags |= GEN6_PTE_GFDT;
	}

	/* gen6 has bit11-4 for physical addr bit39-32 */
	addr |= (addr >> 28) & 0xff0;
	writel(addr | pte_flags, intel_private.gtt + entry);
}

1232 1233 1234 1235
static void gen6_cleanup(void)
{
}

1236
static int i9xx_setup(void)
1237
{
1238
	u32 reg_addr;
1239

1240
	pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1241

1242
	reg_addr &= 0xfff80000;
1243

1244
	intel_private.registers = ioremap(reg_addr, 128 * 4096);
1245
	if (!intel_private.registers)
1246 1247
		return -ENOMEM;

1248 1249
	if (INTEL_GTT_GEN == 3) {
		u32 gtt_addr;
1250

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
		pci_read_config_dword(intel_private.pcidev,
				      I915_PTEADDR, &gtt_addr);
		intel_private.gtt_bus_addr = gtt_addr;
	} else {
		u32 gtt_offset;

		switch (INTEL_GTT_GEN) {
		case 5:
		case 6:
			gtt_offset = MB(2);
			break;
		case 4:
		default:
			gtt_offset =  KB(512);
			break;
		}
		intel_private.gtt_bus_addr = reg_addr + gtt_offset;
	}

1270 1271 1272
	intel_private.pte_bus_addr =
		readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;

1273 1274 1275 1276 1277
	intel_i9xx_setup_flush();

	return 0;
}

1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
static const struct agp_bridge_driver intel_810_driver = {
	.owner			= THIS_MODULE,
	.aperture_sizes		= intel_i810_sizes,
	.size_type		= FIXED_APER_SIZE,
	.num_aperture_sizes	= 2,
	.needs_scratch_page	= true,
	.configure		= intel_i810_configure,
	.fetch_size		= intel_i810_fetch_size,
	.cleanup		= intel_i810_cleanup,
	.mask_memory		= intel_i810_mask_memory,
	.masks			= intel_i810_masks,
1289
	.agp_enable		= intel_fake_agp_enable,
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
	.cache_flush		= global_cache_flush,
	.create_gatt_table	= agp_generic_create_gatt_table,
	.free_gatt_table	= agp_generic_free_gatt_table,
	.insert_memory		= intel_i810_insert_entries,
	.remove_memory		= intel_i810_remove_entries,
	.alloc_by_type		= intel_i810_alloc_by_type,
	.free_by_type		= intel_i810_free_by_type,
	.agp_alloc_page		= agp_generic_alloc_page,
	.agp_alloc_pages        = agp_generic_alloc_pages,
	.agp_destroy_page	= agp_generic_destroy_page,
	.agp_destroy_pages      = agp_generic_destroy_pages,
	.agp_type_to_mask_type  = agp_generic_type_to_mask_type,
};

1304
static const struct agp_bridge_driver intel_fake_agp_driver = {
1305 1306
	.owner			= THIS_MODULE,
	.size_type		= FIXED_APER_SIZE,
1307 1308
	.aperture_sizes		= intel_fake_agp_sizes,
	.num_aperture_sizes	= ARRAY_SIZE(intel_fake_agp_sizes),
1309
	.configure		= intel_fake_agp_configure,
1310
	.fetch_size		= intel_fake_agp_fetch_size,
1311
	.cleanup		= intel_gtt_cleanup,
1312
	.agp_enable		= intel_fake_agp_enable,
1313
	.cache_flush		= global_cache_flush,
1314
	.create_gatt_table	= intel_fake_agp_create_gatt_table,
1315
	.free_gatt_table	= intel_fake_agp_free_gatt_table,
1316 1317
	.insert_memory		= intel_fake_agp_insert_entries,
	.remove_memory		= intel_fake_agp_remove_entries,
1318
	.alloc_by_type		= intel_fake_agp_alloc_by_type,
1319 1320 1321 1322 1323
	.free_by_type		= intel_i810_free_by_type,
	.agp_alloc_page		= agp_generic_alloc_page,
	.agp_alloc_pages        = agp_generic_alloc_pages,
	.agp_destroy_page	= agp_generic_destroy_page,
	.agp_destroy_pages      = agp_generic_destroy_pages,
1324
	.chipset_flush		= intel_fake_agp_chipset_flush,
1325
};
1326

1327 1328
static const struct intel_gtt_driver i81x_gtt_driver = {
	.gen = 1,
1329
	.dma_mask_size = 32,
1330
};
1331 1332
static const struct intel_gtt_driver i8xx_gtt_driver = {
	.gen = 2,
1333
	.setup = i830_setup,
1334
	.cleanup = i830_cleanup,
1335
	.write_entry = i830_write_entry,
1336
	.dma_mask_size = 32,
1337
	.check_flags = i830_check_flags,
1338
	.chipset_flush = i830_chipset_flush,
1339 1340 1341
};
static const struct intel_gtt_driver i915_gtt_driver = {
	.gen = 3,
1342
	.setup = i9xx_setup,
1343
	.cleanup = i9xx_cleanup,
1344 1345
	/* i945 is the last gpu to need phys mem (for overlay and cursors). */
	.write_entry = i830_write_entry, 
1346
	.dma_mask_size = 32,
1347
	.check_flags = i830_check_flags,
1348
	.chipset_flush = i9xx_chipset_flush,
1349 1350 1351 1352
};
static const struct intel_gtt_driver g33_gtt_driver = {
	.gen = 3,
	.is_g33 = 1,
1353
	.setup = i9xx_setup,
1354
	.cleanup = i9xx_cleanup,
1355
	.write_entry = i965_write_entry,
1356
	.dma_mask_size = 36,
1357
	.check_flags = i830_check_flags,
1358
	.chipset_flush = i9xx_chipset_flush,
1359 1360 1361 1362
};
static const struct intel_gtt_driver pineview_gtt_driver = {
	.gen = 3,
	.is_pineview = 1, .is_g33 = 1,
1363
	.setup = i9xx_setup,
1364
	.cleanup = i9xx_cleanup,
1365
	.write_entry = i965_write_entry,
1366
	.dma_mask_size = 36,
1367
	.check_flags = i830_check_flags,
1368
	.chipset_flush = i9xx_chipset_flush,
1369 1370 1371
};
static const struct intel_gtt_driver i965_gtt_driver = {
	.gen = 4,
1372
	.setup = i9xx_setup,
1373
	.cleanup = i9xx_cleanup,
1374
	.write_entry = i965_write_entry,
1375
	.dma_mask_size = 36,
1376
	.check_flags = i830_check_flags,
1377
	.chipset_flush = i9xx_chipset_flush,
1378 1379 1380
};
static const struct intel_gtt_driver g4x_gtt_driver = {
	.gen = 5,
1381
	.setup = i9xx_setup,
1382
	.cleanup = i9xx_cleanup,
1383
	.write_entry = i965_write_entry,
1384
	.dma_mask_size = 36,
1385
	.check_flags = i830_check_flags,
1386
	.chipset_flush = i9xx_chipset_flush,
1387 1388 1389 1390
};
static const struct intel_gtt_driver ironlake_gtt_driver = {
	.gen = 5,
	.is_ironlake = 1,
1391
	.setup = i9xx_setup,
1392
	.cleanup = i9xx_cleanup,
1393
	.write_entry = i965_write_entry,
1394
	.dma_mask_size = 36,
1395
	.check_flags = i830_check_flags,
1396
	.chipset_flush = i9xx_chipset_flush,
1397 1398 1399
};
static const struct intel_gtt_driver sandybridge_gtt_driver = {
	.gen = 6,
1400
	.setup = i9xx_setup,
1401
	.cleanup = gen6_cleanup,
1402
	.write_entry = gen6_write_entry,
1403
	.dma_mask_size = 40,
1404
	.check_flags = gen6_check_flags,
1405
	.chipset_flush = i9xx_chipset_flush,
1406 1407
};

1408 1409 1410 1411 1412 1413 1414 1415
/* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
 * driver and gmch_driver must be non-null, and find_gmch will determine
 * which one should be used if a gmch_chip_id is present.
 */
static const struct intel_gtt_driver_description {
	unsigned int gmch_chip_id;
	char *name;
	const struct agp_bridge_driver *gmch_driver;
1416
	const struct intel_gtt_driver *gtt_driver;
1417
} intel_gtt_chipsets[] = {
1418 1419 1420 1421 1422 1423 1424 1425
	{ PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver,
		&i81x_gtt_driver},
	{ PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver,
		&i81x_gtt_driver},
	{ PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver,
		&i81x_gtt_driver},
	{ PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver,
		&i81x_gtt_driver},
1426
	{ PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1427
		&intel_fake_agp_driver, &i8xx_gtt_driver},
1428
	{ PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1429
		&intel_fake_agp_driver, &i8xx_gtt_driver},
1430
	{ PCI_DEVICE_ID_INTEL_82854_IG, "854",
1431
		&intel_fake_agp_driver, &i8xx_gtt_driver},
1432
	{ PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1433
		&intel_fake_agp_driver, &i8xx_gtt_driver},
1434
	{ PCI_DEVICE_ID_INTEL_82865_IG, "865",
1435
		&intel_fake_agp_driver, &i8xx_gtt_driver},
1436
	{ PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1437
		&intel_fake_agp_driver, &i915_gtt_driver },
1438
	{ PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1439
		&intel_fake_agp_driver, &i915_gtt_driver },
1440
	{ PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1441
		&intel_fake_agp_driver, &i915_gtt_driver },
1442
	{ PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1443
		&intel_fake_agp_driver, &i915_gtt_driver },
1444
	{ PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1445
		&intel_fake_agp_driver, &i915_gtt_driver },
1446
	{ PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1447
		&intel_fake_agp_driver, &i915_gtt_driver },
1448
	{ PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1449
		&intel_fake_agp_driver, &i965_gtt_driver },
1450
	{ PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1451
		&intel_fake_agp_driver, &i965_gtt_driver },
1452
	{ PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1453
		&intel_fake_agp_driver, &i965_gtt_driver },
1454
	{ PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1455
		&intel_fake_agp_driver, &i965_gtt_driver },
1456
	{ PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1457
		&intel_fake_agp_driver, &i965_gtt_driver },
1458
	{ PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1459
		&intel_fake_agp_driver, &i965_gtt_driver },
1460
	{ PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1461
		&intel_fake_agp_driver, &g33_gtt_driver },
1462
	{ PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1463
		&intel_fake_agp_driver, &g33_gtt_driver },
1464
	{ PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1465
		&intel_fake_agp_driver, &g33_gtt_driver },
1466
	{ PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1467
		&intel_fake_agp_driver, &pineview_gtt_driver },
1468
	{ PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1469
		&intel_fake_agp_driver, &pineview_gtt_driver },
1470
	{ PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1471
		&intel_fake_agp_driver, &g4x_gtt_driver },
1472
	{ PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1473
		&intel_fake_agp_driver, &g4x_gtt_driver },
1474
	{ PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1475
		&intel_fake_agp_driver, &g4x_gtt_driver },
1476
	{ PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1477
		&intel_fake_agp_driver, &g4x_gtt_driver },
1478
	{ PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1479
		&intel_fake_agp_driver, &g4x_gtt_driver },
1480
	{ PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1481
		&intel_fake_agp_driver, &g4x_gtt_driver },
1482
	{ PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1483
		&intel_fake_agp_driver, &g4x_gtt_driver },
1484
	{ PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1485
	    "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1486
	{ PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1487
	    "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1488
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1489
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1490
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1491
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1492
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1493
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1494
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1495
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1496
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1497
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1498
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1499
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1500
	{ PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1501
	    "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
	{ 0, NULL, NULL }
};

static int find_gmch(u16 device)
{
	struct pci_dev *gmch_device;

	gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
	if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
		gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
					     device, gmch_device);
	}

	if (!gmch_device)
		return 0;

	intel_private.pcidev = gmch_device;
	return 1;
}

1522
int intel_gmch_probe(struct pci_dev *pdev,
1523 1524 1525 1526 1527 1528 1529 1530 1531
				      struct agp_bridge_data *bridge)
{
	int i, mask;
	bridge->driver = NULL;

	for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
		if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
			bridge->driver =
				intel_gtt_chipsets[i].gmch_driver;
1532 1533
			intel_private.driver = 
				intel_gtt_chipsets[i].gtt_driver;
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
			break;
		}
	}

	if (!bridge->driver)
		return 0;

	bridge->dev_private_data = &intel_private;
	bridge->dev = pdev;

1544 1545
	intel_private.bridge_dev = pci_dev_get(pdev);

1546 1547
	dev_info(&pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);

1548
	mask = intel_private.driver->dma_mask_size;
1549 1550 1551 1552 1553 1554 1555
	if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
		dev_err(&intel_private.pcidev->dev,
			"set gfx device dma mask %d-bit failed!\n", mask);
	else
		pci_set_consistent_dma_mask(intel_private.pcidev,
					    DMA_BIT_MASK(mask));

1556 1557 1558
	if (bridge->driver == &intel_810_driver)
		return 1;

1559 1560
	if (intel_gtt_init() != 0)
		return 0;
1561

1562 1563
	return 1;
}
1564
EXPORT_SYMBOL(intel_gmch_probe);
1565

1566 1567 1568 1569 1570 1571
struct intel_gtt *intel_gtt_get(void)
{
	return &intel_private.base;
}
EXPORT_SYMBOL(intel_gtt_get);

1572
void intel_gmch_remove(struct pci_dev *pdev)
1573 1574 1575
{
	if (intel_private.pcidev)
		pci_dev_put(intel_private.pcidev);
1576 1577
	if (intel_private.bridge_dev)
		pci_dev_put(intel_private.bridge_dev);
1578
}
1579 1580 1581 1582
EXPORT_SYMBOL(intel_gmch_remove);

MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
MODULE_LICENSE("GPL and additional rights");