nv50_graph.c 26.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright (C) 2007 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include "drmP.h"
#include "drm.h"
#include "nouveau_drv.h"
30
#include "nouveau_ramht.h"
31
#include "nouveau_grctx.h"
32
#include "nouveau_dma.h"
B
Ben Skeggs 已提交
33
#include "nouveau_vm.h"
34
#include "nv50_evo.h"
35

36 37
static int  nv50_graph_register(struct drm_device *);
static void nv50_graph_isr(struct drm_device *);
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
static void
nv50_graph_init_reset(struct drm_device *dev)
{
	uint32_t pmc_e = NV_PMC_ENABLE_PGRAPH | (1 << 21);

	NV_DEBUG(dev, "\n");

	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & ~pmc_e);
	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |  pmc_e);
}

static void
nv50_graph_init_intr(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");

55
	nouveau_irq_register(dev, 12, nv50_graph_isr);
56 57 58 59 60 61 62 63
	nv_wr32(dev, NV03_PGRAPH_INTR, 0xffffffff);
	nv_wr32(dev, 0x400138, 0xffffffff);
	nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xffffffff);
}

static void
nv50_graph_init_regs__nv(struct drm_device *dev)
{
64 65 66 67
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	uint32_t units = nv_rd32(dev, 0x1540);
	int i;

68 69 70 71 72
	NV_DEBUG(dev, "\n");

	nv_wr32(dev, 0x400804, 0xc0000000);
	nv_wr32(dev, 0x406800, 0xc0000000);
	nv_wr32(dev, 0x400c04, 0xc0000000);
73
	nv_wr32(dev, 0x401800, 0xc0000000);
74 75 76
	nv_wr32(dev, 0x405018, 0xc0000000);
	nv_wr32(dev, 0x402000, 0xc0000000);

77 78 79 80 81 82 83 84 85 86 87 88 89 90
	for (i = 0; i < 16; i++) {
		if (units & 1 << i) {
			if (dev_priv->chipset < 0xa0) {
				nv_wr32(dev, 0x408900 + (i << 12), 0xc0000000);
				nv_wr32(dev, 0x408e08 + (i << 12), 0xc0000000);
				nv_wr32(dev, 0x408314 + (i << 12), 0xc0000000);
			} else {
				nv_wr32(dev, 0x408600 + (i << 11), 0xc0000000);
				nv_wr32(dev, 0x408708 + (i << 11), 0xc0000000);
				nv_wr32(dev, 0x40831c + (i << 11), 0xc0000000);
			}
		}
	}

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
	nv_wr32(dev, 0x400108, 0xffffffff);

	nv_wr32(dev, 0x400824, 0x00004000);
	nv_wr32(dev, 0x400500, 0x00010001);
}

static void
nv50_graph_init_regs(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");

	nv_wr32(dev, NV04_PGRAPH_DEBUG_3,
				(1 << 2) /* HW_CONTEXT_SWITCH_ENABLED */);
	nv_wr32(dev, 0x402ca8, 0x800);
}

static int
nv50_graph_init_ctxctl(struct drm_device *dev)
{
110
	struct drm_nouveau_private *dev_priv = dev->dev_private;
111 112 113
	struct nouveau_grctx ctx = {};
	uint32_t *cp;
	int i;
114

115 116
	NV_DEBUG(dev, "\n");

117 118 119 120 121
	cp = kmalloc(512 * 4, GFP_KERNEL);
	if (!cp) {
		NV_ERROR(dev, "failed to allocate ctxprog\n");
		dev_priv->engine.graph.accel_blocked = true;
		return 0;
122
	}
123 124 125 126 127 128 129 130 131 132 133 134 135

	ctx.dev = dev;
	ctx.mode = NOUVEAU_GRCTX_PROG;
	ctx.data = cp;
	ctx.ctxprog_max = 512;
	if (!nv50_grctx_init(&ctx)) {
		dev_priv->engine.graph.grctx_size = ctx.ctxvals_pos * 4;

		nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_INDEX, 0);
		for (i = 0; i < ctx.ctxprog_len; i++)
			nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_DATA, cp[i]);
	} else {
		dev_priv->engine.graph.accel_blocked = true;
136
	}
137
	kfree(cp);
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159

	nv_wr32(dev, 0x400320, 4);
	nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, 0);
	nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, 0);
	return 0;
}

int
nv50_graph_init(struct drm_device *dev)
{
	int ret;

	NV_DEBUG(dev, "\n");

	nv50_graph_init_reset(dev);
	nv50_graph_init_regs__nv(dev);
	nv50_graph_init_regs(dev);

	ret = nv50_graph_init_ctxctl(dev);
	if (ret)
		return ret;

160 161 162 163
	ret = nv50_graph_register(dev);
	if (ret)
		return ret;
	nv50_graph_init_intr(dev);
164 165 166 167 168 169 170
	return 0;
}

void
nv50_graph_takedown(struct drm_device *dev)
{
	NV_DEBUG(dev, "\n");
171 172
	nv_wr32(dev, 0x40013c, 0x00000000);
	nouveau_irq_unregister(dev, 12);
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
}

void
nv50_graph_fifo_access(struct drm_device *dev, bool enabled)
{
	const uint32_t mask = 0x00010001;

	if (enabled)
		nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) | mask);
	else
		nv_wr32(dev, 0x400500, nv_rd32(dev, 0x400500) & ~mask);
}

struct nouveau_channel *
nv50_graph_channel(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	uint32_t inst;
	int i;

193 194 195
	/* Be sure we're not in the middle of a context switch or bad things
	 * will happen, such as unloading the wrong pgraph context.
	 */
196
	if (!nv_wait(dev, 0x400300, 0x00000001, 0x00000000))
197 198
		NV_ERROR(dev, "Ctxprog is still running\n");

199 200 201 202 203 204
	inst = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
	if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
		return NULL;
	inst = (inst & NV50_PGRAPH_CTXCTL_CUR_INSTANCE) << 12;

	for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
205
		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
206

207
		if (chan && chan->ramin && chan->ramin->vinst == inst)
208 209 210 211 212 213 214 215 216 217 218
			return chan;
	}

	return NULL;
}

int
nv50_graph_create_context(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
219
	struct nouveau_gpuobj *ramin = chan->ramin;
220
	struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
221
	struct nouveau_grctx ctx = {};
222 223 224 225
	int hdr, ret;

	NV_DEBUG(dev, "ch%d\n", chan->id);

226
	ret = nouveau_gpuobj_new(dev, chan, pgraph->grctx_size, 0,
227 228
				 NVOBJ_FLAG_ZERO_ALLOC |
				 NVOBJ_FLAG_ZERO_FREE, &chan->ramin_grctx);
229 230 231
	if (ret)
		return ret;

B
Ben Skeggs 已提交
232
	hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
233
	nv_wo32(ramin, hdr + 0x00, 0x00190002);
234
	nv_wo32(ramin, hdr + 0x04, chan->ramin_grctx->vinst +
235
				   pgraph->grctx_size - 1);
236
	nv_wo32(ramin, hdr + 0x08, chan->ramin_grctx->vinst);
237 238 239
	nv_wo32(ramin, hdr + 0x0c, 0);
	nv_wo32(ramin, hdr + 0x10, 0);
	nv_wo32(ramin, hdr + 0x14, 0x00010000);
240

241 242
	ctx.dev = chan->dev;
	ctx.mode = NOUVEAU_GRCTX_VALS;
243
	ctx.data = chan->ramin_grctx;
244 245
	nv50_grctx_init(&ctx);

246
	nv_wo32(chan->ramin_grctx, 0x00000, chan->ramin->vinst >> 12);
247

248
	dev_priv->engine.instmem.flush(dev);
249
	atomic_inc(&chan->vm->pgraph_refs);
250 251 252 253 254 255 256 257
	return 0;
}

void
nv50_graph_destroy_context(struct nouveau_channel *chan)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
258
	struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
B
Ben Skeggs 已提交
259
	int i, hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
260
	unsigned long flags;
261 262 263

	NV_DEBUG(dev, "ch%d\n", chan->id);

264
	if (!chan->ramin)
265 266
		return;

267 268 269 270 271 272
	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
	pgraph->fifo_access(dev, false);

	if (pgraph->channel(dev) == chan)
		pgraph->unload_context(dev);

273
	for (i = hdr; i < hdr + 24; i += 4)
274
		nv_wo32(chan->ramin, i, 0);
275
	dev_priv->engine.instmem.flush(dev);
276

277 278 279
	pgraph->fifo_access(dev, true);
	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);

280
	nouveau_gpuobj_ref(NULL, &chan->ramin_grctx);
281 282

	atomic_dec(&chan->vm->pgraph_refs);
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
}

static int
nv50_graph_do_load_context(struct drm_device *dev, uint32_t inst)
{
	uint32_t fifo = nv_rd32(dev, 0x400500);

	nv_wr32(dev, 0x400500, fifo & ~1);
	nv_wr32(dev, 0x400784, inst);
	nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x40);
	nv_wr32(dev, 0x400320, nv_rd32(dev, 0x400320) | 0x11);
	nv_wr32(dev, 0x400040, 0xffffffff);
	(void)nv_rd32(dev, 0x400040);
	nv_wr32(dev, 0x400040, 0x00000000);
	nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 1);

	if (nouveau_wait_for_idle(dev))
		nv_wr32(dev, 0x40032c, inst | (1<<31));
	nv_wr32(dev, 0x400500, fifo);

	return 0;
}

int
nv50_graph_load_context(struct nouveau_channel *chan)
{
309
	uint32_t inst = chan->ramin->vinst >> 12;
310 311 312 313 314 315 316 317

	NV_DEBUG(chan->dev, "ch%d\n", chan->id);
	return nv50_graph_do_load_context(chan->dev, inst);
}

int
nv50_graph_unload_context(struct drm_device *dev)
{
318
	uint32_t inst;
319 320 321 322 323 324

	inst  = nv_rd32(dev, NV50_PGRAPH_CTXCTL_CUR);
	if (!(inst & NV50_PGRAPH_CTXCTL_CUR_LOADED))
		return 0;
	inst &= NV50_PGRAPH_CTXCTL_CUR_INSTANCE;

325
	nouveau_wait_for_idle(dev);
326 327 328 329 330 331 332 333 334
	nv_wr32(dev, 0x400784, inst);
	nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) | 0x20);
	nv_wr32(dev, 0x400304, nv_rd32(dev, 0x400304) | 0x01);
	nouveau_wait_for_idle(dev);

	nv_wr32(dev, NV50_PGRAPH_CTXCTL_CUR, inst);
	return 0;
}

335
static void
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
nv50_graph_context_switch(struct drm_device *dev)
{
	uint32_t inst;

	nv50_graph_unload_context(dev);

	inst  = nv_rd32(dev, NV50_PGRAPH_CTXCTL_NEXT);
	inst &= NV50_PGRAPH_CTXCTL_NEXT_INSTANCE;
	nv50_graph_do_load_context(dev, inst);

	nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
		NV40_PGRAPH_INTR_EN) | NV_PGRAPH_INTR_CONTEXT_SWITCH);
}

static int
351 352
nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan,
			   u32 class, u32 mthd, u32 data)
353
{
354
	struct nouveau_gpuobj *gpuobj;
355

356 357
	gpuobj = nouveau_ramht_find(chan, data);
	if (!gpuobj)
358 359
		return -ENOENT;

360
	if (nouveau_notifier_offset(gpuobj, NULL))
361 362
		return -EINVAL;

363
	chan->nvsw.vblsem = gpuobj;
364 365 366 367 368
	chan->nvsw.vblsem_offset = ~0;
	return 0;
}

static int
369 370
nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan,
			      u32 class, u32 mthd, u32 data)
371 372 373 374 375 376 377 378 379
{
	if (nouveau_notifier_offset(chan->nvsw.vblsem, &data))
		return -ERANGE;

	chan->nvsw.vblsem_offset = data >> 2;
	return 0;
}

static int
380 381
nv50_graph_nvsw_vblsem_release_val(struct nouveau_channel *chan,
				   u32 class, u32 mthd, u32 data)
382 383 384 385 386 387
{
	chan->nvsw.vblsem_rval = data;
	return 0;
}

static int
388 389
nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan,
			       u32 class, u32 mthd, u32 data)
390 391 392 393 394 395 396
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;

	if (!chan->nvsw.vblsem || chan->nvsw.vblsem_offset == ~0 || data > 1)
		return -EINVAL;

397
	drm_vblank_get(dev, data);
398 399

	chan->nvsw.vblsem_head = data;
400
	list_add(&chan->nvsw.vbl_wait, &dev_priv->vbl_waiting);
401

402 403 404
	return 0;
}

405 406 407 408 409 410 411 412 413 414 415 416 417
static int
nv50_graph_nvsw_mthd_page_flip(struct nouveau_channel *chan,
			       u32 class, u32 mthd, u32 data)
{
	struct nouveau_page_flip_state s;

	if (!nouveau_finish_page_flip(chan, &s)) {
		/* XXX - Do something here */
	}

	return 0;
}

418 419 420 421 422 423 424 425 426 427 428 429 430
static int
nv50_graph_register(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;

	if (dev_priv->engine.graph.registered)
		return 0;

	NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
	NVOBJ_MTHD (dev, 0x506e, 0x018c, nv50_graph_nvsw_dma_vblsem);
	NVOBJ_MTHD (dev, 0x506e, 0x0400, nv50_graph_nvsw_vblsem_offset);
	NVOBJ_MTHD (dev, 0x506e, 0x0404, nv50_graph_nvsw_vblsem_release_val);
	NVOBJ_MTHD (dev, 0x506e, 0x0408, nv50_graph_nvsw_vblsem_release);
431
	NVOBJ_MTHD (dev, 0x506e, 0x0500, nv50_graph_nvsw_mthd_page_flip);
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460

	NVOBJ_CLASS(dev, 0x0030, GR); /* null */
	NVOBJ_CLASS(dev, 0x5039, GR); /* m2mf */
	NVOBJ_CLASS(dev, 0x502d, GR); /* 2d */

	/* tesla */
	if (dev_priv->chipset == 0x50)
		NVOBJ_CLASS(dev, 0x5097, GR); /* tesla (nv50) */
	else
	if (dev_priv->chipset < 0xa0)
		NVOBJ_CLASS(dev, 0x8297, GR); /* tesla (nv8x/nv9x) */
	else {
		switch (dev_priv->chipset) {
		case 0xa0:
		case 0xaa:
		case 0xac:
			NVOBJ_CLASS(dev, 0x8397, GR);
			break;
		case 0xa3:
		case 0xa5:
		case 0xa8:
			NVOBJ_CLASS(dev, 0x8597, GR);
			break;
		case 0xaf:
			NVOBJ_CLASS(dev, 0x8697, GR);
			break;
		}
	}

B
Ben Skeggs 已提交
461
	/* compute */
462 463 464 465
	NVOBJ_CLASS(dev, 0x50c0, GR);
	if (dev_priv->chipset  > 0xa0 &&
	    dev_priv->chipset != 0xaa &&
	    dev_priv->chipset != 0xac)
B
Ben Skeggs 已提交
466 467
		NVOBJ_CLASS(dev, 0x85c0, GR);

468 469 470
	dev_priv->engine.graph.registered = true;
	return 0;
}
471 472 473 474

void
nv50_graph_tlb_flush(struct drm_device *dev)
{
B
Ben Skeggs 已提交
475
	nv50_vm_flush_engine(dev, 0);
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
}

void
nv86_graph_tlb_flush(struct drm_device *dev)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
	bool idle, timeout = false;
	unsigned long flags;
	u64 start;
	u32 tmp;

	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
	nv_mask(dev, 0x400500, 0x00000001, 0x00000000);

	start = ptimer->read(dev);
	do {
		idle = true;

		for (tmp = nv_rd32(dev, 0x400380); tmp && idle; tmp >>= 3) {
			if ((tmp & 7) == 1)
				idle = false;
		}

		for (tmp = nv_rd32(dev, 0x400384); tmp && idle; tmp >>= 3) {
			if ((tmp & 7) == 1)
				idle = false;
		}

		for (tmp = nv_rd32(dev, 0x400388); tmp && idle; tmp >>= 3) {
			if ((tmp & 7) == 1)
				idle = false;
		}
	} while (!idle && !(timeout = ptimer->read(dev) - start > 2000000000));

	if (timeout) {
		NV_ERROR(dev, "PGRAPH TLB flush idle timeout fail: "
			      "0x%08x 0x%08x 0x%08x 0x%08x\n",
			 nv_rd32(dev, 0x400700), nv_rd32(dev, 0x400380),
			 nv_rd32(dev, 0x400384), nv_rd32(dev, 0x400388));
	}

B
Ben Skeggs 已提交
518
	nv50_vm_flush_engine(dev, 0);
519 520 521 522

	nv_mask(dev, 0x400500, 0x00000001, 0x00000001);
	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
}
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014

static struct nouveau_enum nv50_mp_exec_error_names[] =
{
	{ 3, "STACK_UNDERFLOW" },
	{ 4, "QUADON_ACTIVE" },
	{ 8, "TIMEOUT" },
	{ 0x10, "INVALID_OPCODE" },
	{ 0x40, "BREAKPOINT" },
	{}
};

static struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
	{ 0x00000001, "NOTIFY" },
	{ 0x00000002, "IN" },
	{ 0x00000004, "OUT" },
	{}
};

static struct nouveau_bitfield nv50_graph_trap_vfetch[] = {
	{ 0x00000001, "FAULT" },
	{}
};

static struct nouveau_bitfield nv50_graph_trap_strmout[] = {
	{ 0x00000001, "FAULT" },
	{}
};

static struct nouveau_bitfield nv50_graph_trap_ccache[] = {
	{ 0x00000001, "FAULT" },
	{}
};

/* There must be a *lot* of these. Will take some time to gather them up. */
static struct nouveau_enum nv50_data_error_names[] = {
	{ 4,	"INVALID_VALUE" },
	{ 5,	"INVALID_ENUM" },
	{ 8,	"INVALID_OBJECT" },
	{ 0xc,	"INVALID_BITFIELD" },
	{ 0x28,	"MP_NO_REG_SPACE" },
	{ 0x2b,	"MP_BLOCK_SIZE_MISMATCH" },
	{}
};

static struct nouveau_bitfield nv50_graph_intr[] = {
	{ 0x00000001, "NOTIFY" },
	{ 0x00000002, "COMPUTE_QUERY" },
	{ 0x00000010, "ILLEGAL_MTHD" },
	{ 0x00000020, "ILLEGAL_CLASS" },
	{ 0x00000040, "DOUBLE_NOTIFY" },
	{ 0x00001000, "CONTEXT_SWITCH" },
	{ 0x00010000, "BUFFER_NOTIFY" },
	{ 0x00100000, "DATA_ERROR" },
	{ 0x00200000, "TRAP" },
	{ 0x01000000, "SINGLE_STEP" },
	{}
};

static void
nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	uint32_t units = nv_rd32(dev, 0x1540);
	uint32_t addr, mp10, status, pc, oplow, ophigh;
	int i;
	int mps = 0;
	for (i = 0; i < 4; i++) {
		if (!(units & 1 << (i+24)))
			continue;
		if (dev_priv->chipset < 0xa0)
			addr = 0x408200 + (tpid << 12) + (i << 7);
		else
			addr = 0x408100 + (tpid << 11) + (i << 7);
		mp10 = nv_rd32(dev, addr + 0x10);
		status = nv_rd32(dev, addr + 0x14);
		if (!status)
			continue;
		if (display) {
			nv_rd32(dev, addr + 0x20);
			pc = nv_rd32(dev, addr + 0x24);
			oplow = nv_rd32(dev, addr + 0x70);
			ophigh= nv_rd32(dev, addr + 0x74);
			NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
					"TP %d MP %d: ", tpid, i);
			nouveau_enum_print(nv50_mp_exec_error_names, status);
			printk(" at %06x warp %d, opcode %08x %08x\n",
					pc&0xffffff, pc >> 24,
					oplow, ophigh);
		}
		nv_wr32(dev, addr + 0x10, mp10);
		nv_wr32(dev, addr + 0x14, 0);
		mps++;
	}
	if (!mps && display)
		NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
				"No MPs claiming errors?\n", tpid);
}

static void
nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
		uint32_t ustatus_new, int display, const char *name)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	int tps = 0;
	uint32_t units = nv_rd32(dev, 0x1540);
	int i, r;
	uint32_t ustatus_addr, ustatus;
	for (i = 0; i < 16; i++) {
		if (!(units & (1 << i)))
			continue;
		if (dev_priv->chipset < 0xa0)
			ustatus_addr = ustatus_old + (i << 12);
		else
			ustatus_addr = ustatus_new + (i << 11);
		ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
		if (!ustatus)
			continue;
		tps++;
		switch (type) {
		case 6: /* texture error... unknown for now */
			nv50_fb_vm_trap(dev, display, name);
			if (display) {
				NV_ERROR(dev, "magic set %d:\n", i);
				for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
					NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
						nv_rd32(dev, r));
			}
			break;
		case 7: /* MP error */
			if (ustatus & 0x00010000) {
				nv50_pgraph_mp_trap(dev, i, display);
				ustatus &= ~0x00010000;
			}
			break;
		case 8: /* TPDMA error */
			{
			uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
			uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
			uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
			uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
			uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
			uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
			uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
			nv50_fb_vm_trap(dev, display, name);
			/* 2d engine destination */
			if (ustatus & 0x00000010) {
				if (display) {
					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
							i, e14, e10);
					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
							i, e0c, e18, e1c, e20, e24);
				}
				ustatus &= ~0x00000010;
			}
			/* Render target */
			if (ustatus & 0x00000040) {
				if (display) {
					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
							i, e14, e10);
					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
							i, e0c, e18, e1c, e20, e24);
				}
				ustatus &= ~0x00000040;
			}
			/* CUDA memory: l[], g[] or stack. */
			if (ustatus & 0x00000080) {
				if (display) {
					if (e18 & 0x80000000) {
						/* g[] read fault? */
						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
								i, e14, e10 | ((e18 >> 24) & 0x1f));
						e18 &= ~0x1f000000;
					} else if (e18 & 0xc) {
						/* g[] write fault? */
						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
								i, e14, e10 | ((e18 >> 7) & 0x1f));
						e18 &= ~0x00000f80;
					} else {
						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
								i, e14, e10);
					}
					NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
							i, e0c, e18, e1c, e20, e24);
				}
				ustatus &= ~0x00000080;
			}
			}
			break;
		}
		if (ustatus) {
			if (display)
				NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
		}
		nv_wr32(dev, ustatus_addr, 0xc0000000);
	}

	if (!tps && display)
		NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
}

static int
nv50_pgraph_trap_handler(struct drm_device *dev, u32 display, u64 inst, u32 chid)
{
	u32 status = nv_rd32(dev, 0x400108);
	u32 ustatus;

	if (!status && display) {
		NV_INFO(dev, "PGRAPH - TRAP: no units reporting traps?\n");
		return 1;
	}

	/* DISPATCH: Relays commands to other units and handles NOTIFY,
	 * COND, QUERY. If you get a trap from it, the command is still stuck
	 * in DISPATCH and you need to do something about it. */
	if (status & 0x001) {
		ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
		if (!ustatus && display) {
			NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
		}

		nv_wr32(dev, 0x400500, 0x00000000);

		/* Known to be triggered by screwed up NOTIFY and COND... */
		if (ustatus & 0x00000001) {
			u32 addr = nv_rd32(dev, 0x400808);
			u32 subc = (addr & 0x00070000) >> 16;
			u32 mthd = (addr & 0x00001ffc);
			u32 datal = nv_rd32(dev, 0x40080c);
			u32 datah = nv_rd32(dev, 0x400810);
			u32 class = nv_rd32(dev, 0x400814);
			u32 r848 = nv_rd32(dev, 0x400848);

			NV_INFO(dev, "PGRAPH - TRAP DISPATCH_FAULT\n");
			if (display && (addr & 0x80000000)) {
				NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
					     "subc %d class 0x%04x mthd 0x%04x "
					     "data 0x%08x%08x "
					     "400808 0x%08x 400848 0x%08x\n",
					chid, inst, subc, class, mthd, datah,
					datal, addr, r848);
			} else
			if (display) {
				NV_INFO(dev, "PGRAPH - no stuck command?\n");
			}

			nv_wr32(dev, 0x400808, 0);
			nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
			nv_wr32(dev, 0x400848, 0);
			ustatus &= ~0x00000001;
		}

		if (ustatus & 0x00000002) {
			u32 addr = nv_rd32(dev, 0x40084c);
			u32 subc = (addr & 0x00070000) >> 16;
			u32 mthd = (addr & 0x00001ffc);
			u32 data = nv_rd32(dev, 0x40085c);
			u32 class = nv_rd32(dev, 0x400814);

			NV_INFO(dev, "PGRAPH - TRAP DISPATCH_QUERY\n");
			if (display && (addr & 0x80000000)) {
				NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
					     "subc %d class 0x%04x mthd 0x%04x "
					     "data 0x%08x 40084c 0x%08x\n",
					chid, inst, subc, class, mthd,
					data, addr);
			} else
			if (display) {
				NV_INFO(dev, "PGRAPH - no stuck command?\n");
			}

			nv_wr32(dev, 0x40084c, 0);
			ustatus &= ~0x00000002;
		}

		if (ustatus && display) {
			NV_INFO(dev, "PGRAPH - TRAP_DISPATCH (unknown "
				      "0x%08x)\n", ustatus);
		}

		nv_wr32(dev, 0x400804, 0xc0000000);
		nv_wr32(dev, 0x400108, 0x001);
		status &= ~0x001;
		if (!status)
			return 0;
	}

	/* M2MF: Memory to memory copy engine. */
	if (status & 0x002) {
		u32 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
		if (display) {
			NV_INFO(dev, "PGRAPH - TRAP_M2MF");
			nouveau_bitfield_print(nv50_graph_trap_m2mf, ustatus);
			printk("\n");
			NV_INFO(dev, "PGRAPH - TRAP_M2MF %08x %08x %08x %08x\n",
				nv_rd32(dev, 0x406804), nv_rd32(dev, 0x406808),
				nv_rd32(dev, 0x40680c), nv_rd32(dev, 0x406810));

		}

		/* No sane way found yet -- just reset the bugger. */
		nv_wr32(dev, 0x400040, 2);
		nv_wr32(dev, 0x400040, 0);
		nv_wr32(dev, 0x406800, 0xc0000000);
		nv_wr32(dev, 0x400108, 0x002);
		status &= ~0x002;
	}

	/* VFETCH: Fetches data from vertex buffers. */
	if (status & 0x004) {
		u32 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
		if (display) {
			NV_INFO(dev, "PGRAPH - TRAP_VFETCH");
			nouveau_bitfield_print(nv50_graph_trap_vfetch, ustatus);
			printk("\n");
			NV_INFO(dev, "PGRAPH - TRAP_VFETCH %08x %08x %08x %08x\n",
				nv_rd32(dev, 0x400c00), nv_rd32(dev, 0x400c08),
				nv_rd32(dev, 0x400c0c), nv_rd32(dev, 0x400c10));
		}

		nv_wr32(dev, 0x400c04, 0xc0000000);
		nv_wr32(dev, 0x400108, 0x004);
		status &= ~0x004;
	}

	/* STRMOUT: DirectX streamout / OpenGL transform feedback. */
	if (status & 0x008) {
		ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
		if (display) {
			NV_INFO(dev, "PGRAPH - TRAP_STRMOUT");
			nouveau_bitfield_print(nv50_graph_trap_strmout, ustatus);
			printk("\n");
			NV_INFO(dev, "PGRAPH - TRAP_STRMOUT %08x %08x %08x %08x\n",
				nv_rd32(dev, 0x401804), nv_rd32(dev, 0x401808),
				nv_rd32(dev, 0x40180c), nv_rd32(dev, 0x401810));

		}

		/* No sane way found yet -- just reset the bugger. */
		nv_wr32(dev, 0x400040, 0x80);
		nv_wr32(dev, 0x400040, 0);
		nv_wr32(dev, 0x401800, 0xc0000000);
		nv_wr32(dev, 0x400108, 0x008);
		status &= ~0x008;
	}

	/* CCACHE: Handles code and c[] caches and fills them. */
	if (status & 0x010) {
		ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
		if (display) {
			NV_INFO(dev, "PGRAPH - TRAP_CCACHE");
			nouveau_bitfield_print(nv50_graph_trap_ccache, ustatus);
			printk("\n");
			NV_INFO(dev, "PGRAPH - TRAP_CCACHE %08x %08x %08x %08x"
				     " %08x %08x %08x\n",
				nv_rd32(dev, 0x405800), nv_rd32(dev, 0x405804),
				nv_rd32(dev, 0x405808), nv_rd32(dev, 0x40580c),
				nv_rd32(dev, 0x405810), nv_rd32(dev, 0x405814),
				nv_rd32(dev, 0x40581c));

		}

		nv_wr32(dev, 0x405018, 0xc0000000);
		nv_wr32(dev, 0x400108, 0x010);
		status &= ~0x010;
	}

	/* Unknown, not seen yet... 0x402000 is the only trap status reg
	 * remaining, so try to handle it anyway. Perhaps related to that
	 * unknown DMA slot on tesla? */
	if (status & 0x20) {
		ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
		if (display)
			NV_INFO(dev, "PGRAPH - TRAP_UNKC04 0x%08x\n", ustatus);
		nv_wr32(dev, 0x402000, 0xc0000000);
		/* no status modifiction on purpose */
	}

	/* TEXTURE: CUDA texturing units */
	if (status & 0x040) {
		nv50_pgraph_tp_trap(dev, 6, 0x408900, 0x408600, display,
				    "PGRAPH - TRAP_TEXTURE");
		nv_wr32(dev, 0x400108, 0x040);
		status &= ~0x040;
	}

	/* MP: CUDA execution engines. */
	if (status & 0x080) {
		nv50_pgraph_tp_trap(dev, 7, 0x408314, 0x40831c, display,
				    "PGRAPH - TRAP_MP");
		nv_wr32(dev, 0x400108, 0x080);
		status &= ~0x080;
	}

	/* TPDMA:  Handles TP-initiated uncached memory accesses:
	 * l[], g[], stack, 2d surfaces, render targets. */
	if (status & 0x100) {
		nv50_pgraph_tp_trap(dev, 8, 0x408e08, 0x408708, display,
				    "PGRAPH - TRAP_TPDMA");
		nv_wr32(dev, 0x400108, 0x100);
		status &= ~0x100;
	}

	if (status) {
		if (display)
			NV_INFO(dev, "PGRAPH - TRAP: unknown 0x%08x\n", status);
		nv_wr32(dev, 0x400108, status);
	}

	return 1;
}

static int
nv50_graph_isr_chid(struct drm_device *dev, u64 inst)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_channel *chan;
	unsigned long flags;
	int i;

	spin_lock_irqsave(&dev_priv->channels.lock, flags);
	for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
		chan = dev_priv->channels.ptr[i];
		if (!chan || !chan->ramin)
			continue;

		if (inst == chan->ramin->vinst)
			break;
	}
	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
	return i;
}

static void
nv50_graph_isr(struct drm_device *dev)
{
	u32 stat;

	while ((stat = nv_rd32(dev, 0x400100))) {
		u64 inst = (u64)(nv_rd32(dev, 0x40032c) & 0x0fffffff) << 12;
		u32 chid = nv50_graph_isr_chid(dev, inst);
		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
		u32 subc = (addr & 0x00070000) >> 16;
		u32 mthd = (addr & 0x00001ffc);
		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
		u32 class = nv_rd32(dev, 0x400814);
		u32 show = stat;

		if (stat & 0x00000010) {
			if (!nouveau_gpuobj_mthd_call2(dev, chid, class,
						       mthd, data))
				show &= ~0x00000010;
		}

		if (stat & 0x00001000) {
			nv_wr32(dev, 0x400500, 0x00000000);
			nv_wr32(dev, 0x400100, 0x00001000);
			nv_mask(dev, 0x40013c, 0x00001000, 0x00000000);
			nv50_graph_context_switch(dev);
			stat &= ~0x00001000;
			show &= ~0x00001000;
		}

		show = (show && nouveau_ratelimit()) ? show : 0;

		if (show & 0x00100000) {
			u32 ecode = nv_rd32(dev, 0x400110);
			NV_INFO(dev, "PGRAPH - DATA_ERROR ");
			nouveau_enum_print(nv50_data_error_names, ecode);
			printk("\n");
		}

		if (stat & 0x00200000) {
			if (!nv50_pgraph_trap_handler(dev, show, inst, chid))
				show &= ~0x00200000;
		}

		nv_wr32(dev, 0x400100, stat);
		nv_wr32(dev, 0x400500, 0x00010001);

		if (show) {
			NV_INFO(dev, "PGRAPH -");
			nouveau_bitfield_print(nv50_graph_intr, show);
			printk("\n");
			NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) subc %d "
				     "class 0x%04x mthd 0x%04x data 0x%08x\n",
				chid, inst, subc, class, mthd, data);
		}
	}

	if (nv_rd32(dev, 0x400824) & (1 << 31))
		nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
}