gf100.c 60.6 KB
Newer Older
1
/*
2
 * Copyright 2012 Red Hat Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
 */
24 25 26 27 28 29
#include "gf100.h"
#include "ctxgf100.h"
#include "fuc/os.h"

#include <core/client.h>
#include <core/option.h>
30
#include <core/firmware.h>
31
#include <subdev/secboot.h>
32 33
#include <subdev/fb.h>
#include <subdev/mc.h>
34
#include <subdev/pmu.h>
35
#include <subdev/therm.h>
36
#include <subdev/timer.h>
37
#include <engine/fifo.h>
38 39

#include <nvif/class.h>
40
#include <nvif/cl9097.h>
41
#include <nvif/if900d.h>
42
#include <nvif/unpack.h>
43

44 45 46 47 48
/*******************************************************************************
 * Zero Bandwidth Clear
 ******************************************************************************/

static void
B
Ben Skeggs 已提交
49
gf100_gr_zbc_clear_color(struct gf100_gr *gr, int zbc)
50
{
51
	struct nvkm_device *device = gr->base.engine.subdev.device;
B
Ben Skeggs 已提交
52
	if (gr->zbc_color[zbc].format) {
53 54 55 56 57 58 59 60
		nvkm_wr32(device, 0x405804, gr->zbc_color[zbc].ds[0]);
		nvkm_wr32(device, 0x405808, gr->zbc_color[zbc].ds[1]);
		nvkm_wr32(device, 0x40580c, gr->zbc_color[zbc].ds[2]);
		nvkm_wr32(device, 0x405810, gr->zbc_color[zbc].ds[3]);
	}
	nvkm_wr32(device, 0x405814, gr->zbc_color[zbc].format);
	nvkm_wr32(device, 0x405820, zbc);
	nvkm_wr32(device, 0x405824, 0x00000004); /* TRIGGER | WRITE | COLOR */
61 62 63
}

static int
B
Ben Skeggs 已提交
64
gf100_gr_zbc_color_get(struct gf100_gr *gr, int format,
65
		       const u32 ds[4], const u32 l2[4])
66
{
67
	struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;
68 69 70
	int zbc = -ENOSPC, i;

	for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
B
Ben Skeggs 已提交
71 72
		if (gr->zbc_color[i].format) {
			if (gr->zbc_color[i].format != format)
73
				continue;
B
Ben Skeggs 已提交
74 75
			if (memcmp(gr->zbc_color[i].ds, ds, sizeof(
				   gr->zbc_color[i].ds)))
76
				continue;
B
Ben Skeggs 已提交
77 78
			if (memcmp(gr->zbc_color[i].l2, l2, sizeof(
				   gr->zbc_color[i].l2))) {
79 80 81 82 83 84 85 86 87
				WARN_ON(1);
				return -EINVAL;
			}
			return i;
		} else {
			zbc = (zbc < 0) ? i : zbc;
		}
	}

88 89 90
	if (zbc < 0)
		return zbc;

B
Ben Skeggs 已提交
91 92 93
	memcpy(gr->zbc_color[zbc].ds, ds, sizeof(gr->zbc_color[zbc].ds));
	memcpy(gr->zbc_color[zbc].l2, l2, sizeof(gr->zbc_color[zbc].l2));
	gr->zbc_color[zbc].format = format;
94
	nvkm_ltc_zbc_color_get(ltc, zbc, l2);
95
	gr->func->zbc->clear_color(gr, zbc);
96 97 98 99
	return zbc;
}

static void
B
Ben Skeggs 已提交
100
gf100_gr_zbc_clear_depth(struct gf100_gr *gr, int zbc)
101
{
102
	struct nvkm_device *device = gr->base.engine.subdev.device;
B
Ben Skeggs 已提交
103
	if (gr->zbc_depth[zbc].format)
104 105 106 107
		nvkm_wr32(device, 0x405818, gr->zbc_depth[zbc].ds);
	nvkm_wr32(device, 0x40581c, gr->zbc_depth[zbc].format);
	nvkm_wr32(device, 0x405820, zbc);
	nvkm_wr32(device, 0x405824, 0x00000005); /* TRIGGER | WRITE | DEPTH */
108 109 110
}

static int
B
Ben Skeggs 已提交
111
gf100_gr_zbc_depth_get(struct gf100_gr *gr, int format,
112
		       const u32 ds, const u32 l2)
113
{
114
	struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;
115 116 117
	int zbc = -ENOSPC, i;

	for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) {
B
Ben Skeggs 已提交
118 119
		if (gr->zbc_depth[i].format) {
			if (gr->zbc_depth[i].format != format)
120
				continue;
B
Ben Skeggs 已提交
121
			if (gr->zbc_depth[i].ds != ds)
122
				continue;
B
Ben Skeggs 已提交
123
			if (gr->zbc_depth[i].l2 != l2) {
124 125 126 127 128 129 130 131 132
				WARN_ON(1);
				return -EINVAL;
			}
			return i;
		} else {
			zbc = (zbc < 0) ? i : zbc;
		}
	}

133 134 135
	if (zbc < 0)
		return zbc;

B
Ben Skeggs 已提交
136 137 138
	gr->zbc_depth[zbc].format = format;
	gr->zbc_depth[zbc].ds = ds;
	gr->zbc_depth[zbc].l2 = l2;
139
	nvkm_ltc_zbc_depth_get(ltc, zbc, l2);
140
	gr->func->zbc->clear_depth(gr, zbc);
141 142 143
	return zbc;
}

144 145 146 147 148 149
const struct gf100_gr_func_zbc
gf100_gr_zbc = {
	.clear_color = gf100_gr_zbc_clear_color,
	.clear_depth = gf100_gr_zbc_clear_depth,
};

150 151 152
/*******************************************************************************
 * Graphics object classes
 ******************************************************************************/
153 154 155 156 157 158
#define gf100_gr_object(p) container_of((p), struct gf100_gr_object, object)

struct gf100_gr_object {
	struct nvkm_object object;
	struct gf100_gr_chan *chan;
};
159

160
static int
161
gf100_fermi_mthd_zbc_color(struct nvkm_object *object, void *data, u32 size)
162
{
163
	struct gf100_gr *gr = gf100_gr(nvkm_gr(object->engine));
164 165 166
	union {
		struct fermi_a_zbc_color_v0 v0;
	} *args = data;
167
	int ret = -ENOSYS;
168

169
	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
		switch (args->v0.format) {
		case FERMI_A_ZBC_COLOR_V0_FMT_ZERO:
		case FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE:
		case FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32:
		case FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16:
		case FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16:
		case FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16:
		case FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16:
		case FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16:
		case FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8:
		case FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8:
		case FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10:
		case FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10:
		case FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8:
		case FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8:
		case FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8:
		case FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8:
		case FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8:
		case FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10:
		case FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11:
B
Ben Skeggs 已提交
190
			ret = gf100_gr_zbc_color_get(gr, args->v0.format,
191 192
							   args->v0.ds,
							   args->v0.l2);
193 194 195 196 197 198 199 200 201 202 203 204 205 206
			if (ret >= 0) {
				args->v0.index = ret;
				return 0;
			}
			break;
		default:
			return -EINVAL;
		}
	}

	return ret;
}

static int
207
gf100_fermi_mthd_zbc_depth(struct nvkm_object *object, void *data, u32 size)
208
{
209
	struct gf100_gr *gr = gf100_gr(nvkm_gr(object->engine));
210 211 212
	union {
		struct fermi_a_zbc_depth_v0 v0;
	} *args = data;
213
	int ret = -ENOSYS;
214

215
	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
216 217
		switch (args->v0.format) {
		case FERMI_A_ZBC_DEPTH_V0_FMT_FP32:
B
Ben Skeggs 已提交
218
			ret = gf100_gr_zbc_depth_get(gr, args->v0.format,
219 220
							   args->v0.ds,
							   args->v0.l2);
221 222 223 224 225 226 227 228 229 230
			return (ret >= 0) ? 0 : -ENOSPC;
		default:
			return -EINVAL;
		}
	}

	return ret;
}

static int
231
gf100_fermi_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
232
{
233
	nvif_ioctl(object, "fermi mthd %08x\n", mthd);
234 235
	switch (mthd) {
	case FERMI_A_ZBC_COLOR:
236
		return gf100_fermi_mthd_zbc_color(object, data, size);
237
	case FERMI_A_ZBC_DEPTH:
238
		return gf100_fermi_mthd_zbc_depth(object, data, size);
239 240 241 242 243 244
	default:
		break;
	}
	return -EINVAL;
}

245 246
const struct nvkm_object_func
gf100_fermi = {
247
	.mthd = gf100_fermi_mthd,
248 249
};

250 251
static void
gf100_gr_mthd_set_shader_exceptions(struct nvkm_device *device, u32 data)
252
{
253 254
	nvkm_wr32(device, 0x419e44, data ? 0xffffffff : 0x00000000);
	nvkm_wr32(device, 0x419e4c, data ? 0xffffffff : 0x00000000);
255 256
}

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
static bool
gf100_gr_mthd_sw(struct nvkm_device *device, u16 class, u32 mthd, u32 data)
{
	switch (class & 0x00ff) {
	case 0x97:
	case 0xc0:
		switch (mthd) {
		case 0x1528:
			gf100_gr_mthd_set_shader_exceptions(device, data);
			return true;
		default:
			break;
		}
		break;
	default:
		break;
	}
	return false;
}
276

277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
static const struct nvkm_object_func
gf100_gr_object_func = {
};

static int
gf100_gr_object_new(const struct nvkm_oclass *oclass, void *data, u32 size,
		    struct nvkm_object **pobject)
{
	struct gf100_gr_chan *chan = gf100_gr_chan(oclass->parent);
	struct gf100_gr_object *object;

	if (!(object = kzalloc(sizeof(*object), GFP_KERNEL)))
		return -ENOMEM;
	*pobject = &object->object;

	nvkm_object_ctor(oclass->base.func ? oclass->base.func :
			 &gf100_gr_object_func, oclass, &object->object);
	object->chan = chan;
	return 0;
}

298 299 300 301 302 303 304 305 306
static int
gf100_gr_object_get(struct nvkm_gr *base, int index, struct nvkm_sclass *sclass)
{
	struct gf100_gr *gr = gf100_gr(base);
	int c = 0;

	while (gr->func->sclass[c].oclass) {
		if (c++ == index) {
			*sclass = gr->func->sclass[index];
307
			sclass->ctor = gf100_gr_object_new;
308 309 310 311 312 313
			return index;
		}
	}

	return c;
}
314 315 316 317

/*******************************************************************************
 * PGRAPH context
 ******************************************************************************/
318

319 320 321
static int
gf100_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent,
		   int align, struct nvkm_gpuobj **pgpuobj)
322
{
323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
	struct gf100_gr_chan *chan = gf100_gr_chan(object);
	struct gf100_gr *gr = chan->gr;
	int ret, i;

	ret = nvkm_gpuobj_new(gr->base.engine.subdev.device, gr->size,
			      align, false, parent, pgpuobj);
	if (ret)
		return ret;

	nvkm_kmap(*pgpuobj);
	for (i = 0; i < gr->size; i += 4)
		nvkm_wo32(*pgpuobj, i, gr->data[i / 4]);

	if (!gr->firmware) {
		nvkm_wo32(*pgpuobj, 0x00, chan->mmio_nr / 2);
338
		nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma->addr >> 8);
339 340 341 342
	} else {
		nvkm_wo32(*pgpuobj, 0xf4, 0);
		nvkm_wo32(*pgpuobj, 0xf8, 0);
		nvkm_wo32(*pgpuobj, 0x10, chan->mmio_nr / 2);
343 344
		nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma->addr));
		nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma->addr));
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
		nvkm_wo32(*pgpuobj, 0x1c, 1);
		nvkm_wo32(*pgpuobj, 0x20, 0);
		nvkm_wo32(*pgpuobj, 0x28, 0);
		nvkm_wo32(*pgpuobj, 0x2c, 0);
	}
	nvkm_done(*pgpuobj);
	return 0;
}

static void *
gf100_gr_chan_dtor(struct nvkm_object *object)
{
	struct gf100_gr_chan *chan = gf100_gr_chan(object);
	int i;

	for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
361
		nvkm_vmm_put(chan->vmm, &chan->data[i].vma);
362
		nvkm_memory_unref(&chan->data[i].mem);
363 364
	}

365
	nvkm_vmm_put(chan->vmm, &chan->mmio_vma);
366
	nvkm_memory_unref(&chan->mmio);
367
	nvkm_vmm_unref(&chan->vmm);
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
	return chan;
}

static const struct nvkm_object_func
gf100_gr_chan = {
	.dtor = gf100_gr_chan_dtor,
	.bind = gf100_gr_chan_bind,
};

static int
gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
		  const struct nvkm_oclass *oclass,
		  struct nvkm_object **pobject)
{
	struct gf100_gr *gr = gf100_gr(base);
B
Ben Skeggs 已提交
383 384
	struct gf100_gr_data *data = gr->mmio_data;
	struct gf100_gr_mmio *mmio = gr->mmio_list;
385
	struct gf100_gr_chan *chan;
386
	struct gf100_vmm_map_v0 args = { .priv = 1 };
387
	struct nvkm_device *device = gr->base.engine.subdev.device;
388 389
	int ret, i;

390 391 392 393
	if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))
		return -ENOMEM;
	nvkm_object_ctor(&gf100_gr_chan, oclass, &chan->object);
	chan->gr = gr;
394
	chan->vmm = nvkm_vmm_ref(fifoch->vmm);
395
	*pobject = &chan->object;
396

397 398 399 400
	/* allocate memory for a "mmio list" buffer that's used by the HUB
	 * fuc to modify some per-context register settings on first load
	 * of the context.
	 */
401 402
	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x100,
			      false, &chan->mmio);
403 404 405
	if (ret)
		return ret;

406
	ret = nvkm_vmm_get(fifoch->vmm, 12, 0x1000, &chan->mmio_vma);
407 408 409
	if (ret)
		return ret;

410
	ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm,
411
			      chan->mmio_vma, &args, sizeof(args));
412 413
	if (ret)
		return ret;
414

415
	/* allocate buffers referenced by mmio list */
B
Ben Skeggs 已提交
416
	for (i = 0; data->size && i < ARRAY_SIZE(gr->mmio_data); i++) {
417 418 419
		ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST,
				      data->size, data->align, false,
				      &chan->data[i].mem);
420 421
		if (ret)
			return ret;
422

423 424 425
		ret = nvkm_vmm_get(fifoch->vmm, 12,
				   nvkm_memory_size(chan->data[i].mem),
				   &chan->data[i].vma);
426 427
		if (ret)
			return ret;
428

429 430 431 432
		args.priv = data->priv;

		ret = nvkm_memory_map(chan->data[i].mem, 0, chan->vmm,
				      chan->data[i].vma, &args, sizeof(args));
433 434 435
		if (ret)
			return ret;

436
		data++;
437 438
	}

439
	/* finally, fill in the mmio list and point the context at it */
440
	nvkm_kmap(chan->mmio);
B
Ben Skeggs 已提交
441
	for (i = 0; mmio->addr && i < ARRAY_SIZE(gr->mmio_list); i++) {
442 443
		u32 addr = mmio->addr;
		u32 data = mmio->data;
444

445
		if (mmio->buffer >= 0) {
446
			u64 info = chan->data[mmio->buffer].vma->addr;
447 448
			data |= info >> mmio->shift;
		}
449

450 451
		nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
		nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
452 453
		mmio++;
	}
454
	nvkm_done(chan->mmio);
455
	return 0;
456 457
}

458
/*******************************************************************************
459
 * PGRAPH register lists
460 461
 ******************************************************************************/

462 463
const struct gf100_gr_init
gf100_gr_init_main_0[] = {
464 465 466 467 468 469 470 471 472 473 474 475 476 477
	{ 0x400080,   1, 0x04, 0x003083c2 },
	{ 0x400088,   1, 0x04, 0x00006fe7 },
	{ 0x40008c,   1, 0x04, 0x00000000 },
	{ 0x400090,   1, 0x04, 0x00000030 },
	{ 0x40013c,   1, 0x04, 0x013901f7 },
	{ 0x400140,   1, 0x04, 0x00000100 },
	{ 0x400144,   1, 0x04, 0x00000000 },
	{ 0x400148,   1, 0x04, 0x00000110 },
	{ 0x400138,   1, 0x04, 0x00000000 },
	{ 0x400130,   2, 0x04, 0x00000000 },
	{ 0x400124,   1, 0x04, 0x00000002 },
	{}
};

478 479
const struct gf100_gr_init
gf100_gr_init_fe_0[] = {
480 481 482 483 484
	{ 0x40415c,   1, 0x04, 0x00000000 },
	{ 0x404170,   1, 0x04, 0x00000000 },
	{}
};

485 486
const struct gf100_gr_init
gf100_gr_init_pri_0[] = {
487 488 489 490
	{ 0x404488,   2, 0x04, 0x00000000 },
	{}
};

491 492
const struct gf100_gr_init
gf100_gr_init_rstr2d_0[] = {
493 494 495 496
	{ 0x407808,   1, 0x04, 0x00000000 },
	{}
};

497 498
const struct gf100_gr_init
gf100_gr_init_pd_0[] = {
499 500 501 502
	{ 0x406024,   1, 0x04, 0x00000000 },
	{}
};

503 504
const struct gf100_gr_init
gf100_gr_init_ds_0[] = {
505 506 507 508 509 510
	{ 0x405844,   1, 0x04, 0x00ffffff },
	{ 0x405850,   1, 0x04, 0x00000000 },
	{ 0x405908,   1, 0x04, 0x00000000 },
	{}
};

511 512
const struct gf100_gr_init
gf100_gr_init_scc_0[] = {
513 514 515 516
	{ 0x40803c,   1, 0x04, 0x00000000 },
	{}
};

517 518
const struct gf100_gr_init
gf100_gr_init_prop_0[] = {
519
	{ 0x4184a0,   1, 0x04, 0x00000000 },
520 521 522
	{}
};

523 524
const struct gf100_gr_init
gf100_gr_init_gpc_unk_0[] = {
525 526 527 528
	{ 0x418604,   1, 0x04, 0x00000000 },
	{ 0x418680,   1, 0x04, 0x00000000 },
	{ 0x418714,   1, 0x04, 0x80000000 },
	{ 0x418384,   1, 0x04, 0x00000000 },
529 530 531
	{}
};

532 533
const struct gf100_gr_init
gf100_gr_init_setup_0[] = {
534
	{ 0x418814,   3, 0x04, 0x00000000 },
535 536 537
	{}
};

538 539
const struct gf100_gr_init
gf100_gr_init_crstr_0[] = {
540
	{ 0x418b04,   1, 0x04, 0x00000000 },
541 542 543
	{}
};

544 545
const struct gf100_gr_init
gf100_gr_init_setup_1[] = {
546 547 548 549
	{ 0x4188c8,   1, 0x04, 0x80000000 },
	{ 0x4188cc,   1, 0x04, 0x00000000 },
	{ 0x4188d0,   1, 0x04, 0x00010000 },
	{ 0x4188d4,   1, 0x04, 0x00000001 },
550 551 552
	{}
};

553 554
const struct gf100_gr_init
gf100_gr_init_zcull_0[] = {
555 556 557 558 559
	{ 0x418910,   1, 0x04, 0x00010001 },
	{ 0x418914,   1, 0x04, 0x00000301 },
	{ 0x418918,   1, 0x04, 0x00800000 },
	{ 0x418980,   1, 0x04, 0x77777770 },
	{ 0x418984,   3, 0x04, 0x77777777 },
560 561 562
	{}
};

563 564
const struct gf100_gr_init
gf100_gr_init_gpm_0[] = {
565 566
	{ 0x418c04,   1, 0x04, 0x00000000 },
	{ 0x418c88,   1, 0x04, 0x00000000 },
567 568 569
	{}
};

570 571
const struct gf100_gr_init
gf100_gr_init_gpc_unk_1[] = {
572 573 574 575
	{ 0x418d00,   1, 0x04, 0x00000000 },
	{ 0x418f08,   1, 0x04, 0x00000000 },
	{ 0x418e00,   1, 0x04, 0x00000050 },
	{ 0x418e08,   1, 0x04, 0x00000000 },
576 577 578
	{}
};

579 580
const struct gf100_gr_init
gf100_gr_init_gcc_0[] = {
581 582 583 584 585
	{ 0x41900c,   1, 0x04, 0x00000000 },
	{ 0x419018,   1, 0x04, 0x00000000 },
	{}
};

586 587
const struct gf100_gr_init
gf100_gr_init_tpccs_0[] = {
588 589
	{ 0x419d08,   2, 0x04, 0x00000000 },
	{ 0x419d10,   1, 0x04, 0x00000014 },
590 591 592
	{}
};

593 594
const struct gf100_gr_init
gf100_gr_init_tex_0[] = {
595 596 597
	{ 0x419ab0,   1, 0x04, 0x00000000 },
	{ 0x419ab8,   1, 0x04, 0x000000e7 },
	{ 0x419abc,   2, 0x04, 0x00000000 },
598 599 600
	{}
};

601 602
const struct gf100_gr_init
gf100_gr_init_pe_0[] = {
603 604 605 606
	{ 0x41980c,   3, 0x04, 0x00000000 },
	{ 0x419844,   1, 0x04, 0x00000000 },
	{ 0x41984c,   1, 0x04, 0x00005bc5 },
	{ 0x419850,   4, 0x04, 0x00000000 },
607 608 609
	{}
};

610 611
const struct gf100_gr_init
gf100_gr_init_l1c_0[] = {
612 613 614 615 616 617
	{ 0x419c98,   1, 0x04, 0x00000000 },
	{ 0x419ca8,   1, 0x04, 0x80000000 },
	{ 0x419cb4,   1, 0x04, 0x00000000 },
	{ 0x419cb8,   1, 0x04, 0x00008bf4 },
	{ 0x419cbc,   1, 0x04, 0x28137606 },
	{ 0x419cc0,   2, 0x04, 0x00000000 },
618 619 620
	{}
};

621 622
const struct gf100_gr_init
gf100_gr_init_wwdx_0[] = {
623 624
	{ 0x419bd4,   1, 0x04, 0x00800000 },
	{ 0x419bdc,   1, 0x04, 0x00000000 },
625 626 627
	{}
};

628 629
const struct gf100_gr_init
gf100_gr_init_tpccs_1[] = {
630
	{ 0x419d2c,   1, 0x04, 0x00000000 },
631 632 633
	{}
};

634 635
const struct gf100_gr_init
gf100_gr_init_mpc_0[] = {
636
	{ 0x419c0c,   1, 0x04, 0x00000000 },
637 638 639
	{}
};

640 641
static const struct gf100_gr_init
gf100_gr_init_sm_0[] = {
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
	{ 0x419e00,   1, 0x04, 0x00000000 },
	{ 0x419ea0,   1, 0x04, 0x00000000 },
	{ 0x419ea4,   1, 0x04, 0x00000100 },
	{ 0x419ea8,   1, 0x04, 0x00001100 },
	{ 0x419eac,   1, 0x04, 0x11100702 },
	{ 0x419eb0,   1, 0x04, 0x00000003 },
	{ 0x419eb4,   4, 0x04, 0x00000000 },
	{ 0x419ec8,   1, 0x04, 0x06060618 },
	{ 0x419ed0,   1, 0x04, 0x0eff0e38 },
	{ 0x419ed4,   1, 0x04, 0x011104f1 },
	{ 0x419edc,   1, 0x04, 0x00000000 },
	{ 0x419f00,   1, 0x04, 0x00000000 },
	{ 0x419f2c,   1, 0x04, 0x00000000 },
	{}
};

658 659
const struct gf100_gr_init
gf100_gr_init_be_0[] = {
660 661 662 663 664 665 666 667 668 669
	{ 0x40880c,   1, 0x04, 0x00000000 },
	{ 0x408910,   9, 0x04, 0x00000000 },
	{ 0x408950,   1, 0x04, 0x00000000 },
	{ 0x408954,   1, 0x04, 0x0000ffff },
	{ 0x408984,   1, 0x04, 0x00000000 },
	{ 0x408988,   1, 0x04, 0x08040201 },
	{ 0x40898c,   1, 0x04, 0x80402010 },
	{}
};

670 671
const struct gf100_gr_init
gf100_gr_init_fe_1[] = {
672 673 674 675
	{ 0x4040f0,   1, 0x04, 0x00000000 },
	{}
};

676 677
const struct gf100_gr_init
gf100_gr_init_pe_1[] = {
678 679 680 681
	{ 0x419880,   1, 0x04, 0x00000002 },
	{}
};

682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
static const struct gf100_gr_pack
gf100_gr_pack_mmio[] = {
	{ gf100_gr_init_main_0 },
	{ gf100_gr_init_fe_0 },
	{ gf100_gr_init_pri_0 },
	{ gf100_gr_init_rstr2d_0 },
	{ gf100_gr_init_pd_0 },
	{ gf100_gr_init_ds_0 },
	{ gf100_gr_init_scc_0 },
	{ gf100_gr_init_prop_0 },
	{ gf100_gr_init_gpc_unk_0 },
	{ gf100_gr_init_setup_0 },
	{ gf100_gr_init_crstr_0 },
	{ gf100_gr_init_setup_1 },
	{ gf100_gr_init_zcull_0 },
	{ gf100_gr_init_gpm_0 },
	{ gf100_gr_init_gpc_unk_1 },
	{ gf100_gr_init_gcc_0 },
	{ gf100_gr_init_tpccs_0 },
	{ gf100_gr_init_tex_0 },
	{ gf100_gr_init_pe_0 },
	{ gf100_gr_init_l1c_0 },
	{ gf100_gr_init_wwdx_0 },
	{ gf100_gr_init_tpccs_1 },
	{ gf100_gr_init_mpc_0 },
	{ gf100_gr_init_sm_0 },
	{ gf100_gr_init_be_0 },
	{ gf100_gr_init_fe_1 },
	{ gf100_gr_init_pe_1 },
M
Maarten Lankhorst 已提交
711 712 713
	{}
};

714 715 716 717
/*******************************************************************************
 * PGRAPH engine/subdev functions
 ******************************************************************************/

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
static int
gf100_gr_fecs_discover_image_size(struct gf100_gr *gr, u32 *psize)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;

	nvkm_wr32(device, 0x409840, 0xffffffff);
	nvkm_wr32(device, 0x409500, 0x00000000);
	nvkm_wr32(device, 0x409504, 0x00000010);
	nvkm_msec(device, 2000,
		if ((*psize = nvkm_rd32(device, 0x409800)))
			return 0;
	);

	return -ETIMEDOUT;
}

734 735 736 737 738 739 740 741 742 743
static void
gf100_gr_fecs_set_watchdog_timeout(struct gf100_gr *gr, u32 timeout)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;

	nvkm_wr32(device, 0x409840, 0xffffffff);
	nvkm_wr32(device, 0x409500, timeout);
	nvkm_wr32(device, 0x409504, 0x00000021);
}

744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
static bool
gf100_gr_chsw_load(struct nvkm_gr *base)
{
	struct gf100_gr *gr = gf100_gr(base);
	if (!gr->firmware) {
		u32 trace = nvkm_rd32(gr->base.engine.subdev.device, 0x40981c);
		if (trace & 0x00000040)
			return true;
	} else {
		u32 mthd = nvkm_rd32(gr->base.engine.subdev.device, 0x409808);
		if (mthd & 0x00080000)
			return true;
	}
	return false;
}

760 761 762 763 764 765 766
int
gf100_gr_rops(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	return (nvkm_rd32(device, 0x409604) & 0x001f0000) >> 16;
}

767
void
B
Ben Skeggs 已提交
768
gf100_gr_zbc_init(struct gf100_gr *gr)
769 770 771 772 773 774 775 776 777
{
	const u32  zero[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,
			      0x00000000, 0x00000000, 0x00000000, 0x00000000 };
	const u32   one[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
			      0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff };
	const u32 f32_0[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,
			      0x00000000, 0x00000000, 0x00000000, 0x00000000 };
	const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
			      0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 };
778
	struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;
779
	int index, c = ltc->zbc_min, d = ltc->zbc_min, s = ltc->zbc_min;
780

B
Ben Skeggs 已提交
781
	if (!gr->zbc_color[0].format) {
782 783 784 785 786 787
		gf100_gr_zbc_color_get(gr, 1,  & zero[0],   &zero[4]); c++;
		gf100_gr_zbc_color_get(gr, 2,  &  one[0],    &one[4]); c++;
		gf100_gr_zbc_color_get(gr, 4,  &f32_0[0],  &f32_0[4]); c++;
		gf100_gr_zbc_color_get(gr, 4,  &f32_1[0],  &f32_1[4]); c++;
		gf100_gr_zbc_depth_get(gr, 1, 0x00000000, 0x00000000); d++;
		gf100_gr_zbc_depth_get(gr, 1, 0x3f800000, 0x3f800000); d++;
788 789 790 791 792
		if (gr->func->zbc->stencil_get) {
			gr->func->zbc->stencil_get(gr, 1, 0x00, 0x00); s++;
			gr->func->zbc->stencil_get(gr, 1, 0x01, 0x01); s++;
			gr->func->zbc->stencil_get(gr, 1, 0xff, 0xff); s++;
		}
793 794 795 796 797 798
	}

	for (index = c; index <= ltc->zbc_max; index++)
		gr->func->zbc->clear_color(gr, index);
	for (index = d; index <= ltc->zbc_max; index++)
		gr->func->zbc->clear_depth(gr, index);
799 800 801 802 803

	if (gr->func->zbc->clear_stencil) {
		for (index = s; index <= ltc->zbc_max; index++)
			gr->func->zbc->clear_stencil(gr, index);
	}
804 805
}

806 807 808 809 810 811
/**
 * Wait until GR goes idle. GR is considered idle if it is disabled by the
 * MC (0x200) register, or GR is not busy and a context switch is not in
 * progress.
 */
int
B
Ben Skeggs 已提交
812
gf100_gr_wait_idle(struct gf100_gr *gr)
813
{
814 815
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
816 817 818 819 820 821 822 823
	unsigned long end_jiffies = jiffies + msecs_to_jiffies(2000);
	bool gr_enabled, ctxsw_active, gr_busy;

	do {
		/*
		 * required to make sure FIFO_ENGINE_STATUS (0x2640) is
		 * up-to-date
		 */
824
		nvkm_rd32(device, 0x400700);
825

826 827 828
		gr_enabled = nvkm_rd32(device, 0x200) & 0x1000;
		ctxsw_active = nvkm_rd32(device, 0x2640) & 0x8000;
		gr_busy = nvkm_rd32(device, 0x40060c) & 0x1;
829 830 831 832 833

		if (!gr_enabled || (!gr_busy && !ctxsw_active))
			return 0;
	} while (time_before(jiffies, end_jiffies));

834 835 836
	nvkm_error(subdev,
		   "wait for idle timeout (en: %d, ctxsw: %d, busy: %d)\n",
		   gr_enabled, ctxsw_active, gr_busy);
837 838 839
	return -EAGAIN;
}

840
void
B
Ben Skeggs 已提交
841
gf100_gr_mmio(struct gf100_gr *gr, const struct gf100_gr_pack *p)
842
{
843
	struct nvkm_device *device = gr->base.engine.subdev.device;
844 845
	const struct gf100_gr_pack *pack;
	const struct gf100_gr_init *init;
846 847 848 849 850

	pack_for_each_init(init, pack, p) {
		u32 next = init->addr + init->count * init->pitch;
		u32 addr = init->addr;
		while (addr < next) {
851
			nvkm_wr32(device, addr, init->data);
852 853 854
			addr += init->pitch;
		}
	}
855 856 857
}

void
B
Ben Skeggs 已提交
858
gf100_gr_icmd(struct gf100_gr *gr, const struct gf100_gr_pack *p)
859
{
860
	struct nvkm_device *device = gr->base.engine.subdev.device;
861 862
	const struct gf100_gr_pack *pack;
	const struct gf100_gr_init *init;
863
	u32 data = 0;
864

865
	nvkm_wr32(device, 0x400208, 0x80000000);
866 867 868 869 870 871

	pack_for_each_init(init, pack, p) {
		u32 next = init->addr + init->count * init->pitch;
		u32 addr = init->addr;

		if ((pack == p && init == p->init) || data != init->data) {
872
			nvkm_wr32(device, 0x400204, init->data);
873 874
			data = init->data;
		}
875

876
		while (addr < next) {
877
			nvkm_wr32(device, 0x400200, addr);
878 879 880 881 882
			/**
			 * Wait for GR to go idle after submitting a
			 * GO_IDLE bundle
			 */
			if ((addr & 0xffff) == 0xe100)
B
Ben Skeggs 已提交
883
				gf100_gr_wait_idle(gr);
884 885 886 887
			nvkm_msec(device, 2000,
				if (!(nvkm_rd32(device, 0x400700) & 0x00000004))
					break;
			);
888 889 890
			addr += init->pitch;
		}
	}
891

892
	nvkm_wr32(device, 0x400208, 0x00000000);
893 894 895
}

void
B
Ben Skeggs 已提交
896
gf100_gr_mthd(struct gf100_gr *gr, const struct gf100_gr_pack *p)
897
{
898
	struct nvkm_device *device = gr->base.engine.subdev.device;
899 900
	const struct gf100_gr_pack *pack;
	const struct gf100_gr_init *init;
901
	u32 data = 0;
902

903 904 905 906 907 908
	pack_for_each_init(init, pack, p) {
		u32 ctrl = 0x80000000 | pack->type;
		u32 next = init->addr + init->count * init->pitch;
		u32 addr = init->addr;

		if ((pack == p && init == p->init) || data != init->data) {
909
			nvkm_wr32(device, 0x40448c, init->data);
910 911 912 913
			data = init->data;
		}

		while (addr < next) {
914
			nvkm_wr32(device, 0x404488, ctrl | (addr << 14));
915
			addr += init->pitch;
916 917 918 919 920
		}
	}
}

u64
921
gf100_gr_units(struct nvkm_gr *base)
922
{
923
	struct gf100_gr *gr = gf100_gr(base);
924 925
	u64 cfg;

B
Ben Skeggs 已提交
926 927 928
	cfg  = (u32)gr->gpc_nr;
	cfg |= (u32)gr->tpc_total << 8;
	cfg |= (u64)gr->rop_nr << 32;
929 930

	return cfg;
931 932
}

933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
static const struct nvkm_bitfield gf100_dispatch_error[] = {
	{ 0x00000001, "INJECTED_BUNDLE_ERROR" },
	{ 0x00000002, "CLASS_SUBCH_MISMATCH" },
	{ 0x00000004, "SUBCHSW_DURING_NOTIFY" },
	{}
};

static const struct nvkm_bitfield gf100_m2mf_error[] = {
	{ 0x00000001, "PUSH_TOO_MUCH_DATA" },
	{ 0x00000002, "PUSH_NOT_ENOUGH_DATA" },
	{}
};

static const struct nvkm_bitfield gf100_unk6_error[] = {
	{ 0x00000001, "TEMP_TOO_SMALL" },
	{}
};

static const struct nvkm_bitfield gf100_ccache_error[] = {
	{ 0x00000001, "INTR" },
	{ 0x00000002, "LDCONST_OOB" },
	{}
};

static const struct nvkm_bitfield gf100_macro_error[] = {
	{ 0x00000001, "TOO_FEW_PARAMS" },
	{ 0x00000002, "TOO_MANY_PARAMS" },
	{ 0x00000004, "ILLEGAL_OPCODE" },
	{ 0x00000008, "DOUBLE_BRANCH" },
	{ 0x00000010, "WATCHDOG" },
	{}
};

966
static const struct nvkm_bitfield gk104_sked_error[] = {
967
	{ 0x00000040, "CTA_RESUME" },
968 969 970 971 972 973 974 975 976
	{ 0x00000080, "CONSTANT_BUFFER_SIZE" },
	{ 0x00000200, "LOCAL_MEMORY_SIZE_POS" },
	{ 0x00000400, "LOCAL_MEMORY_SIZE_NEG" },
	{ 0x00000800, "WARP_CSTACK_SIZE" },
	{ 0x00001000, "TOTAL_TEMP_SIZE" },
	{ 0x00002000, "REGISTER_COUNT" },
	{ 0x00040000, "TOTAL_THREADS" },
	{ 0x00100000, "PROGRAM_OFFSET" },
	{ 0x00200000, "SHARED_MEMORY_SIZE" },
977 978
	{ 0x00800000, "CTA_THREAD_DIMENSION_ZERO" },
	{ 0x01000000, "MEMORY_WINDOW_OVERLAP" },
979 980
	{ 0x02000000, "SHARED_CONFIG_TOO_SMALL" },
	{ 0x04000000, "TOTAL_REGISTER_COUNT" },
981 982 983
	{}
};

984 985 986 987 988 989 990
static const struct nvkm_bitfield gf100_gpc_rop_error[] = {
	{ 0x00000002, "RT_PITCH_OVERRUN" },
	{ 0x00000010, "RT_WIDTH_OVERRUN" },
	{ 0x00000020, "RT_HEIGHT_OVERRUN" },
	{ 0x00000080, "ZETA_STORAGE_TYPE_MISMATCH" },
	{ 0x00000100, "RT_STORAGE_TYPE_MISMATCH" },
	{ 0x00000400, "RT_LINEAR_MISMATCH" },
991 992 993
	{}
};

994
static void
B
Ben Skeggs 已提交
995
gf100_gr_trap_gpc_rop(struct gf100_gr *gr, int gpc)
996
{
997 998 999
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
	char error[128];
1000
	u32 trap[4];
1001

1002
	trap[0] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0420)) & 0x3fffffff;
1003 1004 1005
	trap[1] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0434));
	trap[2] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0438));
	trap[3] = nvkm_rd32(device, GPC_UNIT(gpc, 0x043c));
1006

1007
	nvkm_snprintbf(error, sizeof(error), gf100_gpc_rop_error, trap[0]);
1008

1009 1010 1011 1012
	nvkm_error(subdev, "GPC%d/PROP trap: %08x [%s] x = %u, y = %u, "
			   "format = %x, storage type = %x\n",
		   gpc, trap[0], error, trap[1] & 0xffff, trap[1] >> 16,
		   (trap[2] >> 8) & 0x3f, trap[3] & 0xff);
1013
	nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
1014 1015
}

1016
const struct nvkm_enum gf100_mp_warp_error[] = {
1017 1018 1019 1020
	{ 0x01, "STACK_ERROR" },
	{ 0x02, "API_STACK_ERROR" },
	{ 0x03, "RET_EMPTY_STACK_ERROR" },
	{ 0x04, "PC_WRAP" },
1021
	{ 0x05, "MISALIGNED_PC" },
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	{ 0x06, "PC_OVERFLOW" },
	{ 0x07, "MISALIGNED_IMMC_ADDR" },
	{ 0x08, "MISALIGNED_REG" },
	{ 0x09, "ILLEGAL_INSTR_ENCODING" },
	{ 0x0a, "ILLEGAL_SPH_INSTR_COMBO" },
	{ 0x0b, "ILLEGAL_INSTR_PARAM" },
	{ 0x0c, "INVALID_CONST_ADDR" },
	{ 0x0d, "OOR_REG" },
	{ 0x0e, "OOR_ADDR" },
	{ 0x0f, "MISALIGNED_ADDR" },
1032
	{ 0x10, "INVALID_ADDR_SPACE" },
1033 1034 1035 1036 1037
	{ 0x11, "ILLEGAL_INSTR_PARAM2" },
	{ 0x12, "INVALID_CONST_ADDR_LDC" },
	{ 0x13, "GEOMETRY_SM_ERROR" },
	{ 0x14, "DIVERGENT" },
	{ 0x15, "WARP_EXIT" },
1038 1039 1040
	{}
};

1041
const struct nvkm_bitfield gf100_mp_global_error[] = {
1042 1043
	{ 0x00000001, "SM_TO_SM_FAULT" },
	{ 0x00000002, "L1_ERROR" },
1044
	{ 0x00000004, "MULTIPLE_WARP_ERRORS" },
1045 1046 1047 1048 1049 1050 1051
	{ 0x00000008, "PHYSICAL_STACK_OVERFLOW" },
	{ 0x00000010, "BPT_INT" },
	{ 0x00000020, "BPT_PAUSE" },
	{ 0x00000040, "SINGLE_STEP_COMPLETE" },
	{ 0x20000000, "ECC_SEC_ERROR" },
	{ 0x40000000, "ECC_DED_ERROR" },
	{ 0x80000000, "TIMEOUT" },
1052 1053 1054
	{}
};

1055
void
B
Ben Skeggs 已提交
1056
gf100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)
1057
{
1058 1059
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1060 1061
	u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x648));
	u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x650));
1062 1063
	const struct nvkm_enum *warp;
	char glob[128];
1064

1065 1066 1067 1068 1069 1070
	nvkm_snprintbf(glob, sizeof(glob), gf100_mp_global_error, gerr);
	warp = nvkm_enum_find(gf100_mp_warp_error, werr & 0xffff);

	nvkm_error(subdev, "GPC%i/TPC%i/MP trap: "
			   "global %08x [%s] warp %04x [%s]\n",
		   gpc, tpc, gerr, glob, werr, warp ? warp->name : "");
1071

1072 1073
	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);
	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x650), gerr);
1074 1075
}

1076
static void
B
Ben Skeggs 已提交
1077
gf100_gr_trap_tpc(struct gf100_gr *gr, int gpc, int tpc)
1078
{
1079 1080
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1081
	u32 stat = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0508));
1082 1083

	if (stat & 0x00000001) {
1084
		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0224));
1085
		nvkm_error(subdev, "GPC%d/TPC%d/TEX: %08x\n", gpc, tpc, trap);
1086
		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
1087 1088 1089 1090
		stat &= ~0x00000001;
	}

	if (stat & 0x00000002) {
1091
		gr->func->trap_mp(gr, gpc, tpc);
1092 1093 1094 1095
		stat &= ~0x00000002;
	}

	if (stat & 0x00000004) {
1096
		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0084));
1097
		nvkm_error(subdev, "GPC%d/TPC%d/POLY: %08x\n", gpc, tpc, trap);
1098
		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
1099 1100 1101 1102
		stat &= ~0x00000004;
	}

	if (stat & 0x00000008) {
1103
		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x048c));
1104
		nvkm_error(subdev, "GPC%d/TPC%d/L1C: %08x\n", gpc, tpc, trap);
1105
		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
1106 1107 1108
		stat &= ~0x00000008;
	}

1109 1110 1111 1112 1113 1114 1115
	if (stat & 0x00000010) {
		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0430));
		nvkm_error(subdev, "GPC%d/TPC%d/MPC: %08x\n", gpc, tpc, trap);
		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0430), 0xc0000000);
		stat &= ~0x00000010;
	}

1116
	if (stat) {
1117
		nvkm_error(subdev, "GPC%d/TPC%d/%08x: unknown\n", gpc, tpc, stat);
1118 1119 1120 1121
	}
}

static void
B
Ben Skeggs 已提交
1122
gf100_gr_trap_gpc(struct gf100_gr *gr, int gpc)
1123
{
1124 1125
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1126
	u32 stat = nvkm_rd32(device, GPC_UNIT(gpc, 0x2c90));
1127 1128 1129
	int tpc;

	if (stat & 0x00000001) {
B
Ben Skeggs 已提交
1130
		gf100_gr_trap_gpc_rop(gr, gpc);
1131 1132 1133 1134
		stat &= ~0x00000001;
	}

	if (stat & 0x00000002) {
1135
		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0900));
1136
		nvkm_error(subdev, "GPC%d/ZCULL: %08x\n", gpc, trap);
1137
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
1138 1139 1140 1141
		stat &= ~0x00000002;
	}

	if (stat & 0x00000004) {
1142
		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x1028));
1143
		nvkm_error(subdev, "GPC%d/CCACHE: %08x\n", gpc, trap);
1144
		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
1145 1146 1147 1148
		stat &= ~0x00000004;
	}

	if (stat & 0x00000008) {
1149
		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0824));
1150
		nvkm_error(subdev, "GPC%d/ESETUP: %08x\n", gpc, trap);
1151
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
1152 1153 1154
		stat &= ~0x00000009;
	}

B
Ben Skeggs 已提交
1155
	for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
1156 1157
		u32 mask = 0x00010000 << tpc;
		if (stat & mask) {
B
Ben Skeggs 已提交
1158
			gf100_gr_trap_tpc(gr, gpc, tpc);
1159
			nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), mask);
1160 1161 1162 1163 1164
			stat &= ~mask;
		}
	}

	if (stat) {
1165
		nvkm_error(subdev, "GPC%d/%08x: unknown\n", gpc, stat);
1166 1167 1168 1169
	}
}

static void
B
Ben Skeggs 已提交
1170
gf100_gr_trap_intr(struct gf100_gr *gr)
1171
{
1172 1173
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1174
	char error[128];
1175
	u32 trap = nvkm_rd32(device, 0x400108);
1176
	int rop, gpc;
1177 1178

	if (trap & 0x00000001) {
1179
		u32 stat = nvkm_rd32(device, 0x404000);
1180 1181 1182 1183

		nvkm_snprintbf(error, sizeof(error), gf100_dispatch_error,
			       stat & 0x3fffffff);
		nvkm_error(subdev, "DISPATCH %08x [%s]\n", stat, error);
1184 1185
		nvkm_wr32(device, 0x404000, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000001);
1186 1187 1188 1189
		trap &= ~0x00000001;
	}

	if (trap & 0x00000002) {
1190
		u32 stat = nvkm_rd32(device, 0x404600);
1191 1192 1193 1194 1195

		nvkm_snprintbf(error, sizeof(error), gf100_m2mf_error,
			       stat & 0x3fffffff);
		nvkm_error(subdev, "M2MF %08x [%s]\n", stat, error);

1196 1197
		nvkm_wr32(device, 0x404600, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000002);
1198 1199 1200 1201
		trap &= ~0x00000002;
	}

	if (trap & 0x00000008) {
1202
		u32 stat = nvkm_rd32(device, 0x408030);
1203

1204
		nvkm_snprintbf(error, sizeof(error), gf100_ccache_error,
1205 1206
			       stat & 0x3fffffff);
		nvkm_error(subdev, "CCACHE %08x [%s]\n", stat, error);
1207 1208
		nvkm_wr32(device, 0x408030, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000008);
1209 1210 1211 1212
		trap &= ~0x00000008;
	}

	if (trap & 0x00000010) {
1213
		u32 stat = nvkm_rd32(device, 0x405840);
1214 1215
		nvkm_error(subdev, "SHADER %08x, sph: 0x%06x, stage: 0x%02x\n",
			   stat, stat & 0xffffff, (stat >> 24) & 0x3f);
1216 1217
		nvkm_wr32(device, 0x405840, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000010);
1218 1219 1220 1221
		trap &= ~0x00000010;
	}

	if (trap & 0x00000040) {
1222
		u32 stat = nvkm_rd32(device, 0x40601c);
1223 1224 1225 1226 1227

		nvkm_snprintbf(error, sizeof(error), gf100_unk6_error,
			       stat & 0x3fffffff);
		nvkm_error(subdev, "UNK6 %08x [%s]\n", stat, error);

1228 1229
		nvkm_wr32(device, 0x40601c, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000040);
1230 1231 1232 1233
		trap &= ~0x00000040;
	}

	if (trap & 0x00000080) {
1234
		u32 stat = nvkm_rd32(device, 0x404490);
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
		u32 pc = nvkm_rd32(device, 0x404494);
		u32 op = nvkm_rd32(device, 0x40449c);

		nvkm_snprintbf(error, sizeof(error), gf100_macro_error,
			       stat & 0x1fffffff);
		nvkm_error(subdev, "MACRO %08x [%s], pc: 0x%03x%s, op: 0x%08x\n",
			   stat, error, pc & 0x7ff,
			   (pc & 0x10000000) ? "" : " (invalid)",
			   op);

1245 1246
		nvkm_wr32(device, 0x404490, 0xc0000000);
		nvkm_wr32(device, 0x400108, 0x00000080);
1247 1248 1249
		trap &= ~0x00000080;
	}

1250
	if (trap & 0x00000100) {
1251
		u32 stat = nvkm_rd32(device, 0x407020) & 0x3fffffff;
1252

1253 1254
		nvkm_snprintbf(error, sizeof(error), gk104_sked_error, stat);
		nvkm_error(subdev, "SKED: %08x [%s]\n", stat, error);
1255

1256
		if (stat)
1257 1258
			nvkm_wr32(device, 0x407020, 0x40000000);
		nvkm_wr32(device, 0x400108, 0x00000100);
1259 1260 1261
		trap &= ~0x00000100;
	}

1262
	if (trap & 0x01000000) {
1263
		u32 stat = nvkm_rd32(device, 0x400118);
B
Ben Skeggs 已提交
1264
		for (gpc = 0; stat && gpc < gr->gpc_nr; gpc++) {
1265 1266
			u32 mask = 0x00000001 << gpc;
			if (stat & mask) {
B
Ben Skeggs 已提交
1267
				gf100_gr_trap_gpc(gr, gpc);
1268
				nvkm_wr32(device, 0x400118, mask);
1269 1270 1271
				stat &= ~mask;
			}
		}
1272
		nvkm_wr32(device, 0x400108, 0x01000000);
1273 1274 1275 1276
		trap &= ~0x01000000;
	}

	if (trap & 0x02000000) {
B
Ben Skeggs 已提交
1277
		for (rop = 0; rop < gr->rop_nr; rop++) {
1278 1279
			u32 statz = nvkm_rd32(device, ROP_UNIT(rop, 0x070));
			u32 statc = nvkm_rd32(device, ROP_UNIT(rop, 0x144));
1280
			nvkm_error(subdev, "ROP%d %08x %08x\n",
1281
				 rop, statz, statc);
1282 1283
			nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0xc0000000);
			nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0xc0000000);
1284
		}
1285
		nvkm_wr32(device, 0x400108, 0x02000000);
1286 1287 1288 1289
		trap &= ~0x02000000;
	}

	if (trap) {
1290
		nvkm_error(subdev, "TRAP UNHANDLED %08x\n", trap);
1291
		nvkm_wr32(device, 0x400108, trap);
1292 1293 1294
	}
}

1295
static void
B
Ben Skeggs 已提交
1296
gf100_gr_ctxctl_debug_unit(struct gf100_gr *gr, u32 base)
1297
{
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
	nvkm_error(subdev, "%06x - done %08x\n", base,
		   nvkm_rd32(device, base + 0x400));
	nvkm_error(subdev, "%06x - stat %08x %08x %08x %08x\n", base,
		   nvkm_rd32(device, base + 0x800),
		   nvkm_rd32(device, base + 0x804),
		   nvkm_rd32(device, base + 0x808),
		   nvkm_rd32(device, base + 0x80c));
	nvkm_error(subdev, "%06x - stat %08x %08x %08x %08x\n", base,
		   nvkm_rd32(device, base + 0x810),
		   nvkm_rd32(device, base + 0x814),
		   nvkm_rd32(device, base + 0x818),
		   nvkm_rd32(device, base + 0x81c));
1312 1313 1314
}

void
B
Ben Skeggs 已提交
1315
gf100_gr_ctxctl_debug(struct gf100_gr *gr)
1316
{
1317 1318
	struct nvkm_device *device = gr->base.engine.subdev.device;
	u32 gpcnr = nvkm_rd32(device, 0x409604) & 0xffff;
1319 1320
	u32 gpc;

B
Ben Skeggs 已提交
1321
	gf100_gr_ctxctl_debug_unit(gr, 0x409000);
1322
	for (gpc = 0; gpc < gpcnr; gpc++)
B
Ben Skeggs 已提交
1323
		gf100_gr_ctxctl_debug_unit(gr, 0x502000 + (gpc * 0x8000));
1324 1325 1326
}

static void
B
Ben Skeggs 已提交
1327
gf100_gr_ctxctl_isr(struct gf100_gr *gr)
1328
{
1329 1330
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1331
	u32 stat = nvkm_rd32(device, 0x409c18);
1332

1333
	if (!gr->firmware && (stat & 0x00000001)) {
1334
		u32 code = nvkm_rd32(device, 0x409814);
1335
		if (code == E_BAD_FWMTHD) {
1336 1337
			u32 class = nvkm_rd32(device, 0x409808);
			u32  addr = nvkm_rd32(device, 0x40980c);
1338 1339
			u32  subc = (addr & 0x00070000) >> 16;
			u32  mthd = (addr & 0x00003ffc);
1340
			u32  data = nvkm_rd32(device, 0x409810);
1341

1342 1343 1344
			nvkm_error(subdev, "FECS MTHD subc %d class %04x "
					   "mthd %04x data %08x\n",
				   subc, class, mthd, data);
1345
		} else {
1346
			nvkm_error(subdev, "FECS ucode error %d\n", code);
1347
		}
1348 1349
		nvkm_wr32(device, 0x409c20, 0x00000001);
		stat &= ~0x00000001;
1350
	}
1351

1352
	if (!gr->firmware && (stat & 0x00080000)) {
1353
		nvkm_error(subdev, "FECS watchdog timeout\n");
B
Ben Skeggs 已提交
1354
		gf100_gr_ctxctl_debug(gr);
1355
		nvkm_wr32(device, 0x409c20, 0x00080000);
1356 1357 1358 1359
		stat &= ~0x00080000;
	}

	if (stat) {
1360
		nvkm_error(subdev, "FECS %08x\n", stat);
B
Ben Skeggs 已提交
1361
		gf100_gr_ctxctl_debug(gr);
1362
		nvkm_wr32(device, 0x409c20, stat);
1363
	}
1364 1365
}

1366
static void
1367
gf100_gr_intr(struct nvkm_gr *base)
1368
{
1369 1370 1371
	struct gf100_gr *gr = gf100_gr(base);
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1372 1373
	struct nvkm_fifo_chan *chan;
	unsigned long flags;
1374 1375 1376
	u64 inst = nvkm_rd32(device, 0x409b00) & 0x0fffffff;
	u32 stat = nvkm_rd32(device, 0x400100);
	u32 addr = nvkm_rd32(device, 0x400704);
1377 1378
	u32 mthd = (addr & 0x00003ffc);
	u32 subc = (addr & 0x00070000) >> 16;
1379 1380
	u32 data = nvkm_rd32(device, 0x400708);
	u32 code = nvkm_rd32(device, 0x400110);
1381
	u32 class;
1382 1383
	const char *name = "unknown";
	int chid = -1;
1384

1385
	chan = nvkm_fifo_chan_inst(device->fifo, (u64)inst << 12, &flags);
1386 1387 1388 1389
	if (chan) {
		name = chan->object.client->name;
		chid = chan->chid;
	}
1390

1391
	if (device->card_type < NV_E0 || subc < 4)
1392
		class = nvkm_rd32(device, 0x404200 + (subc * 4));
1393 1394 1395
	else
		class = 0x0000;

1396 1397 1398 1399 1400
	if (stat & 0x00000001) {
		/*
		 * notifier interrupt, only needed for cyclestats
		 * can be safely ignored
		 */
1401
		nvkm_wr32(device, 0x400100, 0x00000001);
1402 1403 1404
		stat &= ~0x00000001;
	}

1405
	if (stat & 0x00000010) {
1406
		if (!gf100_gr_mthd_sw(device, class, mthd, data)) {
1407 1408
			nvkm_error(subdev, "ILLEGAL_MTHD ch %d [%010llx %s] "
				   "subc %d class %04x mthd %04x data %08x\n",
1409 1410
				   chid, inst << 12, name, subc,
				   class, mthd, data);
1411
		}
1412
		nvkm_wr32(device, 0x400100, 0x00000010);
1413 1414 1415 1416
		stat &= ~0x00000010;
	}

	if (stat & 0x00000020) {
1417 1418
		nvkm_error(subdev, "ILLEGAL_CLASS ch %d [%010llx %s] "
			   "subc %d class %04x mthd %04x data %08x\n",
1419
			   chid, inst << 12, name, subc, class, mthd, data);
1420
		nvkm_wr32(device, 0x400100, 0x00000020);
1421 1422 1423 1424
		stat &= ~0x00000020;
	}

	if (stat & 0x00100000) {
1425 1426 1427 1428 1429
		const struct nvkm_enum *en =
			nvkm_enum_find(nv50_data_error_names, code);
		nvkm_error(subdev, "DATA_ERROR %08x [%s] ch %d [%010llx %s] "
				   "subc %d class %04x mthd %04x data %08x\n",
			   code, en ? en->name : "", chid, inst << 12,
1430
			   name, subc, class, mthd, data);
1431
		nvkm_wr32(device, 0x400100, 0x00100000);
1432 1433 1434 1435
		stat &= ~0x00100000;
	}

	if (stat & 0x00200000) {
1436
		nvkm_error(subdev, "TRAP ch %d [%010llx %s]\n",
1437
			   chid, inst << 12, name);
B
Ben Skeggs 已提交
1438
		gf100_gr_trap_intr(gr);
1439
		nvkm_wr32(device, 0x400100, 0x00200000);
1440 1441 1442 1443
		stat &= ~0x00200000;
	}

	if (stat & 0x00080000) {
B
Ben Skeggs 已提交
1444
		gf100_gr_ctxctl_isr(gr);
1445
		nvkm_wr32(device, 0x400100, 0x00080000);
1446 1447 1448 1449
		stat &= ~0x00080000;
	}

	if (stat) {
1450
		nvkm_error(subdev, "intr %08x\n", stat);
1451
		nvkm_wr32(device, 0x400100, stat);
1452 1453
	}

1454
	nvkm_wr32(device, 0x400500, 0x00010001);
1455
	nvkm_fifo_chan_put(device->fifo, flags, &chan);
1456 1457
}

1458
static void
1459
gf100_gr_init_fw(struct nvkm_falcon *falcon,
1460
		 struct gf100_gr_fuc *code, struct gf100_gr_fuc *data)
1461
{
1462 1463
	nvkm_falcon_load_dmem(falcon, data->data, 0x0, data->size, 0);
	nvkm_falcon_load_imem(falcon, code->data, 0x0, code->size, 0, 0, false);
1464 1465
}

1466
static void
B
Ben Skeggs 已提交
1467
gf100_gr_init_csdata(struct gf100_gr *gr,
1468 1469
		     const struct gf100_gr_pack *pack,
		     u32 falcon, u32 starstar, u32 base)
1470
{
1471
	struct nvkm_device *device = gr->base.engine.subdev.device;
1472 1473
	const struct gf100_gr_pack *iter;
	const struct gf100_gr_init *init;
1474
	u32 addr = ~0, prev = ~0, xfer = 0;
1475 1476
	u32 star, temp;

1477 1478 1479
	nvkm_wr32(device, falcon + 0x01c0, 0x02000000 + starstar);
	star = nvkm_rd32(device, falcon + 0x01c4);
	temp = nvkm_rd32(device, falcon + 0x01c4);
1480 1481
	if (temp > star)
		star = temp;
1482
	nvkm_wr32(device, falcon + 0x01c0, 0x01000000 + star);
1483

1484 1485 1486 1487 1488 1489 1490
	pack_for_each_init(init, iter, pack) {
		u32 head = init->addr - base;
		u32 tail = head + init->count * init->pitch;
		while (head < tail) {
			if (head != prev + 4 || xfer >= 32) {
				if (xfer) {
					u32 data = ((--xfer << 26) | addr);
1491
					nvkm_wr32(device, falcon + 0x01c4, data);
1492 1493 1494 1495
					star += 4;
				}
				addr = head;
				xfer = 0;
1496
			}
1497 1498 1499
			prev = head;
			xfer = xfer + 1;
			head = head + init->pitch;
1500
		}
1501
	}
1502

1503 1504 1505
	nvkm_wr32(device, falcon + 0x01c4, (--xfer << 26) | addr);
	nvkm_wr32(device, falcon + 0x01c0, 0x01000004 + starstar);
	nvkm_wr32(device, falcon + 0x01c4, star + 4);
1506 1507
}

1508 1509 1510
/* Initialize context from an external (secure or not) firmware */
static int
gf100_gr_init_ctxctl_ext(struct gf100_gr *gr)
1511
{
1512 1513
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1514
	struct nvkm_secboot *sb = device->secboot;
1515
	u32 secboot_mask = 0;
1516
	int ret;
1517

1518 1519
	/* load fuc microcode */
	nvkm_mc_unk260(device, 0);
1520

1521 1522
	/* securely-managed falcons must be reset using secure boot */
	if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_FECS))
1523
		secboot_mask |= BIT(NVKM_SECBOOT_FALCON_FECS);
1524
	else
1525
		gf100_gr_init_fw(gr->fecs, &gr->fuc409c, &gr->fuc409d);
1526

1527
	if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_GPCCS))
1528
		secboot_mask |= BIT(NVKM_SECBOOT_FALCON_GPCCS);
1529
	else
1530
		gf100_gr_init_fw(gr->gpccs, &gr->fuc41ac, &gr->fuc41ad);
1531 1532 1533 1534 1535 1536

	if (secboot_mask != 0) {
		int ret = nvkm_secboot_reset(sb, secboot_mask);
		if (ret)
			return ret;
	}
1537

1538 1539 1540 1541 1542 1543 1544
	nvkm_mc_unk260(device, 1);

	/* start both of them running */
	nvkm_wr32(device, 0x409840, 0xffffffff);
	nvkm_wr32(device, 0x41a10c, 0x00000000);
	nvkm_wr32(device, 0x40910c, 0x00000000);

1545 1546 1547
	nvkm_falcon_start(gr->gpccs);
	nvkm_falcon_start(gr->fecs);

1548 1549 1550 1551 1552 1553
	if (nvkm_msec(device, 2000,
		if (nvkm_rd32(device, 0x409800) & 0x00000001)
			break;
	) < 0)
		return -EBUSY;

1554
	gf100_gr_fecs_set_watchdog_timeout(gr, 0x7fffffff);
1555

1556 1557 1558 1559
	/* Determine how much memory is required to store main context image. */
	ret = gf100_gr_fecs_discover_image_size(gr, &gr->size);
	if (ret)
		return ret;
1560 1561 1562 1563 1564 1565 1566 1567 1568

	nvkm_wr32(device, 0x409840, 0xffffffff);
	nvkm_wr32(device, 0x409500, 0x00000000);
	nvkm_wr32(device, 0x409504, 0x00000016);
	if (nvkm_msec(device, 2000,
		if (nvkm_rd32(device, 0x409800))
			break;
	) < 0)
		return -EBUSY;
B
Ben Skeggs 已提交
1569

1570 1571 1572 1573 1574 1575 1576 1577
	nvkm_wr32(device, 0x409840, 0xffffffff);
	nvkm_wr32(device, 0x409500, 0x00000000);
	nvkm_wr32(device, 0x409504, 0x00000025);
	if (nvkm_msec(device, 2000,
		if (nvkm_rd32(device, 0x409800))
			break;
	) < 0)
		return -EBUSY;
B
Ben Skeggs 已提交
1578

1579 1580 1581 1582
	if (device->chipset >= 0xe0) {
		nvkm_wr32(device, 0x409800, 0x00000000);
		nvkm_wr32(device, 0x409500, 0x00000001);
		nvkm_wr32(device, 0x409504, 0x00000030);
1583
		if (nvkm_msec(device, 2000,
1584
			if (nvkm_rd32(device, 0x409800))
1585 1586
				break;
		) < 0)
1587
			return -EBUSY;
1588

1589 1590 1591 1592
		nvkm_wr32(device, 0x409810, 0xb00095c8);
		nvkm_wr32(device, 0x409800, 0x00000000);
		nvkm_wr32(device, 0x409500, 0x00000001);
		nvkm_wr32(device, 0x409504, 0x00000031);
1593 1594 1595 1596
		if (nvkm_msec(device, 2000,
			if (nvkm_rd32(device, 0x409800))
				break;
		) < 0)
1597 1598
			return -EBUSY;

1599 1600 1601 1602
		nvkm_wr32(device, 0x409810, 0x00080420);
		nvkm_wr32(device, 0x409800, 0x00000000);
		nvkm_wr32(device, 0x409500, 0x00000001);
		nvkm_wr32(device, 0x409504, 0x00000032);
1603 1604 1605 1606
		if (nvkm_msec(device, 2000,
			if (nvkm_rd32(device, 0x409800))
				break;
		) < 0)
1607 1608
			return -EBUSY;

1609 1610 1611 1612
		nvkm_wr32(device, 0x409614, 0x00000070);
		nvkm_wr32(device, 0x409614, 0x00000770);
		nvkm_wr32(device, 0x40802c, 0x00000001);
	}
1613

1614 1615 1616 1617 1618
	if (gr->data == NULL) {
		int ret = gf100_grctx_generate(gr);
		if (ret) {
			nvkm_error(subdev, "failed to construct context\n");
			return ret;
1619
		}
1620
	}
1621

1622 1623 1624 1625 1626 1627 1628 1629 1630
	return 0;
}

static int
gf100_gr_init_ctxctl_int(struct gf100_gr *gr)
{
	const struct gf100_grctx_func *grctx = gr->func->grctx;
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1631

1632
	if (!gr->func->fecs.ucode) {
1633
		return -ENOSYS;
1634
	}
1635

1636
	/* load HUB microcode */
1637
	nvkm_mc_unk260(device, 0);
1638 1639 1640 1641
	nvkm_falcon_load_dmem(gr->fecs, gr->func->fecs.ucode->data.data, 0x0,
			      gr->func->fecs.ucode->data.size, 0);
	nvkm_falcon_load_imem(gr->fecs, gr->func->fecs.ucode->code.data, 0x0,
			      gr->func->fecs.ucode->code.size, 0, 0, false);
1642 1643

	/* load GPC microcode */
1644 1645 1646 1647
	nvkm_falcon_load_dmem(gr->gpccs, gr->func->gpccs.ucode->data.data, 0x0,
			      gr->func->gpccs.ucode->data.size, 0);
	nvkm_falcon_load_imem(gr->gpccs, gr->func->gpccs.ucode->code.data, 0x0,
			      gr->func->gpccs.ucode->code.size, 0, 0, false);
1648
	nvkm_mc_unk260(device, 1);
1649

1650
	/* load register lists */
1651
	gf100_gr_init_csdata(gr, grctx->hub, 0x409000, 0x000, 0x000000);
1652 1653
	gf100_gr_init_csdata(gr, grctx->gpc_0, 0x41a000, 0x000, 0x418000);
	gf100_gr_init_csdata(gr, grctx->gpc_1, 0x41a000, 0x000, 0x418000);
1654 1655
	gf100_gr_init_csdata(gr, grctx->tpc, 0x41a000, 0x004, 0x419800);
	gf100_gr_init_csdata(gr, grctx->ppc, 0x41a000, 0x008, 0x41be00);
1656

1657
	/* start HUB ucode running, it'll init the GPCs */
1658 1659
	nvkm_wr32(device, 0x40910c, 0x00000000);
	nvkm_wr32(device, 0x409100, 0x00000002);
1660 1661 1662 1663
	if (nvkm_msec(device, 2000,
		if (nvkm_rd32(device, 0x409800) & 0x80000000)
			break;
	) < 0) {
B
Ben Skeggs 已提交
1664
		gf100_gr_ctxctl_debug(gr);
1665 1666 1667
		return -EBUSY;
	}

1668
	gr->size = nvkm_rd32(device, 0x409804);
B
Ben Skeggs 已提交
1669 1670
	if (gr->data == NULL) {
		int ret = gf100_grctx_generate(gr);
1671
		if (ret) {
1672
			nvkm_error(subdev, "failed to construct context\n");
1673 1674
			return ret;
		}
1675 1676 1677
	}

	return 0;
1678 1679
}

1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
int
gf100_gr_init_ctxctl(struct gf100_gr *gr)
{
	int ret;

	if (gr->firmware)
		ret = gf100_gr_init_ctxctl_ext(gr);
	else
		ret = gf100_gr_init_ctxctl_int(gr);

	return ret;
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707
void
gf100_gr_oneinit_sm_id(struct gf100_gr *gr)
{
	int tpc, gpc;
	for (tpc = 0; tpc < gr->tpc_max; tpc++) {
		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
			if (tpc < gr->tpc_nr[gpc]) {
				gr->sm[gr->sm_nr].gpc = gpc;
				gr->sm[gr->sm_nr].tpc = tpc;
				gr->sm_nr++;
			}
		}
	}
}

1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
void
gf100_gr_oneinit_tiles(struct gf100_gr *gr)
{
	static const u8 primes[] = {
		3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61
	};
	int init_frac[GPC_MAX], init_err[GPC_MAX], run_err[GPC_MAX], i, j;
	u32 mul_factor, comm_denom;
	u8  gpc_map[GPC_MAX];
	bool sorted;

	switch (gr->tpc_total) {
	case 15: gr->screen_tile_row_offset = 0x06; break;
	case 14: gr->screen_tile_row_offset = 0x05; break;
	case 13: gr->screen_tile_row_offset = 0x02; break;
	case 11: gr->screen_tile_row_offset = 0x07; break;
	case 10: gr->screen_tile_row_offset = 0x06; break;
	case  7:
	case  5: gr->screen_tile_row_offset = 0x01; break;
	case  3: gr->screen_tile_row_offset = 0x02; break;
	case  2:
	case  1: gr->screen_tile_row_offset = 0x01; break;
	default: gr->screen_tile_row_offset = 0x03;
		for (i = 0; i < ARRAY_SIZE(primes); i++) {
			if (gr->tpc_total % primes[i]) {
				gr->screen_tile_row_offset = primes[i];
				break;
			}
		}
		break;
	}

	/* Sort GPCs by TPC count, highest-to-lowest. */
	for (i = 0; i < gr->gpc_nr; i++)
		gpc_map[i] = i;
	sorted = false;

	while (!sorted) {
		for (sorted = true, i = 0; i < gr->gpc_nr - 1; i++) {
			if (gr->tpc_nr[gpc_map[i + 1]] >
			    gr->tpc_nr[gpc_map[i + 0]]) {
				u8 swap = gpc_map[i];
				gpc_map[i + 0] = gpc_map[i + 1];
				gpc_map[i + 1] = swap;
				sorted = false;
			}
		}
	}

	/* Determine tile->GPC mapping */
	mul_factor = gr->gpc_nr * gr->tpc_max;
	if (mul_factor & 1)
		mul_factor = 2;
	else
		mul_factor = 1;

	comm_denom = gr->gpc_nr * gr->tpc_max * mul_factor;

	for (i = 0; i < gr->gpc_nr; i++) {
		init_frac[i] = gr->tpc_nr[gpc_map[i]] * gr->gpc_nr * mul_factor;
		 init_err[i] = i * gr->tpc_max * mul_factor - comm_denom/2;
		  run_err[i] = init_frac[i] + init_err[i];
	}

	for (i = 0; i < gr->tpc_total;) {
		for (j = 0; j < gr->gpc_nr; j++) {
			if ((run_err[j] * 2) >= comm_denom) {
				gr->tile[i++] = gpc_map[j];
				run_err[j] += init_frac[j] - comm_denom;
			} else {
				run_err[j] += init_frac[j];
			}
		}
	}
}

1784 1785 1786 1787
static int
gf100_gr_oneinit(struct nvkm_gr *base)
{
	struct gf100_gr *gr = gf100_gr(base);
1788 1789
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
1790
	int i, j;
1791 1792 1793 1794 1795 1796 1797 1798 1799
	int ret;

	ret = nvkm_falcon_v1_new(subdev, "FECS", 0x409000, &gr->fecs);
	if (ret)
		return ret;

	ret = nvkm_falcon_v1_new(subdev, "GPCCS", 0x41a000, &gr->gpccs);
	if (ret)
		return ret;
1800 1801 1802

	nvkm_pmu_pgob(device->pmu, false);

1803 1804
	gr->rop_nr = gr->func->rops(gr);
	gr->gpc_nr = nvkm_rd32(device, 0x409604) & 0x0000001f;
1805 1806
	for (i = 0; i < gr->gpc_nr; i++) {
		gr->tpc_nr[i]  = nvkm_rd32(device, GPC_UNIT(i, 0x2608));
1807
		gr->tpc_max = max(gr->tpc_max, gr->tpc_nr[i]);
1808 1809 1810
		gr->tpc_total += gr->tpc_nr[i];
		gr->ppc_nr[i]  = gr->func->ppc_nr;
		for (j = 0; j < gr->ppc_nr[i]; j++) {
1811 1812 1813 1814 1815 1816
			gr->ppc_tpc_mask[i][j] =
				nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4)));
			if (gr->ppc_tpc_mask[i][j] == 0)
				continue;
			gr->ppc_mask[i] |= (1 << j);
			gr->ppc_tpc_nr[i][j] = hweight8(gr->ppc_tpc_mask[i][j]);
1817 1818 1819
			if (gr->ppc_tpc_min == 0 ||
			    gr->ppc_tpc_min > gr->ppc_tpc_nr[i][j])
				gr->ppc_tpc_min = gr->ppc_tpc_nr[i][j];
1820 1821
			if (gr->ppc_tpc_max < gr->ppc_tpc_nr[i][j])
				gr->ppc_tpc_max = gr->ppc_tpc_nr[i][j];
1822 1823 1824
		}
	}

1825 1826
	memset(gr->tile, 0xff, sizeof(gr->tile));
	gr->func->oneinit_tiles(gr);
1827
	gr->func->oneinit_sm_id(gr);
1828 1829 1830
	return 0;
}

1831
static int
1832 1833 1834
gf100_gr_init_(struct nvkm_gr *base)
{
	struct gf100_gr *gr = gf100_gr(base);
1835 1836 1837
	struct nvkm_subdev *subdev = &base->engine.subdev;
	u32 ret;

1838
	nvkm_pmu_pgob(gr->base.engine.subdev.device->pmu, false);
1839 1840 1841 1842 1843 1844 1845 1846 1847

	ret = nvkm_falcon_get(gr->fecs, subdev);
	if (ret)
		return ret;

	ret = nvkm_falcon_get(gr->gpccs, subdev);
	if (ret)
		return ret;

1848 1849 1850
	return gr->func->init(gr);
}

1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
static int
gf100_gr_fini_(struct nvkm_gr *base, bool suspend)
{
	struct gf100_gr *gr = gf100_gr(base);
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	nvkm_falcon_put(gr->gpccs, subdev);
	nvkm_falcon_put(gr->fecs, subdev);
	return 0;
}

1861 1862 1863 1864 1865 1866 1867
void
gf100_gr_dtor_fw(struct gf100_gr_fuc *fuc)
{
	kfree(fuc->data);
	fuc->data = NULL;
}

1868 1869 1870 1871 1872 1873
static void
gf100_gr_dtor_init(struct gf100_gr_pack *pack)
{
	vfree(pack);
}

1874 1875 1876 1877 1878 1879 1880 1881 1882
void *
gf100_gr_dtor(struct nvkm_gr *base)
{
	struct gf100_gr *gr = gf100_gr(base);

	if (gr->func->dtor)
		gr->func->dtor(gr);
	kfree(gr->data);

1883 1884 1885
	nvkm_falcon_del(&gr->gpccs);
	nvkm_falcon_del(&gr->fecs);

1886 1887 1888 1889 1890
	gf100_gr_dtor_fw(&gr->fuc409c);
	gf100_gr_dtor_fw(&gr->fuc409d);
	gf100_gr_dtor_fw(&gr->fuc41ac);
	gf100_gr_dtor_fw(&gr->fuc41ad);

1891 1892 1893 1894 1895
	gf100_gr_dtor_init(gr->fuc_bundle);
	gf100_gr_dtor_init(gr->fuc_method);
	gf100_gr_dtor_init(gr->fuc_sw_ctx);
	gf100_gr_dtor_init(gr->fuc_sw_nonctx);

1896 1897 1898 1899 1900 1901 1902 1903
	return gr;
}

static const struct nvkm_gr_func
gf100_gr_ = {
	.dtor = gf100_gr_dtor,
	.oneinit = gf100_gr_oneinit,
	.init = gf100_gr_init_,
1904
	.fini = gf100_gr_fini_,
1905 1906 1907 1908
	.intr = gf100_gr_intr,
	.units = gf100_gr_units,
	.chan_new = gf100_gr_chan_new,
	.object_get = gf100_gr_object_get,
1909
	.chsw_load = gf100_gr_chsw_load,
1910 1911
};

1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
int
gf100_gr_ctor_fw_legacy(struct gf100_gr *gr, const char *fwname,
			struct gf100_gr_fuc *fuc, int ret)
{
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
	const struct firmware *fw;
	char f[32];

	/* see if this firmware has a legacy path */
	if (!strcmp(fwname, "fecs_inst"))
		fwname = "fuc409c";
	else if (!strcmp(fwname, "fecs_data"))
		fwname = "fuc409d";
	else if (!strcmp(fwname, "gpccs_inst"))
		fwname = "fuc41ac";
	else if (!strcmp(fwname, "gpccs_data"))
		fwname = "fuc41ad";
	else {
		/* nope, let's just return the error we got */
		nvkm_error(subdev, "failed to load %s\n", fwname);
		return ret;
	}

	/* yes, try to load from the legacy path */
	nvkm_debug(subdev, "%s: falling back to legacy path\n", fwname);

	snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
	ret = request_firmware(&fw, f, device->dev);
	if (ret) {
		snprintf(f, sizeof(f), "nouveau/%s", fwname);
		ret = request_firmware(&fw, f, device->dev);
		if (ret) {
			nvkm_error(subdev, "failed to load %s\n", fwname);
			return ret;
		}
	}

	fuc->size = fw->size;
	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
	release_firmware(fw);
	return (fuc->data != NULL) ? 0 : -ENOMEM;
}

1956 1957 1958 1959 1960 1961 1962 1963 1964
int
gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname,
		 struct gf100_gr_fuc *fuc)
{
	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
	struct nvkm_device *device = subdev->device;
	const struct firmware *fw;
	int ret;

1965
	ret = nvkm_firmware_get(device, fwname, &fw);
1966 1967 1968 1969 1970 1971
	if (ret) {
		ret = gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret);
		if (ret)
			return -ENODEV;
		return 0;
	}
1972 1973 1974

	fuc->size = fw->size;
	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
1975
	nvkm_firmware_put(fw);
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986
	return (fuc->data != NULL) ? 0 : -ENOMEM;
}

int
gf100_gr_ctor(const struct gf100_gr_func *func, struct nvkm_device *device,
	      int index, struct gf100_gr *gr)
{
	gr->func = func;
	gr->firmware = nvkm_boolopt(device->cfgopt, "NvGrUseFW",
				    func->fecs.ucode == NULL);

1987 1988 1989
	return nvkm_gr_ctor(&gf100_gr_, device, index,
			    gr->firmware || func->fecs.ucode != NULL,
			    &gr->base);
1990 1991
}

1992
int
1993 1994 1995 1996
gf100_gr_new_(const struct gf100_gr_func *func, struct nvkm_device *device,
	      int index, struct nvkm_gr **pgr)
{
	struct gf100_gr *gr;
1997 1998
	int ret;

1999 2000 2001
	if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL)))
		return -ENOMEM;
	*pgr = &gr->base;
2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015

	ret = gf100_gr_ctor(func, device, index, gr);
	if (ret)
		return ret;

	if (gr->firmware) {
		if (gf100_gr_ctor_fw(gr, "fecs_inst", &gr->fuc409c) ||
		    gf100_gr_ctor_fw(gr, "fecs_data", &gr->fuc409d) ||
		    gf100_gr_ctor_fw(gr, "gpccs_inst", &gr->fuc41ac) ||
		    gf100_gr_ctor_fw(gr, "gpccs_data", &gr->fuc41ad))
			return -ENODEV;
	}

	return 0;
2016 2017
}

2018 2019 2020 2021 2022 2023
void
gf100_gr_init_400054(struct gf100_gr *gr)
{
	nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464);
}

2024
void
2025 2026 2027 2028 2029 2030 2031 2032 2033
gf100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
}

void
gf100_gr_init_tex_hww_esr(struct gf100_gr *gr, int gpc, int tpc)
2034 2035 2036 2037 2038
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
}

2039 2040 2041 2042 2043 2044 2045
void
gf100_gr_init_419eb4(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);
}

2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
void
gf100_gr_init_419cc0(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	int gpc, tpc;

	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);

	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++)
			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
	}
}

2060 2061 2062 2063 2064 2065
void
gf100_gr_init_40601c(struct gf100_gr *gr)
{
	nvkm_wr32(gr->base.engine.subdev.device, 0x40601c, 0xc0000000);
}

2066 2067 2068 2069 2070 2071 2072
void
gf100_gr_init_fecs_exceptions(struct gf100_gr *gr)
{
	const u32 data = gr->firmware ? 0x000e0000 : 0x000e0001;
	nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, data);
}

2073 2074 2075 2076 2077 2078
void
gf100_gr_init_gpc_mmu(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	struct nvkm_fb *fb = device->fb;

2079
	nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001);
2080
	nvkm_wr32(device, 0x4188a4, 0x03000000);
2081 2082 2083 2084 2085 2086 2087 2088
	nvkm_wr32(device, 0x418888, 0x00000000);
	nvkm_wr32(device, 0x41888c, 0x00000000);
	nvkm_wr32(device, 0x418890, 0x00000000);
	nvkm_wr32(device, 0x418894, 0x00000000);
	nvkm_wr32(device, 0x4188b4, nvkm_memory_addr(fb->mmu_wr) >> 8);
	nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8);
}

2089 2090 2091 2092 2093 2094 2095
void
gf100_gr_init_num_active_ltcs(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
}

2096 2097 2098 2099 2100
void
gf100_gr_init_zcull(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
	const u8 tile_nr = ALIGN(gr->tpc_total, 32);
	u8 bank[GPC_MAX] = {}, gpc, i, j;
	u32 data;

	for (i = 0; i < tile_nr; i += 8) {
		for (data = 0, j = 0; j < 8 && i + j < gr->tpc_total; j++) {
			data |= bank[gr->tile[i + j]] << (j * 4);
			bank[gr->tile[i + j]]++;
		}
		nvkm_wr32(device, GPC_BCAST(0x0980 + ((i / 8) * 4)), data);
2111 2112
	}

2113 2114 2115 2116 2117 2118 2119
	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
			  gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
							 gr->tpc_total);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
	}
2120 2121 2122 2123

	nvkm_wr32(device, GPC_BCAST(0x1bd4), magicgpc918);
}

2124 2125 2126 2127 2128 2129 2130
void
gf100_gr_init_vsc_stream_master(struct gf100_gr *gr)
{
	struct nvkm_device *device = gr->base.engine.subdev.device;
	nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001);
}

2131 2132
int
gf100_gr_init(struct gf100_gr *gr)
2133
{
2134
	struct nvkm_device *device = gr->base.engine.subdev.device;
2135
	int gpc, tpc, rop;
2136

2137 2138 2139
	if (gr->func->init_419bd8)
		gr->func->init_419bd8(gr);

2140
	gr->func->init_gpc_mmu(gr);
2141

2142 2143 2144 2145
	if (gr->fuc_sw_nonctx)
		gf100_gr_mmio(gr, gr->fuc_sw_nonctx);
	else
		gf100_gr_mmio(gr, gr->func->mmio);
2146

2147 2148
	gf100_gr_wait_idle(gr);

2149 2150 2151
	if (gr->func->init_r405a14)
		gr->func->init_r405a14(gr);

2152 2153 2154
	if (gr->func->clkgate_pack)
		nvkm_therm_clkgate_init(device->therm, gr->func->clkgate_pack);

2155 2156 2157
	if (gr->func->init_bios)
		gr->func->init_bios(gr);

2158
	gr->func->init_vsc_stream_master(gr);
2159
	gr->func->init_zcull(gr);
2160
	gr->func->init_num_active_ltcs(gr);
2161 2162
	if (gr->func->init_rop_active_fbps)
		gr->func->init_rop_active_fbps(gr);
2163 2164
	if (gr->func->init_bios_2)
		gr->func->init_bios_2(gr);
2165 2166
	if (gr->func->init_swdx_pes_mask)
		gr->func->init_swdx_pes_mask(gr);
B
Ben Skeggs 已提交
2167

2168
	nvkm_wr32(device, 0x400500, 0x00010001);
B
Ben Skeggs 已提交
2169

2170 2171
	nvkm_wr32(device, 0x400100, 0xffffffff);
	nvkm_wr32(device, 0x40013c, 0xffffffff);
2172
	nvkm_wr32(device, 0x400124, 0x00000002);
B
Ben Skeggs 已提交
2173

2174
	gr->func->init_fecs_exceptions(gr);
2175 2176
	if (gr->func->init_ds_hww_esr_2)
		gr->func->init_ds_hww_esr_2(gr);
2177

2178 2179 2180
	nvkm_wr32(device, 0x404000, 0xc0000000);
	nvkm_wr32(device, 0x404600, 0xc0000000);
	nvkm_wr32(device, 0x408030, 0xc0000000);
2181 2182 2183 2184

	if (gr->func->init_40601c)
		gr->func->init_40601c(gr);

2185 2186
	nvkm_wr32(device, 0x404490, 0xc0000000);
	nvkm_wr32(device, 0x406018, 0xc0000000);
2187 2188 2189 2190

	if (gr->func->init_sked_hww_esr)
		gr->func->init_sked_hww_esr(gr);

2191 2192
	nvkm_wr32(device, 0x405840, 0xc0000000);
	nvkm_wr32(device, 0x405844, 0x00ffffff);
2193 2194 2195

	if (gr->func->init_419cc0)
		gr->func->init_419cc0(gr);
2196 2197
	if (gr->func->init_419eb4)
		gr->func->init_419eb4(gr);
2198 2199
	if (gr->func->init_419c9c)
		gr->func->init_419c9c(gr);
B
Ben Skeggs 已提交
2200

2201 2202 2203
	if (gr->func->init_ppc_exceptions)
		gr->func->init_ppc_exceptions(gr);

B
Ben Skeggs 已提交
2204
	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
2205 2206 2207 2208
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
B
Ben Skeggs 已提交
2209
		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
2210 2211
			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
2212 2213
			if (gr->func->init_tex_hww_esr)
				gr->func->init_tex_hww_esr(gr, gpc, tpc);
2214
			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
2215 2216
			if (gr->func->init_504430)
				gr->func->init_504430(gr, gpc, tpc);
2217
			gr->func->init_shader_exceptions(gr, gpc, tpc);
2218
		}
2219 2220
		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
2221 2222
	}

B
Ben Skeggs 已提交
2223
	for (rop = 0; rop < gr->rop_nr; rop++) {
2224 2225
		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
2226 2227
		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
2228
	}
2229

2230 2231 2232 2233 2234 2235
	nvkm_wr32(device, 0x400108, 0xffffffff);
	nvkm_wr32(device, 0x400138, 0xffffffff);
	nvkm_wr32(device, 0x400118, 0xffffffff);
	nvkm_wr32(device, 0x400130, 0xffffffff);
	nvkm_wr32(device, 0x40011c, 0xffffffff);
	nvkm_wr32(device, 0x400134, 0xffffffff);
2236

2237 2238
	if (gr->func->init_400054)
		gr->func->init_400054(gr);
2239

B
Ben Skeggs 已提交
2240
	gf100_gr_zbc_init(gr);
2241

2242 2243 2244
	if (gr->func->init_4188a4)
		gr->func->init_4188a4(gr);

B
Ben Skeggs 已提交
2245
	return gf100_gr_init_ctxctl(gr);
2246 2247
}

2248
#include "fuc/hubgf100.fuc3.h"
2249

2250 2251 2252 2253 2254 2255
struct gf100_gr_ucode
gf100_gr_fecs_ucode = {
	.code.data = gf100_grhub_code,
	.code.size = sizeof(gf100_grhub_code),
	.data.data = gf100_grhub_data,
	.data.size = sizeof(gf100_grhub_data),
2256 2257
};

2258
#include "fuc/gpcgf100.fuc3.h"
2259

2260 2261 2262 2263 2264 2265
struct gf100_gr_ucode
gf100_gr_gpccs_ucode = {
	.code.data = gf100_grgpc_code,
	.code.size = sizeof(gf100_grgpc_code),
	.data.data = gf100_grgpc_data,
	.data.size = sizeof(gf100_grgpc_data),
2266 2267
};

2268 2269
static const struct gf100_gr_func
gf100_gr = {
2270
	.oneinit_tiles = gf100_gr_oneinit_tiles,
2271
	.oneinit_sm_id = gf100_gr_oneinit_sm_id,
2272
	.init = gf100_gr_init,
2273
	.init_gpc_mmu = gf100_gr_init_gpc_mmu,
2274
	.init_vsc_stream_master = gf100_gr_init_vsc_stream_master,
2275
	.init_zcull = gf100_gr_init_zcull,
2276
	.init_num_active_ltcs = gf100_gr_init_num_active_ltcs,
2277
	.init_fecs_exceptions = gf100_gr_init_fecs_exceptions,
2278
	.init_40601c = gf100_gr_init_40601c,
2279
	.init_419cc0 = gf100_gr_init_419cc0,
2280
	.init_419eb4 = gf100_gr_init_419eb4,
2281
	.init_tex_hww_esr = gf100_gr_init_tex_hww_esr,
2282
	.init_shader_exceptions = gf100_gr_init_shader_exceptions,
2283
	.init_400054 = gf100_gr_init_400054,
2284
	.trap_mp = gf100_gr_trap_mp,
2285 2286 2287
	.mmio = gf100_gr_pack_mmio,
	.fecs.ucode = &gf100_gr_fecs_ucode,
	.gpccs.ucode = &gf100_gr_gpccs_ucode,
2288
	.rops = gf100_gr_rops,
2289
	.grctx = &gf100_grctx,
2290
	.zbc = &gf100_gr_zbc,
2291 2292 2293 2294 2295 2296 2297 2298 2299
	.sclass = {
		{ -1, -1, FERMI_TWOD_A },
		{ -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },
		{ -1, -1, FERMI_A, &gf100_fermi },
		{ -1, -1, FERMI_COMPUTE_A },
		{}
	}
};

2300 2301 2302 2303 2304
int
gf100_gr_new(struct nvkm_device *device, int index, struct nvkm_gr **pgr)
{
	return gf100_gr_new_(&gf100_gr, device, index, pgr);
}