mic_x100.c 15.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7 8 9 10
/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2013 Intel Corporation.
 *
 * Intel MIC Host driver.
 */
#include <linux/fs.h>
#include <linux/pci.h>
11 12 13
#include <linux/sched.h>
#include <linux/firmware.h>
#include <linux/delay.h>
14

S
Sudeep Dutt 已提交
15
#include "../common/mic_dev.h"
16 17
#include "mic_device.h"
#include "mic_x100.h"
18
#include "mic_smpt.h"
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33

/**
 * mic_x100_write_spad - write to the scratchpad register
 * @mdev: pointer to mic_device instance
 * @idx: index to the scratchpad register, 0 based
 * @val: the data value to put into the register
 *
 * This function allows writing of a 32bit value to the indexed scratchpad
 * register.
 *
 * RETURNS: none.
 */
static void
mic_x100_write_spad(struct mic_device *mdev, unsigned int idx, u32 val)
{
34
	dev_dbg(&mdev->pdev->dev, "Writing 0x%x to scratch pad index %d\n",
35 36
		val, idx);
	mic_mmio_write(&mdev->mmio, val,
37 38
		       MIC_X100_SBOX_BASE_ADDRESS +
		       MIC_X100_SBOX_SPAD0 + idx * 4);
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
}

/**
 * mic_x100_read_spad - read from the scratchpad register
 * @mdev: pointer to mic_device instance
 * @idx: index to scratchpad register, 0 based
 *
 * This function allows reading of the 32bit scratchpad register.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static u32
mic_x100_read_spad(struct mic_device *mdev, unsigned int idx)
{
	u32 val = mic_mmio_read(&mdev->mmio,
		MIC_X100_SBOX_BASE_ADDRESS +
		MIC_X100_SBOX_SPAD0 + idx * 4);

57
	dev_dbg(&mdev->pdev->dev,
58 59 60 61
		"Reading 0x%x from scratch pad index %d\n", val, idx);
	return val;
}

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
/**
 * mic_x100_enable_interrupts - Enable interrupts.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_enable_interrupts(struct mic_device *mdev)
{
	u32 reg;
	struct mic_mw *mw = &mdev->mmio;
	u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
	u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;

	reg = mic_mmio_read(mw, sice0);
	reg |= MIC_X100_SBOX_DBR_BITS(0xf) | MIC_X100_SBOX_DMA_BITS(0xff);
	mic_mmio_write(mw, reg, sice0);

	/*
	 * Enable auto-clear when enabling interrupts. Applicable only for
	 * MSI-x. Legacy and MSI mode cannot have auto-clear enabled.
	 */
	if (mdev->irq_info.num_vectors > 1) {
		reg = mic_mmio_read(mw, siac0);
		reg |= MIC_X100_SBOX_DBR_BITS(0xf) |
			MIC_X100_SBOX_DMA_BITS(0xff);
		mic_mmio_write(mw, reg, siac0);
	}
}

/**
 * mic_x100_disable_interrupts - Disable interrupts.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_disable_interrupts(struct mic_device *mdev)
{
	u32 reg;
	struct mic_mw *mw = &mdev->mmio;
	u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
	u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;
	u32 sicc0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICC0;

	reg = mic_mmio_read(mw, sice0);
	mic_mmio_write(mw, reg, sicc0);

	if (mdev->irq_info.num_vectors > 1) {
		reg = mic_mmio_read(mw, siac0);
		reg &= ~(MIC_X100_SBOX_DBR_BITS(0xf) |
			MIC_X100_SBOX_DMA_BITS(0xff));
		mic_mmio_write(mw, reg, siac0);
	}
}

/**
 * mic_x100_send_sbox_intr - Send an MIC_X100_SBOX interrupt to MIC.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_send_sbox_intr(struct mic_device *mdev,
117
				    int doorbell)
118 119 120
{
	struct mic_mw *mw = &mdev->mmio;
	u64 apic_icr_offset = MIC_X100_SBOX_APICICR0 + doorbell * 8;
121 122
	u32 apicicr_low = mic_mmio_read(mw, MIC_X100_SBOX_BASE_ADDRESS +
					apic_icr_offset);
123 124 125 126 127 128 129

	/* for MIC we need to make sure we "hit" the send_icr bit (13) */
	apicicr_low = (apicicr_low | (1 << 13));

	/* Ensure that the interrupt is ordered w.r.t. previous stores. */
	wmb();
	mic_mmio_write(mw, apicicr_low,
130
		       MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
131 132 133 134 135 136 137
}

/**
 * mic_x100_send_rdmasr_intr - Send an RDMASR interrupt to MIC.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_send_rdmasr_intr(struct mic_device *mdev,
138
				      int doorbell)
139 140 141 142 143
{
	int rdmasr_offset = MIC_X100_SBOX_RDMASR0 + (doorbell << 2);
	/* Ensure that the interrupt is ordered w.r.t. previous stores. */
	wmb();
	mic_mmio_write(&mdev->mmio, 0,
144
		       MIC_X100_SBOX_BASE_ADDRESS + rdmasr_offset);
145 146 147 148 149 150 151 152 153 154 155 156 157
}

/**
 * __mic_x100_send_intr - Send interrupt to MIC.
 * @mdev: pointer to mic_device instance
 * @doorbell: doorbell number.
 */
static void mic_x100_send_intr(struct mic_device *mdev, int doorbell)
{
	int rdmasr_db;
	if (doorbell < MIC_X100_NUM_SBOX_IRQ) {
		mic_x100_send_sbox_intr(mdev, doorbell);
	} else {
158
		rdmasr_db = doorbell - MIC_X100_NUM_SBOX_IRQ;
159 160 161 162 163
		mic_x100_send_rdmasr_intr(mdev, rdmasr_db);
	}
}

/**
164 165 166
 * mic_x100_ack_interrupt - Read the interrupt sources register and
 * clear it. This function will be called in the MSI/INTx case.
 * @mdev: Pointer to mic_device instance.
167
 *
168
 * Returns: bitmask of interrupt sources triggered.
169 170 171 172
 */
static u32 mic_x100_ack_interrupt(struct mic_device *mdev)
{
	u32 sicr0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICR0;
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	u32 reg = mic_mmio_read(&mdev->mmio, sicr0);
	mic_mmio_write(&mdev->mmio, reg, sicr0);
	return reg;
}

/**
 * mic_x100_intr_workarounds - These hardware specific workarounds are
 * to be invoked everytime an interrupt is handled.
 * @mdev: Pointer to mic_device instance.
 *
 * Returns: none
 */
static void mic_x100_intr_workarounds(struct mic_device *mdev)
{
	struct mic_mw *mw = &mdev->mmio;
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204

	/* Clear pending bit array. */
	if (MIC_A0_STEP == mdev->stepping)
		mic_mmio_write(mw, 1, MIC_X100_SBOX_BASE_ADDRESS +
			MIC_X100_SBOX_MSIXPBACR);

	if (mdev->stepping >= MIC_B0_STEP)
		mdev->intr_ops->enable_interrupts(mdev);
}

/**
 * mic_x100_hw_intr_init - Initialize h/w specific interrupt
 * information.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_hw_intr_init(struct mic_device *mdev)
{
205
	mdev->intr_info = (struct mic_intr_info *)mic_x100_intr_init;
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
}

/**
 * mic_x100_read_msi_to_src_map - read from the MSI mapping registers
 * @mdev: pointer to mic_device instance
 * @idx: index to the mapping register, 0 based
 *
 * This function allows reading of the 32bit MSI mapping register.
 *
 * RETURNS: The value in the register.
 */
static u32
mic_x100_read_msi_to_src_map(struct mic_device *mdev, int idx)
{
	return mic_mmio_read(&mdev->mmio,
		MIC_X100_SBOX_BASE_ADDRESS +
		MIC_X100_SBOX_MXAR0 + idx * 4);
}

/**
 * mic_x100_program_msi_to_src_map - program the MSI mapping registers
 * @mdev: pointer to mic_device instance
 * @idx: index to the mapping register, 0 based
 * @offset: The bit offset in the register that needs to be updated.
 * @set: boolean specifying if the bit in the specified offset needs
 * to be set or cleared.
 *
 * RETURNS: None.
 */
static void
mic_x100_program_msi_to_src_map(struct mic_device *mdev,
237
				int idx, int offset, bool set)
238 239 240 241 242 243 244 245 246 247 248 249 250 251
{
	unsigned long reg;
	struct mic_mw *mw = &mdev->mmio;
	u32 mxar = MIC_X100_SBOX_BASE_ADDRESS +
		MIC_X100_SBOX_MXAR0 + idx * 4;

	reg = mic_mmio_read(mw, mxar);
	if (set)
		__set_bit(offset, &reg);
	else
		__clear_bit(offset, &reg);
	mic_mmio_write(mw, reg, mxar);
}

252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
/*
 * mic_x100_reset_fw_ready - Reset Firmware ready status field.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_reset_fw_ready(struct mic_device *mdev)
{
	mdev->ops->write_spad(mdev, MIC_X100_DOWNLOAD_INFO, 0);
}

/*
 * mic_x100_is_fw_ready - Check if firmware is ready.
 * @mdev: pointer to mic_device instance
 */
static bool mic_x100_is_fw_ready(struct mic_device *mdev)
{
	u32 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
	return MIC_X100_SPAD2_DOWNLOAD_STATUS(scratch2) ? true : false;
}

/**
 * mic_x100_get_apic_id - Get bootstrap APIC ID.
 * @mdev: pointer to mic_device instance
 */
static u32 mic_x100_get_apic_id(struct mic_device *mdev)
{
	u32 scratch2 = 0;

	scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
	return MIC_X100_SPAD2_APIC_ID(scratch2);
}

/**
 * mic_x100_send_firmware_intr - Send an interrupt to the firmware on MIC.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_send_firmware_intr(struct mic_device *mdev)
{
	u32 apicicr_low;
	u64 apic_icr_offset = MIC_X100_SBOX_APICICR7;
	int vector = MIC_X100_BSP_INTERRUPT_VECTOR;
	struct mic_mw *mw = &mdev->mmio;

	/*
	 * For MIC we need to make sure we "hit"
	 * the send_icr bit (13).
	 */
	apicicr_low = (vector | (1 << 13));

	mic_mmio_write(mw, mic_x100_get_apic_id(mdev),
301
		       MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset + 4);
302 303 304 305

	/* Ensure that the interrupt is ordered w.r.t. previous stores. */
	wmb();
	mic_mmio_write(mw, apicicr_low,
306
		       MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
}

/**
 * mic_x100_hw_reset - Reset the MIC device.
 * @mdev: pointer to mic_device instance
 */
static void mic_x100_hw_reset(struct mic_device *mdev)
{
	u32 reset_reg;
	u32 rgcr = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_RGCR;
	struct mic_mw *mw = &mdev->mmio;

	/* Ensure that the reset is ordered w.r.t. previous loads and stores */
	mb();
	/* Trigger reset */
	reset_reg = mic_mmio_read(mw, rgcr);
	reset_reg |= 0x1;
	mic_mmio_write(mw, reset_reg, rgcr);
	/*
	 * It seems we really want to delay at least 1 second
	 * after touching reset to prevent a lot of problems.
	 */
	msleep(1000);
}

/**
 * mic_x100_load_command_line - Load command line to MIC.
 * @mdev: pointer to mic_device instance
 * @fw: the firmware image
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int
mic_x100_load_command_line(struct mic_device *mdev, const struct firmware *fw)
{
	u32 len = 0;
	u32 boot_mem;
	char *buf;
	void __iomem *cmd_line_va = mdev->aper.va + mdev->bootaddr + fw->size;
#define CMDLINE_SIZE 2048

	boot_mem = mdev->aper.len >> 20;
	buf = kzalloc(CMDLINE_SIZE, GFP_KERNEL);
350
	if (!buf)
351
		return -ENOMEM;
352

353 354
	len += snprintf(buf, CMDLINE_SIZE - len,
		" mem=%dM", boot_mem);
355 356 357
	if (mdev->cosm_dev->cmdline)
		snprintf(buf + len, CMDLINE_SIZE - len, " %s",
			 mdev->cosm_dev->cmdline);
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
	memcpy_toio(cmd_line_va, buf, strlen(buf) + 1);
	kfree(buf);
	return 0;
}

/**
 * mic_x100_load_ramdisk - Load ramdisk to MIC.
 * @mdev: pointer to mic_device instance
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int
mic_x100_load_ramdisk(struct mic_device *mdev)
{
	const struct firmware *fw;
	int rc;
	struct boot_params __iomem *bp = mdev->aper.va + mdev->bootaddr;

376
	rc = request_firmware(&fw, mdev->cosm_dev->ramdisk, &mdev->pdev->dev);
377
	if (rc < 0) {
378
		dev_err(&mdev->pdev->dev,
379
			"ramdisk request_firmware failed: %d %s\n",
380
			rc, mdev->cosm_dev->ramdisk);
381 382 383 384 385 386
		goto error;
	}
	/*
	 * Typically the bootaddr for card OS is 64M
	 * so copy over the ramdisk @ 128M.
	 */
387
	memcpy_toio(mdev->aper.va + (mdev->bootaddr << 1), fw->data, fw->size);
A
Ashutosh Dixit 已提交
388 389
	iowrite32(mdev->bootaddr << 1, &bp->hdr.ramdisk_image);
	iowrite32(fw->size, &bp->hdr.ramdisk_size);
390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
	release_firmware(fw);
error:
	return rc;
}

/**
 * mic_x100_get_boot_addr - Get MIC boot address.
 * @mdev: pointer to mic_device instance
 *
 * This function is called during firmware load to determine
 * the address at which the OS should be downloaded in card
 * memory i.e. GDDR.
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int
mic_x100_get_boot_addr(struct mic_device *mdev)
{
	u32 scratch2, boot_addr;
	int rc = 0;

	scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
	boot_addr = MIC_X100_SPAD2_DOWNLOAD_ADDR(scratch2);
412
	dev_dbg(&mdev->pdev->dev, "%s %d boot_addr 0x%x\n",
413 414
		__func__, __LINE__, boot_addr);
	if (boot_addr > (1 << 31)) {
415
		dev_err(&mdev->pdev->dev,
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
			"incorrect bootaddr 0x%x\n",
			boot_addr);
		rc = -EINVAL;
		goto error;
	}
	mdev->bootaddr = boot_addr;
error:
	return rc;
}

/**
 * mic_x100_load_firmware - Load firmware to MIC.
 * @mdev: pointer to mic_device instance
 * @buf: buffer containing boot string including firmware/ramdisk path.
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
static int
mic_x100_load_firmware(struct mic_device *mdev, const char *buf)
{
	int rc;
	const struct firmware *fw;

	rc = mic_x100_get_boot_addr(mdev);
	if (rc)
441
		return rc;
442
	/* load OS */
443
	rc = request_firmware(&fw, mdev->cosm_dev->firmware, &mdev->pdev->dev);
444
	if (rc < 0) {
445
		dev_err(&mdev->pdev->dev,
446
			"ramdisk request_firmware failed: %d %s\n",
447
			rc, mdev->cosm_dev->firmware);
448
		return rc;
449 450 451
	}
	if (mdev->bootaddr > mdev->aper.len - fw->size) {
		rc = -EINVAL;
452
		dev_err(&mdev->pdev->dev, "%s %d rc %d bootaddr 0x%x\n",
453
			__func__, __LINE__, rc, mdev->bootaddr);
454
		goto error;
455 456 457
	}
	memcpy_toio(mdev->aper.va + mdev->bootaddr, fw->data, fw->size);
	mdev->ops->write_spad(mdev, MIC_X100_FW_SIZE, fw->size);
458 459 460 461
	if (!strcmp(mdev->cosm_dev->bootmode, "flash")) {
		rc = -EINVAL;
		dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
			__func__, __LINE__, rc);
462
		goto error;
463
	}
464 465 466
	/* load command line */
	rc = mic_x100_load_command_line(mdev, fw);
	if (rc) {
467
		dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
468
			__func__, __LINE__, rc);
469
		goto error;
470 471 472
	}
	release_firmware(fw);
	/* load ramdisk */
473
	if (mdev->cosm_dev->ramdisk)
474
		rc = mic_x100_load_ramdisk(mdev);
475

476 477 478 479
	return rc;

error:
	release_firmware(fw);
480 481 482 483 484 485 486 487 488 489 490 491 492 493
	return rc;
}

/**
 * mic_x100_get_postcode - Get postcode status from firmware.
 * @mdev: pointer to mic_device instance
 *
 * RETURNS: postcode.
 */
static u32 mic_x100_get_postcode(struct mic_device *mdev)
{
	return mic_mmio_read(&mdev->mmio, MIC_X100_POSTCODE);
}

494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
/**
 * mic_x100_smpt_set - Update an SMPT entry with a DMA address.
 * @mdev: pointer to mic_device instance
 *
 * RETURNS: none.
 */
static void
mic_x100_smpt_set(struct mic_device *mdev, dma_addr_t dma_addr, u8 index)
{
#define SNOOP_ON	(0 << 0)
#define SNOOP_OFF	(1 << 0)
/*
 * Sbox Smpt Reg Bits:
 * Bits	31:2	Host address
 * Bits	1	RSVD
 * Bits	0	No snoop
 */
#define BUILD_SMPT(NO_SNOOP, HOST_ADDR)  \
	(u32)(((HOST_ADDR) << 2) | ((NO_SNOOP) & 0x01))

	uint32_t smpt_reg_val = BUILD_SMPT(SNOOP_ON,
			dma_addr >> mdev->smpt->info.page_shift);
	mic_mmio_write(&mdev->mmio, smpt_reg_val,
517 518
		       MIC_X100_SBOX_BASE_ADDRESS +
		       MIC_X100_SBOX_SMPT00 + (4 * index));
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
}

/**
 * mic_x100_smpt_hw_init - Initialize SMPT X100 specific fields.
 * @mdev: pointer to mic_device instance
 *
 * RETURNS: none.
 */
static void mic_x100_smpt_hw_init(struct mic_device *mdev)
{
	struct mic_smpt_hw_info *info = &mdev->smpt->info;

	info->num_reg = 32;
	info->page_shift = 34;
	info->page_size = (1ULL << info->page_shift);
	info->base = 0x8000000000ULL;
}

struct mic_smpt_ops mic_x100_smpt_ops = {
	.init = mic_x100_smpt_hw_init,
	.set = mic_x100_smpt_set,
};

542 543 544 545 546 547 548
static bool mic_x100_dma_filter(struct dma_chan *chan, void *param)
{
	if (chan->device->dev->parent == (struct device *)param)
		return true;
	return false;
}

549 550 551 552 553
struct mic_hw_ops mic_x100_ops = {
	.aper_bar = MIC_X100_APER_BAR,
	.mmio_bar = MIC_X100_MMIO_BAR,
	.read_spad = mic_x100_read_spad,
	.write_spad = mic_x100_write_spad,
554 555
	.send_intr = mic_x100_send_intr,
	.ack_interrupt = mic_x100_ack_interrupt,
556
	.intr_workarounds = mic_x100_intr_workarounds,
557 558 559 560 561 562
	.reset = mic_x100_hw_reset,
	.reset_fw_ready = mic_x100_reset_fw_ready,
	.is_fw_ready = mic_x100_is_fw_ready,
	.send_firmware_intr = mic_x100_send_firmware_intr,
	.load_mic_fw = mic_x100_load_firmware,
	.get_postcode = mic_x100_get_postcode,
563
	.dma_filter = mic_x100_dma_filter,
564 565 566 567 568 569 570 571
};

struct mic_hw_intr_ops mic_x100_intr_ops = {
	.intr_init = mic_x100_hw_intr_init,
	.enable_interrupts = mic_x100_enable_interrupts,
	.disable_interrupts = mic_x100_disable_interrupts,
	.program_msi_to_src_map = mic_x100_program_msi_to_src_map,
	.read_msi_to_src_map = mic_x100_read_msi_to_src_map,
572
};