native.c 29.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright 2014 IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mutex.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <asm/synch.h>
18
#include <misc/cxl-base.h>
19 20

#include "cxl.h"
I
Ian Munsie 已提交
21
#include "trace.h"
22 23 24 25 26 27

static int afu_control(struct cxl_afu *afu, u64 command,
		       u64 result, u64 mask, bool enabled)
{
	u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
28
	int rc = 0;
29 30 31 32

	spin_lock(&afu->afu_cntl_lock);
	pr_devel("AFU command starting: %llx\n", command);

I
Ian Munsie 已提交
33 34
	trace_cxl_afu_ctrl(afu, command);

35 36 37 38 39 40
	cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command);

	AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	while ((AFU_Cntl & mask) != result) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
I
Ian Munsie 已提交
41 42
			rc = -EBUSY;
			goto out;
43
		}
44

45
		if (!cxl_ops->link_ok(afu->adapter, afu)) {
46 47 48 49 50
			afu->enabled = enabled;
			rc = -EIO;
			goto out;
		}

51
		pr_devel_ratelimited("AFU control... (0x%016llx)\n",
52 53 54 55 56 57
				     AFU_Cntl | command);
		cpu_relax();
		AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	};
	pr_devel("AFU command complete: %llx\n", command);
	afu->enabled = enabled;
I
Ian Munsie 已提交
58 59
out:
	trace_cxl_afu_ctrl_done(afu, command, rc);
60 61
	spin_unlock(&afu->afu_cntl_lock);

I
Ian Munsie 已提交
62
	return rc;
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
}

static int afu_enable(struct cxl_afu *afu)
{
	pr_devel("AFU enable request\n");

	return afu_control(afu, CXL_AFU_Cntl_An_E,
			   CXL_AFU_Cntl_An_ES_Enabled,
			   CXL_AFU_Cntl_An_ES_MASK, true);
}

int cxl_afu_disable(struct cxl_afu *afu)
{
	pr_devel("AFU disable request\n");

	return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled,
			   CXL_AFU_Cntl_An_ES_MASK, false);
}

/* This will disable as well as reset */
83
static int native_afu_reset(struct cxl_afu *afu)
84 85 86 87 88 89 90 91 92
{
	pr_devel("AFU reset request\n");

	return afu_control(afu, CXL_AFU_Cntl_An_RA,
			   CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
			   CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
			   false);
}

93
static int native_afu_check_and_enable(struct cxl_afu *afu)
94
{
95
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
96 97 98
		WARN(1, "Refusing to enable afu while link down!\n");
		return -EIO;
	}
99 100 101 102 103 104 105 106 107 108 109 110
	if (afu->enabled)
		return 0;
	return afu_enable(afu);
}

int cxl_psl_purge(struct cxl_afu *afu)
{
	u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
	u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	u64 dsisr, dar;
	u64 start, end;
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
111 112 113
	int rc = 0;

	trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
114 115 116

	pr_devel("PSL purge request\n");

117
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
118 119 120 121 122
		dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
		rc = -EIO;
		goto out;
	}

123 124 125 126 127 128 129 130 131 132 133 134 135
	if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
		WARN(1, "psl_purge request while AFU not disabled!\n");
		cxl_afu_disable(afu);
	}

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
		       PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
	start = local_clock();
	PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
	while ((PSL_CNTL &  CXL_PSL_SCNTL_An_Ps_MASK)
			== CXL_PSL_SCNTL_An_Ps_Pending) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
I
Ian Munsie 已提交
136 137
			rc = -EBUSY;
			goto out;
138
		}
139
		if (!cxl_ops->link_ok(afu->adapter, afu)) {
140 141 142 143
			rc = -EIO;
			goto out;
		}

144
		dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
145
		pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx  PSL_DSISR: 0x%016llx\n", PSL_CNTL, dsisr);
146 147
		if (dsisr & CXL_PSL_DSISR_TRANS) {
			dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
148
			dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", dsisr, dar);
149 150
			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
		} else if (dsisr) {
151
			dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", dsisr);
152 153 154 155 156 157 158 159 160 161 162
			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
		} else {
			cpu_relax();
		}
		PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
	};
	end = local_clock();
	pr_devel("PSL purged in %lld ns\n", end - start);

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
		       PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
I
Ian Munsie 已提交
163 164 165
out:
	trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc);
	return rc;
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
}

static int spa_max_procs(int spa_size)
{
	/*
	 * From the CAIA:
	 *    end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
	 * Most of that junk is really just an overly-complicated way of saying
	 * the last 256 bytes are __aligned(128), so it's really:
	 *    end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
	 * and
	 *    end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
	 * so
	 *    sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
	 * Ignore the alignment (which is safe in this case as long as we are
	 * careful with our rounding) and solve for n:
	 */
	return ((spa_size / 8) - 96) / 17;
}

186
int cxl_alloc_spa(struct cxl_afu *afu)
187 188
{
	/* Work out how many pages to allocate */
189
	afu->native->spa_order = 0;
190
	do {
191 192 193 194
		afu->native->spa_order++;
		afu->native->spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;
		afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);
	} while (afu->native->spa_max_procs < afu->num_procs);
195

196
	WARN_ON(afu->native->spa_size > 0x100000); /* Max size supported by the hardware */
197

198 199
	if (!(afu->native->spa = (struct cxl_process_element *)
	      __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {
200 201 202 203
		pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
		return -ENOMEM;
	}
	pr_devel("spa pages: %i afu->spa_max_procs: %i   afu->num_procs: %i\n",
204
		 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);
205

206 207 208 209 210 211 212
	return 0;
}

static void attach_spa(struct cxl_afu *afu)
{
	u64 spap;

213 214
	afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +
					    ((afu->native->spa_max_procs + 3) * 128));
215

216 217
	spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;
	spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
218
	spap |= CXL_PSL_SPAP_V;
219 220 221
	pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n",
		afu->native->spa, afu->native->spa_max_procs,
		afu->native->sw_command_status, spap);
222 223 224
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
}

225
static inline void detach_spa(struct cxl_afu *afu)
226
{
227
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);
228 229 230 231
}

void cxl_release_spa(struct cxl_afu *afu)
{
232 233 234 235
	if (afu->native->spa) {
		free_pages((unsigned long) afu->native->spa,
			afu->native->spa_order);
		afu->native->spa = NULL;
236
	}
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
}

int cxl_tlb_slb_invalidate(struct cxl *adapter)
{
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);

	pr_devel("CXL adapter wide TLBIA & SLBIA\n");

	cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);

	cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
	while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
			return -EBUSY;
		}
253
		if (!cxl_ops->link_ok(adapter, NULL))
254
			return -EIO;
255 256 257 258 259 260 261 262 263
		cpu_relax();
	}

	cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
	while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
			return -EBUSY;
		}
264
		if (!cxl_ops->link_ok(adapter, NULL))
265
			return -EIO;
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
		cpu_relax();
	}
	return 0;
}

static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
{
	int rc;

	/* 1. Disable SSTP by writing 0 to SSTP1[V] */
	cxl_p2n_write(afu, CXL_SSTP1_An, 0);

	/* 2. Invalidate all SLB entries */
	if ((rc = cxl_afu_slbia(afu)))
		return rc;

	/* 3. Set SSTP0_An */
	cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);

	/* 4. Set SSTP1_An */
	cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);

	return 0;
}

/* Using per slice version may improve performance here. (ie. SLBIA_An) */
static void slb_invalid(struct cxl_context *ctx)
{
	struct cxl *adapter = ctx->afu->adapter;
	u64 slbia;

297
	WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));
298 299 300 301 302 303 304

	cxl_p1_write(adapter, CXL_PSL_LBISEL,
			((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
			be32_to_cpu(ctx->elem->lpid));
	cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);

	while (1) {
305
		if (!cxl_ops->link_ok(adapter, NULL))
306
			break;
307 308 309 310 311 312 313 314 315 316 317
		slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
		if (!(slbia & CXL_TLB_SLB_P))
			break;
		cpu_relax();
	}
}

static int do_process_element_cmd(struct cxl_context *ctx,
				  u64 cmd, u64 pe_state)
{
	u64 state;
318
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
319 320 321
	int rc = 0;

	trace_cxl_llcmd(ctx, cmd);
322 323 324 325 326

	WARN_ON(!ctx->afu->enabled);

	ctx->elem->software_state = cpu_to_be32(pe_state);
	smp_wmb();
327
	*(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
328 329 330
	smp_mb();
	cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
	while (1) {
331 332
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
I
Ian Munsie 已提交
333 334
			rc = -EBUSY;
			goto out;
335
		}
336
		if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
337 338 339 340
			dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
			rc = -EIO;
			goto out;
		}
341
		state = be64_to_cpup(ctx->afu->native->sw_command_status);
342 343
		if (state == ~0ULL) {
			pr_err("cxl: Error adding process element to AFU\n");
I
Ian Munsie 已提交
344 345
			rc = -1;
			goto out;
346 347 348 349 350 351 352 353 354 355 356 357 358 359
		}
		if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK  | CXL_SPA_SW_LINK_MASK)) ==
		    (cmd | (cmd >> 16) | ctx->pe))
			break;
		/*
		 * The command won't finish in the PSL if there are
		 * outstanding DSIs.  Hence we need to yield here in
		 * case there are outstanding DSIs that we need to
		 * service.  Tuning possiblity: we could wait for a
		 * while before sched
		 */
		schedule();

	}
I
Ian Munsie 已提交
360 361 362
out:
	trace_cxl_llcmd_done(ctx, cmd, rc);
	return rc;
363 364 365 366 367 368
}

static int add_process_element(struct cxl_context *ctx)
{
	int rc = 0;

369
	mutex_lock(&ctx->afu->native->spa_mutex);
370 371 372 373
	pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
	if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
		ctx->pe_inserted = true;
	pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
374
	mutex_unlock(&ctx->afu->native->spa_mutex);
375 376 377 378 379 380 381 382 383 384 385
	return rc;
}

static int terminate_process_element(struct cxl_context *ctx)
{
	int rc = 0;

	/* fast path terminate if it's already invalid */
	if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
		return rc;

386
	mutex_lock(&ctx->afu->native->spa_mutex);
387
	pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
388 389 390 391
	/* We could be asked to terminate when the hw is down. That
	 * should always succeed: it's not running if the hw has gone
	 * away and is being reset.
	 */
392
	if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
393 394
		rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
					    CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
395 396
	ctx->elem->software_state = 0;	/* Remove Valid bit */
	pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
397
	mutex_unlock(&ctx->afu->native->spa_mutex);
398 399 400 401 402 403 404
	return rc;
}

static int remove_process_element(struct cxl_context *ctx)
{
	int rc = 0;

405
	mutex_lock(&ctx->afu->native->spa_mutex);
406
	pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
407 408 409 410

	/* We could be asked to remove when the hw is down. Again, if
	 * the hw is down, the PE is gone, so we succeed.
	 */
411
	if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
412 413 414
		rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0);

	if (!rc)
415 416 417
		ctx->pe_inserted = false;
	slb_invalid(ctx);
	pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
418
	mutex_unlock(&ctx->afu->native->spa_mutex);
419 420 421 422 423

	return rc;
}


M
Michael Neuling 已提交
424
void cxl_assign_psn_space(struct cxl_context *ctx)
425 426 427 428 429 430
{
	if (!ctx->afu->pp_size || ctx->master) {
		ctx->psn_phys = ctx->afu->psn_phys;
		ctx->psn_size = ctx->afu->adapter->ps_size;
	} else {
		ctx->psn_phys = ctx->afu->psn_phys +
431
			(ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);
432 433 434 435 436 437 438 439 440 441
		ctx->psn_size = ctx->afu->pp_size;
	}
}

static int activate_afu_directed(struct cxl_afu *afu)
{
	int rc;

	dev_info(&afu->dev, "Activating AFU directed mode\n");

442
	afu->num_procs = afu->max_procs_virtualised;
443
	if (afu->native->spa == NULL) {
444 445 446 447
		if (cxl_alloc_spa(afu))
			return -ENOMEM;
	}
	attach_spa(afu);
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
	cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
	cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);

	afu->current_mode = CXL_MODE_DIRECTED;

	if ((rc = cxl_chardev_m_afu_add(afu)))
		return rc;

	if ((rc = cxl_sysfs_afu_m_add(afu)))
		goto err;

	if ((rc = cxl_chardev_s_afu_add(afu)))
		goto err1;

	return 0;
err1:
	cxl_sysfs_afu_m_remove(afu);
err:
	cxl_chardev_afu_remove(afu);
	return rc;
}

#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
#else
#define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
#endif

478 479 480 481
static u64 calculate_sr(struct cxl_context *ctx)
{
	u64 sr = 0;

482
	set_endian(sr);
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
	if (ctx->master)
		sr |= CXL_PSL_SR_An_MP;
	if (mfspr(SPRN_LPCR) & LPCR_TC)
		sr |= CXL_PSL_SR_An_TC;
	if (ctx->kernel) {
		sr |= CXL_PSL_SR_An_R | (mfmsr() & MSR_SF);
		sr |= CXL_PSL_SR_An_HV;
	} else {
		sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
		sr &= ~(CXL_PSL_SR_An_HV);
		if (!test_tsk_thread_flag(current, TIF_32BIT))
			sr |= CXL_PSL_SR_An_SF;
	}
	return sr;
}

499 500
static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
{
501
	u32 pid;
502 503
	int r, result;

M
Michael Neuling 已提交
504
	cxl_assign_psn_space(ctx);
505 506 507 508 509 510

	ctx->elem->ctxtime = 0; /* disable */
	ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
	ctx->elem->haurp = 0; /* disable */
	ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1));

511 512 513
	pid = current->pid;
	if (ctx->kernel)
		pid = 0;
514
	ctx->elem->common.tid = 0;
515 516 517
	ctx->elem->common.pid = cpu_to_be32(pid);

	ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536

	ctx->elem->common.csrp = 0; /* disable */
	ctx->elem->common.aurp0 = 0; /* disable */
	ctx->elem->common.aurp1 = 0; /* disable */

	cxl_prefault(ctx, wed);

	ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0);
	ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1);

	for (r = 0; r < CXL_IRQ_RANGES; r++) {
		ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
		ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
	}

	ctx->elem->common.amr = cpu_to_be64(amr);
	ctx->elem->common.wed = cpu_to_be64(wed);

	/* first guy needs to enable */
537
	if ((result = cxl_ops->afu_check_and_enable(ctx->afu)))
538 539
		return result;

540
	return add_process_element(ctx);
541 542 543 544 545 546 547 548 549 550 551 552
}

static int deactivate_afu_directed(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Deactivating AFU directed mode\n");

	afu->current_mode = 0;
	afu->num_procs = 0;

	cxl_sysfs_afu_m_remove(afu);
	cxl_chardev_afu_remove(afu);

553
	cxl_ops->afu_reset(afu);
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
	cxl_afu_disable(afu);
	cxl_psl_purge(afu);

	return 0;
}

static int activate_dedicated_process(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Activating dedicated process mode\n");

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);

	cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);    /* disable */
	cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
	cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
	cxl_p1n_write(afu, CXL_HAURP_An, 0);       /* disable */
	cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));

	cxl_p2n_write(afu, CXL_CSRP_An, 0);        /* disable */
	cxl_p2n_write(afu, CXL_AURP0_An, 0);       /* disable */
	cxl_p2n_write(afu, CXL_AURP1_An, 0);       /* disable */

	afu->current_mode = CXL_MODE_DEDICATED;
	afu->num_procs = 1;

	return cxl_chardev_d_afu_add(afu);
}

static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr)
{
	struct cxl_afu *afu = ctx->afu;
586
	u64 pid;
587 588
	int rc;

589 590 591 592 593 594
	pid = (u64)current->pid << 32;
	if (ctx->kernel)
		pid = 0;
	cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid);

	cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx));
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614

	if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
		return rc;

	cxl_prefault(ctx, wed);

	cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
		       (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
		       (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
		       (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
			((u64)ctx->irqs.offset[3] & 0xffff));
	cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
		       (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
		       (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
		       (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
			((u64)ctx->irqs.range[3] & 0xffff));

	cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);

	/* master only context for dedicated */
M
Michael Neuling 已提交
615
	cxl_assign_psn_space(ctx);
616

617
	if ((rc = cxl_ops->afu_reset(afu)))
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
		return rc;

	cxl_p2n_write(afu, CXL_PSL_WED_An, wed);

	return afu_enable(afu);
}

static int deactivate_dedicated_process(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Deactivating dedicated process mode\n");

	afu->current_mode = 0;
	afu->num_procs = 0;

	cxl_chardev_afu_remove(afu);

	return 0;
}

637
static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode)
638 639 640 641 642 643 644 645
{
	if (mode == CXL_MODE_DIRECTED)
		return deactivate_afu_directed(afu);
	if (mode == CXL_MODE_DEDICATED)
		return deactivate_dedicated_process(afu);
	return 0;
}

646
static int native_afu_activate_mode(struct cxl_afu *afu, int mode)
647 648 649 650 651 652
{
	if (!mode)
		return 0;
	if (!(mode & afu->modes_supported))
		return -EINVAL;

653
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
654 655 656 657
		WARN(1, "Device link is down, refusing to activate!\n");
		return -EIO;
	}

658 659 660 661 662 663 664 665
	if (mode == CXL_MODE_DIRECTED)
		return activate_afu_directed(afu);
	if (mode == CXL_MODE_DEDICATED)
		return activate_dedicated_process(afu);

	return -EINVAL;
}

666 667
static int native_attach_process(struct cxl_context *ctx, bool kernel,
				u64 wed, u64 amr)
668
{
669
	if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
670 671 672 673
		WARN(1, "Device link is down, refusing to attach process!\n");
		return -EIO;
	}

674 675 676 677 678 679 680 681 682 683 684 685
	ctx->kernel = kernel;
	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
		return attach_afu_directed(ctx, wed, amr);

	if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
		return attach_dedicated(ctx, wed, amr);

	return -EINVAL;
}

static inline int detach_process_native_dedicated(struct cxl_context *ctx)
{
686
	cxl_ops->afu_reset(ctx->afu);
687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
	cxl_afu_disable(ctx->afu);
	cxl_psl_purge(ctx->afu);
	return 0;
}

static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
{
	if (!ctx->pe_inserted)
		return 0;
	if (terminate_process_element(ctx))
		return -1;
	if (remove_process_element(ctx))
		return -1;

	return 0;
}

704
static int native_detach_process(struct cxl_context *ctx)
705
{
I
Ian Munsie 已提交
706 707
	trace_cxl_detach(ctx);

708 709 710 711 712 713
	if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
		return detach_process_native_dedicated(ctx);

	return detach_process_native_afu_directed(ctx);
}

714
static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
715 716 717
{
	u64 pidtid;

718 719 720
	/* If the adapter has gone away, we can't get any meaningful
	 * information.
	 */
721
	if (!cxl_ops->link_ok(afu->adapter, afu))
722 723
		return -EIO;

724 725 726 727
	info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
	info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
	info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
	pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An);
728 729
	info->pid = pidtid >> 32;
	info->tid = pidtid & 0xffffffff;
730 731
	info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
	info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
732
	info->proc_handle = 0;
733 734 735 736

	return 0;
}

737 738
static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
						u64 dsisr, u64 errstat)
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757
{
	u64 fir1, fir2, fir_slice, serr, afu_debug;

	fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
	fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
	fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
	serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
	afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);

	dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
	dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
	dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
	dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
	dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
	dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);

	dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
	cxl_stop_trace(ctx->afu->adapter);

758
	return cxl_ops->ack_irq(ctx, 0, errstat);
759 760 761 762 763 764 765 766 767 768 769 770
}

static irqreturn_t fail_psl_irq(struct cxl_afu *afu, struct cxl_irq_info *irq_info)
{
	if (irq_info->dsisr & CXL_PSL_DSISR_TRANS)
		cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
	else
		cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);

	return IRQ_HANDLED;
}

771
static irqreturn_t native_irq_multiplexed(int irq, void *data)
772 773 774 775 776 777 778
{
	struct cxl_afu *afu = data;
	struct cxl_context *ctx;
	struct cxl_irq_info irq_info;
	int ph = cxl_p2n_read(afu, CXL_PSL_PEHandle_An) & 0xffff;
	int ret;

779
	if ((ret = native_get_irq_info(afu, &irq_info))) {
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
		WARN(1, "Unable to get CXL IRQ Info: %i\n", ret);
		return fail_psl_irq(afu, &irq_info);
	}

	rcu_read_lock();
	ctx = idr_find(&afu->contexts_idr, ph);
	if (ctx) {
		ret = cxl_irq(irq, ctx, &irq_info);
		rcu_read_unlock();
		return ret;
	}
	rcu_read_unlock();

	WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR"
		" %016llx\n(Possible AFU HW issue - was a term/remove acked"
		" with outstanding transactions?)\n", ph, irq_info.dsisr,
		irq_info.dar);
	return fail_psl_irq(afu, &irq_info);
}

800
static irqreturn_t native_slice_irq_err(int irq, void *data)
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
{
	struct cxl_afu *afu = data;
	u64 fir_slice, errstat, serr, afu_debug;

	WARN(irq, "CXL SLICE ERROR interrupt %i\n", irq);

	serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
	fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
	errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
	afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
	dev_crit(&afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
	dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
	dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat);
	dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);

	cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);

	return IRQ_HANDLED;
}

821
static irqreturn_t native_irq_err(int irq, void *data)
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
{
	struct cxl *adapter = data;
	u64 fir1, fir2, err_ivte;

	WARN(1, "CXL ERROR interrupt %i\n", irq);

	err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
	dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte);

	dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
	cxl_stop_trace(adapter);

	fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
	fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);

	dev_crit(&adapter->dev, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1, fir2);

	return IRQ_HANDLED;
}

842
int cxl_native_register_psl_err_irq(struct cxl *adapter)
843 844 845 846 847 848 849 850
{
	int rc;

	adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
				      dev_name(&adapter->dev));
	if (!adapter->irq_name)
		return -ENOMEM;

851
	if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter,
852 853
				       &adapter->native->err_hwirq,
				       &adapter->native->err_virq,
854 855 856 857 858 859
				       adapter->irq_name))) {
		kfree(adapter->irq_name);
		adapter->irq_name = NULL;
		return rc;
	}

860
	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff);
861 862 863 864

	return 0;
}

865
void cxl_native_release_psl_err_irq(struct cxl *adapter)
866
{
867
	if (adapter->native->err_virq != irq_find_mapping(NULL, adapter->native->err_hwirq))
868 869 870
		return;

	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
871 872
	cxl_unmap_irq(adapter->native->err_virq, adapter);
	cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq);
873 874 875
	kfree(adapter->irq_name);
}

876
int cxl_native_register_serr_irq(struct cxl_afu *afu)
877 878 879 880 881 882 883 884 885
{
	u64 serr;
	int rc;

	afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
				      dev_name(&afu->dev));
	if (!afu->err_irq_name)
		return -ENOMEM;

886
	if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu,
887 888 889 890 891 892 893 894 895 896 897 898 899 900
				       &afu->serr_hwirq,
				       &afu->serr_virq, afu->err_irq_name))) {
		kfree(afu->err_irq_name);
		afu->err_irq_name = NULL;
		return rc;
	}

	serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
	serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
	cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);

	return 0;
}

901
void cxl_native_release_serr_irq(struct cxl_afu *afu)
902 903 904 905 906 907
{
	if (afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq))
		return;

	cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
	cxl_unmap_irq(afu->serr_virq, afu);
908
	cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
909 910 911
	kfree(afu->err_irq_name);
}

912
int cxl_native_register_psl_irq(struct cxl_afu *afu)
913 914 915 916 917 918 919 920
{
	int rc;

	afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s",
				      dev_name(&afu->dev));
	if (!afu->psl_irq_name)
		return -ENOMEM;

921 922
	if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed,
				    afu, &afu->native->psl_hwirq, &afu->native->psl_virq,
923 924 925 926 927 928 929
				    afu->psl_irq_name))) {
		kfree(afu->psl_irq_name);
		afu->psl_irq_name = NULL;
	}
	return rc;
}

930
void cxl_native_release_psl_irq(struct cxl_afu *afu)
931
{
932
	if (afu->native->psl_virq != irq_find_mapping(NULL, afu->native->psl_hwirq))
933 934
		return;

935 936
	cxl_unmap_irq(afu->native->psl_virq, afu);
	cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);
937 938 939
	kfree(afu->psl_irq_name);
}

940 941 942 943
static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
{
	u64 dsisr;

944
	pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat);
945 946 947 948 949 950 951 952 953

	/* Clear PSL_DSISR[PE] */
	dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
	cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);

	/* Write 1s to clear error status bits */
	cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
}

954
static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
955
{
I
Ian Munsie 已提交
956
	trace_cxl_psl_irq_ack(ctx, tfc);
957 958 959 960 961 962 963 964 965 966 967 968
	if (tfc)
		cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
	if (psl_reset_mask)
		recover_psl_err(ctx->afu, psl_reset_mask);

	return 0;
}

int cxl_check_error(struct cxl_afu *afu)
{
	return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
}
969

970 971 972 973 974 975
static bool native_support_attributes(const char *attr_name,
				      enum cxl_attrs type)
{
	return true;
}

976
static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out)
977
{
978
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
979 980 981
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
982
	*out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +
983 984
		(cr * afu->crs_len) + off);
	return 0;
985 986
}

987
static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out)
988
{
989
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
990 991 992
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
993
	*out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +
994 995
		(cr * afu->crs_len) + off);
	return 0;
996 997
}

998
static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out)
999 1000 1001
{
	u64 aligned_off = off & ~0x3L;
	u32 val;
1002
	int rc;
1003

1004
	rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1005 1006 1007
	if (!rc)
		*out = (val >> ((off & 0x3) * 8)) & 0xffff;
	return rc;
1008 1009
}

1010
static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out)
1011 1012 1013
{
	u64 aligned_off = off & ~0x3L;
	u32 val;
1014
	int rc;
1015

1016
	rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1017 1018 1019
	if (!rc)
		*out = (val >> ((off & 0x3) * 8)) & 0xff;
	return rc;
1020
}
1021

1022 1023
static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
{
1024
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
	out_le32(afu->native->afu_desc_mmio + afu->crs_offset +
		(cr * afu->crs_len) + off, in);
	return 0;
}

static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
{
	u64 aligned_off = off & ~0x3L;
	u32 val32, mask, shift;
	int rc;

	rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
	if (rc)
		return rc;
	shift = (off & 0x3) * 8;
	WARN_ON(shift == 24);
	mask = 0xffff << shift;
	val32 = (val32 & ~mask) | (in << shift);

	rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
	return rc;
}

static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
{
	u64 aligned_off = off & ~0x3L;
	u32 val32, mask, shift;
	int rc;

	rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
	if (rc)
		return rc;
	shift = (off & 0x3) * 8;
	mask = 0xff << shift;
	val32 = (val32 & ~mask) | (in << shift);

	rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
	return rc;
}

1068 1069
const struct cxl_backend_ops cxl_native_ops = {
	.module = THIS_MODULE,
1070 1071 1072 1073 1074 1075 1076
	.adapter_reset = cxl_pci_reset,
	.alloc_one_irq = cxl_pci_alloc_one_irq,
	.release_one_irq = cxl_pci_release_one_irq,
	.alloc_irq_ranges = cxl_pci_alloc_irq_ranges,
	.release_irq_ranges = cxl_pci_release_irq_ranges,
	.setup_irq = cxl_pci_setup_irq,
	.handle_psl_slice_error = native_handle_psl_slice_error,
1077
	.psl_interrupt = NULL,
1078 1079 1080
	.ack_irq = native_ack_irq,
	.attach_process = native_attach_process,
	.detach_process = native_detach_process,
1081
	.support_attributes = native_support_attributes,
1082
	.link_ok = cxl_adapter_link_ok,
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
	.release_afu = cxl_pci_release_afu,
	.afu_read_err_buffer = cxl_pci_afu_read_err_buffer,
	.afu_check_and_enable = native_afu_check_and_enable,
	.afu_activate_mode = native_afu_activate_mode,
	.afu_deactivate_mode = native_afu_deactivate_mode,
	.afu_reset = native_afu_reset,
	.afu_cr_read8 = native_afu_cr_read8,
	.afu_cr_read16 = native_afu_cr_read16,
	.afu_cr_read32 = native_afu_cr_read32,
	.afu_cr_read64 = native_afu_cr_read64,
1093 1094 1095 1096
	.afu_cr_write8 = native_afu_cr_write8,
	.afu_cr_write16 = native_afu_cr_write16,
	.afu_cr_write32 = native_afu_cr_write32,
	.read_adapter_vpd = cxl_pci_read_adapter_vpd,
1097
};