native.c 42.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Copyright 2014 IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/spinlock.h>
#include <linux/sched.h>
12
#include <linux/sched/clock.h>
13 14 15 16
#include <linux/slab.h>
#include <linux/mutex.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
17
#include <linux/delay.h>
18
#include <asm/synch.h>
19
#include <misc/cxl-base.h>
20 21

#include "cxl.h"
I
Ian Munsie 已提交
22
#include "trace.h"
23

24
static int afu_control(struct cxl_afu *afu, u64 command, u64 clear,
25 26
		       u64 result, u64 mask, bool enabled)
{
27
	u64 AFU_Cntl;
28
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
29
	int rc = 0;
30 31 32 33

	spin_lock(&afu->afu_cntl_lock);
	pr_devel("AFU command starting: %llx\n", command);

I
Ian Munsie 已提交
34 35
	trace_cxl_afu_ctrl(afu, command);

36 37
	AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	cxl_p2n_write(afu, CXL_AFU_Cntl_An, (AFU_Cntl & ~clear) | command);
38 39 40 41 42

	AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	while ((AFU_Cntl & mask) != result) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
I
Ian Munsie 已提交
43 44
			rc = -EBUSY;
			goto out;
45
		}
46

47
		if (!cxl_ops->link_ok(afu->adapter, afu)) {
48 49 50 51 52
			afu->enabled = enabled;
			rc = -EIO;
			goto out;
		}

53
		pr_devel_ratelimited("AFU control... (0x%016llx)\n",
54 55 56
				     AFU_Cntl | command);
		cpu_relax();
		AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
A
Andrew Donnellan 已提交
57
	}
58 59 60 61 62 63 64 65 66 67

	if (AFU_Cntl & CXL_AFU_Cntl_An_RA) {
		/*
		 * Workaround for a bug in the XSL used in the Mellanox CX4
		 * that fails to clear the RA bit after an AFU reset,
		 * preventing subsequent AFU resets from working.
		 */
		cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl & ~CXL_AFU_Cntl_An_RA);
	}

68 69
	pr_devel("AFU command complete: %llx\n", command);
	afu->enabled = enabled;
I
Ian Munsie 已提交
70 71
out:
	trace_cxl_afu_ctrl_done(afu, command, rc);
72 73
	spin_unlock(&afu->afu_cntl_lock);

I
Ian Munsie 已提交
74
	return rc;
75 76 77 78 79 80
}

static int afu_enable(struct cxl_afu *afu)
{
	pr_devel("AFU enable request\n");

81
	return afu_control(afu, CXL_AFU_Cntl_An_E, 0,
82 83 84 85 86 87 88 89
			   CXL_AFU_Cntl_An_ES_Enabled,
			   CXL_AFU_Cntl_An_ES_MASK, true);
}

int cxl_afu_disable(struct cxl_afu *afu)
{
	pr_devel("AFU disable request\n");

90 91
	return afu_control(afu, 0, CXL_AFU_Cntl_An_E,
			   CXL_AFU_Cntl_An_ES_Disabled,
92 93 94 95
			   CXL_AFU_Cntl_An_ES_MASK, false);
}

/* This will disable as well as reset */
96
static int native_afu_reset(struct cxl_afu *afu)
97 98 99
{
	pr_devel("AFU reset request\n");

100
	return afu_control(afu, CXL_AFU_Cntl_An_RA, 0,
101 102 103 104 105
			   CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
			   CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
			   false);
}

106
static int native_afu_check_and_enable(struct cxl_afu *afu)
107
{
108
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
109 110 111
		WARN(1, "Refusing to enable afu while link down!\n");
		return -EIO;
	}
112 113 114 115 116 117 118 119 120 121 122
	if (afu->enabled)
		return 0;
	return afu_enable(afu);
}

int cxl_psl_purge(struct cxl_afu *afu)
{
	u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
	u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
	u64 dsisr, dar;
	u64 start, end;
C
Christophe Lombard 已提交
123
	u64 trans_fault = 0x0ULL;
124
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
125 126 127
	int rc = 0;

	trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
128 129 130

	pr_devel("PSL purge request\n");

C
Christophe Lombard 已提交
131 132 133 134 135
	if (cxl_is_psl8(afu))
		trans_fault = CXL_PSL_DSISR_TRANS;
	if (cxl_is_psl9(afu))
		trans_fault = CXL_PSL9_DSISR_An_TF;

136
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
137 138 139 140 141
		dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
		rc = -EIO;
		goto out;
	}

142 143 144 145 146 147 148 149 150 151 152 153 154
	if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
		WARN(1, "psl_purge request while AFU not disabled!\n");
		cxl_afu_disable(afu);
	}

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
		       PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
	start = local_clock();
	PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
	while ((PSL_CNTL &  CXL_PSL_SCNTL_An_Ps_MASK)
			== CXL_PSL_SCNTL_An_Ps_Pending) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
I
Ian Munsie 已提交
155 156
			rc = -EBUSY;
			goto out;
157
		}
158
		if (!cxl_ops->link_ok(afu->adapter, afu)) {
159 160 161 162
			rc = -EIO;
			goto out;
		}

163
		dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
164 165 166
		pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx  PSL_DSISR: 0x%016llx\n",
				     PSL_CNTL, dsisr);

C
Christophe Lombard 已提交
167
		if (dsisr & trans_fault) {
168
			dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
169 170
			dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n",
				   dsisr, dar);
171 172
			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
		} else if (dsisr) {
173 174
			dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n",
				   dsisr);
175 176 177 178 179
			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
		} else {
			cpu_relax();
		}
		PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
A
Andrew Donnellan 已提交
180
	}
181 182 183 184 185
	end = local_clock();
	pr_devel("PSL purged in %lld ns\n", end - start);

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
		       PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
I
Ian Munsie 已提交
186 187 188
out:
	trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc);
	return rc;
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
}

static int spa_max_procs(int spa_size)
{
	/*
	 * From the CAIA:
	 *    end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
	 * Most of that junk is really just an overly-complicated way of saying
	 * the last 256 bytes are __aligned(128), so it's really:
	 *    end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
	 * and
	 *    end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
	 * so
	 *    sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
	 * Ignore the alignment (which is safe in this case as long as we are
	 * careful with our rounding) and solve for n:
	 */
	return ((spa_size / 8) - 96) / 17;
}

C
Christophe Lombard 已提交
209
static int cxl_alloc_spa(struct cxl_afu *afu, int mode)
210
{
211 212
	unsigned spa_size;

213
	/* Work out how many pages to allocate */
214
	afu->native->spa_order = -1;
215
	do {
216
		afu->native->spa_order++;
217 218 219 220 221
		spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;

		if (spa_size > 0x100000) {
			dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n",
					afu->native->spa_max_procs, afu->native->spa_size);
C
Christophe Lombard 已提交
222 223
			if (mode != CXL_MODE_DEDICATED)
				afu->num_procs = afu->native->spa_max_procs;
224 225 226 227
			break;
		}

		afu->native->spa_size = spa_size;
228 229
		afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);
	} while (afu->native->spa_max_procs < afu->num_procs);
230

231 232
	if (!(afu->native->spa = (struct cxl_process_element *)
	      __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {
233 234 235 236
		pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
		return -ENOMEM;
	}
	pr_devel("spa pages: %i afu->spa_max_procs: %i   afu->num_procs: %i\n",
237
		 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);
238

239 240 241 242 243 244 245
	return 0;
}

static void attach_spa(struct cxl_afu *afu)
{
	u64 spap;

246 247
	afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +
					    ((afu->native->spa_max_procs + 3) * 128));
248

249 250
	spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;
	spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
251
	spap |= CXL_PSL_SPAP_V;
252 253 254
	pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n",
		afu->native->spa, afu->native->spa_max_procs,
		afu->native->sw_command_status, spap);
255 256 257
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
}

258
static inline void detach_spa(struct cxl_afu *afu)
259
{
260
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);
261 262 263 264
}

void cxl_release_spa(struct cxl_afu *afu)
{
265 266 267 268
	if (afu->native->spa) {
		free_pages((unsigned long) afu->native->spa,
			afu->native->spa_order);
		afu->native->spa = NULL;
269
	}
270 271
}

C
Christophe Lombard 已提交
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
/*
 * Invalidation of all ERAT entries is no longer required by CAIA2. Use
 * only for debug.
 */
int cxl_invalidate_all_psl9(struct cxl *adapter)
{
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
	u64 ierat;

	pr_devel("CXL adapter - invalidation of all ERAT entries\n");

	/* Invalidates all ERAT entries for Radix or HPT */
	ierat = CXL_XSL9_IERAT_IALL;
	if (radix_enabled())
		ierat |= CXL_XSL9_IERAT_INVR;
	cxl_p1_write(adapter, CXL_XSL9_IERAT, ierat);

	while (cxl_p1_read(adapter, CXL_XSL9_IERAT) & CXL_XSL9_IERAT_IINPROG) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev,
			"WARNING: CXL adapter invalidation of all ERAT entries timed out!\n");
			return -EBUSY;
		}
		if (!cxl_ops->link_ok(adapter, NULL))
			return -EIO;
		cpu_relax();
	}
	return 0;
}

302
int cxl_invalidate_all_psl8(struct cxl *adapter)
303 304 305 306 307 308 309 310 311 312 313 314 315
{
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);

	pr_devel("CXL adapter wide TLBIA & SLBIA\n");

	cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);

	cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
	while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
			return -EBUSY;
		}
316
		if (!cxl_ops->link_ok(adapter, NULL))
317
			return -EIO;
318 319 320 321 322 323 324 325 326
		cpu_relax();
	}

	cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
	while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
			return -EBUSY;
		}
327
		if (!cxl_ops->link_ok(adapter, NULL))
328
			return -EIO;
329 330 331 332 333
		cpu_relax();
	}
	return 0;
}

334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
int cxl_data_cache_flush(struct cxl *adapter)
{
	u64 reg;
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);

	pr_devel("Flushing data cache\n");

	reg = cxl_p1_read(adapter, CXL_PSL_Control);
	reg |= CXL_PSL_Control_Fr;
	cxl_p1_write(adapter, CXL_PSL_Control, reg);

	reg = cxl_p1_read(adapter, CXL_PSL_Control);
	while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) {
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n");
			return -EBUSY;
		}

		if (!cxl_ops->link_ok(adapter, NULL)) {
			dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n");
			return -EIO;
		}
		cpu_relax();
		reg = cxl_p1_read(adapter, CXL_PSL_Control);
	}

	reg &= ~CXL_PSL_Control_Fr;
	cxl_p1_write(adapter, CXL_PSL_Control, reg);
	return 0;
}

365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
{
	int rc;

	/* 1. Disable SSTP by writing 0 to SSTP1[V] */
	cxl_p2n_write(afu, CXL_SSTP1_An, 0);

	/* 2. Invalidate all SLB entries */
	if ((rc = cxl_afu_slbia(afu)))
		return rc;

	/* 3. Set SSTP0_An */
	cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);

	/* 4. Set SSTP1_An */
	cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);

	return 0;
}

/* Using per slice version may improve performance here. (ie. SLBIA_An) */
static void slb_invalid(struct cxl_context *ctx)
{
	struct cxl *adapter = ctx->afu->adapter;
	u64 slbia;

391
	WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));
392 393 394 395 396 397 398

	cxl_p1_write(adapter, CXL_PSL_LBISEL,
			((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
			be32_to_cpu(ctx->elem->lpid));
	cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);

	while (1) {
399
		if (!cxl_ops->link_ok(adapter, NULL))
400
			break;
401 402 403 404 405 406 407 408 409 410 411
		slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
		if (!(slbia & CXL_TLB_SLB_P))
			break;
		cpu_relax();
	}
}

static int do_process_element_cmd(struct cxl_context *ctx,
				  u64 cmd, u64 pe_state)
{
	u64 state;
412
	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
I
Ian Munsie 已提交
413 414 415
	int rc = 0;

	trace_cxl_llcmd(ctx, cmd);
416 417 418 419 420

	WARN_ON(!ctx->afu->enabled);

	ctx->elem->software_state = cpu_to_be32(pe_state);
	smp_wmb();
421
	*(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
422 423 424
	smp_mb();
	cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
	while (1) {
425 426
		if (time_after_eq(jiffies, timeout)) {
			dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
I
Ian Munsie 已提交
427 428
			rc = -EBUSY;
			goto out;
429
		}
430
		if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
431 432 433 434
			dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
			rc = -EIO;
			goto out;
		}
435
		state = be64_to_cpup(ctx->afu->native->sw_command_status);
436 437
		if (state == ~0ULL) {
			pr_err("cxl: Error adding process element to AFU\n");
I
Ian Munsie 已提交
438 439
			rc = -1;
			goto out;
440 441 442 443 444 445 446 447 448 449 450 451 452 453
		}
		if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK  | CXL_SPA_SW_LINK_MASK)) ==
		    (cmd | (cmd >> 16) | ctx->pe))
			break;
		/*
		 * The command won't finish in the PSL if there are
		 * outstanding DSIs.  Hence we need to yield here in
		 * case there are outstanding DSIs that we need to
		 * service.  Tuning possiblity: we could wait for a
		 * while before sched
		 */
		schedule();

	}
I
Ian Munsie 已提交
454 455 456
out:
	trace_cxl_llcmd_done(ctx, cmd, rc);
	return rc;
457 458 459 460 461 462
}

static int add_process_element(struct cxl_context *ctx)
{
	int rc = 0;

463
	mutex_lock(&ctx->afu->native->spa_mutex);
464 465 466 467
	pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
	if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
		ctx->pe_inserted = true;
	pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
468
	mutex_unlock(&ctx->afu->native->spa_mutex);
469 470 471 472 473 474 475 476 477 478 479
	return rc;
}

static int terminate_process_element(struct cxl_context *ctx)
{
	int rc = 0;

	/* fast path terminate if it's already invalid */
	if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
		return rc;

480
	mutex_lock(&ctx->afu->native->spa_mutex);
481
	pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
482 483 484 485
	/* We could be asked to terminate when the hw is down. That
	 * should always succeed: it's not running if the hw has gone
	 * away and is being reset.
	 */
486
	if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
487 488
		rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
					    CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
489 490
	ctx->elem->software_state = 0;	/* Remove Valid bit */
	pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
491
	mutex_unlock(&ctx->afu->native->spa_mutex);
492 493 494 495 496 497 498
	return rc;
}

static int remove_process_element(struct cxl_context *ctx)
{
	int rc = 0;

499
	mutex_lock(&ctx->afu->native->spa_mutex);
500
	pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
501 502 503 504

	/* We could be asked to remove when the hw is down. Again, if
	 * the hw is down, the PE is gone, so we succeed.
	 */
505
	if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
506 507 508
		rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0);

	if (!rc)
509
		ctx->pe_inserted = false;
510 511
	if (cxl_is_power8())
		slb_invalid(ctx);
512
	pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
513
	mutex_unlock(&ctx->afu->native->spa_mutex);
514 515 516 517

	return rc;
}

M
Michael Neuling 已提交
518
void cxl_assign_psn_space(struct cxl_context *ctx)
519 520 521 522 523 524
{
	if (!ctx->afu->pp_size || ctx->master) {
		ctx->psn_phys = ctx->afu->psn_phys;
		ctx->psn_size = ctx->afu->adapter->ps_size;
	} else {
		ctx->psn_phys = ctx->afu->psn_phys +
525
			(ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);
526 527 528 529 530 531 532 533 534 535
		ctx->psn_size = ctx->afu->pp_size;
	}
}

static int activate_afu_directed(struct cxl_afu *afu)
{
	int rc;

	dev_info(&afu->dev, "Activating AFU directed mode\n");

536
	afu->num_procs = afu->max_procs_virtualised;
537
	if (afu->native->spa == NULL) {
C
Christophe Lombard 已提交
538
		if (cxl_alloc_spa(afu, CXL_MODE_DIRECTED))
539 540 541
			return -ENOMEM;
	}
	attach_spa(afu);
542 543

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
544 545
	if (cxl_is_power8())
		cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
	cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);

	afu->current_mode = CXL_MODE_DIRECTED;

	if ((rc = cxl_chardev_m_afu_add(afu)))
		return rc;

	if ((rc = cxl_sysfs_afu_m_add(afu)))
		goto err;

	if ((rc = cxl_chardev_s_afu_add(afu)))
		goto err1;

	return 0;
err1:
	cxl_sysfs_afu_m_remove(afu);
err:
	cxl_chardev_afu_remove(afu);
	return rc;
}

#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
#else
#define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
#endif

573 574 575 576
static u64 calculate_sr(struct cxl_context *ctx)
{
	u64 sr = 0;

577
	set_endian(sr);
578 579 580 581 582
	if (ctx->master)
		sr |= CXL_PSL_SR_An_MP;
	if (mfspr(SPRN_LPCR) & LPCR_TC)
		sr |= CXL_PSL_SR_An_TC;
	if (ctx->kernel) {
583 584 585
		if (!ctx->real_mode)
			sr |= CXL_PSL_SR_An_R;
		sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV;
586 587
	} else {
		sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
C
Christophe Lombard 已提交
588 589 590 591
		if (radix_enabled())
			sr |= CXL_PSL_SR_An_HV;
		else
			sr &= ~(CXL_PSL_SR_An_HV);
592 593 594
		if (!test_tsk_thread_flag(current, TIF_32BIT))
			sr |= CXL_PSL_SR_An_SF;
	}
C
Christophe Lombard 已提交
595 596 597 598 599 600
	if (cxl_is_psl9(ctx->afu)) {
		if (radix_enabled())
			sr |= CXL_PSL_SR_An_XLAT_ror;
		else
			sr |= CXL_PSL_SR_An_XLAT_hpt;
	}
601 602 603
	return sr;
}

604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
static void update_ivtes_directed(struct cxl_context *ctx)
{
	bool need_update = (ctx->status == STARTED);
	int r;

	if (need_update) {
		WARN_ON(terminate_process_element(ctx));
		WARN_ON(remove_process_element(ctx));
	}

	for (r = 0; r < CXL_IRQ_RANGES; r++) {
		ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
		ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
	}

	/*
	 * Theoretically we could use the update llcmd, instead of a
	 * terminate/remove/add (or if an atomic update was required we could
	 * do a suspend/update/resume), however it seems there might be issues
	 * with the update llcmd on some cards (including those using an XSL on
	 * an ASIC) so for now it's safest to go with the commands that are
	 * known to work. In the future if we come across a situation where the
	 * card may be performing transactions using the same PE while we are
	 * doing this update we might need to revisit this.
	 */
	if (need_update)
		WARN_ON(add_process_element(ctx));
}

C
Christophe Lombard 已提交
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
{
	u32 pid;

	cxl_assign_psn_space(ctx);

	ctx->elem->ctxtime = 0; /* disable */
	ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
	ctx->elem->haurp = 0; /* disable */

	if (ctx->kernel)
		pid = 0;
	else {
		if (ctx->mm == NULL) {
			pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
				__func__, ctx->pe, pid_nr(ctx->pid));
			return -EINVAL;
		}
		pid = ctx->mm->context.id;
	}

	ctx->elem->common.tid = 0;
	ctx->elem->common.pid = cpu_to_be32(pid);

	ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));

	ctx->elem->common.csrp = 0; /* disable */

	cxl_prefault(ctx, wed);

	/*
	 * Ensure we have the multiplexed PSL interrupt set up to take faults
	 * for kernel contexts that may not have allocated any AFU IRQs at all:
	 */
	if (ctx->irqs.range[0] == 0) {
		ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
		ctx->irqs.range[0] = 1;
	}

	ctx->elem->common.amr = cpu_to_be64(amr);
	ctx->elem->common.wed = cpu_to_be64(wed);

	return 0;
}

int cxl_attach_afu_directed_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
{
	int result;

	/* fill the process element entry */
	result = process_element_entry_psl9(ctx, wed, amr);
	if (result)
		return result;

	update_ivtes_directed(ctx);

	/* first guy needs to enable */
	result = cxl_ops->afu_check_and_enable(ctx->afu);
	if (result)
		return result;

	return add_process_element(ctx);
}

697
int cxl_attach_afu_directed_psl8(struct cxl_context *ctx, u64 wed, u64 amr)
698
{
699
	u32 pid;
700
	int result;
701

M
Michael Neuling 已提交
702
	cxl_assign_psn_space(ctx);
703 704 705 706

	ctx->elem->ctxtime = 0; /* disable */
	ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
	ctx->elem->haurp = 0; /* disable */
C
Christophe Lombard 已提交
707
	ctx->elem->u.sdr = cpu_to_be64(mfspr(SPRN_SDR1));
708

709 710 711
	pid = current->pid;
	if (ctx->kernel)
		pid = 0;
712
	ctx->elem->common.tid = 0;
713 714 715
	ctx->elem->common.pid = cpu_to_be32(pid);

	ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
716 717

	ctx->elem->common.csrp = 0; /* disable */
C
Christophe Lombard 已提交
718 719
	ctx->elem->common.u.psl8.aurp0 = 0; /* disable */
	ctx->elem->common.u.psl8.aurp1 = 0; /* disable */
720 721 722

	cxl_prefault(ctx, wed);

C
Christophe Lombard 已提交
723 724
	ctx->elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0);
	ctx->elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1);
725

726 727 728 729 730 731 732 733 734
	/*
	 * Ensure we have the multiplexed PSL interrupt set up to take faults
	 * for kernel contexts that may not have allocated any AFU IRQs at all:
	 */
	if (ctx->irqs.range[0] == 0) {
		ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
		ctx->irqs.range[0] = 1;
	}

735
	update_ivtes_directed(ctx);
736 737 738 739 740

	ctx->elem->common.amr = cpu_to_be64(amr);
	ctx->elem->common.wed = cpu_to_be64(wed);

	/* first guy needs to enable */
741
	if ((result = cxl_ops->afu_check_and_enable(ctx->afu)))
742 743
		return result;

744
	return add_process_element(ctx);
745 746 747 748 749 750 751 752 753 754 755 756
}

static int deactivate_afu_directed(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Deactivating AFU directed mode\n");

	afu->current_mode = 0;
	afu->num_procs = 0;

	cxl_sysfs_afu_m_remove(afu);
	cxl_chardev_afu_remove(afu);

757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
	/*
	 * The CAIA section 2.2.1 indicates that the procedure for starting and
	 * stopping an AFU in AFU directed mode is AFU specific, which is not
	 * ideal since this code is generic and with one exception has no
	 * knowledge of the AFU. This is in contrast to the procedure for
	 * disabling a dedicated process AFU, which is documented to just
	 * require a reset. The architecture does indicate that both an AFU
	 * reset and an AFU disable should result in the AFU being disabled and
	 * we do both followed by a PSL purge for safety.
	 *
	 * Notably we used to have some issues with the disable sequence on PSL
	 * cards, which is why we ended up using this heavy weight procedure in
	 * the first place, however a bug was discovered that had rendered the
	 * disable operation ineffective, so it is conceivable that was the
	 * sole explanation for those difficulties. Careful regression testing
	 * is recommended if anyone attempts to remove or reorder these
	 * operations.
	 *
	 * The XSL on the Mellanox CX4 behaves a little differently from the
	 * PSL based cards and will time out an AFU reset if the AFU is still
	 * enabled. That card is special in that we do have a means to identify
	 * it from this code, so in that case we skip the reset and just use a
	 * disable/purge to avoid the timeout and corresponding noise in the
	 * kernel log.
	 */
	if (afu->adapter->native->sl_ops->needs_reset_before_disable)
		cxl_ops->afu_reset(afu);
784 785 786 787 788 789
	cxl_afu_disable(afu);
	cxl_psl_purge(afu);

	return 0;
}

C
Christophe Lombard 已提交
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
int cxl_activate_dedicated_process_psl9(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Activating dedicated process mode\n");

	/*
	 * If XSL is set to dedicated mode (Set in PSL_SCNTL reg), the
	 * XSL and AFU are programmed to work with a single context.
	 * The context information should be configured in the SPA area
	 * index 0 (so PSL_SPAP must be configured before enabling the
	 * AFU).
	 */
	afu->num_procs = 1;
	if (afu->native->spa == NULL) {
		if (cxl_alloc_spa(afu, CXL_MODE_DEDICATED))
			return -ENOMEM;
	}
	attach_spa(afu);

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
	cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);

	afu->current_mode = CXL_MODE_DEDICATED;

	return cxl_chardev_d_afu_add(afu);
}

816
int cxl_activate_dedicated_process_psl8(struct cxl_afu *afu)
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
{
	dev_info(&afu->dev, "Activating dedicated process mode\n");

	cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);

	cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);    /* disable */
	cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
	cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
	cxl_p1n_write(afu, CXL_HAURP_An, 0);       /* disable */
	cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));

	cxl_p2n_write(afu, CXL_CSRP_An, 0);        /* disable */
	cxl_p2n_write(afu, CXL_AURP0_An, 0);       /* disable */
	cxl_p2n_write(afu, CXL_AURP1_An, 0);       /* disable */

	afu->current_mode = CXL_MODE_DEDICATED;
	afu->num_procs = 1;

	return cxl_chardev_d_afu_add(afu);
}

C
Christophe Lombard 已提交
839 840 841 842 843 844 845 846 847 848
void cxl_update_dedicated_ivtes_psl9(struct cxl_context *ctx)
{
	int r;

	for (r = 0; r < CXL_IRQ_RANGES; r++) {
		ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
		ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
	}
}

849
void cxl_update_dedicated_ivtes_psl8(struct cxl_context *ctx)
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864
{
	struct cxl_afu *afu = ctx->afu;

	cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
		       (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
		       (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
		       (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
			((u64)ctx->irqs.offset[3] & 0xffff));
	cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
		       (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
		       (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
		       (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
			((u64)ctx->irqs.range[3] & 0xffff));
}

C
Christophe Lombard 已提交
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
int cxl_attach_dedicated_process_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
{
	struct cxl_afu *afu = ctx->afu;
	int result;

	/* fill the process element entry */
	result = process_element_entry_psl9(ctx, wed, amr);
	if (result)
		return result;

	if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
		afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);

	result = cxl_ops->afu_reset(afu);
	if (result)
		return result;

	return afu_enable(afu);
}

885
int cxl_attach_dedicated_process_psl8(struct cxl_context *ctx, u64 wed, u64 amr)
886 887
{
	struct cxl_afu *afu = ctx->afu;
888
	u64 pid;
889 890
	int rc;

891 892 893 894 895 896
	pid = (u64)current->pid << 32;
	if (ctx->kernel)
		pid = 0;
	cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid);

	cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx));
897 898 899 900 901 902

	if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
		return rc;

	cxl_prefault(ctx, wed);

903 904
	if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
		afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
905 906 907 908

	cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);

	/* master only context for dedicated */
M
Michael Neuling 已提交
909
	cxl_assign_psn_space(ctx);
910

911
	if ((rc = cxl_ops->afu_reset(afu)))
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
		return rc;

	cxl_p2n_write(afu, CXL_PSL_WED_An, wed);

	return afu_enable(afu);
}

static int deactivate_dedicated_process(struct cxl_afu *afu)
{
	dev_info(&afu->dev, "Deactivating dedicated process mode\n");

	afu->current_mode = 0;
	afu->num_procs = 0;

	cxl_chardev_afu_remove(afu);

	return 0;
}

931
static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode)
932 933 934 935 936 937 938 939
{
	if (mode == CXL_MODE_DIRECTED)
		return deactivate_afu_directed(afu);
	if (mode == CXL_MODE_DEDICATED)
		return deactivate_dedicated_process(afu);
	return 0;
}

940
static int native_afu_activate_mode(struct cxl_afu *afu, int mode)
941 942 943 944 945 946
{
	if (!mode)
		return 0;
	if (!(mode & afu->modes_supported))
		return -EINVAL;

947
	if (!cxl_ops->link_ok(afu->adapter, afu)) {
948 949 950 951
		WARN(1, "Device link is down, refusing to activate!\n");
		return -EIO;
	}

952 953
	if (mode == CXL_MODE_DIRECTED)
		return activate_afu_directed(afu);
954 955 956
	if ((mode == CXL_MODE_DEDICATED) &&
	    (afu->adapter->native->sl_ops->activate_dedicated_process))
		return afu->adapter->native->sl_ops->activate_dedicated_process(afu);
957 958 959 960

	return -EINVAL;
}

961 962
static int native_attach_process(struct cxl_context *ctx, bool kernel,
				u64 wed, u64 amr)
963
{
964
	if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
965 966 967 968
		WARN(1, "Device link is down, refusing to attach process!\n");
		return -EIO;
	}

969
	ctx->kernel = kernel;
970 971 972
	if ((ctx->afu->current_mode == CXL_MODE_DIRECTED) &&
	    (ctx->afu->adapter->native->sl_ops->attach_afu_directed))
		return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr);
973

974 975 976
	if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
	    (ctx->afu->adapter->native->sl_ops->attach_dedicated_process))
		return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr);
977 978 979 980 981 982

	return -EINVAL;
}

static inline int detach_process_native_dedicated(struct cxl_context *ctx)
{
983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
	/*
	 * The CAIA section 2.1.1 indicates that we need to do an AFU reset to
	 * stop the AFU in dedicated mode (we therefore do not make that
	 * optional like we do in the afu directed path). It does not indicate
	 * that we need to do an explicit disable (which should occur
	 * implicitly as part of the reset) or purge, but we do these as well
	 * to be on the safe side.
	 *
	 * Notably we used to have some issues with the disable sequence
	 * (before the sequence was spelled out in the architecture) which is
	 * why we were so heavy weight in the first place, however a bug was
	 * discovered that had rendered the disable operation ineffective, so
	 * it is conceivable that was the sole explanation for those
	 * difficulties. Point is, we should be careful and do some regression
	 * testing if we ever attempt to remove any part of this procedure.
	 */
999
	cxl_ops->afu_reset(ctx->afu);
1000 1001 1002 1003 1004
	cxl_afu_disable(ctx->afu);
	cxl_psl_purge(ctx->afu);
	return 0;
}

1005 1006 1007 1008
static void native_update_ivtes(struct cxl_context *ctx)
{
	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
		return update_ivtes_directed(ctx);
1009 1010 1011
	if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
	    (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes))
		return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
1012 1013 1014
	WARN(1, "native_update_ivtes: Bad mode\n");
}

1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
{
	if (!ctx->pe_inserted)
		return 0;
	if (terminate_process_element(ctx))
		return -1;
	if (remove_process_element(ctx))
		return -1;

	return 0;
}

1027
static int native_detach_process(struct cxl_context *ctx)
1028
{
I
Ian Munsie 已提交
1029 1030
	trace_cxl_detach(ctx);

1031 1032 1033 1034 1035 1036
	if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
		return detach_process_native_dedicated(ctx);

	return detach_process_native_afu_directed(ctx);
}

1037
static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
1038
{
1039 1040 1041
	/* If the adapter has gone away, we can't get any meaningful
	 * information.
	 */
1042
	if (!cxl_ops->link_ok(afu->adapter, afu))
1043 1044
		return -EIO;

1045 1046
	info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
	info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
1047 1048
	if (cxl_is_power8())
		info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
1049 1050
	info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
	info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1051
	info->proc_handle = 0;
1052 1053 1054 1055

	return 0;
}

C
Christophe Lombard 已提交
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
void cxl_native_irq_dump_regs_psl9(struct cxl_context *ctx)
{
	u64 fir1, fir2, serr;

	fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR1);
	fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR2);

	dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
	dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
	if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
		serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
		cxl_afu_decode_psl_serr(ctx->afu, serr);
	}
}

1071
void cxl_native_irq_dump_regs_psl8(struct cxl_context *ctx)
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
{
	u64 fir1, fir2, fir_slice, serr, afu_debug;

	fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
	fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
	fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
	afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);

	dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
	dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
1082 1083
	if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
		serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
1084
		cxl_afu_decode_psl_serr(ctx->afu, serr);
1085
	}
1086 1087
	dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
	dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
1088 1089 1090 1091 1092 1093 1094
}

static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
						u64 dsisr, u64 errstat)
{

	dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
1095

1096 1097 1098 1099 1100 1101 1102
	if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers)
		ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx);

	if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) {
		dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
		ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter);
	}
1103

1104
	return cxl_ops->ack_irq(ctx, 0, errstat);
1105 1106
}

C
Christophe Lombard 已提交
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
static bool cxl_is_translation_fault(struct cxl_afu *afu, u64 dsisr)
{
	if ((cxl_is_psl8(afu)) && (dsisr & CXL_PSL_DSISR_TRANS))
		return true;

	if ((cxl_is_psl9(afu)) && (dsisr & CXL_PSL9_DSISR_An_TF))
		return true;

	return false;
}

1118
irqreturn_t cxl_fail_irq_psl(struct cxl_afu *afu, struct cxl_irq_info *irq_info)
1119
{
C
Christophe Lombard 已提交
1120
	if (cxl_is_translation_fault(afu, irq_info->dsisr))
1121 1122 1123 1124 1125 1126 1127
		cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
	else
		cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);

	return IRQ_HANDLED;
}

1128
static irqreturn_t native_irq_multiplexed(int irq, void *data)
1129 1130 1131 1132
{
	struct cxl_afu *afu = data;
	struct cxl_context *ctx;
	struct cxl_irq_info irq_info;
1133
	u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An);
1134
	int ph, ret = IRQ_HANDLED, res;
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144

	/* check if eeh kicked in while the interrupt was in flight */
	if (unlikely(phreg == ~0ULL)) {
		dev_warn(&afu->dev,
			 "Ignoring slice interrupt(%d) due to fenced card",
			 irq);
		return IRQ_HANDLED;
	}
	/* Mask the pe-handle from register value */
	ph = phreg & 0xffff;
1145 1146 1147 1148 1149
	if ((res = native_get_irq_info(afu, &irq_info))) {
		WARN(1, "Unable to get CXL IRQ Info: %i\n", res);
		if (afu->adapter->native->sl_ops->fail_irq)
			return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
		return ret;
1150 1151 1152 1153 1154
	}

	rcu_read_lock();
	ctx = idr_find(&afu->contexts_idr, ph);
	if (ctx) {
1155 1156
		if (afu->adapter->native->sl_ops->handle_interrupt)
			ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info);
1157 1158 1159 1160 1161 1162 1163 1164 1165
		rcu_read_unlock();
		return ret;
	}
	rcu_read_unlock();

	WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR"
		" %016llx\n(Possible AFU HW issue - was a term/remove acked"
		" with outstanding transactions?)\n", ph, irq_info.dsisr,
		irq_info.dar);
1166 1167 1168
	if (afu->adapter->native->sl_ops->fail_irq)
		ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
	return ret;
1169 1170
}

A
Andrew Donnellan 已提交
1171
static void native_irq_wait(struct cxl_context *ctx)
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
{
	u64 dsisr;
	int timeout = 1000;
	int ph;

	/*
	 * Wait until no further interrupts are presented by the PSL
	 * for this context.
	 */
	while (timeout--) {
		ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff;
		if (ph != ctx->pe)
			return;
		dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An);
1186 1187
		if (cxl_is_psl8(ctx->afu) &&
		   ((dsisr & CXL_PSL_DSISR_PENDING) == 0))
1188
			return;
C
Christophe Lombard 已提交
1189 1190 1191
		if (cxl_is_psl9(ctx->afu) &&
		   ((dsisr & CXL_PSL9_DSISR_PENDING) == 0))
			return;
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
		/*
		 * We are waiting for the workqueue to process our
		 * irq, so need to let that run here.
		 */
		msleep(1);
	}

	dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i"
		 " DSISR %016llx!\n", ph, dsisr);
	return;
}

1204
static irqreturn_t native_slice_irq_err(int irq, void *data)
1205 1206
{
	struct cxl_afu *afu = data;
1207 1208
	u64 errstat, serr, afu_error, dsisr;
	u64 fir_slice, afu_debug;
1209

1210 1211 1212
	/*
	 * slice err interrupt is only used with full PSL (no XSL)
	 */
1213 1214
	serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
	errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1215 1216 1217
	afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
	dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
	cxl_afu_decode_psl_serr(afu, serr);
1218 1219 1220 1221 1222 1223 1224

	if (cxl_is_power8()) {
		fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
		afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
		dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
		dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
	}
1225
	dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat);
1226 1227
	dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
	dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
1228 1229 1230 1231 1232 1233

	cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);

	return IRQ_HANDLED;
}

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
void cxl_native_err_irq_dump_regs(struct cxl *adapter)
{
	u64 fir1, fir2;

	fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
	fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);

	dev_crit(&adapter->dev, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1, fir2);
}

1244
static irqreturn_t native_irq_err(int irq, void *data)
1245 1246
{
	struct cxl *adapter = data;
1247
	u64 err_ivte;
1248 1249 1250 1251 1252 1253

	WARN(1, "CXL ERROR interrupt %i\n", irq);

	err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
	dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte);

1254 1255 1256 1257
	if (adapter->native->sl_ops->debugfs_stop_trace) {
		dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
		adapter->native->sl_ops->debugfs_stop_trace(adapter);
	}
1258

1259 1260
	if (adapter->native->sl_ops->err_irq_dump_registers)
		adapter->native->sl_ops->err_irq_dump_registers(adapter);
1261 1262 1263 1264

	return IRQ_HANDLED;
}

1265
int cxl_native_register_psl_err_irq(struct cxl *adapter)
1266 1267 1268 1269 1270 1271 1272 1273
{
	int rc;

	adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
				      dev_name(&adapter->dev));
	if (!adapter->irq_name)
		return -ENOMEM;

1274
	if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter,
1275 1276
				       &adapter->native->err_hwirq,
				       &adapter->native->err_virq,
1277 1278 1279 1280 1281 1282
				       adapter->irq_name))) {
		kfree(adapter->irq_name);
		adapter->irq_name = NULL;
		return rc;
	}

1283
	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff);
1284 1285 1286 1287

	return 0;
}

1288
void cxl_native_release_psl_err_irq(struct cxl *adapter)
1289
{
1290
	if (adapter->native->err_virq != irq_find_mapping(NULL, adapter->native->err_hwirq))
1291 1292 1293
		return;

	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
1294 1295
	cxl_unmap_irq(adapter->native->err_virq, adapter);
	cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq);
1296 1297 1298
	kfree(adapter->irq_name);
}

1299
int cxl_native_register_serr_irq(struct cxl_afu *afu)
1300 1301 1302 1303 1304 1305 1306 1307 1308
{
	u64 serr;
	int rc;

	afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
				      dev_name(&afu->dev));
	if (!afu->err_irq_name)
		return -ENOMEM;

1309
	if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu,
1310 1311 1312 1313 1314 1315 1316 1317
				       &afu->serr_hwirq,
				       &afu->serr_virq, afu->err_irq_name))) {
		kfree(afu->err_irq_name);
		afu->err_irq_name = NULL;
		return rc;
	}

	serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
1318 1319
	if (cxl_is_power8())
		serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
C
Christophe Lombard 已提交
1320 1321 1322 1323 1324 1325 1326
	if (cxl_is_power9()) {
		/*
		 * By default, all errors are masked. So don't set all masks.
		 * Slice errors will be transfered.
		 */
		serr = (serr & ~0xff0000007fffffffULL) | (afu->serr_hwirq & 0xffff);
	}
1327 1328 1329 1330 1331
	cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);

	return 0;
}

1332
void cxl_native_release_serr_irq(struct cxl_afu *afu)
1333 1334 1335 1336 1337 1338
{
	if (afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq))
		return;

	cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
	cxl_unmap_irq(afu->serr_virq, afu);
1339
	cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
1340 1341 1342
	kfree(afu->err_irq_name);
}

1343
int cxl_native_register_psl_irq(struct cxl_afu *afu)
1344 1345 1346 1347 1348 1349 1350 1351
{
	int rc;

	afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s",
				      dev_name(&afu->dev));
	if (!afu->psl_irq_name)
		return -ENOMEM;

1352 1353
	if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed,
				    afu, &afu->native->psl_hwirq, &afu->native->psl_virq,
1354 1355 1356 1357 1358 1359 1360
				    afu->psl_irq_name))) {
		kfree(afu->psl_irq_name);
		afu->psl_irq_name = NULL;
	}
	return rc;
}

1361
void cxl_native_release_psl_irq(struct cxl_afu *afu)
1362
{
1363
	if (afu->native->psl_virq != irq_find_mapping(NULL, afu->native->psl_hwirq))
1364 1365
		return;

1366 1367
	cxl_unmap_irq(afu->native->psl_virq, afu);
	cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);
1368 1369 1370
	kfree(afu->psl_irq_name);
}

1371 1372 1373 1374
static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
{
	u64 dsisr;

1375
	pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat);
1376 1377 1378 1379 1380 1381 1382 1383 1384

	/* Clear PSL_DSISR[PE] */
	dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
	cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);

	/* Write 1s to clear error status bits */
	cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
}

1385
static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
1386
{
I
Ian Munsie 已提交
1387
	trace_cxl_psl_irq_ack(ctx, tfc);
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
	if (tfc)
		cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
	if (psl_reset_mask)
		recover_psl_err(ctx->afu, psl_reset_mask);

	return 0;
}

int cxl_check_error(struct cxl_afu *afu)
{
	return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
}
1400

1401 1402 1403 1404 1405 1406
static bool native_support_attributes(const char *attr_name,
				      enum cxl_attrs type)
{
	return true;
}

1407
static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out)
1408
{
1409
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1410 1411 1412
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
1413
	*out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +
1414 1415
		(cr * afu->crs_len) + off);
	return 0;
1416 1417
}

1418
static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out)
1419
{
1420
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1421 1422 1423
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
1424
	*out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1425 1426
		(cr * afu->crs_len) + off);
	return 0;
1427 1428
}

1429
static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out)
1430 1431 1432
{
	u64 aligned_off = off & ~0x3L;
	u32 val;
1433
	int rc;
1434

1435
	rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1436 1437 1438
	if (!rc)
		*out = (val >> ((off & 0x3) * 8)) & 0xffff;
	return rc;
1439 1440
}

1441
static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out)
1442 1443 1444
{
	u64 aligned_off = off & ~0x3L;
	u32 val;
1445
	int rc;
1446

1447
	rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1448 1449 1450
	if (!rc)
		*out = (val >> ((off & 0x3) * 8)) & 0xff;
	return rc;
1451
}
1452

1453 1454
static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
{
1455
	if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
		return -EIO;
	if (unlikely(off >= afu->crs_len))
		return -ERANGE;
	out_le32(afu->native->afu_desc_mmio + afu->crs_offset +
		(cr * afu->crs_len) + off, in);
	return 0;
}

static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
{
	u64 aligned_off = off & ~0x3L;
	u32 val32, mask, shift;
	int rc;

	rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
	if (rc)
		return rc;
	shift = (off & 0x3) * 8;
	WARN_ON(shift == 24);
	mask = 0xffff << shift;
	val32 = (val32 & ~mask) | (in << shift);

	rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
	return rc;
}

static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
{
	u64 aligned_off = off & ~0x3L;
	u32 val32, mask, shift;
	int rc;

	rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
	if (rc)
		return rc;
	shift = (off & 0x3) * 8;
	mask = 0xff << shift;
	val32 = (val32 & ~mask) | (in << shift);

	rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
	return rc;
}

1499 1500
const struct cxl_backend_ops cxl_native_ops = {
	.module = THIS_MODULE,
1501 1502 1503 1504 1505 1506 1507
	.adapter_reset = cxl_pci_reset,
	.alloc_one_irq = cxl_pci_alloc_one_irq,
	.release_one_irq = cxl_pci_release_one_irq,
	.alloc_irq_ranges = cxl_pci_alloc_irq_ranges,
	.release_irq_ranges = cxl_pci_release_irq_ranges,
	.setup_irq = cxl_pci_setup_irq,
	.handle_psl_slice_error = native_handle_psl_slice_error,
1508
	.psl_interrupt = NULL,
1509
	.ack_irq = native_ack_irq,
1510
	.irq_wait = native_irq_wait,
1511 1512
	.attach_process = native_attach_process,
	.detach_process = native_detach_process,
1513
	.update_ivtes = native_update_ivtes,
1514
	.support_attributes = native_support_attributes,
1515
	.link_ok = cxl_adapter_link_ok,
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
	.release_afu = cxl_pci_release_afu,
	.afu_read_err_buffer = cxl_pci_afu_read_err_buffer,
	.afu_check_and_enable = native_afu_check_and_enable,
	.afu_activate_mode = native_afu_activate_mode,
	.afu_deactivate_mode = native_afu_deactivate_mode,
	.afu_reset = native_afu_reset,
	.afu_cr_read8 = native_afu_cr_read8,
	.afu_cr_read16 = native_afu_cr_read16,
	.afu_cr_read32 = native_afu_cr_read32,
	.afu_cr_read64 = native_afu_cr_read64,
1526 1527 1528 1529
	.afu_cr_write8 = native_afu_cr_write8,
	.afu_cr_write16 = native_afu_cr_write16,
	.afu_cr_write32 = native_afu_cr_write32,
	.read_adapter_vpd = cxl_pci_read_adapter_vpd,
1530
};