driver.c 36.4 KB
Newer Older
M
Mike Marciniszyn 已提交
1
/*
J
Jubin John 已提交
2
 * Copyright(c) 2015, 2016 Intel Corporation.
M
Mike Marciniszyn 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * BSD LICENSE
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  - Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  - Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  - Neither the name of Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/prefetch.h>
56
#include <rdma/ib_verbs.h>
M
Mike Marciniszyn 已提交
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77

#include "hfi.h"
#include "trace.h"
#include "qp.h"
#include "sdma.h"

#undef pr_fmt
#define pr_fmt(fmt) DRIVER_NAME ": " fmt

/*
 * The size has to be longer than this string, so we can append
 * board/chip information to it in the initialization code.
 */
const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";

DEFINE_SPINLOCK(hfi1_devs_lock);
LIST_HEAD(hfi1_dev_list);
DEFINE_MUTEX(hfi1_mutex);	/* general driver use */

unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
78 79
MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
		 HFI1_DEFAULT_MAX_MTU));
M
Mike Marciniszyn 已提交
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159

unsigned int hfi1_cu = 1;
module_param_named(cu, hfi1_cu, uint, S_IRUGO);
MODULE_PARM_DESC(cu, "Credit return units");

unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
static int hfi1_caps_set(const char *, const struct kernel_param *);
static int hfi1_caps_get(char *, const struct kernel_param *);
static const struct kernel_param_ops cap_ops = {
	.set = hfi1_caps_set,
	.get = hfi1_caps_get
};
module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");

MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
MODULE_VERSION(HFI1_DRIVER_VERSION);

/*
 * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
 */
#define MAX_PKT_RECV 64
#define EGR_HEAD_UPDATE_THRESHOLD 16

struct hfi1_ib_stats hfi1_stats;

static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
{
	int ret = 0;
	unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
		cap_mask = *cap_mask_ptr, value, diff,
		write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
			      HFI1_CAP_WRITABLE_MASK);

	ret = kstrtoul(val, 0, &value);
	if (ret) {
		pr_warn("Invalid module parameter value for 'cap_mask'\n");
		goto done;
	}
	/* Get the changed bits (except the locked bit) */
	diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);

	/* Remove any bits that are not allowed to change after driver load */
	if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
		pr_warn("Ignoring non-writable capability bits %#lx\n",
			diff & ~write_mask);
		diff &= write_mask;
	}

	/* Mask off any reserved bits */
	diff &= ~HFI1_CAP_RESERVED_MASK;
	/* Clear any previously set and changing bits */
	cap_mask &= ~diff;
	/* Update the bits with the new capability */
	cap_mask |= (value & diff);
	/* Check for any kernel/user restrictions */
	diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
		((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
	cap_mask &= ~diff;
	/* Set the bitmask to the final set */
	*cap_mask_ptr = cap_mask;
done:
	return ret;
}

static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
{
	unsigned long cap_mask = *(unsigned long *)kp->arg;

	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
	cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);

	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
}

const char *get_unit_name(int unit)
{
	static char iname[16];

160
	snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit);
M
Mike Marciniszyn 已提交
161 162 163
	return iname;
}

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
const char *get_card_name(struct rvt_dev_info *rdi)
{
	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
	struct hfi1_devdata *dd = container_of(ibdev,
					       struct hfi1_devdata, verbs_dev);
	return get_unit_name(dd->unit);
}

struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
{
	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
	struct hfi1_devdata *dd = container_of(ibdev,
					       struct hfi1_devdata, verbs_dev);
	return dd->pcidev;
}

M
Mike Marciniszyn 已提交
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
/*
 * Return count of units with at least one port ACTIVE.
 */
int hfi1_count_active_units(void)
{
	struct hfi1_devdata *dd;
	struct hfi1_pportdata *ppd;
	unsigned long flags;
	int pidx, nunits_active = 0;

	spin_lock_irqsave(&hfi1_devs_lock, flags);
	list_for_each_entry(dd, &hfi1_dev_list, list) {
		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase)
			continue;
		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
			ppd = dd->pport + pidx;
			if (ppd->lid && ppd->linkup) {
				nunits_active++;
				break;
			}
		}
	}
	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
	return nunits_active;
}

/*
 * Return count of all units, optionally return in arguments
 * the number of usable (present) units, and the number of
 * ports that are up.
 */
int hfi1_count_units(int *npresentp, int *nupp)
{
	int nunits = 0, npresent = 0, nup = 0;
	struct hfi1_devdata *dd;
	unsigned long flags;
	int pidx;
	struct hfi1_pportdata *ppd;

	spin_lock_irqsave(&hfi1_devs_lock, flags);

	list_for_each_entry(dd, &hfi1_dev_list, list) {
		nunits++;
		if ((dd->flags & HFI1_PRESENT) && dd->kregbase)
			npresent++;
		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
			ppd = dd->pport + pidx;
			if (ppd->lid && ppd->linkup)
				nup++;
		}
	}

	spin_unlock_irqrestore(&hfi1_devs_lock, flags);

	if (npresentp)
		*npresentp = npresent;
	if (nupp)
		*nupp = nup;

	return nunits;
}

/*
 * Get address of eager buffer from it's index (allocated in chunks, not
 * contiguous).
 */
static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
			       u8 *update)
{
	u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);

	*update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
	return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
			(offset * RCV_BUF_BLOCK_SIZE));
}

/*
 * Validate and encode the a given RcvArray Buffer size.
 * The function will check whether the given size falls within
 * allowed size ranges for the respective type and, optionally,
 * return the proper encoding.
 */
inline int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
{
264
	if (unlikely(!PAGE_ALIGNED(size)))
M
Mike Marciniszyn 已提交
265 266 267 268 269 270 271 272 273 274 275 276 277 278
		return 0;
	if (unlikely(size < MIN_EAGER_BUFFER))
		return 0;
	if (size >
	    (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
		return 0;
	if (encoded)
		*encoded = ilog2(size / PAGE_SIZE) + 1;
	return 1;
}

static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
		       struct hfi1_packet *packet)
{
279
	struct ib_header *rhdr = packet->hdr;
M
Mike Marciniszyn 已提交
280 281 282
	u32 rte = rhf_rcv_type_err(packet->rhf);
	int lnh = be16_to_cpu(rhdr->lrh[0]) & 3;
	struct hfi1_ibport *ibp = &ppd->ibport_data;
283 284
	struct hfi1_devdata *dd = ppd->dd;
	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
M
Mike Marciniszyn 已提交
285 286 287 288 289 290

	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
		return;

	if (packet->rhf & RHF_TID_ERR) {
		/* For TIDERR and RC QPs preemptively schedule a NAK */
291
		struct ib_other_headers *ohdr = NULL;
M
Mike Marciniszyn 已提交
292
		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
293
		u16 lid  = be16_to_cpu(rhdr->lrh[1]);
M
Mike Marciniszyn 已提交
294 295 296 297 298 299 300 301
		u32 qp_num;
		u32 rcv_flags = 0;

		/* Sanity check packet */
		if (tlen < 24)
			goto drop;

		/* Check for GRH */
302
		if (lnh == HFI1_LRH_BTH) {
303
			ohdr = &rhdr->u.oth;
304
		} else if (lnh == HFI1_LRH_GRH) {
M
Mike Marciniszyn 已提交
305 306
			u32 vtf;

307 308
			ohdr = &rhdr->u.l.oth;
			if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
M
Mike Marciniszyn 已提交
309
				goto drop;
310
			vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow);
M
Mike Marciniszyn 已提交
311 312 313
			if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
				goto drop;
			rcv_flags |= HFI1_HAS_GRH;
314
		} else {
M
Mike Marciniszyn 已提交
315
			goto drop;
316
		}
M
Mike Marciniszyn 已提交
317
		/* Get the destination QP number. */
318
		qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
319
		if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
320
			struct rvt_qp *qp;
321
			unsigned long flags;
M
Mike Marciniszyn 已提交
322 323

			rcu_read_lock();
324
			qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
M
Mike Marciniszyn 已提交
325 326 327 328 329 330 331 332 333
			if (!qp) {
				rcu_read_unlock();
				goto drop;
			}

			/*
			 * Handle only RC QPs - for other QP types drop error
			 * packet.
			 */
334
			spin_lock_irqsave(&qp->r_lock, flags);
M
Mike Marciniszyn 已提交
335 336

			/* Check for valid receive state. */
337 338
			if (!(ib_rvt_state_ops[qp->state] &
			      RVT_PROCESS_RECV_OK)) {
339
				ibp->rvp.n_pkt_drops++;
M
Mike Marciniszyn 已提交
340 341 342 343 344 345
			}

			switch (qp->ibqp.qp_type) {
			case IB_QPT_RC:
				hfi1_rc_hdrerr(
					rcd,
346
					rhdr,
M
Mike Marciniszyn 已提交
347 348 349 350 351 352 353 354
					rcv_flags,
					qp);
				break;
			default:
				/* For now don't handle any other QP types */
				break;
			}

355
			spin_unlock_irqrestore(&qp->r_lock, flags);
M
Mike Marciniszyn 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
			rcu_read_unlock();
		} /* Unicast QP */
	} /* Valid packet with TIDErr */

	/* handle "RcvTypeErr" flags */
	switch (rte) {
	case RHF_RTE_ERROR_OP_CODE_ERR:
	{
		u32 opcode;
		void *ebuf = NULL;
		__be32 *bth = NULL;

		if (rhf_use_egr_bfr(packet->rhf))
			ebuf = packet->ebuf;

371
		if (!ebuf)
M
Mike Marciniszyn 已提交
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
			goto drop; /* this should never happen */

		if (lnh == HFI1_LRH_BTH)
			bth = (__be32 *)ebuf;
		else if (lnh == HFI1_LRH_GRH)
			bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh));
		else
			goto drop;

		opcode = be32_to_cpu(bth[0]) >> 24;
		opcode &= 0xff;

		if (opcode == IB_OPCODE_CNP) {
			/*
			 * Only in pre-B0 h/w is the CNP_OPCODE handled
387
			 * via this code path.
M
Mike Marciniszyn 已提交
388
			 */
389
			struct rvt_qp *qp = NULL;
M
Mike Marciniszyn 已提交
390 391 392 393
			u32 lqpn, rqpn;
			u16 rlid;
			u8 svc_type, sl, sc5;

394
			sc5 = hdr2sc(rhdr, packet->rhf);
M
Mike Marciniszyn 已提交
395 396
			sl = ibp->sc_to_sl[sc5];

397
			lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
M
Mike Marciniszyn 已提交
398
			rcu_read_lock();
399
			qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
400
			if (!qp) {
M
Mike Marciniszyn 已提交
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
				rcu_read_unlock();
				goto drop;
			}

			switch (qp->ibqp.qp_type) {
			case IB_QPT_UD:
				rlid = 0;
				rqpn = 0;
				svc_type = IB_CC_SVCTYPE_UD;
				break;
			case IB_QPT_UC:
				rlid = be16_to_cpu(rhdr->lrh[3]);
				rqpn = qp->remote_qpn;
				svc_type = IB_CC_SVCTYPE_UC;
				break;
			default:
				goto drop;
			}

			process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
			rcu_read_unlock();
		}

		packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
		break;
	}
	default:
		break;
	}

drop:
	return;
}

static inline void init_packet(struct hfi1_ctxtdata *rcd,
436
			       struct hfi1_packet *packet)
M
Mike Marciniszyn 已提交
437 438 439 440 441 442
{
	packet->rsize = rcd->rcvhdrqentsize; /* words */
	packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
	packet->rcd = rcd;
	packet->updegr = 0;
	packet->etail = -1;
443
	packet->rhf_addr = get_rhf_addr(rcd);
M
Mike Marciniszyn 已提交
444 445 446 447 448 449
	packet->rhf = rhf_to_cpu(packet->rhf_addr);
	packet->rhqoff = rcd->head;
	packet->numpkt = 0;
	packet->rcv_flags = 0;
}

450 451
void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
			       bool do_cnp)
M
Mike Marciniszyn 已提交
452 453
{
	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
454 455
	struct ib_header *hdr = pkt->hdr;
	struct ib_other_headers *ohdr = pkt->ohdr;
456 457 458 459 460 461 462 463
	struct ib_grh *grh = NULL;
	u32 rqpn = 0, bth1;
	u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
	u8 sc, svc_type;
	bool is_mcast = false;

	if (pkt->rcv_flags & HFI1_HAS_GRH)
		grh = &hdr->u.l.grh;
M
Mike Marciniszyn 已提交
464 465

	switch (qp->ibqp.qp_type) {
466 467
	case IB_QPT_SMI:
	case IB_QPT_GSI:
M
Mike Marciniszyn 已提交
468
	case IB_QPT_UD:
469
		rlid = be16_to_cpu(hdr->lrh[3]);
470
		rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
M
Mike Marciniszyn 已提交
471
		svc_type = IB_CC_SVCTYPE_UD;
472 473
		is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
			(dlid != be16_to_cpu(IB_LID_PERMISSIVE));
M
Mike Marciniszyn 已提交
474
		break;
475 476 477 478 479 480 481 482 483 484
	case IB_QPT_UC:
		rlid = qp->remote_ah_attr.dlid;
		rqpn = qp->remote_qpn;
		svc_type = IB_CC_SVCTYPE_UC;
		break;
	case IB_QPT_RC:
		rlid = qp->remote_ah_attr.dlid;
		rqpn = qp->remote_qpn;
		svc_type = IB_CC_SVCTYPE_RC;
		break;
M
Mike Marciniszyn 已提交
485 486 487 488
	default:
		return;
	}

489
	sc = hdr2sc(hdr, pkt->rhf);
M
Mike Marciniszyn 已提交
490

491 492
	bth1 = be32_to_cpu(ohdr->bth[1]);
	if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
M
Mike Marciniszyn 已提交
493 494
		u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);

495
		return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
M
Mike Marciniszyn 已提交
496 497
	}

498
	if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) {
M
Mike Marciniszyn 已提交
499
		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
500
		u32 lqpn = bth1 & RVT_QPN_MASK;
501
		u8 sl = ibp->sc_to_sl[sc];
M
Mike Marciniszyn 已提交
502

503
		process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
M
Mike Marciniszyn 已提交
504
	}
505

M
Mike Marciniszyn 已提交
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
}

struct ps_mdata {
	struct hfi1_ctxtdata *rcd;
	u32 rsize;
	u32 maxcnt;
	u32 ps_head;
	u32 ps_tail;
	u32 ps_seq;
};

static inline void init_ps_mdata(struct ps_mdata *mdata,
				 struct hfi1_packet *packet)
{
	struct hfi1_ctxtdata *rcd = packet->rcd;

	mdata->rcd = rcd;
	mdata->rsize = packet->rsize;
	mdata->maxcnt = packet->maxcnt;
525
	mdata->ps_head = packet->rhqoff;
M
Mike Marciniszyn 已提交
526

527
	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
528
		mdata->ps_tail = get_rcvhdrtail(rcd);
529 530 531 532
		if (rcd->ctxt == HFI1_CTRL_CTXT)
			mdata->ps_seq = rcd->seq_cnt;
		else
			mdata->ps_seq = 0; /* not used with DMA_RTAIL */
M
Mike Marciniszyn 已提交
533 534 535 536 537 538
	} else {
		mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
		mdata->ps_seq = rcd->seq_cnt;
	}
}

539 540
static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
			  struct hfi1_ctxtdata *rcd)
M
Mike Marciniszyn 已提交
541
{
542
	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
M
Mike Marciniszyn 已提交
543 544 545 546
		return mdata->ps_head == mdata->ps_tail;
	return mdata->ps_seq != rhf_rcv_seq(rhf);
}

547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
			  struct hfi1_ctxtdata *rcd)
{
	/*
	 * Control context can potentially receive an invalid rhf.
	 * Drop such packets.
	 */
	if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
		return mdata->ps_seq != rhf_rcv_seq(rhf);

	return 0;
}

static inline void update_ps_mdata(struct ps_mdata *mdata,
				   struct hfi1_ctxtdata *rcd)
M
Mike Marciniszyn 已提交
562 563
{
	mdata->ps_head += mdata->rsize;
564
	if (mdata->ps_head >= mdata->maxcnt)
M
Mike Marciniszyn 已提交
565
		mdata->ps_head = 0;
566 567 568 569

	/* Control context must do seq counting */
	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
	    (rcd->ctxt == HFI1_CTRL_CTXT)) {
M
Mike Marciniszyn 已提交
570 571 572 573 574 575 576 577 578 579
		if (++mdata->ps_seq > 13)
			mdata->ps_seq = 1;
	}
}

/*
 * prescan_rxq - search through the receive queue looking for packets
 * containing Excplicit Congestion Notifications (FECNs, or BECNs).
 * When an ECN is found, process the Congestion Notification, and toggle
 * it off.
580 581
 * This is declared as a macro to allow quick checking of the port to avoid
 * the overhead of a function call if not enabled.
M
Mike Marciniszyn 已提交
582
 */
583 584 585 586 587 588
#define prescan_rxq(rcd, packet) \
	do { \
		if (rcd->ppd->cc_prescan) \
			__prescan_rxq(packet); \
	} while (0)
static void __prescan_rxq(struct hfi1_packet *packet)
M
Mike Marciniszyn 已提交
589 590 591 592 593 594 595 596 597
{
	struct hfi1_ctxtdata *rcd = packet->rcd;
	struct ps_mdata mdata;

	init_ps_mdata(&mdata, packet);

	while (1) {
		struct hfi1_devdata *dd = rcd->dd;
		struct hfi1_ibport *ibp = &rcd->ppd->ibport_data;
598
		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
M
Mike Marciniszyn 已提交
599
					 dd->rhf_offset;
600
		struct rvt_qp *qp;
601
		struct ib_header *hdr;
602
		struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
M
Mike Marciniszyn 已提交
603
		u64 rhf = rhf_to_cpu(rhf_addr);
604
		u32 etype = rhf_rcv_type(rhf), qpn, bth1;
M
Mike Marciniszyn 已提交
605 606 607
		int is_ecn = 0;
		u8 lnh;

608
		if (ps_done(&mdata, rhf, rcd))
M
Mike Marciniszyn 已提交
609 610
			break;

611 612 613
		if (ps_skip(&mdata, rhf, rcd))
			goto next;

M
Mike Marciniszyn 已提交
614 615 616
		if (etype != RHF_RCV_TYPE_IB)
			goto next;

617 618
		packet->hdr = hfi1_get_msgheader(dd, rhf_addr);
		hdr = packet->hdr;
619

M
Mike Marciniszyn 已提交
620 621
		lnh = be16_to_cpu(hdr->lrh[0]) & 3;

622 623 624 625 626 627
		if (lnh == HFI1_LRH_BTH) {
			packet->ohdr = &hdr->u.oth;
		} else if (lnh == HFI1_LRH_GRH) {
			packet->ohdr = &hdr->u.l.oth;
			packet->rcv_flags |= HFI1_HAS_GRH;
		} else {
M
Mike Marciniszyn 已提交
628
			goto next; /* just in case */
629
		}
630

631
		bth1 = be32_to_cpu(packet->ohdr->bth[1]);
632
		is_ecn = !!(bth1 & (HFI1_FECN_SMASK | HFI1_BECN_SMASK));
M
Mike Marciniszyn 已提交
633 634 635 636

		if (!is_ecn)
			goto next;

637
		qpn = bth1 & RVT_QPN_MASK;
M
Mike Marciniszyn 已提交
638
		rcu_read_lock();
639
		qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
M
Mike Marciniszyn 已提交
640

641
		if (!qp) {
M
Mike Marciniszyn 已提交
642 643 644 645
			rcu_read_unlock();
			goto next;
		}

646
		process_ecn(qp, packet, true);
M
Mike Marciniszyn 已提交
647
		rcu_read_unlock();
648 649 650

		/* turn off BECN, FECN */
		bth1 &= ~(HFI1_FECN_SMASK | HFI1_BECN_SMASK);
651
		packet->ohdr->bth[1] = cpu_to_be32(bth1);
M
Mike Marciniszyn 已提交
652
next:
653
		update_ps_mdata(&mdata, rcd);
M
Mike Marciniszyn 已提交
654 655
	}
}
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681

static inline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
{
	int ret = RCV_PKT_OK;

	/* Set up for the next packet */
	packet->rhqoff += packet->rsize;
	if (packet->rhqoff >= packet->maxcnt)
		packet->rhqoff = 0;

	packet->numpkt++;
	if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) {
		if (thread) {
			cond_resched();
		} else {
			ret = RCV_PKT_LIMIT;
			this_cpu_inc(*packet->rcd->dd->rcv_limit);
		}
	}

	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
				     packet->rcd->dd->rhf_offset;
	packet->rhf = rhf_to_cpu(packet->rhf_addr);

	return ret;
}
M
Mike Marciniszyn 已提交
682

683
static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
M
Mike Marciniszyn 已提交
684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
{
	int ret = RCV_PKT_OK;

	packet->hdr = hfi1_get_msgheader(packet->rcd->dd,
					 packet->rhf_addr);
	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
	packet->etype = rhf_rcv_type(packet->rhf);
	/* total length */
	packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
	/* retrieve eager buffer details */
	packet->ebuf = NULL;
	if (rhf_use_egr_bfr(packet->rhf)) {
		packet->etail = rhf_egr_index(packet->rhf);
		packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
				 &packet->updegr);
		/*
		 * Prefetch the contents of the eager buffer.  It is
		 * OK to send a negative length to prefetch_range().
		 * The +2 is the size of the RHF.
		 */
		prefetch_range(packet->ebuf,
705 706 707
			       packet->tlen - ((packet->rcd->rcvhdrqentsize -
					       (rhf_hdrq_offset(packet->rhf)
						+ 2)) * 4));
M
Mike Marciniszyn 已提交
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
	}

	/*
	 * Call a type specific handler for the packet. We
	 * should be able to trust that etype won't be beyond
	 * the range of valid indexes. If so something is really
	 * wrong and we can probably just let things come
	 * crashing down. There is no need to eat another
	 * comparison in this performance critical code.
	 */
	packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet);
	packet->numpkt++;

	/* Set up for the next packet */
	packet->rhqoff += packet->rsize;
	if (packet->rhqoff >= packet->maxcnt)
		packet->rhqoff = 0;

726 727 728 729 730 731 732
	if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) {
		if (thread) {
			cond_resched();
		} else {
			ret = RCV_PKT_LIMIT;
			this_cpu_inc(*packet->rcd->dd->rcv_limit);
		}
M
Mike Marciniszyn 已提交
733 734
	}

735
	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
M
Mike Marciniszyn 已提交
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
				      packet->rcd->dd->rhf_offset;
	packet->rhf = rhf_to_cpu(packet->rhf_addr);

	return ret;
}

static inline void process_rcv_update(int last, struct hfi1_packet *packet)
{
	/*
	 * Update head regs etc., every 16 packets, if not last pkt,
	 * to help prevent rcvhdrq overflows, when many packets
	 * are processed and queue is nearly full.
	 * Don't request an interrupt for intermediate updates.
	 */
	if (!last && !(packet->numpkt & 0xf)) {
		update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
			       packet->etail, 0, 0);
		packet->updegr = 0;
	}
	packet->rcv_flags = 0;
}

static inline void finish_packet(struct hfi1_packet *packet)
{
	/*
	 * Nothing we need to free for the packet.
	 *
	 * The only thing we need to do is a final update and call for an
	 * interrupt
	 */
	update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
		       packet->etail, rcv_intr_dynamic, packet->numpkt);
}

static inline void process_rcv_qp_work(struct hfi1_packet *packet)
{
	struct hfi1_ctxtdata *rcd;
773
	struct rvt_qp *qp, *nqp;
M
Mike Marciniszyn 已提交
774 775 776 777 778 779 780 781 782 783

	rcd = packet->rcd;
	rcd->head = packet->rhqoff;

	/*
	 * Iterate over all QPs waiting to respond.
	 * The list won't change since the IRQ is only run on one CPU.
	 */
	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
		list_del_init(&qp->rspwait);
784 785
		if (qp->r_flags & RVT_R_RSP_NAK) {
			qp->r_flags &= ~RVT_R_RSP_NAK;
M
Mike Marciniszyn 已提交
786 787
			hfi1_send_rc_ack(rcd, qp, 0);
		}
788
		if (qp->r_flags & RVT_R_RSP_SEND) {
M
Mike Marciniszyn 已提交
789 790
			unsigned long flags;

791
			qp->r_flags &= ~RVT_R_RSP_SEND;
M
Mike Marciniszyn 已提交
792
			spin_lock_irqsave(&qp->s_lock, flags);
793 794
			if (ib_rvt_state_ops[qp->state] &
					RVT_PROCESS_OR_FLUSH_SEND)
M
Mike Marciniszyn 已提交
795 796 797
				hfi1_schedule_send(qp);
			spin_unlock_irqrestore(&qp->s_lock, flags);
		}
798
		rvt_put_qp(qp);
M
Mike Marciniszyn 已提交
799 800 801 802 803 804
	}
}

/*
 * Handle receive interrupts when using the no dma rtail option.
 */
805
int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
M
Mike Marciniszyn 已提交
806 807
{
	u32 seq;
808
	int last = RCV_PKT_OK;
M
Mike Marciniszyn 已提交
809 810 811 812
	struct hfi1_packet packet;

	init_packet(rcd, &packet);
	seq = rhf_rcv_seq(packet.rhf);
813 814
	if (seq != rcd->seq_cnt) {
		last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
815
		goto bail;
816
	}
M
Mike Marciniszyn 已提交
817

818
	prescan_rxq(rcd, &packet);
M
Mike Marciniszyn 已提交
819

820 821
	while (last == RCV_PKT_OK) {
		last = process_rcv_packet(&packet, thread);
M
Mike Marciniszyn 已提交
822 823 824 825
		seq = rhf_rcv_seq(packet.rhf);
		if (++rcd->seq_cnt > 13)
			rcd->seq_cnt = 1;
		if (seq != rcd->seq_cnt)
826
			last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
827 828 829 830 831
		process_rcv_update(last, &packet);
	}
	process_rcv_qp_work(&packet);
bail:
	finish_packet(&packet);
832
	return last;
M
Mike Marciniszyn 已提交
833 834
}

835
int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
M
Mike Marciniszyn 已提交
836 837
{
	u32 hdrqtail;
838
	int last = RCV_PKT_OK;
M
Mike Marciniszyn 已提交
839 840 841 842
	struct hfi1_packet packet;

	init_packet(rcd, &packet);
	hdrqtail = get_rcvhdrtail(rcd);
843 844
	if (packet.rhqoff == hdrqtail) {
		last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
845
		goto bail;
846
	}
M
Mike Marciniszyn 已提交
847 848
	smp_rmb();  /* prevent speculative reads of dma'ed hdrq */

849
	prescan_rxq(rcd, &packet);
M
Mike Marciniszyn 已提交
850

851 852
	while (last == RCV_PKT_OK) {
		last = process_rcv_packet(&packet, thread);
M
Mike Marciniszyn 已提交
853
		if (packet.rhqoff == hdrqtail)
854
			last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
855 856 857 858 859
		process_rcv_update(last, &packet);
	}
	process_rcv_qp_work(&packet);
bail:
	finish_packet(&packet);
860
	return last;
M
Mike Marciniszyn 已提交
861 862 863 864 865 866
}

static inline void set_all_nodma_rtail(struct hfi1_devdata *dd)
{
	int i;

867
	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
M
Mike Marciniszyn 已提交
868 869 870 871 872 873 874 875
		dd->rcd[i]->do_interrupt =
			&handle_receive_interrupt_nodma_rtail;
}

static inline void set_all_dma_rtail(struct hfi1_devdata *dd)
{
	int i;

876
	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
M
Mike Marciniszyn 已提交
877 878 879 880
		dd->rcd[i]->do_interrupt =
			&handle_receive_interrupt_dma_rtail;
}

881 882 883 884 885 886 887 888 889 890
void set_all_slowpath(struct hfi1_devdata *dd)
{
	int i;

	/* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
		dd->rcd[i]->do_interrupt = &handle_receive_interrupt;
}

static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
891
				      struct hfi1_packet *packet,
892 893 894
				      struct hfi1_devdata *dd)
{
	struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
895 896
	struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
						   packet->rhf_addr);
897
	u8 etype = rhf_rcv_type(packet->rhf);
898

899
	if (etype == RHF_RCV_TYPE_IB && hdr2sc(hdr, packet->rhf) != 0xf) {
900 901 902 903 904 905 906 907 908 909 910 911 912
		int hwstate = read_logical_state(dd);

		if (hwstate != LSTATE_ACTIVE) {
			dd_dev_info(dd, "Unexpected link state %d\n", hwstate);
			return 0;
		}

		queue_work(rcd->ppd->hfi1_wq, lsaw);
		return 1;
	}
	return 0;
}

M
Mike Marciniszyn 已提交
913 914 915 916 917 918 919
/*
 * handle_receive_interrupt - receive a packet
 * @rcd: the context
 *
 * Called from interrupt handler for errors or receive interrupt.
 * This is the slow path interrupt handler.
 */
920
int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
M
Mike Marciniszyn 已提交
921 922 923
{
	struct hfi1_devdata *dd = rcd->dd;
	u32 hdrqtail;
924
	int needset, last = RCV_PKT_OK;
M
Mike Marciniszyn 已提交
925
	struct hfi1_packet packet;
926 927 928 929
	int skip_pkt = 0;

	/* Control context will always use the slow path interrupt handler */
	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
M
Mike Marciniszyn 已提交
930 931 932

	init_packet(rcd, &packet);

933
	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
M
Mike Marciniszyn 已提交
934 935
		u32 seq = rhf_rcv_seq(packet.rhf);

936 937
		if (seq != rcd->seq_cnt) {
			last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
938
			goto bail;
939
		}
M
Mike Marciniszyn 已提交
940 941 942
		hdrqtail = 0;
	} else {
		hdrqtail = get_rcvhdrtail(rcd);
943 944
		if (packet.rhqoff == hdrqtail) {
			last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
945
			goto bail;
946
		}
M
Mike Marciniszyn 已提交
947
		smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
948 949 950 951 952 953 954 955 956 957 958

		/*
		 * Control context can potentially receive an invalid
		 * rhf. Drop such packets.
		 */
		if (rcd->ctxt == HFI1_CTRL_CTXT) {
			u32 seq = rhf_rcv_seq(packet.rhf);

			if (seq != rcd->seq_cnt)
				skip_pkt = 1;
		}
M
Mike Marciniszyn 已提交
959 960
	}

961
	prescan_rxq(rcd, &packet);
M
Mike Marciniszyn 已提交
962

963
	while (last == RCV_PKT_OK) {
964 965 966
		if (unlikely(dd->do_drop &&
			     atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
			     DROP_PACKET_ON)) {
M
Mike Marciniszyn 已提交
967 968 969 970
			dd->do_drop = 0;

			/* On to the next packet */
			packet.rhqoff += packet.rsize;
971
			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
M
Mike Marciniszyn 已提交
972 973 974 975
					  packet.rhqoff +
					  dd->rhf_offset;
			packet.rhf = rhf_to_cpu(packet.rhf_addr);

976 977 978
		} else if (skip_pkt) {
			last = skip_rcv_packet(&packet, thread);
			skip_pkt = 0;
M
Mike Marciniszyn 已提交
979
		} else {
980 981 982
			/* Auto activate link on non-SC15 packet receive */
			if (unlikely(rcd->ppd->host_link_state ==
				     HLS_UP_ARMED) &&
983
			    set_armed_to_active(rcd, &packet, dd))
984
				goto bail;
985
			last = process_rcv_packet(&packet, thread);
M
Mike Marciniszyn 已提交
986 987
		}

988
		if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
M
Mike Marciniszyn 已提交
989 990 991 992 993
			u32 seq = rhf_rcv_seq(packet.rhf);

			if (++rcd->seq_cnt > 13)
				rcd->seq_cnt = 1;
			if (seq != rcd->seq_cnt)
994
				last = RCV_PKT_DONE;
M
Mike Marciniszyn 已提交
995
			if (needset) {
996
				dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
M
Mike Marciniszyn 已提交
997 998 999 1000 1001
				set_all_nodma_rtail(dd);
				needset = 0;
			}
		} else {
			if (packet.rhqoff == hdrqtail)
1002
				last = RCV_PKT_DONE;
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
			/*
			 * Control context can potentially receive an invalid
			 * rhf. Drop such packets.
			 */
			if (rcd->ctxt == HFI1_CTRL_CTXT) {
				u32 seq = rhf_rcv_seq(packet.rhf);

				if (++rcd->seq_cnt > 13)
					rcd->seq_cnt = 1;
				if (!last && (seq != rcd->seq_cnt))
					skip_pkt = 1;
			}

M
Mike Marciniszyn 已提交
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
			if (needset) {
				dd_dev_info(dd,
					    "Switching to DMA_RTAIL\n");
				set_all_dma_rtail(dd);
				needset = 0;
			}
		}

		process_rcv_update(last, &packet);
	}

	process_rcv_qp_work(&packet);

bail:
	/*
	 * Always write head at end, and setup rcv interrupt, even
	 * if no packets were processed.
	 */
	finish_packet(&packet);
1035
	return last;
M
Mike Marciniszyn 已提交
1036 1037
}

1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
/*
 * We may discover in the interrupt that the hardware link state has
 * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
 * and we need to update the driver's notion of the link state.  We cannot
 * run set_link_state from interrupt context, so we queue this function on
 * a workqueue.
 *
 * We delay the regular interrupt processing until after the state changes
 * so that the link will be in the correct state by the time any application
 * we wake up attempts to send a reply to any message it received.
 * (Subsequent receive interrupts may possibly force the wakeup before we
 * update the link state.)
 *
 * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
 * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
 * so we're safe from use-after-free of the rcd.
 */
void receive_interrupt_work(struct work_struct *work)
{
	struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
						  linkstate_active_work);
	struct hfi1_devdata *dd = ppd->dd;
	int i;

	/* Received non-SC15 packet implies neighbor_normal */
	ppd->neighbor_normal = 1;
	set_link_state(ppd, HLS_UP_ACTIVE);

	/*
	 * Interrupt all kernel contexts that could have had an
	 * interrupt during auto activation.
	 */
	for (i = HFI1_CTRL_CTXT; i < dd->first_user_ctxt; i++)
		force_recv_intr(dd->rcd[i]);
}

M
Mike Marciniszyn 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
/*
 * Convert a given MTU size to the on-wire MAD packet enumeration.
 * Return -1 if the size is invalid.
 */
int mtu_to_enum(u32 mtu, int default_if_bad)
{
	switch (mtu) {
	case     0: return OPA_MTU_0;
	case   256: return OPA_MTU_256;
	case   512: return OPA_MTU_512;
	case  1024: return OPA_MTU_1024;
	case  2048: return OPA_MTU_2048;
	case  4096: return OPA_MTU_4096;
	case  8192: return OPA_MTU_8192;
	case 10240: return OPA_MTU_10240;
	}
	return default_if_bad;
}

u16 enum_to_mtu(int mtu)
{
	switch (mtu) {
	case OPA_MTU_0:     return 0;
	case OPA_MTU_256:   return 256;
	case OPA_MTU_512:   return 512;
	case OPA_MTU_1024:  return 1024;
	case OPA_MTU_2048:  return 2048;
	case OPA_MTU_4096:  return 4096;
	case OPA_MTU_8192:  return 8192;
	case OPA_MTU_10240: return 10240;
	default: return 0xffff;
	}
}

/*
 * set_mtu - set the MTU
 * @ppd: the per port data
 *
 * We can handle "any" incoming size, the issue here is whether we
 * need to restrict our outgoing size.  We do not deal with what happens
 * to programs that are already running when the size changes.
 */
int set_mtu(struct hfi1_pportdata *ppd)
{
	struct hfi1_devdata *dd = ppd->dd;
	int i, drain, ret = 0, is_up = 0;

	ppd->ibmtu = 0;
	for (i = 0; i < ppd->vls_supported; i++)
		if (ppd->ibmtu < dd->vld[i].mtu)
			ppd->ibmtu = dd->vld[i].mtu;
	ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);

	mutex_lock(&ppd->hls_lock);
1128 1129 1130
	if (ppd->host_link_state == HLS_UP_INIT ||
	    ppd->host_link_state == HLS_UP_ARMED ||
	    ppd->host_link_state == HLS_UP_ACTIVE)
M
Mike Marciniszyn 已提交
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
		is_up = 1;

	drain = !is_ax(dd) && is_up;

	if (drain)
		/*
		 * MTU is specified per-VL. To ensure that no packet gets
		 * stuck (due, e.g., to the MTU for the packet's VL being
		 * reduced), empty the per-VL FIFOs before adjusting MTU.
		 */
		ret = stop_drain_data_vls(dd);

	if (ret) {
		dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
			   __func__);
		goto err;
	}

	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);

	if (drain)
		open_fill_data_vls(dd); /* reopen all VLs */

err:
	mutex_unlock(&ppd->hls_lock);

	return ret;
}

int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
{
	struct hfi1_devdata *dd = ppd->dd;

	ppd->lid = lid;
	ppd->lmc = lmc;
	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);

1168
	dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
M
Mike Marciniszyn 已提交
1169 1170 1171 1172

	return 0;
}

1173 1174 1175 1176
void shutdown_led_override(struct hfi1_pportdata *ppd)
{
	struct hfi1_devdata *dd = ppd->dd;

1177
	/*
1178 1179 1180
	 * This pairs with the memory barrier in hfi1_start_led_override to
	 * ensure that we read the correct state of LED beaconing represented
	 * by led_override_timer_active
1181
	 */
1182
	smp_rmb();
1183 1184 1185
	if (atomic_read(&ppd->led_override_timer_active)) {
		del_timer_sync(&ppd->led_override_timer);
		atomic_set(&ppd->led_override_timer_active, 0);
1186 1187
		/* Ensure the atomic_set is visible to all CPUs */
		smp_wmb();
1188 1189
	}

1190 1191
	/* Hand control of the LED to the DC for normal operation */
	write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1192
}
M
Mike Marciniszyn 已提交
1193 1194 1195 1196 1197

static void run_led_override(unsigned long opaque)
{
	struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)opaque;
	struct hfi1_devdata *dd = ppd->dd;
1198 1199
	unsigned long timeout;
	int phase_idx;
M
Mike Marciniszyn 已提交
1200 1201 1202 1203

	if (!(dd->flags & HFI1_INITTED))
		return;

1204
	phase_idx = ppd->led_override_phase & 1;
M
Mike Marciniszyn 已提交
1205

1206 1207 1208
	setextled(dd, phase_idx);

	timeout = ppd->led_override_vals[phase_idx];
1209

1210 1211
	/* Set up for next phase */
	ppd->led_override_phase = !ppd->led_override_phase;
M
Mike Marciniszyn 已提交
1212

1213
	mod_timer(&ppd->led_override_timer, jiffies + timeout);
M
Mike Marciniszyn 已提交
1214 1215
}

1216 1217
/*
 * To have the LED blink in a particular pattern, provide timeon and timeoff
1218 1219 1220
 * in milliseconds.
 * To turn off custom blinking and return to normal operation, use
 * shutdown_led_override()
1221
 */
1222 1223
void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
			     unsigned int timeoff)
M
Mike Marciniszyn 已提交
1224
{
1225
	if (!(ppd->dd->flags & HFI1_INITTED))
M
Mike Marciniszyn 已提交
1226 1227
		return;

1228 1229 1230
	/* Convert to jiffies for direct use in timer */
	ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
	ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
M
Mike Marciniszyn 已提交
1231

1232 1233
	/* Arbitrarily start from LED on phase */
	ppd->led_override_phase = 1;
M
Mike Marciniszyn 已提交
1234 1235 1236

	/*
	 * If the timer has not already been started, do so. Use a "quick"
1237
	 * timeout so the handler will be called soon to look at our request.
M
Mike Marciniszyn 已提交
1238
	 */
1239
	if (!timer_pending(&ppd->led_override_timer)) {
1240
		setup_timer(&ppd->led_override_timer, run_led_override,
1241
			    (unsigned long)ppd);
M
Mike Marciniszyn 已提交
1242 1243
		ppd->led_override_timer.expires = jiffies + 1;
		add_timer(&ppd->led_override_timer);
1244 1245 1246
		atomic_set(&ppd->led_override_timer_active, 1);
		/* Ensure the atomic_set is visible to all CPUs */
		smp_wmb();
M
Mike Marciniszyn 已提交
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
	}
}

/**
 * hfi1_reset_device - reset the chip if possible
 * @unit: the device to reset
 *
 * Whether or not reset is successful, we attempt to re-initialize the chip
 * (that is, much like a driver unload/reload).  We clear the INITTED flag
 * so that the various entry points will fail until we reinitialize.  For
 * now, we only allow this if no user contexts are open that use chip resources
 */
int hfi1_reset_device(int unit)
{
	int ret, i;
	struct hfi1_devdata *dd = hfi1_lookup(unit);
	struct hfi1_pportdata *ppd;
	unsigned long flags;
	int pidx;

	if (!dd) {
		ret = -ENODEV;
		goto bail;
	}

	dd_dev_info(dd, "Reset on unit %u requested\n", unit);

	if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) {
		dd_dev_info(dd,
1276 1277
			    "Invalid unit number %u or not initialized or not present\n",
			    unit);
M
Mike Marciniszyn 已提交
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
		ret = -ENXIO;
		goto bail;
	}

	spin_lock_irqsave(&dd->uctxt_lock, flags);
	if (dd->rcd)
		for (i = dd->first_user_ctxt; i < dd->num_rcv_contexts; i++) {
			if (!dd->rcd[i] || !dd->rcd[i]->cnt)
				continue;
			spin_unlock_irqrestore(&dd->uctxt_lock, flags);
			ret = -EBUSY;
			goto bail;
		}
	spin_unlock_irqrestore(&dd->uctxt_lock, flags);

	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
		ppd = dd->pport + pidx;

1296
		shutdown_led_override(ppd);
M
Mike Marciniszyn 已提交
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
	}
	if (dd->flags & HFI1_HAS_SEND_DMA)
		sdma_exit(dd);

	hfi1_reset_cpu_counters(dd);

	ret = hfi1_init(dd, 1);

	if (ret)
		dd_dev_err(dd,
1307 1308
			   "Reinitialize unit %u after reset failed with %d\n",
			   unit, ret);
M
Mike Marciniszyn 已提交
1309 1310
	else
		dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1311
			    unit);
M
Mike Marciniszyn 已提交
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322

bail:
	return ret;
}

void handle_eflags(struct hfi1_packet *packet)
{
	struct hfi1_ctxtdata *rcd = packet->rcd;
	u32 rte = rhf_rcv_type_err(packet->rhf);

	rcv_hdrerr(rcd, rcd->ppd, packet);
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
	if (rhf_err_flags(packet->rhf))
		dd_dev_err(rcd->dd,
			   "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
			   rcd->ctxt, packet->rhf,
			   packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
			   packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
			   packet->rhf & RHF_DC_ERR ? "dc " : "",
			   packet->rhf & RHF_TID_ERR ? "tid " : "",
			   packet->rhf & RHF_LEN_ERR ? "len " : "",
			   packet->rhf & RHF_ECC_ERR ? "ecc " : "",
			   packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
			   packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
			   rte);
M
Mike Marciniszyn 已提交
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
}

/*
 * The following functions are called by the interrupt handler. They are type
 * specific handlers for each packet type.
 */
int process_receive_ib(struct hfi1_packet *packet)
{
	trace_hfi1_rcvhdr(packet->rcd->ppd->dd,
			  packet->rcd->ctxt,
			  rhf_err_flags(packet->rhf),
			  RHF_RCV_TYPE_IB,
			  packet->hlen,
			  packet->tlen,
			  packet->updegr,
			  rhf_egr_index(packet->rhf));

	if (unlikely(rhf_err_flags(packet->rhf))) {
		handle_eflags(packet);
		return RHF_RCV_CONTINUE;
	}

	hfi1_ib_rcv(packet);
	return RHF_RCV_CONTINUE;
}

int process_receive_bypass(struct hfi1_packet *packet)
{
1364 1365
	struct hfi1_devdata *dd = packet->rcd->dd;

M
Mike Marciniszyn 已提交
1366 1367 1368
	if (unlikely(rhf_err_flags(packet->rhf)))
		handle_eflags(packet);

1369
	dd_dev_err(dd,
1370
		   "Bypass packets are not supported in normal operation. Dropping\n");
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
	incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
	if (!(dd->err_info_rcvport.status_and_code & OPA_EI_STATUS_SMASK)) {
		u64 *flits = packet->ebuf;

		if (flits && !(packet->rhf & RHF_LEN_ERR)) {
			dd->err_info_rcvport.packet_flit1 = flits[0];
			dd->err_info_rcvport.packet_flit2 =
				packet->tlen > sizeof(flits[0]) ? flits[1] : 0;
		}
		dd->err_info_rcvport.status_and_code |=
			(OPA_EI_STATUS_SMASK | BAD_L2_ERR);
	}
M
Mike Marciniszyn 已提交
1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
	return RHF_RCV_CONTINUE;
}

int process_receive_error(struct hfi1_packet *packet)
{
	handle_eflags(packet);

	if (unlikely(rhf_err_flags(packet->rhf)))
		dd_dev_err(packet->rcd->dd,
			   "Unhandled error packet received. Dropping.\n");

	return RHF_RCV_CONTINUE;
}

int kdeth_process_expected(struct hfi1_packet *packet)
{
	if (unlikely(rhf_err_flags(packet->rhf)))
		handle_eflags(packet);

	dd_dev_err(packet->rcd->dd,
		   "Unhandled expected packet received. Dropping.\n");
	return RHF_RCV_CONTINUE;
}

int kdeth_process_eager(struct hfi1_packet *packet)
{
	if (unlikely(rhf_err_flags(packet->rhf)))
		handle_eflags(packet);

	dd_dev_err(packet->rcd->dd,
		   "Unhandled eager packet received. Dropping.\n");
	return RHF_RCV_CONTINUE;
}

int process_receive_invalid(struct hfi1_packet *packet)
{
	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1420
		   rhf_rcv_type(packet->rhf));
M
Mike Marciniszyn 已提交
1421 1422
	return RHF_RCV_CONTINUE;
}