ud.c 23.6 KB
Newer Older
M
Mike Marciniszyn 已提交
1
/*
J
Jubin John 已提交
2
 * Copyright(c) 2015, 2016 Intel Corporation.
M
Mike Marciniszyn 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * BSD LICENSE
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  - Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  - Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  - Neither the name of Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <linux/net.h>
#include <rdma/ib_smi.h>

#include "hfi.h"
#include "mad.h"
53
#include "verbs_txreq.h"
54
#include "qp.h"
M
Mike Marciniszyn 已提交
55 56 57 58 59 60 61 62 63 64 65

/**
 * ud_loopback - handle send on loopback QPs
 * @sqp: the sending QP
 * @swqe: the send work request
 *
 * This is called from hfi1_make_ud_req() to forward a WQE addressed
 * to the same HFI.
 * Note that the receive interrupt handler may be calling hfi1_ud_rcv()
 * while this is being called.
 */
66
static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
M
Mike Marciniszyn 已提交
67 68 69
{
	struct hfi1_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num);
	struct hfi1_pportdata *ppd;
70
	struct rvt_qp *qp;
71
	struct rdma_ah_attr *ah_attr;
M
Mike Marciniszyn 已提交
72
	unsigned long flags;
73 74
	struct rvt_sge_state ssge;
	struct rvt_sge *sge;
M
Mike Marciniszyn 已提交
75 76 77 78 79 80
	struct ib_wc wc;
	u32 length;
	enum ib_qp_type sqptype, dqptype;

	rcu_read_lock();

81 82
	qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp,
			    swqe->ud_wr.remote_qpn);
M
Mike Marciniszyn 已提交
83
	if (!qp) {
84
		ibp->rvp.n_pkt_drops++;
M
Mike Marciniszyn 已提交
85 86 87 88 89 90 91 92 93 94
		rcu_read_unlock();
		return;
	}

	sqptype = sqp->ibqp.qp_type == IB_QPT_GSI ?
			IB_QPT_UD : sqp->ibqp.qp_type;
	dqptype = qp->ibqp.qp_type == IB_QPT_GSI ?
			IB_QPT_UD : qp->ibqp.qp_type;

	if (dqptype != sqptype ||
95
	    !(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
96
		ibp->rvp.n_pkt_drops++;
M
Mike Marciniszyn 已提交
97 98 99
		goto drop;
	}

100
	ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr;
M
Mike Marciniszyn 已提交
101 102 103 104 105
	ppd = ppd_from_ibp(ibp);

	if (qp->ibqp.qp_num > 1) {
		u16 pkey;
		u16 slid;
106
		u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
M
Mike Marciniszyn 已提交
107 108

		pkey = hfi1_get_pkey(ibp, sqp->s_pkey_index);
109
		slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
M
Mike Marciniszyn 已提交
110 111
				   ((1 << ppd->lmc) - 1));
		if (unlikely(ingress_pkey_check(ppd, pkey, sc5,
112 113
						qp->s_pkey_index,
						slid, false))) {
114 115 116 117
			hfi1_bad_pkey(ibp, pkey,
				      rdma_ah_get_sl(ah_attr),
				      sqp->ibqp.qp_num, qp->ibqp.qp_num,
				      slid, rdma_ah_get_dlid(ah_attr));
M
Mike Marciniszyn 已提交
118 119 120 121 122 123 124 125 126 127 128 129
			goto drop;
		}
	}

	/*
	 * Check that the qkey matches (except for QP0, see 9.6.1.4.1).
	 * Qkeys with the high order bit set mean use the
	 * qkey from the QP context instead of the WR (see 10.2.5).
	 */
	if (qp->ibqp.qp_num) {
		u32 qkey;

C
Christoph Hellwig 已提交
130 131
		qkey = (int)swqe->ud_wr.remote_qkey < 0 ?
			sqp->qkey : swqe->ud_wr.remote_qkey;
132 133
		if (unlikely(qkey != qp->qkey))
			goto drop; /* silently drop per IBTA spec */
M
Mike Marciniszyn 已提交
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
	}

	/*
	 * A GRH is expected to precede the data even if not
	 * present on the wire.
	 */
	length = swqe->length;
	memset(&wc, 0, sizeof(wc));
	wc.byte_len = length + sizeof(struct ib_grh);

	if (swqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
		wc.wc_flags = IB_WC_WITH_IMM;
		wc.ex.imm_data = swqe->wr.ex.imm_data;
	}

	spin_lock_irqsave(&qp->r_lock, flags);

	/*
	 * Get the next work request entry to find where to put the data.
	 */
154
	if (qp->r_flags & RVT_R_REUSE_SGE) {
155
		qp->r_flags &= ~RVT_R_REUSE_SGE;
156
	} else {
M
Mike Marciniszyn 已提交
157 158
		int ret;

159
		ret = hfi1_rvt_get_rwqe(qp, 0);
M
Mike Marciniszyn 已提交
160
		if (ret < 0) {
161
			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
M
Mike Marciniszyn 已提交
162 163 164 165
			goto bail_unlock;
		}
		if (!ret) {
			if (qp->ibqp.qp_num == 0)
166
				ibp->rvp.n_vl15_dropped++;
M
Mike Marciniszyn 已提交
167 168 169 170 171
			goto bail_unlock;
		}
	}
	/* Silently drop packets which are too big. */
	if (unlikely(wc.byte_len > qp->r_len)) {
172
		qp->r_flags |= RVT_R_REUSE_SGE;
173
		ibp->rvp.n_pkt_drops++;
M
Mike Marciniszyn 已提交
174 175 176
		goto bail_unlock;
	}

177
	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
178
		struct ib_grh grh;
179
		const struct ib_global_route *grd = rdma_ah_read_grh(ah_attr);
180

181
		hfi1_make_grh(ibp, &grh, grd, 0, 0);
182
		hfi1_copy_sge(&qp->r_sge, &grh,
183
			      sizeof(grh), true, false);
M
Mike Marciniszyn 已提交
184
		wc.wc_flags |= IB_WC_GRH;
185
	} else {
186
		rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
187
	}
M
Mike Marciniszyn 已提交
188 189 190 191 192 193 194 195 196 197 198 199
	ssge.sg_list = swqe->sg_list + 1;
	ssge.sge = *swqe->sg_list;
	ssge.num_sge = swqe->wr.num_sge;
	sge = &ssge.sge;
	while (length) {
		u32 len = sge->length;

		if (len > length)
			len = length;
		if (len > sge->sge_length)
			len = sge->sge_length;
		WARN_ON_ONCE(len == 0);
200
		hfi1_copy_sge(&qp->r_sge, sge->vaddr, len, true, false);
M
Mike Marciniszyn 已提交
201 202 203 204 205 206 207
		sge->vaddr += len;
		sge->length -= len;
		sge->sge_length -= len;
		if (sge->sge_length == 0) {
			if (--ssge.num_sge)
				*sge = *ssge.sg_list++;
		} else if (sge->length == 0 && sge->mr->lkey) {
208
			if (++sge->n >= RVT_SEGSZ) {
M
Mike Marciniszyn 已提交
209 210 211 212 213 214 215 216 217 218 219
				if (++sge->m >= sge->mr->mapsz)
					break;
				sge->n = 0;
			}
			sge->vaddr =
				sge->mr->map[sge->m]->segs[sge->n].vaddr;
			sge->length =
				sge->mr->map[sge->m]->segs[sge->n].length;
		}
		length -= len;
	}
220
	rvt_put_ss(&qp->r_sge);
221
	if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
M
Mike Marciniszyn 已提交
222 223 224 225 226 227 228 229 230
		goto bail_unlock;
	wc.wr_id = qp->r_wr_id;
	wc.status = IB_WC_SUCCESS;
	wc.opcode = IB_WC_RECV;
	wc.qp = &qp->ibqp;
	wc.src_qp = sqp->ibqp.qp_num;
	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) {
		if (sqp->ibqp.qp_type == IB_QPT_GSI ||
		    sqp->ibqp.qp_type == IB_QPT_SMI)
C
Christoph Hellwig 已提交
231
			wc.pkey_index = swqe->ud_wr.pkey_index;
M
Mike Marciniszyn 已提交
232 233 234 235 236
		else
			wc.pkey_index = sqp->s_pkey_index;
	} else {
		wc.pkey_index = 0;
	}
237 238
	wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
			      ((1 << ppd->lmc) - 1));
M
Mike Marciniszyn 已提交
239 240
	/* Check for loopback when the port lid is not set */
	if (wc.slid == 0 && sqp->ibqp.qp_type == IB_QPT_GSI)
241
		wc.slid = be16_to_cpu(IB_LID_PERMISSIVE);
242 243
	wc.sl = rdma_ah_get_sl(ah_attr);
	wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1);
M
Mike Marciniszyn 已提交
244 245
	wc.port_num = qp->port_num;
	/* Signal completion event if the solicited bit is set. */
246 247
	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
		     swqe->wr.send_flags & IB_SEND_SOLICITED);
248
	ibp->rvp.n_loop_pkts++;
M
Mike Marciniszyn 已提交
249 250 251 252 253 254 255 256 257 258
bail_unlock:
	spin_unlock_irqrestore(&qp->r_lock, flags);
drop:
	rcu_read_unlock();
}

/**
 * hfi1_make_ud_req - construct a UD request packet
 * @qp: the QP
 *
259 260
 * Assume s_lock is held.
 *
M
Mike Marciniszyn 已提交
261 262
 * Return 1 if constructed; otherwise, return 0.
 */
263
int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
M
Mike Marciniszyn 已提交
264
{
265
	struct hfi1_qp_priv *priv = qp->priv;
266
	struct ib_other_headers *ohdr;
267
	struct rdma_ah_attr *ah_attr;
M
Mike Marciniszyn 已提交
268 269
	struct hfi1_pportdata *ppd;
	struct hfi1_ibport *ibp;
270
	struct rvt_swqe *wqe;
M
Mike Marciniszyn 已提交
271 272 273 274 275 276 277 278
	u32 nwords;
	u32 extra_bytes;
	u32 bth0;
	u16 lrh0;
	u16 lid;
	int next_cur;
	u8 sc5;

279 280 281 282
	ps->s_txreq = get_txreq(ps->dev, qp);
	if (IS_ERR(ps->s_txreq))
		goto bail_no_tx;

283 284
	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) {
		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
M
Mike Marciniszyn 已提交
285 286
			goto bail;
		/* We are in the error state, flush the work request. */
287 288
		smp_read_barrier_depends(); /* see post_one_send */
		if (qp->s_last == ACCESS_ONCE(qp->s_head))
M
Mike Marciniszyn 已提交
289 290
			goto bail;
		/* If DMAs are in progress, we can't flush immediately. */
291
		if (iowait_sdma_pending(&priv->s_iowait)) {
292
			qp->s_flags |= RVT_S_WAIT_DMA;
M
Mike Marciniszyn 已提交
293 294
			goto bail;
		}
295
		wqe = rvt_get_swqe_ptr(qp, qp->s_last);
M
Mike Marciniszyn 已提交
296
		hfi1_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
297
		goto done_free_tx;
M
Mike Marciniszyn 已提交
298 299
	}

300 301 302
	/* see post_one_send() */
	smp_read_barrier_depends();
	if (qp->s_cur == ACCESS_ONCE(qp->s_head))
M
Mike Marciniszyn 已提交
303 304
		goto bail;

305
	wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
M
Mike Marciniszyn 已提交
306 307 308 309 310 311 312
	next_cur = qp->s_cur + 1;
	if (next_cur >= qp->s_size)
		next_cur = 0;

	/* Construct the header. */
	ibp = to_iport(qp->ibqp.device, qp->port_num);
	ppd = ppd_from_ibp(ibp);
313
	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
314 315 316
	if (rdma_ah_get_dlid(ah_attr) < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
	    rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
		lid = rdma_ah_get_dlid(ah_attr) & ~((1 << ppd->lmc) - 1);
317 318 319 320
		if (unlikely(!loopback &&
			     (lid == ppd->lid ||
			      (lid == be16_to_cpu(IB_LID_PERMISSIVE) &&
			      qp->ibqp.qp_type == IB_QPT_GSI)))) {
321
			unsigned long tflags = ps->flags;
M
Mike Marciniszyn 已提交
322 323 324 325 326 327 328
			/*
			 * If DMAs are in progress, we can't generate
			 * a completion for the loopback packet since
			 * it would be out of order.
			 * Instead of waiting, we could queue a
			 * zero length descriptor so we get a callback.
			 */
329
			if (iowait_sdma_pending(&priv->s_iowait)) {
330
				qp->s_flags |= RVT_S_WAIT_DMA;
M
Mike Marciniszyn 已提交
331 332 333
				goto bail;
			}
			qp->s_cur = next_cur;
334
			spin_unlock_irqrestore(&qp->s_lock, tflags);
M
Mike Marciniszyn 已提交
335
			ud_loopback(qp, wqe);
336 337
			spin_lock_irqsave(&qp->s_lock, tflags);
			ps->flags = tflags;
M
Mike Marciniszyn 已提交
338
			hfi1_send_complete(qp, wqe, IB_WC_SUCCESS);
339
			goto done_free_tx;
M
Mike Marciniszyn 已提交
340 341 342 343 344 345 346 347 348
		}
	}

	qp->s_cur = next_cur;
	extra_bytes = -wqe->length & 3;
	nwords = (wqe->length + extra_bytes) >> 2;

	/* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
	qp->s_hdrwords = 7;
349
	ps->s_txreq->s_cur_size = wqe->length;
350
	ps->s_txreq->ss = &qp->s_sge;
351
	qp->s_srate = rdma_ah_get_static_rate(ah_attr);
M
Mike Marciniszyn 已提交
352 353 354 355 356 357 358
	qp->srate_mbps = ib_rate_to_mbps(qp->s_srate);
	qp->s_wqe = wqe;
	qp->s_sge.sge = wqe->sg_list[0];
	qp->s_sge.sg_list = wqe->sg_list + 1;
	qp->s_sge.num_sge = wqe->wr.num_sge;
	qp->s_sge.total_len = wqe->length;

359
	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
M
Mike Marciniszyn 已提交
360
		/* Header size in 32-bit words. */
361 362 363 364 365
		qp->s_hdrwords +=
			hfi1_make_grh(ibp,
				      &ps->s_txreq->phdr.hdr.ibh.u.l.grh,
				      rdma_ah_read_grh(ah_attr),
				      qp->s_hdrwords, nwords);
M
Mike Marciniszyn 已提交
366
		lrh0 = HFI1_LRH_GRH;
367
		ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
M
Mike Marciniszyn 已提交
368 369 370 371 372 373 374
		/*
		 * Don't worry about sending to locally attached multicast
		 * QPs.  It is unspecified by the spec. what happens.
		 */
	} else {
		/* Header size in 32-bit words. */
		lrh0 = HFI1_LRH_BTH;
375
		ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
M
Mike Marciniszyn 已提交
376 377 378 379 380
	}
	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
		qp->s_hdrwords++;
		ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
		bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
381
	} else {
M
Mike Marciniszyn 已提交
382
		bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
383
	}
384 385
	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
M
Mike Marciniszyn 已提交
386 387
	if (qp->ibqp.qp_type == IB_QPT_SMI) {
		lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
388
		priv->s_sc = 0xf;
M
Mike Marciniszyn 已提交
389 390
	} else {
		lrh0 |= (sc5 & 0xf) << 12;
391
		priv->s_sc = sc5;
M
Mike Marciniszyn 已提交
392
	}
393
	priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
394
	ps->s_txreq->sde = priv->s_sde;
395
	priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
396
	ps->s_txreq->psc = priv->s_sendcontext;
397 398
	ps->s_txreq->phdr.hdr.ibh.lrh[0] = cpu_to_be16(lrh0);
	ps->s_txreq->phdr.hdr.ibh.lrh[1] =
399
		cpu_to_be16(rdma_ah_get_dlid(ah_attr));
400
	ps->s_txreq->phdr.hdr.ibh.lrh[2] =
M
Mike Marciniszyn 已提交
401
		cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
402
	if (rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
403
		ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE;
404
	} else {
M
Mike Marciniszyn 已提交
405 406
		lid = ppd->lid;
		if (lid) {
407 408
			lid |= rdma_ah_get_path_bits(ah_attr) &
				((1 << ppd->lmc) - 1);
409
			ps->s_txreq->phdr.hdr.ibh.lrh[3] = cpu_to_be16(lid);
410
		} else {
411
			ps->s_txreq->phdr.hdr.ibh.lrh[3] = IB_LID_PERMISSIVE;
412
		}
M
Mike Marciniszyn 已提交
413 414 415 416 417
	}
	if (wqe->wr.send_flags & IB_SEND_SOLICITED)
		bth0 |= IB_BTH_SOLICITED;
	bth0 |= extra_bytes << 20;
	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI)
C
Christoph Hellwig 已提交
418
		bth0 |= hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
M
Mike Marciniszyn 已提交
419 420 421
	else
		bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index);
	ohdr->bth[0] = cpu_to_be32(bth0);
C
Christoph Hellwig 已提交
422
	ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn);
423
	ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn));
M
Mike Marciniszyn 已提交
424 425 426 427
	/*
	 * Qkeys with the high order bit set mean use the
	 * qkey from the QP context instead of the WR (see 10.2.5).
	 */
C
Christoph Hellwig 已提交
428 429
	ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ?
					 qp->qkey : wqe->ud_wr.remote_qkey);
M
Mike Marciniszyn 已提交
430 431
	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
	/* disarm any ahg */
432 433 434
	priv->s_ahg->ahgcount = 0;
	priv->s_ahg->ahgidx = 0;
	priv->s_ahg->tx_flags = 0;
435 436
	/* pbc */
	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
M
Mike Marciniszyn 已提交
437

438
	return 1;
439 440 441 442 443 444

done_free_tx:
	hfi1_put_txreq(ps->s_txreq);
	ps->s_txreq = NULL;
	return 1;

M
Mike Marciniszyn 已提交
445
bail:
446 447 448 449
	hfi1_put_txreq(ps->s_txreq);

bail_no_tx:
	ps->s_txreq = NULL;
450
	qp->s_flags &= ~RVT_S_BUSY;
451 452
	qp->s_hdrwords = 0;
	return 0;
M
Mike Marciniszyn 已提交
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
}

/*
 * Hardware can't check this so we do it here.
 *
 * This is a slightly different algorithm than the standard pkey check.  It
 * special cases the management keys and allows for 0x7fff and 0xffff to be in
 * the table at the same time.
 *
 * @returns the index found or -1 if not found
 */
int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey)
{
	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
	unsigned i;

	if (pkey == FULL_MGMT_P_KEY || pkey == LIM_MGMT_P_KEY) {
		unsigned lim_idx = -1;

		for (i = 0; i < ARRAY_SIZE(ppd->pkeys); ++i) {
			/* here we look for an exact match */
			if (ppd->pkeys[i] == pkey)
				return i;
			if (ppd->pkeys[i] == LIM_MGMT_P_KEY)
				lim_idx = i;
		}

		/* did not find 0xffff return 0x7fff idx if found */
		if (pkey == FULL_MGMT_P_KEY)
			return lim_idx;

		/* no match...  */
		return -1;
	}

	pkey &= 0x7fff; /* remove limited/full membership bit */

	for (i = 0; i < ARRAY_SIZE(ppd->pkeys); ++i)
		if ((ppd->pkeys[i] & 0x7fff) == pkey)
			return i;

	/*
	 * Should not get here, this means hardware failed to validate pkeys.
	 */
	return -1;
}

500
void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
M
Mike Marciniszyn 已提交
501 502 503 504 505 506 507
		u32 pkey, u32 slid, u32 dlid, u8 sc5,
		const struct ib_grh *old_grh)
{
	u64 pbc, pbc_flags = 0;
	u32 bth0, plen, vl, hwords = 5;
	u16 lrh0;
	u8 sl = ibp->sc_to_sl[sc5];
508 509
	struct ib_header hdr;
	struct ib_other_headers *ohdr;
M
Mike Marciniszyn 已提交
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
	struct pio_buf *pbuf;
	struct send_context *ctxt = qp_to_send_context(qp, sc5);
	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);

	if (old_grh) {
		struct ib_grh *grh = &hdr.u.l.grh;

		grh->version_tclass_flow = old_grh->version_tclass_flow;
		grh->paylen = cpu_to_be16((hwords - 2 + SIZE_OF_CRC) << 2);
		grh->hop_limit = 0xff;
		grh->sgid = old_grh->dgid;
		grh->dgid = old_grh->sgid;
		ohdr = &hdr.u.l.oth;
		lrh0 = HFI1_LRH_GRH;
		hwords += sizeof(struct ib_grh) / sizeof(u32);
	} else {
		ohdr = &hdr.u.oth;
		lrh0 = HFI1_LRH_BTH;
	}

	lrh0 |= (sc5 & 0xf) << 12 | sl << 4;

	bth0 = pkey | (IB_OPCODE_CNP << 24);
	ohdr->bth[0] = cpu_to_be32(bth0);

535
	ohdr->bth[1] = cpu_to_be32(remote_qpn | (1 << IB_BECN_SHIFT));
M
Mike Marciniszyn 已提交
536 537 538 539 540 541 542 543
	ohdr->bth[2] = 0; /* PSN 0 */

	hdr.lrh[0] = cpu_to_be16(lrh0);
	hdr.lrh[1] = cpu_to_be16(dlid);
	hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
	hdr.lrh[3] = cpu_to_be16(slid);

	plen = 2 /* PBC */ + hwords;
544
	pbc_flags |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
M
Mike Marciniszyn 已提交
545 546 547 548 549 550 551 552 553 554 555 556
	vl = sc_to_vlt(ppd->dd, sc5);
	pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
	if (ctxt) {
		pbuf = sc_buffer_alloc(ctxt, plen, NULL, NULL);
		if (pbuf)
			ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc,
						 &hdr, hwords);
	}
}

/*
 * opa_smp_check() - Do the regular pkey checking, and the additional
557 558
 * checks for SMPs specified in OPAv1 rev 1.0, 9/19/2016 update, section
 * 9.10.25 ("SMA Packet Checks").
M
Mike Marciniszyn 已提交
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
 *
 * Note that:
 *   - Checks are done using the pkey directly from the packet's BTH,
 *     and specifically _not_ the pkey that we attach to the completion,
 *     which may be different.
 *   - These checks are specifically for "non-local" SMPs (i.e., SMPs
 *     which originated on another node). SMPs which are sent from, and
 *     destined to this node are checked in opa_local_smp_check().
 *
 * At the point where opa_smp_check() is called, we know:
 *   - destination QP is QP0
 *
 * opa_smp_check() returns 0 if all checks succeed, 1 otherwise.
 */
static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
574
			 struct rvt_qp *qp, u16 slid, struct opa_smp *smp)
M
Mike Marciniszyn 已提交
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
{
	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);

	/*
	 * I don't think it's possible for us to get here with sc != 0xf,
	 * but check it to be certain.
	 */
	if (sc5 != 0xf)
		return 1;

	if (rcv_pkey_check(ppd, pkey, sc5, slid))
		return 1;

	/*
	 * At this point we know (and so don't need to check again) that
	 * the pkey is either LIM_MGMT_P_KEY, or FULL_MGMT_P_KEY
	 * (see ingress_pkey_check).
	 */
	if (smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE &&
	    smp->mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED) {
		ingress_pkey_table_fail(ppd, pkey, slid);
		return 1;
	}

	/*
	 * SMPs fall into one of four (disjoint) categories:
601 602 603
	 * SMA request, SMA response, SMA trap, or SMA trap repress.
	 * Our response depends, in part, on which type of SMP we're
	 * processing.
M
Mike Marciniszyn 已提交
604
	 *
605 606 607
	 * If this is an SMA response, skip the check here.
	 *
	 * If this is an SMA request or SMA trap repress:
M
Mike Marciniszyn 已提交
608 609
	 *   - pkey != FULL_MGMT_P_KEY =>
	 *       increment port recv constraint errors, drop MAD
610 611 612 613 614 615 616 617
	 *
	 * Otherwise:
	 *    - accept if the port is running an SM
	 *    - drop MAD if it's an SMA trap
	 *    - pkey == FULL_MGMT_P_KEY =>
	 *        reply with unsupported method
	 *    - pkey != FULL_MGMT_P_KEY =>
	 *	  increment port recv constraint errors, drop MAD
M
Mike Marciniszyn 已提交
618 619
	 */
	switch (smp->method) {
620 621 622
	case IB_MGMT_METHOD_GET_RESP:
	case IB_MGMT_METHOD_REPORT_RESP:
		break;
M
Mike Marciniszyn 已提交
623 624 625 626 627 628 629 630 631
	case IB_MGMT_METHOD_GET:
	case IB_MGMT_METHOD_SET:
	case IB_MGMT_METHOD_REPORT:
	case IB_MGMT_METHOD_TRAP_REPRESS:
		if (pkey != FULL_MGMT_P_KEY) {
			ingress_pkey_table_fail(ppd, pkey, slid);
			return 1;
		}
		break;
632
	default:
633
		if (ibp->rvp.port_cap_flags & IB_PORT_SM)
M
Mike Marciniszyn 已提交
634
			return 0;
635 636
		if (smp->method == IB_MGMT_METHOD_TRAP)
			return 1;
M
Mike Marciniszyn 已提交
637 638 639 640
		if (pkey == FULL_MGMT_P_KEY) {
			smp->status |= IB_SMP_UNSUP_METHOD;
			return 0;
		}
641 642
		ingress_pkey_table_fail(ppd, pkey, slid);
		return 1;
M
Mike Marciniszyn 已提交
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661
	}
	return 0;
}

/**
 * hfi1_ud_rcv - receive an incoming UD packet
 * @ibp: the port the packet came in on
 * @hdr: the packet header
 * @rcv_flags: flags relevant to rcv processing
 * @data: the packet data
 * @tlen: the packet length
 * @qp: the QP the packet came on
 *
 * This is called from qp_rcv() to process an incoming UD packet
 * for the given QP.
 * Called at interrupt level.
 */
void hfi1_ud_rcv(struct hfi1_packet *packet)
{
662
	struct ib_other_headers *ohdr = packet->ohdr;
M
Mike Marciniszyn 已提交
663 664 665 666
	u32 hdrsize = packet->hlen;
	struct ib_wc wc;
	u32 qkey;
	u32 src_qp;
667
	u16 pkey;
M
Mike Marciniszyn 已提交
668
	int mgmt_pkey_idx = -1;
669
	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
670
	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
671
	struct ib_header *hdr = packet->hdr;
672
	void *data = packet->payload;
M
Mike Marciniszyn 已提交
673
	u32 tlen = packet->tlen;
674
	struct rvt_qp *qp = packet->qp;
675
	u8 sc5 = hfi1_9B_get_sc5(hdr, packet->rhf);
676 677 678 679 680 681
	u8 sl_from_sc;
	u8 extra_bytes = packet->pad;
	u8 opcode = packet->opcode;
	u8 sl = packet->sl;
	u32 dlid = packet->dlid;
	u32 slid = packet->slid;
M
Mike Marciniszyn 已提交
682

683 684
	qkey = ib_get_qkey(ohdr);
	src_qp = ib_get_sqpn(ohdr);
685
	pkey = ib_bth_get_pkey(ohdr);
686 687
	extra_bytes += (SIZE_OF_CRC << 2);
	sl_from_sc = ibp->sc_to_sl[sc5];
M
Mike Marciniszyn 已提交
688

689
	process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
M
Mike Marciniszyn 已提交
690 691 692 693
	/*
	 * Get the number of bytes the message was padded by
	 * and drop incomplete packets.
	 */
694
	if (unlikely(tlen < (hdrsize + extra_bytes)))
M
Mike Marciniszyn 已提交
695 696
		goto drop;

697
	tlen -= hdrsize + extra_bytes;
M
Mike Marciniszyn 已提交
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714

	/*
	 * Check that the permissive LID is only used on QP0
	 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
	 */
	if (qp->ibqp.qp_num) {
		if (unlikely(hdr->lrh[1] == IB_LID_PERMISSIVE ||
			     hdr->lrh[3] == IB_LID_PERMISSIVE))
			goto drop;
		if (qp->ibqp.qp_num > 1) {
			if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) {
				/*
				 * Traps will not be sent for packets dropped
				 * by the HW. This is fine, as sending trap
				 * for invalid pkeys is optional according to
				 * IB spec (release 1.3, section 10.9.4)
				 */
715 716 717 718
				hfi1_bad_pkey(ibp,
					      pkey, sl,
					      src_qp, qp->ibqp.qp_num,
					      slid, dlid);
M
Mike Marciniszyn 已提交
719 720 721 722 723 724 725 726
				return;
			}
		} else {
			/* GSI packet */
			mgmt_pkey_idx = hfi1_lookup_pkey_idx(ibp, pkey);
			if (mgmt_pkey_idx < 0)
				goto drop;
		}
727
		if (unlikely(qkey != qp->qkey)) /* Silent drop */
M
Mike Marciniszyn 已提交
728
			return;
729

M
Mike Marciniszyn 已提交
730 731
		/* Drop invalid MAD packets (see 13.5.3.1). */
		if (unlikely(qp->ibqp.qp_num == 1 &&
732
			     (tlen > 2048 || (sc5 == 0xF))))
M
Mike Marciniszyn 已提交
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
			goto drop;
	} else {
		/* Received on QP0, and so by definition, this is an SMP */
		struct opa_smp *smp = (struct opa_smp *)data;

		if (opa_smp_check(ibp, pkey, sc5, qp, slid, smp))
			goto drop;

		if (tlen > 2048)
			goto drop;
		if ((hdr->lrh[1] == IB_LID_PERMISSIVE ||
		     hdr->lrh[3] == IB_LID_PERMISSIVE) &&
		    smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
			goto drop;

		/* look up SMI pkey */
		mgmt_pkey_idx = hfi1_lookup_pkey_idx(ibp, pkey);
		if (mgmt_pkey_idx < 0)
			goto drop;
	}

	if (qp->ibqp.qp_num > 1 &&
	    opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) {
		wc.ex.imm_data = ohdr->u.ud.imm_data;
		wc.wc_flags = IB_WC_WITH_IMM;
		tlen -= sizeof(u32);
	} else if (opcode == IB_OPCODE_UD_SEND_ONLY) {
		wc.ex.imm_data = 0;
		wc.wc_flags = 0;
762
	} else {
M
Mike Marciniszyn 已提交
763
		goto drop;
764
	}
M
Mike Marciniszyn 已提交
765 766 767 768 769 770 771 772 773 774

	/*
	 * A GRH is expected to precede the data even if not
	 * present on the wire.
	 */
	wc.byte_len = tlen + sizeof(struct ib_grh);

	/*
	 * Get the next work request entry to find where to put the data.
	 */
775
	if (qp->r_flags & RVT_R_REUSE_SGE) {
776
		qp->r_flags &= ~RVT_R_REUSE_SGE;
777
	} else {
M
Mike Marciniszyn 已提交
778 779
		int ret;

780
		ret = hfi1_rvt_get_rwqe(qp, 0);
M
Mike Marciniszyn 已提交
781
		if (ret < 0) {
782
			rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
M
Mike Marciniszyn 已提交
783 784 785 786
			return;
		}
		if (!ret) {
			if (qp->ibqp.qp_num == 0)
787
				ibp->rvp.n_vl15_dropped++;
M
Mike Marciniszyn 已提交
788 789 790 791 792
			return;
		}
	}
	/* Silently drop packets which are too big. */
	if (unlikely(wc.byte_len > qp->r_len)) {
793
		qp->r_flags |= RVT_R_REUSE_SGE;
M
Mike Marciniszyn 已提交
794 795
		goto drop;
	}
796
	if (packet->grh) {
M
Mike Marciniszyn 已提交
797
		hfi1_copy_sge(&qp->r_sge, &hdr->u.l.grh,
798
			      sizeof(struct ib_grh), true, false);
M
Mike Marciniszyn 已提交
799
		wc.wc_flags |= IB_WC_GRH;
800
	} else {
801
		rvt_skip_sge(&qp->r_sge, sizeof(struct ib_grh), true);
802
	}
803
	hfi1_copy_sge(&qp->r_sge, data, wc.byte_len - sizeof(struct ib_grh),
804
		      true, false);
805
	rvt_put_ss(&qp->r_sge);
806
	if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
M
Mike Marciniszyn 已提交
807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
		return;
	wc.wr_id = qp->r_wr_id;
	wc.status = IB_WC_SUCCESS;
	wc.opcode = IB_WC_RECV;
	wc.vendor_err = 0;
	wc.qp = &qp->ibqp;
	wc.src_qp = src_qp;

	if (qp->ibqp.qp_type == IB_QPT_GSI ||
	    qp->ibqp.qp_type == IB_QPT_SMI) {
		if (mgmt_pkey_idx < 0) {
			if (net_ratelimit()) {
				struct hfi1_devdata *dd = ppd->dd;

				dd_dev_err(dd, "QP type %d mgmt_pkey_idx < 0 and packet not dropped???\n",
					   qp->ibqp.qp_type);
				mgmt_pkey_idx = 0;
			}
		}
		wc.pkey_index = (unsigned)mgmt_pkey_idx;
827
	} else {
M
Mike Marciniszyn 已提交
828
		wc.pkey_index = 0;
829
	}
M
Mike Marciniszyn 已提交
830

831 832
	wc.slid = slid;
	wc.sl = sl_from_sc;
M
Mike Marciniszyn 已提交
833 834 835 836

	/*
	 * Save the LMC lower bits if the destination LID is a unicast LID.
	 */
837
	wc.dlid_path_bits = dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE) ? 0 :
M
Mike Marciniszyn 已提交
838 839 840
		dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1);
	wc.port_num = qp->port_num;
	/* Signal completion event if the solicited bit is set. */
841 842 843
	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
		     (ohdr->bth[0] &
		      cpu_to_be32(IB_BTH_SOLICITED)) != 0);
M
Mike Marciniszyn 已提交
844 845 846
	return;

drop:
847
	ibp->rvp.n_pkt_drops++;
M
Mike Marciniszyn 已提交
848
}