qp.c 51.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
32 33 34

#include <linux/module.h>

35 36
#include "iw_cxgb4.h"

37 38 39 40
static int db_delay_usecs = 1;
module_param(db_delay_usecs, int, 0644);
MODULE_PARM_DESC(db_delay_usecs, "Usecs to delay awaiting db fifo to drain");

41
static int ocqp_support = 1;
S
Steve Wise 已提交
42
module_param(ocqp_support, int, 0644);
43
MODULE_PARM_DESC(ocqp_support, "Support on-chip SQs (default=1)");
S
Steve Wise 已提交
44

45
int db_fc_threshold = 1000;
46
module_param(db_fc_threshold, int, 0644);
47 48 49 50 51 52 53 54 55
MODULE_PARM_DESC(db_fc_threshold,
		 "QP count/threshold that triggers"
		 " automatic db flow control mode (default = 1000)");

int db_coalescing_threshold;
module_param(db_coalescing_threshold, int, 0644);
MODULE_PARM_DESC(db_coalescing_threshold,
		 "QP count/threshold that triggers"
		 " disabling db coalescing (default = 0)");
56

57 58 59 60
static int max_fr_immd = T4_MAX_FR_IMMD;
module_param(max_fr_immd, int, 0644);
MODULE_PARM_DESC(max_fr_immd, "fastreg threshold for using DSGL instead of immedate");

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
static int alloc_ird(struct c4iw_dev *dev, u32 ird)
{
	int ret = 0;

	spin_lock_irq(&dev->lock);
	if (ird <= dev->avail_ird)
		dev->avail_ird -= ird;
	else
		ret = -ENOMEM;
	spin_unlock_irq(&dev->lock);

	if (ret)
		dev_warn(&dev->rdev.lldi.pdev->dev,
			 "device IRD resources exhausted\n");

	return ret;
}

static void free_ird(struct c4iw_dev *dev, int ird)
{
	spin_lock_irq(&dev->lock);
	dev->avail_ird += ird;
	spin_unlock_irq(&dev->lock);
}

86 87 88 89 90 91 92 93
static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
{
	unsigned long flag;
	spin_lock_irqsave(&qhp->lock, flag);
	qhp->attr.state = state;
	spin_unlock_irqrestore(&qhp->lock, flag);
}

S
Steve Wise 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
static void dealloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	c4iw_ocqp_pool_free(rdev, sq->dma_addr, sq->memsize);
}

static void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	dma_free_coherent(&(rdev->lldi.pdev->dev), sq->memsize, sq->queue,
			  pci_unmap_addr(sq, mapping));
}

static void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	if (t4_sq_onchip(sq))
		dealloc_oc_sq(rdev, sq);
	else
		dealloc_host_sq(rdev, sq);
}

static int alloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
115
	if (!ocqp_support || !ocqp_supported(&rdev->lldi))
S
Steve Wise 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
		return -ENOSYS;
	sq->dma_addr = c4iw_ocqp_pool_alloc(rdev, sq->memsize);
	if (!sq->dma_addr)
		return -ENOMEM;
	sq->phys_addr = rdev->oc_mw_pa + sq->dma_addr -
			rdev->lldi.vr->ocq.start;
	sq->queue = (__force union t4_wr *)(rdev->oc_mw_kva + sq->dma_addr -
					    rdev->lldi.vr->ocq.start);
	sq->flags |= T4_SQ_ONCHIP;
	return 0;
}

static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	sq->queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), sq->memsize,
				       &(sq->dma_addr), GFP_KERNEL);
	if (!sq->queue)
		return -ENOMEM;
	sq->phys_addr = virt_to_phys(sq->queue);
	pci_unmap_addr_set(sq, mapping, sq->dma_addr);
	return 0;
}

139 140 141 142 143 144 145 146 147 148
static int alloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq, int user)
{
	int ret = -ENOSYS;
	if (user)
		ret = alloc_oc_sq(rdev, sq);
	if (ret)
		ret = alloc_host_sq(rdev, sq);
	return ret;
}

149 150 151 152 153 154 155 156 157
static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
		      struct c4iw_dev_ucontext *uctx)
{
	/*
	 * uP clears EQ contexts when the connection exits rdma mode,
	 * so no need to post a RESET WR for these EQs.
	 */
	dma_free_coherent(&(rdev->lldi.pdev->dev),
			  wq->rq.memsize, wq->rq.queue,
158
			  dma_unmap_addr(&wq->rq, mapping));
S
Steve Wise 已提交
159
	dealloc_sq(rdev, &wq->sq);
160 161 162 163 164 165 166 167
	c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
	kfree(wq->rq.sw_rq);
	kfree(wq->sq.sw_sq);
	c4iw_put_qpid(rdev, wq->rq.qid, uctx);
	c4iw_put_qpid(rdev, wq->sq.qid, uctx);
	return 0;
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
/*
 * Determine the BAR2 virtual address and qid. If pbar2_pa is not NULL,
 * then this is a user mapping so compute the page-aligned physical address
 * for mapping.
 */
void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
			      enum cxgb4_bar2_qtype qtype,
			      unsigned int *pbar2_qid, u64 *pbar2_pa)
{
	u64 bar2_qoffset;
	int ret;

	ret = cxgb4_bar2_sge_qregs(rdev->lldi.ports[0], qid, qtype,
				   pbar2_pa ? 1 : 0,
				   &bar2_qoffset, pbar2_qid);
	if (ret)
		return NULL;

	if (pbar2_pa)
		*pbar2_pa = (rdev->bar2_pa + bar2_qoffset) & PAGE_MASK;
188 189 190 191

	if (is_t4(rdev->lldi.adapter_type))
		return NULL;

192 193 194
	return rdev->bar2_kva + bar2_qoffset;
}

195 196 197 198 199 200 201 202 203 204
static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
		     struct t4_cq *rcq, struct t4_cq *scq,
		     struct c4iw_dev_ucontext *uctx)
{
	int user = (uctx != &rdev->uctx);
	struct fw_ri_res_wr *res_wr;
	struct fw_ri_res *res;
	int wr_len;
	struct c4iw_wr_wait wr_wait;
	struct sk_buff *skb;
205
	int ret = 0;
206 207 208 209 210 211 212
	int eqsize;

	wq->sq.qid = c4iw_get_qpid(rdev, uctx);
	if (!wq->sq.qid)
		return -ENOMEM;

	wq->rq.qid = c4iw_get_qpid(rdev, uctx);
213 214 215 216
	if (!wq->rq.qid) {
		ret = -ENOMEM;
		goto free_sq_qid;
	}
217 218 219 220

	if (!user) {
		wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,
				 GFP_KERNEL);
221 222 223 224
		if (!wq->sq.sw_sq) {
			ret = -ENOMEM;
			goto free_rq_qid;
		}
225 226 227

		wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq,
				 GFP_KERNEL);
228 229 230 231
		if (!wq->rq.sw_rq) {
			ret = -ENOMEM;
			goto free_sw_sq;
		}
232 233 234
	}

	/*
235
	 * RQT must be a power of 2 and at least 16 deep.
236
	 */
237
	wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16));
238
	wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size);
239 240 241 242
	if (!wq->rq.rqt_hwaddr) {
		ret = -ENOMEM;
		goto free_sw_rq;
	}
243

244 245 246
	ret = alloc_sq(rdev, &wq->sq, user);
	if (ret)
		goto free_hwaddr;
247
	memset(wq->sq.queue, 0, wq->sq.memsize);
248
	dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);
249 250 251 252

	wq->rq.queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev),
					  wq->rq.memsize, &(wq->rq.dma_addr),
					  GFP_KERNEL);
253 254
	if (!wq->rq.queue) {
		ret = -ENOMEM;
255
		goto free_sq;
256
	}
257 258 259 260 261 262
	PDBG("%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx\n",
		__func__, wq->sq.queue,
		(unsigned long long)virt_to_phys(wq->sq.queue),
		wq->rq.queue,
		(unsigned long long)virt_to_phys(wq->rq.queue));
	memset(wq->rq.queue, 0, wq->rq.memsize);
263
	dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
264 265

	wq->db = rdev->lldi.db_reg;
266

267 268 269 270 271 272 273 274 275 276
	wq->sq.bar2_va = c4iw_bar2_addrs(rdev, wq->sq.qid, T4_BAR2_QTYPE_EGRESS,
					 &wq->sq.bar2_qid,
					 user ? &wq->sq.bar2_pa : NULL);
	wq->rq.bar2_va = c4iw_bar2_addrs(rdev, wq->rq.qid, T4_BAR2_QTYPE_EGRESS,
					 &wq->rq.bar2_qid,
					 user ? &wq->rq.bar2_pa : NULL);

	/*
	 * User mode must have bar2 access.
	 */
277
	if (user && (!wq->sq.bar2_pa || !wq->rq.bar2_pa)) {
278 279 280
		pr_warn(MOD "%s: sqid %u or rqid %u not in BAR2 range.\n",
			pci_name(rdev->lldi.pdev), wq->sq.qid, wq->rq.qid);
		goto free_dma;
281
	}
282

283 284 285 286 287 288
	wq->rdev = rdev;
	wq->rq.msn = 1;

	/* build fw_ri_res_wr */
	wr_len = sizeof *res_wr + 2 * sizeof *res;

289
	skb = alloc_skb(wr_len, GFP_KERNEL);
290 291
	if (!skb) {
		ret = -ENOMEM;
292
		goto free_dma;
293 294 295 296 297 298
	}
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);

	res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len);
	memset(res_wr, 0, wr_len);
	res_wr->op_nres = cpu_to_be32(
299
			FW_WR_OP_V(FW_RI_RES_WR) |
300
			FW_RI_RES_WR_NRES_V(2) |
301
			FW_WR_COMPL_F);
302
	res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
H
Hariprasad S 已提交
303
	res_wr->cookie = (uintptr_t)&wr_wait;
304 305 306 307 308 309 310
	res = res_wr->res;
	res->u.sqrq.restype = FW_RI_RES_TYPE_SQ;
	res->u.sqrq.op = FW_RI_RES_OP_WRITE;

	/*
	 * eqsize is the number of 64B entries plus the status page size.
	 */
311 312
	eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +
		rdev->hw_queue.t4_eq_status_entries;
313 314

	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
315 316 317 318 319
		FW_RI_RES_WR_HOSTFCMODE_V(0) |	/* no host cidx updates */
		FW_RI_RES_WR_CPRIO_V(0) |	/* don't keep in chip cache */
		FW_RI_RES_WR_PCIECHN_V(0) |	/* set by uP at ri_init time */
		(t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_ONCHIP_F : 0) |
		FW_RI_RES_WR_IQID_V(scq->cqid));
320
	res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
321 322 323 324 325 326 327
		FW_RI_RES_WR_DCAEN_V(0) |
		FW_RI_RES_WR_DCACPU_V(0) |
		FW_RI_RES_WR_FBMIN_V(2) |
		FW_RI_RES_WR_FBMAX_V(2) |
		FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
		FW_RI_RES_WR_CIDXFTHRESH_V(0) |
		FW_RI_RES_WR_EQSIZE_V(eqsize));
328 329 330 331 332 333 334 335 336
	res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);
	res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);
	res++;
	res->u.sqrq.restype = FW_RI_RES_TYPE_RQ;
	res->u.sqrq.op = FW_RI_RES_OP_WRITE;

	/*
	 * eqsize is the number of 64B entries plus the status page size.
	 */
337 338
	eqsize = wq->rq.size * T4_RQ_NUM_SLOTS +
		rdev->hw_queue.t4_eq_status_entries;
339
	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
340 341 342 343
		FW_RI_RES_WR_HOSTFCMODE_V(0) |	/* no host cidx updates */
		FW_RI_RES_WR_CPRIO_V(0) |	/* don't keep in chip cache */
		FW_RI_RES_WR_PCIECHN_V(0) |	/* set by uP at ri_init time */
		FW_RI_RES_WR_IQID_V(rcq->cqid));
344
	res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
345 346 347 348 349 350 351
		FW_RI_RES_WR_DCAEN_V(0) |
		FW_RI_RES_WR_DCACPU_V(0) |
		FW_RI_RES_WR_FBMIN_V(2) |
		FW_RI_RES_WR_FBMAX_V(2) |
		FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
		FW_RI_RES_WR_CIDXFTHRESH_V(0) |
		FW_RI_RES_WR_EQSIZE_V(eqsize));
352 353 354 355 356 357 358
	res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid);
	res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr);

	c4iw_init_wr_wait(&wr_wait);

	ret = c4iw_ofld_send(rdev, skb);
	if (ret)
359
		goto free_dma;
360
	ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__);
361
	if (ret)
362
		goto free_dma;
363

364
	PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p sq_bar2_addr %p rq_bar2_addr %p\n",
365
	     __func__, wq->sq.qid, wq->rq.qid, wq->db,
366
	     wq->sq.bar2_va, wq->rq.bar2_va);
367 368

	return 0;
369
free_dma:
370 371
	dma_free_coherent(&(rdev->lldi.pdev->dev),
			  wq->rq.memsize, wq->rq.queue,
372
			  dma_unmap_addr(&wq->rq, mapping));
373
free_sq:
S
Steve Wise 已提交
374
	dealloc_sq(rdev, &wq->sq);
375
free_hwaddr:
376
	c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
377
free_sw_rq:
378
	kfree(wq->rq.sw_rq);
379
free_sw_sq:
380
	kfree(wq->sq.sw_sq);
381
free_rq_qid:
382
	c4iw_put_qpid(rdev, wq->rq.qid, uctx);
383
free_sq_qid:
384
	c4iw_put_qpid(rdev, wq->sq.qid, uctx);
385
	return ret;
386 387
}

388 389
static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,
		      struct ib_send_wr *wr, int max, u32 *plenp)
390
{
391 392
	u8 *dstp, *srcp;
	u32 plen = 0;
393
	int i;
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
	int rem, len;

	dstp = (u8 *)immdp->data;
	for (i = 0; i < wr->num_sge; i++) {
		if ((plen + wr->sg_list[i].length) > max)
			return -EMSGSIZE;
		srcp = (u8 *)(unsigned long)wr->sg_list[i].addr;
		plen += wr->sg_list[i].length;
		rem = wr->sg_list[i].length;
		while (rem) {
			if (dstp == (u8 *)&sq->queue[sq->size])
				dstp = (u8 *)sq->queue;
			if (rem <= (u8 *)&sq->queue[sq->size] - dstp)
				len = rem;
			else
				len = (u8 *)&sq->queue[sq->size] - dstp;
			memcpy(dstp, srcp, len);
			dstp += len;
			srcp += len;
			rem -= len;
		}
	}
S
Steve Wise 已提交
416 417 418
	len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);
	if (len)
		memset(dstp, 0, len);
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
	immdp->op = FW_RI_DATA_IMMD;
	immdp->r1 = 0;
	immdp->r2 = 0;
	immdp->immdlen = cpu_to_be32(plen);
	*plenp = plen;
	return 0;
}

static int build_isgl(__be64 *queue_start, __be64 *queue_end,
		      struct fw_ri_isgl *isglp, struct ib_sge *sg_list,
		      int num_sge, u32 *plenp)

{
	int i;
	u32 plen = 0;
	__be64 *flitp = (__be64 *)isglp->sge;

	for (i = 0; i < num_sge; i++) {
		if ((plen + sg_list[i].length) < plen)
			return -EMSGSIZE;
		plen += sg_list[i].length;
		*flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) |
				     sg_list[i].length);
		if (++flitp == queue_end)
			flitp = queue_start;
		*flitp = cpu_to_be64(sg_list[i].addr);
		if (++flitp == queue_end)
			flitp = queue_start;
	}
S
Steve Wise 已提交
448
	*flitp = (__force __be64)0;
449 450 451 452 453 454 455 456 457 458 459 460
	isglp->op = FW_RI_DATA_ISGL;
	isglp->r1 = 0;
	isglp->nsge = cpu_to_be16(num_sge);
	isglp->r2 = 0;
	if (plenp)
		*plenp = plen;
	return 0;
}

static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
			   struct ib_send_wr *wr, u8 *len16)
{
461 462
	u32 plen;
	int size;
463
	int ret;
464 465 466 467 468 469 470

	if (wr->num_sge > T4_MAX_SEND_SGE)
		return -EINVAL;
	switch (wr->opcode) {
	case IB_WR_SEND:
		if (wr->send_flags & IB_SEND_SOLICITED)
			wqe->send.sendop_pkd = cpu_to_be32(
471
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE));
472 473
		else
			wqe->send.sendop_pkd = cpu_to_be32(
474
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND));
475 476 477 478 479
		wqe->send.stag_inv = 0;
		break;
	case IB_WR_SEND_WITH_INV:
		if (wr->send_flags & IB_SEND_SOLICITED)
			wqe->send.sendop_pkd = cpu_to_be32(
480
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE_INV));
481 482
		else
			wqe->send.sendop_pkd = cpu_to_be32(
483
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_INV));
484 485 486 487 488 489
		wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
		break;

	default:
		return -EINVAL;
	}
490 491
	wqe->send.r3 = 0;
	wqe->send.r4 = 0;
492

493 494 495
	plen = 0;
	if (wr->num_sge) {
		if (wr->send_flags & IB_SEND_INLINE) {
496 497 498 499
			ret = build_immd(sq, wqe->send.u.immd_src, wr,
					 T4_MAX_SEND_INLINE, &plen);
			if (ret)
				return ret;
500 501 502
			size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
			       plen;
		} else {
503 504 505 506 507 508
			ret = build_isgl((__be64 *)sq->queue,
					 (__be64 *)&sq->queue[sq->size],
					 wqe->send.u.isgl_src,
					 wr->sg_list, wr->num_sge, &plen);
			if (ret)
				return ret;
509 510 511 512 513 514 515 516 517
			size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
			       wr->num_sge * sizeof(struct fw_ri_sge);
		}
	} else {
		wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
		wqe->send.u.immd_src[0].r1 = 0;
		wqe->send.u.immd_src[0].r2 = 0;
		wqe->send.u.immd_src[0].immdlen = 0;
		size = sizeof wqe->send + sizeof(struct fw_ri_immd);
518
		plen = 0;
519 520 521 522 523 524
	}
	*len16 = DIV_ROUND_UP(size, 16);
	wqe->send.plen = cpu_to_be32(plen);
	return 0;
}

525 526
static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
			    struct ib_send_wr *wr, u8 *len16)
527 528 529
{
	u32 plen;
	int size;
530
	int ret;
531

532
	if (wr->num_sge > T4_MAX_SEND_SGE)
533 534
		return -EINVAL;
	wqe->write.r2 = 0;
C
Christoph Hellwig 已提交
535 536
	wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey);
	wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr);
537 538
	if (wr->num_sge) {
		if (wr->send_flags & IB_SEND_INLINE) {
539 540 541 542
			ret = build_immd(sq, wqe->write.u.immd_src, wr,
					 T4_MAX_WRITE_INLINE, &plen);
			if (ret)
				return ret;
543 544 545
			size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
			       plen;
		} else {
546 547 548 549 550 551
			ret = build_isgl((__be64 *)sq->queue,
					 (__be64 *)&sq->queue[sq->size],
					 wqe->write.u.isgl_src,
					 wr->sg_list, wr->num_sge, &plen);
			if (ret)
				return ret;
552 553 554 555 556 557 558 559 560
			size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
			       wr->num_sge * sizeof(struct fw_ri_sge);
		}
	} else {
		wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
		wqe->write.u.immd_src[0].r1 = 0;
		wqe->write.u.immd_src[0].r2 = 0;
		wqe->write.u.immd_src[0].immdlen = 0;
		size = sizeof wqe->write + sizeof(struct fw_ri_immd);
561
		plen = 0;
562 563 564 565 566 567 568 569 570 571 572
	}
	*len16 = DIV_ROUND_UP(size, 16);
	wqe->write.plen = cpu_to_be32(plen);
	return 0;
}

static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16)
{
	if (wr->num_sge > 1)
		return -EINVAL;
	if (wr->num_sge) {
C
Christoph Hellwig 已提交
573 574
		wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey);
		wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr
575
							>> 32));
C
Christoph Hellwig 已提交
576
		wqe->read.to_src_lo = cpu_to_be32((u32)rdma_wr(wr)->remote_addr);
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
		wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey);
		wqe->read.plen = cpu_to_be32(wr->sg_list[0].length);
		wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr
							 >> 32));
		wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr));
	} else {
		wqe->read.stag_src = cpu_to_be32(2);
		wqe->read.to_src_hi = 0;
		wqe->read.to_src_lo = 0;
		wqe->read.stag_sink = cpu_to_be32(2);
		wqe->read.plen = 0;
		wqe->read.to_sink_hi = 0;
		wqe->read.to_sink_lo = 0;
	}
	wqe->read.r2 = 0;
	wqe->read.r5 = 0;
	*len16 = DIV_ROUND_UP(sizeof wqe->read, 16);
	return 0;
}

static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
			   struct ib_recv_wr *wr, u8 *len16)
{
600
	int ret;
601

602 603 604 605 606
	ret = build_isgl((__be64 *)qhp->wq.rq.queue,
			 (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size],
			 &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL);
	if (ret)
		return ret;
607 608 609 610 611
	*len16 = DIV_ROUND_UP(sizeof wqe->recv +
			      wr->num_sge * sizeof(struct fw_ri_sge), 16);
	return 0;
}

612
static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
613
			struct ib_reg_wr *wr, u8 *len16, bool dsgl_supported)
614 615 616 617 618 619 620 621
{
	struct c4iw_mr *mhp = to_c4iw_mr(wr->mr);
	struct fw_ri_immd *imdp;
	__be64 *p;
	int i;
	int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32);
	int rem;

622
	if (mhp->mpl_len > t4_max_fr_depth(dsgl_supported && use_dsgl))
623 624 625 626 627 628 629 630 631 632 633 634 635
		return -EINVAL;

	wqe->fr.qpbinde_to_dcacpu = 0;
	wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12;
	wqe->fr.addr_type = FW_RI_VA_BASED_TO;
	wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access);
	wqe->fr.len_hi = 0;
	wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length);
	wqe->fr.stag = cpu_to_be32(wr->key);
	wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
	wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova &
					0xffffffff);

636
	if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) {
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
		struct fw_ri_dsgl *sglp;

		for (i = 0; i < mhp->mpl_len; i++)
			mhp->mpl[i] = (__force u64)cpu_to_be64((u64)mhp->mpl[i]);

		sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1);
		sglp->op = FW_RI_DATA_DSGL;
		sglp->r1 = 0;
		sglp->nsge = cpu_to_be16(1);
		sglp->addr0 = cpu_to_be64(mhp->mpl_addr);
		sglp->len0 = cpu_to_be32(pbllen);

		*len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16);
	} else {
		imdp = (struct fw_ri_immd *)(&wqe->fr + 1);
		imdp->op = FW_RI_DATA_IMMD;
		imdp->r1 = 0;
		imdp->r2 = 0;
		imdp->immdlen = cpu_to_be32(pbllen);
		p = (__be64 *)(imdp + 1);
		rem = pbllen;
		for (i = 0; i < mhp->mpl_len; i++) {
			*p = cpu_to_be64((u64)mhp->mpl[i]);
			rem -= sizeof(*p);
			if (++p == (__be64 *)&sq->queue[sq->size])
				p = (__be64 *)sq->queue;
		}
		BUG_ON(rem < 0);
		while (rem) {
			*p = 0;
			rem -= sizeof(*p);
			if (++p == (__be64 *)&sq->queue[sq->size])
				p = (__be64 *)sq->queue;
		}
		*len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp)
				      + pbllen, 16);
	}
	return 0;
}

677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr,
			  u8 *len16)
{
	wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
	wqe->inv.r2 = 0;
	*len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
	return 0;
}

void c4iw_qp_add_ref(struct ib_qp *qp)
{
	PDBG("%s ib_qp %p\n", __func__, qp);
	atomic_inc(&(to_c4iw_qp(qp)->refcnt));
}

void c4iw_qp_rem_ref(struct ib_qp *qp)
{
	PDBG("%s ib_qp %p\n", __func__, qp);
	if (atomic_dec_and_test(&(to_c4iw_qp(qp)->refcnt)))
		wake_up(&(to_c4iw_qp(qp)->wait));
}

699 700 701 702 703 704 705 706 707 708 709 710
static void add_to_fc_list(struct list_head *head, struct list_head *entry)
{
	if (list_empty(entry))
		list_add_tail(entry, head);
}

static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
{
	unsigned long flags;

	spin_lock_irqsave(&qhp->rhp->lock, flags);
	spin_lock(&qhp->lock);
711
	if (qhp->rhp->db_state == NORMAL)
712
		t4_ring_sq_db(&qhp->wq, inc, NULL);
713
	else {
714 715 716 717 718 719 720 721 722 723 724 725 726 727
		add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
		qhp->wq.sq.wq_pidx_inc += inc;
	}
	spin_unlock(&qhp->lock);
	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
	return 0;
}

static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
{
	unsigned long flags;

	spin_lock_irqsave(&qhp->rhp->lock, flags);
	spin_lock(&qhp->lock);
728
	if (qhp->rhp->db_state == NORMAL)
729
		t4_ring_rq_db(&qhp->wq, inc, NULL);
730
	else {
731 732 733 734 735 736 737 738
		add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
		qhp->wq.rq.wq_pidx_inc += inc;
	}
	spin_unlock(&qhp->lock);
	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
	return 0;
}

739 740 741 742 743 744 745 746
int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
		   struct ib_send_wr **bad_wr)
{
	int err = 0;
	u8 len16 = 0;
	enum fw_wr_opcodes fw_opcode = 0;
	enum fw_ri_wr_flags fw_flags;
	struct c4iw_qp *qhp;
747
	union t4_wr *wqe = NULL;
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
	u32 num_wrs;
	struct t4_swsqe *swsqe;
	unsigned long flag;
	u16 idx = 0;

	qhp = to_c4iw_qp(ibqp);
	spin_lock_irqsave(&qhp->lock, flag);
	if (t4_wq_in_error(&qhp->wq)) {
		spin_unlock_irqrestore(&qhp->lock, flag);
		return -EINVAL;
	}
	num_wrs = t4_sq_avail(&qhp->wq);
	if (num_wrs == 0) {
		spin_unlock_irqrestore(&qhp->lock, flag);
		return -ENOMEM;
	}
	while (wr) {
		if (num_wrs == 0) {
			err = -ENOMEM;
			*bad_wr = wr;
			break;
		}
770 771 772
		wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue +
		      qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE);

773 774 775
		fw_flags = 0;
		if (wr->send_flags & IB_SEND_SOLICITED)
			fw_flags |= FW_RI_SOLICITED_EVENT_FLAG;
776
		if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all)
777 778 779 780 781 782 783 784 785 786 787 788
			fw_flags |= FW_RI_COMPLETION_FLAG;
		swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx];
		switch (wr->opcode) {
		case IB_WR_SEND_WITH_INV:
		case IB_WR_SEND:
			if (wr->send_flags & IB_SEND_FENCE)
				fw_flags |= FW_RI_READ_FENCE_FLAG;
			fw_opcode = FW_RI_SEND_WR;
			if (wr->opcode == IB_WR_SEND)
				swsqe->opcode = FW_RI_SEND;
			else
				swsqe->opcode = FW_RI_SEND_WITH_INV;
789
			err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16);
790 791 792 793
			break;
		case IB_WR_RDMA_WRITE:
			fw_opcode = FW_RI_RDMA_WRITE_WR;
			swsqe->opcode = FW_RI_RDMA_WRITE;
794
			err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16);
795 796
			break;
		case IB_WR_RDMA_READ:
797
		case IB_WR_RDMA_READ_WITH_INV:
798 799
			fw_opcode = FW_RI_RDMA_READ_WR;
			swsqe->opcode = FW_RI_READ_REQ;
800
			if (wr->opcode == IB_WR_RDMA_READ_WITH_INV)
801
				fw_flags = FW_RI_RDMA_READ_INVALIDATE;
802 803
			else
				fw_flags = 0;
804 805 806 807 808 809 810
			err = build_rdma_read(wqe, wr, &len16);
			if (err)
				break;
			swsqe->read_len = wr->sg_list[0].length;
			if (!qhp->wq.sq.oldest_read)
				qhp->wq.sq.oldest_read = swsqe;
			break;
811 812 813 814
		case IB_WR_REG_MR:
			fw_opcode = FW_RI_FR_NSMR_WR;
			swsqe->opcode = FW_RI_FAST_REGISTER;
			err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), &len16,
815
				qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
816
			break;
817
		case IB_WR_LOCAL_INV:
818 819
			if (wr->send_flags & IB_SEND_FENCE)
				fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
			fw_opcode = FW_RI_INV_LSTAG_WR;
			swsqe->opcode = FW_RI_LOCAL_INV;
			err = build_inv_stag(wqe, wr, &len16);
			break;
		default:
			PDBG("%s post of type=%d TBD!\n", __func__,
			     wr->opcode);
			err = -EINVAL;
		}
		if (err) {
			*bad_wr = wr;
			break;
		}
		swsqe->idx = qhp->wq.sq.pidx;
		swsqe->complete = 0;
835 836
		swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) ||
				  qhp->sq_sig_all;
S
Steve Wise 已提交
837
		swsqe->flushed = 0;
838
		swsqe->wr_id = wr->wr_id;
839 840 841 842 843
		if (c4iw_wr_log) {
			swsqe->sge_ts = cxgb4_read_sge_timestamp(
					qhp->rhp->rdev.lldi.ports[0]);
			getnstimeofday(&swsqe->host_ts);
		}
844 845 846 847 848 849 850 851

		init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);

		PDBG("%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u\n",
		     __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx,
		     swsqe->opcode, swsqe->read_len);
		wr = wr->next;
		num_wrs--;
852 853
		t4_sq_produce(&qhp->wq, len16);
		idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
854
	}
855
	if (!qhp->rhp->rdev.status_page->db_off) {
856
		t4_ring_sq_db(&qhp->wq, idx, wqe);
857 858 859 860 861
		spin_unlock_irqrestore(&qhp->lock, flag);
	} else {
		spin_unlock_irqrestore(&qhp->lock, flag);
		ring_kernel_sq_db(qhp, idx);
	}
862 863 864 865 866 867 868 869
	return err;
}

int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
		      struct ib_recv_wr **bad_wr)
{
	int err = 0;
	struct c4iw_qp *qhp;
870
	union t4_recv_wr *wqe = NULL;
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
	u32 num_wrs;
	u8 len16 = 0;
	unsigned long flag;
	u16 idx = 0;

	qhp = to_c4iw_qp(ibqp);
	spin_lock_irqsave(&qhp->lock, flag);
	if (t4_wq_in_error(&qhp->wq)) {
		spin_unlock_irqrestore(&qhp->lock, flag);
		return -EINVAL;
	}
	num_wrs = t4_rq_avail(&qhp->wq);
	if (num_wrs == 0) {
		spin_unlock_irqrestore(&qhp->lock, flag);
		return -ENOMEM;
	}
	while (wr) {
		if (wr->num_sge > T4_MAX_RECV_SGE) {
			err = -EINVAL;
			*bad_wr = wr;
			break;
		}
893 894 895
		wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue +
					   qhp->wq.rq.wq_pidx *
					   T4_EQ_ENTRY_SIZE);
896 897 898 899 900 901 902 903 904 905
		if (num_wrs)
			err = build_rdma_recv(qhp, wqe, wr, &len16);
		else
			err = -ENOMEM;
		if (err) {
			*bad_wr = wr;
			break;
		}

		qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id;
906 907 908 909 910 911 912
		if (c4iw_wr_log) {
			qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].sge_ts =
				cxgb4_read_sge_timestamp(
						qhp->rhp->rdev.lldi.ports[0]);
			getnstimeofday(
				&qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_ts);
		}
913 914 915 916 917 918 919 920 921 922

		wqe->recv.opcode = FW_RI_RECV_WR;
		wqe->recv.r1 = 0;
		wqe->recv.wrid = qhp->wq.rq.pidx;
		wqe->recv.r2[0] = 0;
		wqe->recv.r2[1] = 0;
		wqe->recv.r2[2] = 0;
		wqe->recv.len16 = len16;
		PDBG("%s cookie 0x%llx pidx %u\n", __func__,
		     (unsigned long long) wr->wr_id, qhp->wq.rq.pidx);
923 924
		t4_rq_produce(&qhp->wq, len16);
		idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
925 926 927
		wr = wr->next;
		num_wrs--;
	}
928
	if (!qhp->rhp->rdev.status_page->db_off) {
929
		t4_ring_rq_db(&qhp->wq, idx, wqe);
930 931 932 933 934
		spin_unlock_irqrestore(&qhp->lock, flag);
	} else {
		spin_unlock_irqrestore(&qhp->lock, flag);
		ring_kernel_rq_db(qhp, idx);
	}
935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	return err;
}

static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type,
				    u8 *ecode)
{
	int status;
	int tagged;
	int opcode;
	int rqtype;
	int send_inv;

	if (!err_cqe) {
		*layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
		*ecode = 0;
		return;
	}

	status = CQE_STATUS(err_cqe);
	opcode = CQE_OPCODE(err_cqe);
	rqtype = RQ_TYPE(err_cqe);
	send_inv = (opcode == FW_RI_SEND_WITH_INV) ||
		   (opcode == FW_RI_SEND_WITH_SE_INV);
	tagged = (opcode == FW_RI_RDMA_WRITE) ||
		 (rqtype && (opcode == FW_RI_READ_RESP));

	switch (status) {
	case T4_ERR_STAG:
		if (send_inv) {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
			*ecode = RDMAP_CANT_INV_STAG;
		} else {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
			*ecode = RDMAP_INV_STAG;
		}
		break;
	case T4_ERR_PDID:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		if ((opcode == FW_RI_SEND_WITH_INV) ||
		    (opcode == FW_RI_SEND_WITH_SE_INV))
			*ecode = RDMAP_CANT_INV_STAG;
		else
			*ecode = RDMAP_STAG_NOT_ASSOC;
		break;
	case T4_ERR_QPID:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_STAG_NOT_ASSOC;
		break;
	case T4_ERR_ACCESS:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_ACC_VIOL;
		break;
	case T4_ERR_WRAP:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_TO_WRAP;
		break;
	case T4_ERR_BOUND:
		if (tagged) {
			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
			*ecode = DDPT_BASE_BOUNDS;
		} else {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
			*ecode = RDMAP_BASE_BOUNDS;
		}
		break;
	case T4_ERR_INVALIDATE_SHARED_MR:
	case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_CANT_INV_STAG;
		break;
	case T4_ERR_ECC:
	case T4_ERR_ECC_PSTAG:
	case T4_ERR_INTERNAL_ERR:
		*layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA;
		*ecode = 0;
		break;
	case T4_ERR_OUT_OF_RQE:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MSN_NOBUF;
		break;
	case T4_ERR_PBL_ADDR_BOUND:
		*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
		*ecode = DDPT_BASE_BOUNDS;
		break;
	case T4_ERR_CRC:
		*layer_type = LAYER_MPA|DDP_LLP;
		*ecode = MPA_CRC_ERR;
		break;
	case T4_ERR_MARKER:
		*layer_type = LAYER_MPA|DDP_LLP;
		*ecode = MPA_MARKER_ERR;
		break;
	case T4_ERR_PDU_LEN_ERR:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_MSG_TOOBIG;
		break;
	case T4_ERR_DDP_VERSION:
		if (tagged) {
			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
			*ecode = DDPT_INV_VERS;
		} else {
			*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
			*ecode = DDPU_INV_VERS;
		}
		break;
	case T4_ERR_RDMA_VERSION:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_INV_VERS;
		break;
	case T4_ERR_OPCODE:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_INV_OPCODE;
		break;
	case T4_ERR_DDP_QUEUE_NUM:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_QN;
		break;
	case T4_ERR_MSN:
	case T4_ERR_MSN_GAP:
	case T4_ERR_MSN_RANGE:
	case T4_ERR_IRD_OVERFLOW:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MSN_RANGE;
		break;
	case T4_ERR_TBIT:
		*layer_type = LAYER_DDP|DDP_LOCAL_CATA;
		*ecode = 0;
		break;
	case T4_ERR_MO:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MO;
		break;
	default:
		*layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
		*ecode = 0;
		break;
	}
}

1074 1075
static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
			   gfp_t gfp)
1076 1077 1078 1079 1080 1081 1082 1083
{
	struct fw_ri_wr *wqe;
	struct sk_buff *skb;
	struct terminate_message *term;

	PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,
	     qhp->ep->hwtid);

1084 1085
	skb = skb_dequeue(&qhp->ep->com.ep_skb_list);
	if (WARN_ON(!skb))
1086
		return;
1087

1088 1089 1090 1091
	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);

	wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
	memset(wqe, 0, sizeof *wqe);
1092
	wqe->op_compl = cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR));
1093
	wqe->flowid_len16 = cpu_to_be32(
1094 1095
		FW_WR_FLOWID_V(qhp->ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
1096 1097 1098 1099

	wqe->u.terminate.type = FW_RI_TYPE_TERMINATE;
	wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term);
	term = (struct terminate_message *)wqe->u.terminate.termmsg;
1100 1101 1102 1103 1104
	if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) {
		term->layer_etype = qhp->attr.layer_etype;
		term->ecode = qhp->attr.ecode;
	} else
		build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
1105
	c4iw_ofld_send(&qhp->rhp->rdev, skb);
1106 1107 1108 1109 1110 1111
}

/*
 * Assumes qhp lock is held.
 */
static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
1112
		       struct c4iw_cq *schp)
1113 1114
{
	int count;
1115
	int rq_flushed, sq_flushed;
1116
	unsigned long flag;
1117 1118 1119

	PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp);

1120
	/* locking hierarchy: cq lock first, then qp lock. */
1121
	spin_lock_irqsave(&rchp->lock, flag);
1122
	spin_lock(&qhp->lock);
S
Steve Wise 已提交
1123 1124 1125 1126 1127 1128 1129 1130 1131

	if (qhp->wq.flushed) {
		spin_unlock(&qhp->lock);
		spin_unlock_irqrestore(&rchp->lock, flag);
		return;
	}
	qhp->wq.flushed = 1;

	c4iw_flush_hw_cq(rchp);
1132
	c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count);
1133
	rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count);
1134
	spin_unlock(&qhp->lock);
1135
	spin_unlock_irqrestore(&rchp->lock, flag);
1136

1137
	/* locking hierarchy: cq lock first, then qp lock. */
1138
	spin_lock_irqsave(&schp->lock, flag);
1139
	spin_lock(&qhp->lock);
S
Steve Wise 已提交
1140 1141
	if (schp != rchp)
		c4iw_flush_hw_cq(schp);
1142
	sq_flushed = c4iw_flush_sq(qhp);
1143
	spin_unlock(&qhp->lock);
1144
	spin_unlock_irqrestore(&schp->lock, flag);
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166

	if (schp == rchp) {
		if (t4_clear_cq_armed(&rchp->cq) &&
		    (rq_flushed || sq_flushed)) {
			spin_lock_irqsave(&rchp->comp_handler_lock, flag);
			(*rchp->ibcq.comp_handler)(&rchp->ibcq,
						   rchp->ibcq.cq_context);
			spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
		}
	} else {
		if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) {
			spin_lock_irqsave(&rchp->comp_handler_lock, flag);
			(*rchp->ibcq.comp_handler)(&rchp->ibcq,
						   rchp->ibcq.cq_context);
			spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
		}
		if (t4_clear_cq_armed(&schp->cq) && sq_flushed) {
			spin_lock_irqsave(&schp->comp_handler_lock, flag);
			(*schp->ibcq.comp_handler)(&schp->ibcq,
						   schp->ibcq.cq_context);
			spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
		}
1167
	}
1168 1169
}

1170
static void flush_qp(struct c4iw_qp *qhp)
1171 1172
{
	struct c4iw_cq *rchp, *schp;
1173
	unsigned long flag;
1174

S
Steve Wise 已提交
1175 1176
	rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
	schp = to_c4iw_cq(qhp->ibqp.send_cq);
1177

S
Steve Wise 已提交
1178
	t4_set_wq_in_error(&qhp->wq);
1179 1180
	if (qhp->ibqp.uobject) {
		t4_set_cq_in_error(&rchp->cq);
1181
		spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1182
		(*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
1183
		spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
1184
		if (schp != rchp) {
1185
			t4_set_cq_in_error(&schp->cq);
1186
			spin_lock_irqsave(&schp->comp_handler_lock, flag);
1187 1188
			(*schp->ibcq.comp_handler)(&schp->ibcq,
					schp->ibcq.cq_context);
1189
			spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
1190
		}
1191 1192
		return;
	}
1193
	__flush_qp(qhp, rchp, schp);
1194 1195
}

S
Steve Wise 已提交
1196 1197
static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
		     struct c4iw_ep *ep)
1198 1199 1200 1201 1202 1203
{
	struct fw_ri_wr *wqe;
	int ret;
	struct sk_buff *skb;

	PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,
S
Steve Wise 已提交
1204
	     ep->hwtid);
1205

1206 1207
	skb = skb_dequeue(&ep->com.ep_skb_list);
	if (WARN_ON(!skb))
1208
		return -ENOMEM;
1209

S
Steve Wise 已提交
1210
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
1211 1212 1213 1214

	wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
	memset(wqe, 0, sizeof *wqe);
	wqe->op_compl = cpu_to_be32(
1215 1216
		FW_WR_OP_V(FW_RI_INIT_WR) |
		FW_WR_COMPL_F);
1217
	wqe->flowid_len16 = cpu_to_be32(
1218 1219
		FW_WR_FLOWID_V(ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
H
Hariprasad S 已提交
1220
	wqe->cookie = (uintptr_t)&ep->com.wr_wait;
1221 1222 1223 1224 1225 1226

	wqe->u.fini.type = FW_RI_TYPE_FINI;
	ret = c4iw_ofld_send(&rhp->rdev, skb);
	if (ret)
		goto out;

1227
	ret = c4iw_wait_for_reply(&rhp->rdev, &ep->com.wr_wait, qhp->ep->hwtid,
1228
			     qhp->wq.sq.qid, __func__);
1229 1230 1231 1232 1233 1234 1235
out:
	PDBG("%s ret %d\n", __func__, ret);
	return ret;
}

static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
{
1236
	PDBG("%s p2p_type = %d\n", __func__, p2p_type);
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264
	memset(&init->u, 0, sizeof init->u);
	switch (p2p_type) {
	case FW_RI_INIT_P2PTYPE_RDMA_WRITE:
		init->u.write.opcode = FW_RI_RDMA_WRITE_WR;
		init->u.write.stag_sink = cpu_to_be32(1);
		init->u.write.to_sink = cpu_to_be64(1);
		init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;
		init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write +
						   sizeof(struct fw_ri_immd),
						   16);
		break;
	case FW_RI_INIT_P2PTYPE_READ_REQ:
		init->u.write.opcode = FW_RI_RDMA_READ_WR;
		init->u.read.stag_src = cpu_to_be32(1);
		init->u.read.to_src_lo = cpu_to_be32(1);
		init->u.read.stag_sink = cpu_to_be32(1);
		init->u.read.to_sink_lo = cpu_to_be32(1);
		init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16);
		break;
	}
}

static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
{
	struct fw_ri_wr *wqe;
	int ret;
	struct sk_buff *skb;

1265 1266
	PDBG("%s qhp %p qid 0x%x tid %u ird %u ord %u\n", __func__, qhp,
	     qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord);
1267

1268
	skb = alloc_skb(sizeof *wqe, GFP_KERNEL);
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	if (!skb) {
		ret = -ENOMEM;
		goto out;
	}
	ret = alloc_ird(rhp, qhp->attr.max_ird);
	if (ret) {
		qhp->attr.max_ird = 0;
		kfree_skb(skb);
		goto out;
	}
1279 1280 1281 1282 1283
	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);

	wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
	memset(wqe, 0, sizeof *wqe);
	wqe->op_compl = cpu_to_be32(
1284 1285
		FW_WR_OP_V(FW_RI_INIT_WR) |
		FW_WR_COMPL_F);
1286
	wqe->flowid_len16 = cpu_to_be32(
1287 1288
		FW_WR_FLOWID_V(qhp->ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
1289

H
Hariprasad S 已提交
1290
	wqe->cookie = (uintptr_t)&qhp->ep->com.wr_wait;
1291 1292 1293

	wqe->u.init.type = FW_RI_TYPE_INIT;
	wqe->u.init.mpareqbit_p2ptype =
1294 1295
		FW_RI_WR_MPAREQBIT_V(qhp->attr.mpa_attr.initiator) |
		FW_RI_WR_P2PTYPE_V(qhp->attr.mpa_attr.p2p_type);
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
	wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE;
	if (qhp->attr.mpa_attr.recv_marker_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE;
	if (qhp->attr.mpa_attr.xmit_marker_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE;
	if (qhp->attr.mpa_attr.crc_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE;

	wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE |
			    FW_RI_QP_RDMA_WRITE_ENABLE |
			    FW_RI_QP_BIND_ENABLE;
	if (!qhp->ibqp.uobject)
		wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE |
				     FW_RI_QP_STAG0_ENABLE;
	wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq));
	wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd);
	wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid);
	wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid);
	wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid);
	wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq);
	wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq);
	wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord);
	wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird);
	wqe->u.init.iss = cpu_to_be32(qhp->ep->snd_seq);
	wqe->u.init.irs = cpu_to_be32(qhp->ep->rcv_seq);
	wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size);
	wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr -
					 rhp->rdev.lldi.vr->rq.start);
	if (qhp->attr.mpa_attr.initiator)
		build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init);

	ret = c4iw_ofld_send(&rhp->rdev, skb);
	if (ret)
1329
		goto err1;
1330

1331 1332
	ret = c4iw_wait_for_reply(&rhp->rdev, &qhp->ep->com.wr_wait,
				  qhp->ep->hwtid, qhp->wq.sq.qid, __func__);
1333 1334 1335 1336
	if (!ret)
		goto out;
err1:
	free_ird(rhp, qhp->attr.max_ird);
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
out:
	PDBG("%s ret %d\n", __func__, ret);
	return ret;
}

int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
		   enum c4iw_qp_attr_mask mask,
		   struct c4iw_qp_attributes *attrs,
		   int internal)
{
	int ret = 0;
	struct c4iw_qp_attributes newattr = qhp->attr;
	int disconnect = 0;
	int terminate = 0;
	int abort = 0;
	int free = 0;
	struct c4iw_ep *ep = NULL;

	PDBG("%s qhp %p sqid 0x%x rqid 0x%x ep %p state %d -> %d\n", __func__,
	     qhp, qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep, qhp->attr.state,
	     (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1);

1359
	mutex_lock(&qhp->mutex);
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373

	/* Process attr changes if in IDLE */
	if (mask & C4IW_QP_ATTR_VALID_MODIFY) {
		if (qhp->attr.state != C4IW_QP_STATE_IDLE) {
			ret = -EIO;
			goto out;
		}
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ)
			newattr.enable_rdma_read = attrs->enable_rdma_read;
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE)
			newattr.enable_rdma_write = attrs->enable_rdma_write;
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND)
			newattr.enable_bind = attrs->enable_bind;
		if (mask & C4IW_QP_ATTR_MAX_ORD) {
1374
			if (attrs->max_ord > c4iw_max_read_depth) {
1375 1376 1377 1378 1379 1380
				ret = -EINVAL;
				goto out;
			}
			newattr.max_ord = attrs->max_ord;
		}
		if (mask & C4IW_QP_ATTR_MAX_IRD) {
1381
			if (attrs->max_ird > cur_max_read_depth(rhp)) {
1382 1383 1384 1385 1386 1387 1388 1389
				ret = -EINVAL;
				goto out;
			}
			newattr.max_ird = attrs->max_ird;
		}
		qhp->attr = newattr;
	}

1390
	if (mask & C4IW_QP_ATTR_SQ_DB) {
1391
		ret = ring_kernel_sq_db(qhp, attrs->sq_db_inc);
1392 1393 1394
		goto out;
	}
	if (mask & C4IW_QP_ATTR_RQ_DB) {
1395
		ret = ring_kernel_rq_db(qhp, attrs->rq_db_inc);
1396 1397 1398
		goto out;
	}

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
	if (!(mask & C4IW_QP_ATTR_NEXT_STATE))
		goto out;
	if (qhp->attr.state == attrs->next_state)
		goto out;

	switch (qhp->attr.state) {
	case C4IW_QP_STATE_IDLE:
		switch (attrs->next_state) {
		case C4IW_QP_STATE_RTS:
			if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) {
				ret = -EINVAL;
				goto out;
			}
			if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) {
				ret = -EINVAL;
				goto out;
			}
			qhp->attr.mpa_attr = attrs->mpa_attr;
			qhp->attr.llp_stream_handle = attrs->llp_stream_handle;
			qhp->ep = qhp->attr.llp_stream_handle;
1419
			set_state(qhp, C4IW_QP_STATE_RTS);
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432

			/*
			 * Ref the endpoint here and deref when we
			 * disassociate the endpoint from the QP.  This
			 * happens in CLOSING->IDLE transition or *->ERROR
			 * transition.
			 */
			c4iw_get_ep(&qhp->ep->com);
			ret = rdma_init(rhp, qhp);
			if (ret)
				goto err;
			break;
		case C4IW_QP_STATE_ERROR:
1433 1434
			set_state(qhp, C4IW_QP_STATE_ERROR);
			flush_qp(qhp);
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
			break;
		default:
			ret = -EINVAL;
			goto out;
		}
		break;
	case C4IW_QP_STATE_RTS:
		switch (attrs->next_state) {
		case C4IW_QP_STATE_CLOSING:
			BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);
S
Steve Wise 已提交
1445
			t4_set_wq_in_error(&qhp->wq);
1446
			set_state(qhp, C4IW_QP_STATE_CLOSING);
S
Steve Wise 已提交
1447
			ep = qhp->ep;
1448 1449 1450
			if (!internal) {
				abort = 0;
				disconnect = 1;
1451
				c4iw_get_ep(&qhp->ep->com);
1452
			}
S
Steve Wise 已提交
1453
			ret = rdma_fini(rhp, qhp, ep);
S
Steve Wise 已提交
1454
			if (ret)
1455 1456 1457
				goto err;
			break;
		case C4IW_QP_STATE_TERMINATE:
S
Steve Wise 已提交
1458
			t4_set_wq_in_error(&qhp->wq);
1459
			set_state(qhp, C4IW_QP_STATE_TERMINATE);
1460 1461
			qhp->attr.layer_etype = attrs->layer_etype;
			qhp->attr.ecode = attrs->ecode;
1462
			ep = qhp->ep;
1463 1464
			if (!internal) {
				c4iw_get_ep(&qhp->ep->com);
1465
				terminate = 1;
1466 1467 1468
				disconnect = 1;
			} else {
				terminate = qhp->attr.send_term;
1469 1470 1471 1472
				ret = rdma_fini(rhp, qhp, ep);
				if (ret)
					goto err;
			}
1473 1474
			break;
		case C4IW_QP_STATE_ERROR:
S
Steve Wise 已提交
1475
			t4_set_wq_in_error(&qhp->wq);
S
Steve Wise 已提交
1476
			set_state(qhp, C4IW_QP_STATE_ERROR);
1477 1478 1479 1480
			if (!internal) {
				abort = 1;
				disconnect = 1;
				ep = qhp->ep;
1481
				c4iw_get_ep(&qhp->ep->com);
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
			}
			goto err;
			break;
		default:
			ret = -EINVAL;
			goto out;
		}
		break;
	case C4IW_QP_STATE_CLOSING:
		if (!internal) {
			ret = -EINVAL;
			goto out;
		}
		switch (attrs->next_state) {
		case C4IW_QP_STATE_IDLE:
1497 1498
			flush_qp(qhp);
			set_state(qhp, C4IW_QP_STATE_IDLE);
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
			qhp->attr.llp_stream_handle = NULL;
			c4iw_put_ep(&qhp->ep->com);
			qhp->ep = NULL;
			wake_up(&qhp->wait);
			break;
		case C4IW_QP_STATE_ERROR:
			goto err;
		default:
			ret = -EINVAL;
			goto err;
		}
		break;
	case C4IW_QP_STATE_ERROR:
		if (attrs->next_state != C4IW_QP_STATE_IDLE) {
			ret = -EINVAL;
			goto out;
		}
		if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) {
			ret = -EINVAL;
			goto out;
		}
1520
		set_state(qhp, C4IW_QP_STATE_IDLE);
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
		break;
	case C4IW_QP_STATE_TERMINATE:
		if (!internal) {
			ret = -EINVAL;
			goto out;
		}
		goto err;
		break;
	default:
		printk(KERN_ERR "%s in a bad state %d\n",
		       __func__, qhp->attr.state);
		ret = -EINVAL;
		goto err;
		break;
	}
	goto out;
err:
	PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep,
	     qhp->wq.sq.qid);

	/* disassociate the LLP connection */
	qhp->attr.llp_stream_handle = NULL;
S
Steve Wise 已提交
1543 1544
	if (!ep)
		ep = qhp->ep;
1545
	qhp->ep = NULL;
1546
	set_state(qhp, C4IW_QP_STATE_ERROR);
1547
	free = 1;
1548
	abort = 1;
1549
	BUG_ON(!ep);
1550
	flush_qp(qhp);
1551
	wake_up(&qhp->wait);
1552
out:
1553
	mutex_unlock(&qhp->mutex);
1554 1555

	if (terminate)
1556
		post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL);
1557 1558 1559 1560 1561 1562 1563

	/*
	 * If disconnect is 1, then we need to initiate a disconnect
	 * on the EP.  This can be a normal close (RTS->CLOSING) or
	 * an abnormal close (RTS/CLOSING->ERROR).
	 */
	if (disconnect) {
1564 1565
		c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC :
							 GFP_KERNEL);
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
		c4iw_put_ep(&ep->com);
	}

	/*
	 * If free is 1, then we've disassociated the EP from the QP
	 * and we need to dereference the EP.
	 */
	if (free)
		c4iw_put_ep(&ep->com);
	PDBG("%s exit state %d\n", __func__, qhp->attr.state);
	return ret;
}

int c4iw_destroy_qp(struct ib_qp *ib_qp)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	struct c4iw_qp_attributes attrs;
	struct c4iw_ucontext *ucontext;

	qhp = to_c4iw_qp(ib_qp);
	rhp = qhp->rhp;

	attrs.next_state = C4IW_QP_STATE_ERROR;
1590 1591 1592 1593
	if (qhp->attr.state == C4IW_QP_STATE_TERMINATE)
		c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
	else
		c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
1594 1595
	wait_event(qhp->wait, !qhp->ep);

1596
	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
1597 1598 1599
	atomic_dec(&qhp->refcnt);
	wait_event(qhp->wait, !atomic_read(&qhp->refcnt));

1600 1601 1602 1603
	spin_lock_irq(&rhp->lock);
	if (!list_empty(&qhp->db_fc_entry))
		list_del_init(&qhp->db_fc_entry);
	spin_unlock_irq(&rhp->lock);
1604
	free_ird(rhp, qhp->attr.max_ird);
1605

1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
	ucontext = ib_qp->uobject ?
		   to_c4iw_ucontext(ib_qp->uobject->context) : NULL;
	destroy_qp(&rhp->rdev, &qhp->wq,
		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx);

	PDBG("%s ib_qp %p qpid 0x%0x\n", __func__, ib_qp, qhp->wq.sq.qid);
	kfree(qhp);
	return 0;
}

struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
			     struct ib_udata *udata)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	struct c4iw_pd *php;
	struct c4iw_cq *schp;
	struct c4iw_cq *rchp;
	struct c4iw_create_qp_resp uresp;
1625
	unsigned int sqsize, rqsize;
1626 1627
	struct c4iw_ucontext *ucontext;
	int ret;
1628 1629
	struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;
	struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;
1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645

	PDBG("%s ib_pd %p\n", __func__, pd);

	if (attrs->qp_type != IB_QPT_RC)
		return ERR_PTR(-EINVAL);

	php = to_c4iw_pd(pd);
	rhp = php->rhp;
	schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid);
	rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid);
	if (!schp || !rchp)
		return ERR_PTR(-EINVAL);

	if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
		return ERR_PTR(-EINVAL);

1646
	if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size)
1647
		return ERR_PTR(-E2BIG);
1648 1649 1650
	rqsize = attrs->cap.max_recv_wr + 1;
	if (rqsize < 8)
		rqsize = 8;
1651

1652
	if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size)
1653
		return ERR_PTR(-E2BIG);
1654 1655 1656
	sqsize = attrs->cap.max_send_wr + 1;
	if (sqsize < 8)
		sqsize = 8;
1657 1658 1659 1660 1661 1662 1663

	ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;

	qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
	if (!qhp)
		return ERR_PTR(-ENOMEM);
	qhp->wq.sq.size = sqsize;
1664 1665 1666
	qhp->wq.sq.memsize =
		(sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
		sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64);
S
Steve Wise 已提交
1667
	qhp->wq.sq.flush_cidx = -1;
1668
	qhp->wq.rq.size = rqsize;
1669 1670 1671
	qhp->wq.rq.memsize =
		(rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
		sizeof(*qhp->wq.rq.queue);
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700

	if (ucontext) {
		qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);
		qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE);
	}

	ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq,
			ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
	if (ret)
		goto err1;

	attrs->cap.max_recv_wr = rqsize - 1;
	attrs->cap.max_send_wr = sqsize - 1;
	attrs->cap.max_inline_data = T4_MAX_SEND_INLINE;

	qhp->rhp = rhp;
	qhp->attr.pd = php->pdid;
	qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid;
	qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid;
	qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
	qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
	qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
	qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
	qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
	qhp->attr.state = C4IW_QP_STATE_IDLE;
	qhp->attr.next_state = C4IW_QP_STATE_IDLE;
	qhp->attr.enable_rdma_read = 1;
	qhp->attr.enable_rdma_write = 1;
	qhp->attr.enable_bind = 1;
1701 1702
	qhp->attr.max_ord = 0;
	qhp->attr.max_ird = 0;
1703
	qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1704
	spin_lock_init(&qhp->lock);
S
Steve Wise 已提交
1705 1706
	init_completion(&qhp->sq_drained);
	init_completion(&qhp->rq_drained);
1707
	mutex_init(&qhp->mutex);
1708 1709 1710
	init_waitqueue_head(&qhp->wait);
	atomic_set(&qhp->refcnt, 1);

1711
	ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
1712 1713 1714 1715
	if (ret)
		goto err2;

	if (udata) {
1716 1717
		sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL);
		if (!sq_key_mm) {
1718
			ret = -ENOMEM;
1719
			goto err3;
1720
		}
1721 1722
		rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL);
		if (!rq_key_mm) {
1723
			ret = -ENOMEM;
1724
			goto err4;
1725
		}
1726 1727
		sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL);
		if (!sq_db_key_mm) {
1728
			ret = -ENOMEM;
1729
			goto err5;
1730
		}
1731 1732
		rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL);
		if (!rq_db_key_mm) {
1733
			ret = -ENOMEM;
1734
			goto err6;
1735
		}
S
Steve Wise 已提交
1736
		if (t4_sq_onchip(&qhp->wq.sq)) {
1737 1738 1739
			ma_sync_key_mm = kmalloc(sizeof(*ma_sync_key_mm),
						 GFP_KERNEL);
			if (!ma_sync_key_mm) {
S
Steve Wise 已提交
1740 1741 1742 1743 1744 1745
				ret = -ENOMEM;
				goto err7;
			}
			uresp.flags = C4IW_QPF_ONCHIP;
		} else
			uresp.flags = 0;
1746 1747 1748 1749 1750 1751 1752 1753
		uresp.qid_mask = rhp->rdev.qpmask;
		uresp.sqid = qhp->wq.sq.qid;
		uresp.sq_size = qhp->wq.sq.size;
		uresp.sq_memsize = qhp->wq.sq.memsize;
		uresp.rqid = qhp->wq.rq.qid;
		uresp.rq_size = qhp->wq.rq.size;
		uresp.rq_memsize = qhp->wq.rq.memsize;
		spin_lock(&ucontext->mmap_lock);
1754
		if (ma_sync_key_mm) {
S
Steve Wise 已提交
1755 1756
			uresp.ma_sync_key = ucontext->key;
			ucontext->key += PAGE_SIZE;
1757 1758
		} else {
			uresp.ma_sync_key =  0;
S
Steve Wise 已提交
1759
		}
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
		uresp.sq_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.rq_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.sq_db_gts_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.rq_db_gts_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		spin_unlock(&ucontext->mmap_lock);
		ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
		if (ret)
S
Steve Wise 已提交
1771
			goto err8;
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
		sq_key_mm->key = uresp.sq_key;
		sq_key_mm->addr = qhp->wq.sq.phys_addr;
		sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize);
		insert_mmap(ucontext, sq_key_mm);
		rq_key_mm->key = uresp.rq_key;
		rq_key_mm->addr = virt_to_phys(qhp->wq.rq.queue);
		rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize);
		insert_mmap(ucontext, rq_key_mm);
		sq_db_key_mm->key = uresp.sq_db_gts_key;
		sq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.sq.bar2_pa;
		sq_db_key_mm->len = PAGE_SIZE;
		insert_mmap(ucontext, sq_db_key_mm);
		rq_db_key_mm->key = uresp.rq_db_gts_key;
		rq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.rq.bar2_pa;
		rq_db_key_mm->len = PAGE_SIZE;
		insert_mmap(ucontext, rq_db_key_mm);
		if (ma_sync_key_mm) {
			ma_sync_key_mm->key = uresp.ma_sync_key;
			ma_sync_key_mm->addr =
				(pci_resource_start(rhp->rdev.lldi.pdev, 0) +
				PCIE_MA_SYNC_A) & PAGE_MASK;
			ma_sync_key_mm->len = PAGE_SIZE;
			insert_mmap(ucontext, ma_sync_key_mm);
S
Steve Wise 已提交
1795
		}
1796 1797 1798
	}
	qhp->ibqp.qp_num = qhp->wq.sq.qid;
	init_timer(&(qhp->timer));
1799
	INIT_LIST_HEAD(&qhp->db_fc_entry);
1800 1801 1802 1803 1804
	PDBG("%s sq id %u size %u memsize %zu num_entries %u "
	     "rq id %u size %u memsize %zu num_entries %u\n", __func__,
	     qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize,
	     attrs->cap.max_send_wr, qhp->wq.rq.qid, qhp->wq.rq.size,
	     qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
1805
	return &qhp->ibqp;
S
Steve Wise 已提交
1806
err8:
1807
	kfree(ma_sync_key_mm);
1808
err7:
1809
	kfree(rq_db_key_mm);
1810
err6:
1811
	kfree(sq_db_key_mm);
1812
err5:
1813
	kfree(rq_key_mm);
1814
err4:
1815
	kfree(sq_key_mm);
1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861
err3:
	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
err2:
	destroy_qp(&rhp->rdev, &qhp->wq,
		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
err1:
	kfree(qhp);
	return ERR_PTR(ret);
}

int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		      int attr_mask, struct ib_udata *udata)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	enum c4iw_qp_attr_mask mask = 0;
	struct c4iw_qp_attributes attrs;

	PDBG("%s ib_qp %p\n", __func__, ibqp);

	/* iwarp does not support the RTR state */
	if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
		attr_mask &= ~IB_QP_STATE;

	/* Make sure we still have something left to do */
	if (!attr_mask)
		return 0;

	memset(&attrs, 0, sizeof attrs);
	qhp = to_c4iw_qp(ibqp);
	rhp = qhp->rhp;

	attrs.next_state = c4iw_convert_state(attr->qp_state);
	attrs.enable_rdma_read = (attr->qp_access_flags &
			       IB_ACCESS_REMOTE_READ) ?  1 : 0;
	attrs.enable_rdma_write = (attr->qp_access_flags &
				IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
	attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;


	mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0;
	mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
			(C4IW_QP_ATTR_ENABLE_RDMA_READ |
			 C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
			 C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0;

1862 1863 1864
	/*
	 * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for
	 * ringing the queue db when we're in DB_FULL mode.
1865
	 * Only allow this on T4 devices.
1866 1867 1868 1869 1870
	 */
	attrs.sq_db_inc = attr->sq_psn;
	attrs.rq_db_inc = attr->rq_psn;
	mask |= (attr_mask & IB_QP_SQ_PSN) ? C4IW_QP_ATTR_SQ_DB : 0;
	mask |= (attr_mask & IB_QP_RQ_PSN) ? C4IW_QP_ATTR_RQ_DB : 0;
1871
	if (!is_t4(to_c4iw_qp(ibqp)->rhp->rdev.lldi.adapter_type) &&
1872 1873
	    (mask & (C4IW_QP_ATTR_SQ_DB|C4IW_QP_ATTR_RQ_DB)))
		return -EINVAL;
1874

1875 1876 1877 1878 1879 1880 1881 1882
	return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0);
}

struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn)
{
	PDBG("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
	return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn);
}
V
Vipul Pandya 已提交
1883 1884 1885 1886 1887 1888 1889 1890 1891

int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		     int attr_mask, struct ib_qp_init_attr *init_attr)
{
	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);

	memset(attr, 0, sizeof *attr);
	memset(init_attr, 0, sizeof *init_attr);
	attr->qp_state = to_ib_qp_state(qhp->attr.state);
1892 1893 1894 1895 1896 1897
	init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
	init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
	init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
	init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges;
	init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE;
	init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0;
V
Vipul Pandya 已提交
1898 1899
	return 0;
}
S
Steve Wise 已提交
1900

1901 1902 1903 1904 1905 1906 1907
static void move_qp_to_err(struct c4iw_qp *qp)
{
	struct c4iw_qp_attributes attrs = { .next_state = C4IW_QP_STATE_ERROR };

	(void)c4iw_modify_qp(qp->rhp, qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
}

S
Steve Wise 已提交
1908 1909 1910
void c4iw_drain_sq(struct ib_qp *ibqp)
{
	struct c4iw_qp *qp = to_c4iw_qp(ibqp);
1911 1912 1913
	unsigned long flag;
	bool need_to_wait;

1914
	move_qp_to_err(qp);
1915 1916 1917
	spin_lock_irqsave(&qp->lock, flag);
	need_to_wait = !t4_sq_empty(&qp->wq);
	spin_unlock_irqrestore(&qp->lock, flag);
S
Steve Wise 已提交
1918

1919 1920
	if (need_to_wait)
		wait_for_completion(&qp->sq_drained);
S
Steve Wise 已提交
1921 1922 1923 1924 1925
}

void c4iw_drain_rq(struct ib_qp *ibqp)
{
	struct c4iw_qp *qp = to_c4iw_qp(ibqp);
1926 1927 1928
	unsigned long flag;
	bool need_to_wait;

1929
	move_qp_to_err(qp);
1930 1931 1932
	spin_lock_irqsave(&qp->lock, flag);
	need_to_wait = !t4_rq_empty(&qp->wq);
	spin_unlock_irqrestore(&qp->lock, flag);
S
Steve Wise 已提交
1933

1934 1935
	if (need_to_wait)
		wait_for_completion(&qp->rq_drained);
S
Steve Wise 已提交
1936
}