qp.c 54.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
32 33 34

#include <linux/module.h>

35 36
#include "iw_cxgb4.h"

37 38 39 40
static int db_delay_usecs = 1;
module_param(db_delay_usecs, int, 0644);
MODULE_PARM_DESC(db_delay_usecs, "Usecs to delay awaiting db fifo to drain");

41
static int ocqp_support = 1;
S
Steve Wise 已提交
42
module_param(ocqp_support, int, 0644);
43
MODULE_PARM_DESC(ocqp_support, "Support on-chip SQs (default=1)");
S
Steve Wise 已提交
44

45
int db_fc_threshold = 1000;
46
module_param(db_fc_threshold, int, 0644);
47 48 49 50 51 52 53 54 55
MODULE_PARM_DESC(db_fc_threshold,
		 "QP count/threshold that triggers"
		 " automatic db flow control mode (default = 1000)");

int db_coalescing_threshold;
module_param(db_coalescing_threshold, int, 0644);
MODULE_PARM_DESC(db_coalescing_threshold,
		 "QP count/threshold that triggers"
		 " disabling db coalescing (default = 0)");
56

57 58 59 60
static int max_fr_immd = T4_MAX_FR_IMMD;
module_param(max_fr_immd, int, 0644);
MODULE_PARM_DESC(max_fr_immd, "fastreg threshold for using DSGL instead of immedate");

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
static int alloc_ird(struct c4iw_dev *dev, u32 ird)
{
	int ret = 0;

	spin_lock_irq(&dev->lock);
	if (ird <= dev->avail_ird)
		dev->avail_ird -= ird;
	else
		ret = -ENOMEM;
	spin_unlock_irq(&dev->lock);

	if (ret)
		dev_warn(&dev->rdev.lldi.pdev->dev,
			 "device IRD resources exhausted\n");

	return ret;
}

static void free_ird(struct c4iw_dev *dev, int ird)
{
	spin_lock_irq(&dev->lock);
	dev->avail_ird += ird;
	spin_unlock_irq(&dev->lock);
}

86 87 88 89 90 91 92 93
static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
{
	unsigned long flag;
	spin_lock_irqsave(&qhp->lock, flag);
	qhp->attr.state = state;
	spin_unlock_irqrestore(&qhp->lock, flag);
}

S
Steve Wise 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
static void dealloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	c4iw_ocqp_pool_free(rdev, sq->dma_addr, sq->memsize);
}

static void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	dma_free_coherent(&(rdev->lldi.pdev->dev), sq->memsize, sq->queue,
			  pci_unmap_addr(sq, mapping));
}

static void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	if (t4_sq_onchip(sq))
		dealloc_oc_sq(rdev, sq);
	else
		dealloc_host_sq(rdev, sq);
}

static int alloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
115
	if (!ocqp_support || !ocqp_supported(&rdev->lldi))
S
Steve Wise 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
		return -ENOSYS;
	sq->dma_addr = c4iw_ocqp_pool_alloc(rdev, sq->memsize);
	if (!sq->dma_addr)
		return -ENOMEM;
	sq->phys_addr = rdev->oc_mw_pa + sq->dma_addr -
			rdev->lldi.vr->ocq.start;
	sq->queue = (__force union t4_wr *)(rdev->oc_mw_kva + sq->dma_addr -
					    rdev->lldi.vr->ocq.start);
	sq->flags |= T4_SQ_ONCHIP;
	return 0;
}

static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
{
	sq->queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), sq->memsize,
				       &(sq->dma_addr), GFP_KERNEL);
	if (!sq->queue)
		return -ENOMEM;
	sq->phys_addr = virt_to_phys(sq->queue);
	pci_unmap_addr_set(sq, mapping, sq->dma_addr);
	return 0;
}

139 140 141 142 143 144 145 146 147 148
static int alloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq, int user)
{
	int ret = -ENOSYS;
	if (user)
		ret = alloc_oc_sq(rdev, sq);
	if (ret)
		ret = alloc_host_sq(rdev, sq);
	return ret;
}

149 150 151 152 153 154 155 156 157
static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
		      struct c4iw_dev_ucontext *uctx)
{
	/*
	 * uP clears EQ contexts when the connection exits rdma mode,
	 * so no need to post a RESET WR for these EQs.
	 */
	dma_free_coherent(&(rdev->lldi.pdev->dev),
			  wq->rq.memsize, wq->rq.queue,
158
			  dma_unmap_addr(&wq->rq, mapping));
S
Steve Wise 已提交
159
	dealloc_sq(rdev, &wq->sq);
160 161 162 163 164 165 166 167
	c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
	kfree(wq->rq.sw_rq);
	kfree(wq->sq.sw_sq);
	c4iw_put_qpid(rdev, wq->rq.qid, uctx);
	c4iw_put_qpid(rdev, wq->sq.qid, uctx);
	return 0;
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
/*
 * Determine the BAR2 virtual address and qid. If pbar2_pa is not NULL,
 * then this is a user mapping so compute the page-aligned physical address
 * for mapping.
 */
void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
			      enum cxgb4_bar2_qtype qtype,
			      unsigned int *pbar2_qid, u64 *pbar2_pa)
{
	u64 bar2_qoffset;
	int ret;

	ret = cxgb4_bar2_sge_qregs(rdev->lldi.ports[0], qid, qtype,
				   pbar2_pa ? 1 : 0,
				   &bar2_qoffset, pbar2_qid);
	if (ret)
		return NULL;

	if (pbar2_pa)
		*pbar2_pa = (rdev->bar2_pa + bar2_qoffset) & PAGE_MASK;
188 189 190 191

	if (is_t4(rdev->lldi.adapter_type))
		return NULL;

192 193 194
	return rdev->bar2_kva + bar2_qoffset;
}

195 196 197 198 199 200 201 202 203 204
static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
		     struct t4_cq *rcq, struct t4_cq *scq,
		     struct c4iw_dev_ucontext *uctx)
{
	int user = (uctx != &rdev->uctx);
	struct fw_ri_res_wr *res_wr;
	struct fw_ri_res *res;
	int wr_len;
	struct c4iw_wr_wait wr_wait;
	struct sk_buff *skb;
205
	int ret = 0;
206 207 208 209 210 211 212
	int eqsize;

	wq->sq.qid = c4iw_get_qpid(rdev, uctx);
	if (!wq->sq.qid)
		return -ENOMEM;

	wq->rq.qid = c4iw_get_qpid(rdev, uctx);
213 214 215 216
	if (!wq->rq.qid) {
		ret = -ENOMEM;
		goto free_sq_qid;
	}
217 218 219 220

	if (!user) {
		wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,
				 GFP_KERNEL);
221 222 223 224
		if (!wq->sq.sw_sq) {
			ret = -ENOMEM;
			goto free_rq_qid;
		}
225 226 227

		wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq,
				 GFP_KERNEL);
228 229 230 231
		if (!wq->rq.sw_rq) {
			ret = -ENOMEM;
			goto free_sw_sq;
		}
232 233 234
	}

	/*
235
	 * RQT must be a power of 2 and at least 16 deep.
236
	 */
237
	wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16));
238
	wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size);
239 240 241 242
	if (!wq->rq.rqt_hwaddr) {
		ret = -ENOMEM;
		goto free_sw_rq;
	}
243

244 245 246
	ret = alloc_sq(rdev, &wq->sq, user);
	if (ret)
		goto free_hwaddr;
247
	memset(wq->sq.queue, 0, wq->sq.memsize);
248
	dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);
249 250 251 252

	wq->rq.queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev),
					  wq->rq.memsize, &(wq->rq.dma_addr),
					  GFP_KERNEL);
253 254
	if (!wq->rq.queue) {
		ret = -ENOMEM;
255
		goto free_sq;
256
	}
257 258
	pr_debug("sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx\n",
		 wq->sq.queue,
J
Joe Perches 已提交
259 260 261
		 (unsigned long long)virt_to_phys(wq->sq.queue),
		 wq->rq.queue,
		 (unsigned long long)virt_to_phys(wq->rq.queue));
262
	memset(wq->rq.queue, 0, wq->rq.memsize);
263
	dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
264 265

	wq->db = rdev->lldi.db_reg;
266

267 268 269 270 271 272 273 274 275 276
	wq->sq.bar2_va = c4iw_bar2_addrs(rdev, wq->sq.qid, T4_BAR2_QTYPE_EGRESS,
					 &wq->sq.bar2_qid,
					 user ? &wq->sq.bar2_pa : NULL);
	wq->rq.bar2_va = c4iw_bar2_addrs(rdev, wq->rq.qid, T4_BAR2_QTYPE_EGRESS,
					 &wq->rq.bar2_qid,
					 user ? &wq->rq.bar2_pa : NULL);

	/*
	 * User mode must have bar2 access.
	 */
277
	if (user && (!wq->sq.bar2_pa || !wq->rq.bar2_pa)) {
278
		pr_warn("%s: sqid %u or rqid %u not in BAR2 range\n",
279 280
			pci_name(rdev->lldi.pdev), wq->sq.qid, wq->rq.qid);
		goto free_dma;
281
	}
282

283 284 285 286 287 288
	wq->rdev = rdev;
	wq->rq.msn = 1;

	/* build fw_ri_res_wr */
	wr_len = sizeof *res_wr + 2 * sizeof *res;

289
	skb = alloc_skb(wr_len, GFP_KERNEL);
290 291
	if (!skb) {
		ret = -ENOMEM;
292
		goto free_dma;
293 294 295
	}
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);

296
	res_wr = __skb_put_zero(skb, wr_len);
297
	res_wr->op_nres = cpu_to_be32(
298
			FW_WR_OP_V(FW_RI_RES_WR) |
299
			FW_RI_RES_WR_NRES_V(2) |
300
			FW_WR_COMPL_F);
301
	res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
H
Hariprasad S 已提交
302
	res_wr->cookie = (uintptr_t)&wr_wait;
303 304 305 306 307 308 309
	res = res_wr->res;
	res->u.sqrq.restype = FW_RI_RES_TYPE_SQ;
	res->u.sqrq.op = FW_RI_RES_OP_WRITE;

	/*
	 * eqsize is the number of 64B entries plus the status page size.
	 */
310 311
	eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +
		rdev->hw_queue.t4_eq_status_entries;
312 313

	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
314 315 316 317 318
		FW_RI_RES_WR_HOSTFCMODE_V(0) |	/* no host cidx updates */
		FW_RI_RES_WR_CPRIO_V(0) |	/* don't keep in chip cache */
		FW_RI_RES_WR_PCIECHN_V(0) |	/* set by uP at ri_init time */
		(t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_ONCHIP_F : 0) |
		FW_RI_RES_WR_IQID_V(scq->cqid));
319
	res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
320 321 322
		FW_RI_RES_WR_DCAEN_V(0) |
		FW_RI_RES_WR_DCACPU_V(0) |
		FW_RI_RES_WR_FBMIN_V(2) |
323 324
		(t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_FBMAX_V(2) :
					 FW_RI_RES_WR_FBMAX_V(3)) |
325 326 327
		FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
		FW_RI_RES_WR_CIDXFTHRESH_V(0) |
		FW_RI_RES_WR_EQSIZE_V(eqsize));
328 329 330 331 332 333 334 335 336
	res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);
	res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);
	res++;
	res->u.sqrq.restype = FW_RI_RES_TYPE_RQ;
	res->u.sqrq.op = FW_RI_RES_OP_WRITE;

	/*
	 * eqsize is the number of 64B entries plus the status page size.
	 */
337 338
	eqsize = wq->rq.size * T4_RQ_NUM_SLOTS +
		rdev->hw_queue.t4_eq_status_entries;
339
	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
340 341 342 343
		FW_RI_RES_WR_HOSTFCMODE_V(0) |	/* no host cidx updates */
		FW_RI_RES_WR_CPRIO_V(0) |	/* don't keep in chip cache */
		FW_RI_RES_WR_PCIECHN_V(0) |	/* set by uP at ri_init time */
		FW_RI_RES_WR_IQID_V(rcq->cqid));
344
	res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
345 346 347
		FW_RI_RES_WR_DCAEN_V(0) |
		FW_RI_RES_WR_DCACPU_V(0) |
		FW_RI_RES_WR_FBMIN_V(2) |
348
		FW_RI_RES_WR_FBMAX_V(3) |
349 350 351
		FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
		FW_RI_RES_WR_CIDXFTHRESH_V(0) |
		FW_RI_RES_WR_EQSIZE_V(eqsize));
352 353 354 355 356 357 358
	res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid);
	res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr);

	c4iw_init_wr_wait(&wr_wait);

	ret = c4iw_ofld_send(rdev, skb);
	if (ret)
359
		goto free_dma;
360
	ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__);
361
	if (ret)
362
		goto free_dma;
363

364 365
	pr_debug("sqid 0x%x rqid 0x%x kdb 0x%p sq_bar2_addr %p rq_bar2_addr %p\n",
		 wq->sq.qid, wq->rq.qid, wq->db,
J
Joe Perches 已提交
366
		 wq->sq.bar2_va, wq->rq.bar2_va);
367 368

	return 0;
369
free_dma:
370 371
	dma_free_coherent(&(rdev->lldi.pdev->dev),
			  wq->rq.memsize, wq->rq.queue,
372
			  dma_unmap_addr(&wq->rq, mapping));
373
free_sq:
S
Steve Wise 已提交
374
	dealloc_sq(rdev, &wq->sq);
375
free_hwaddr:
376
	c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
377
free_sw_rq:
378
	kfree(wq->rq.sw_rq);
379
free_sw_sq:
380
	kfree(wq->sq.sw_sq);
381
free_rq_qid:
382
	c4iw_put_qpid(rdev, wq->rq.qid, uctx);
383
free_sq_qid:
384
	c4iw_put_qpid(rdev, wq->sq.qid, uctx);
385
	return ret;
386 387
}

388 389
static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,
		      struct ib_send_wr *wr, int max, u32 *plenp)
390
{
391 392
	u8 *dstp, *srcp;
	u32 plen = 0;
393
	int i;
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
	int rem, len;

	dstp = (u8 *)immdp->data;
	for (i = 0; i < wr->num_sge; i++) {
		if ((plen + wr->sg_list[i].length) > max)
			return -EMSGSIZE;
		srcp = (u8 *)(unsigned long)wr->sg_list[i].addr;
		plen += wr->sg_list[i].length;
		rem = wr->sg_list[i].length;
		while (rem) {
			if (dstp == (u8 *)&sq->queue[sq->size])
				dstp = (u8 *)sq->queue;
			if (rem <= (u8 *)&sq->queue[sq->size] - dstp)
				len = rem;
			else
				len = (u8 *)&sq->queue[sq->size] - dstp;
			memcpy(dstp, srcp, len);
			dstp += len;
			srcp += len;
			rem -= len;
		}
	}
S
Steve Wise 已提交
416 417 418
	len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);
	if (len)
		memset(dstp, 0, len);
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
	immdp->op = FW_RI_DATA_IMMD;
	immdp->r1 = 0;
	immdp->r2 = 0;
	immdp->immdlen = cpu_to_be32(plen);
	*plenp = plen;
	return 0;
}

static int build_isgl(__be64 *queue_start, __be64 *queue_end,
		      struct fw_ri_isgl *isglp, struct ib_sge *sg_list,
		      int num_sge, u32 *plenp)

{
	int i;
	u32 plen = 0;
	__be64 *flitp = (__be64 *)isglp->sge;

	for (i = 0; i < num_sge; i++) {
		if ((plen + sg_list[i].length) < plen)
			return -EMSGSIZE;
		plen += sg_list[i].length;
		*flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) |
				     sg_list[i].length);
		if (++flitp == queue_end)
			flitp = queue_start;
		*flitp = cpu_to_be64(sg_list[i].addr);
		if (++flitp == queue_end)
			flitp = queue_start;
	}
S
Steve Wise 已提交
448
	*flitp = (__force __be64)0;
449 450 451 452 453 454 455 456 457 458 459 460
	isglp->op = FW_RI_DATA_ISGL;
	isglp->r1 = 0;
	isglp->nsge = cpu_to_be16(num_sge);
	isglp->r2 = 0;
	if (plenp)
		*plenp = plen;
	return 0;
}

static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
			   struct ib_send_wr *wr, u8 *len16)
{
461 462
	u32 plen;
	int size;
463
	int ret;
464 465 466 467 468 469 470

	if (wr->num_sge > T4_MAX_SEND_SGE)
		return -EINVAL;
	switch (wr->opcode) {
	case IB_WR_SEND:
		if (wr->send_flags & IB_SEND_SOLICITED)
			wqe->send.sendop_pkd = cpu_to_be32(
471
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE));
472 473
		else
			wqe->send.sendop_pkd = cpu_to_be32(
474
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND));
475 476 477 478 479
		wqe->send.stag_inv = 0;
		break;
	case IB_WR_SEND_WITH_INV:
		if (wr->send_flags & IB_SEND_SOLICITED)
			wqe->send.sendop_pkd = cpu_to_be32(
480
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE_INV));
481 482
		else
			wqe->send.sendop_pkd = cpu_to_be32(
483
				FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_INV));
484 485 486 487 488 489
		wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
		break;

	default:
		return -EINVAL;
	}
490 491
	wqe->send.r3 = 0;
	wqe->send.r4 = 0;
492

493 494 495
	plen = 0;
	if (wr->num_sge) {
		if (wr->send_flags & IB_SEND_INLINE) {
496 497 498 499
			ret = build_immd(sq, wqe->send.u.immd_src, wr,
					 T4_MAX_SEND_INLINE, &plen);
			if (ret)
				return ret;
500 501 502
			size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
			       plen;
		} else {
503 504 505 506 507 508
			ret = build_isgl((__be64 *)sq->queue,
					 (__be64 *)&sq->queue[sq->size],
					 wqe->send.u.isgl_src,
					 wr->sg_list, wr->num_sge, &plen);
			if (ret)
				return ret;
509 510 511 512 513 514 515 516 517
			size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
			       wr->num_sge * sizeof(struct fw_ri_sge);
		}
	} else {
		wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
		wqe->send.u.immd_src[0].r1 = 0;
		wqe->send.u.immd_src[0].r2 = 0;
		wqe->send.u.immd_src[0].immdlen = 0;
		size = sizeof wqe->send + sizeof(struct fw_ri_immd);
518
		plen = 0;
519 520 521 522 523 524
	}
	*len16 = DIV_ROUND_UP(size, 16);
	wqe->send.plen = cpu_to_be32(plen);
	return 0;
}

525 526
static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
			    struct ib_send_wr *wr, u8 *len16)
527 528 529
{
	u32 plen;
	int size;
530
	int ret;
531

532
	if (wr->num_sge > T4_MAX_SEND_SGE)
533 534
		return -EINVAL;
	wqe->write.r2 = 0;
C
Christoph Hellwig 已提交
535 536
	wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey);
	wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr);
537 538
	if (wr->num_sge) {
		if (wr->send_flags & IB_SEND_INLINE) {
539 540 541 542
			ret = build_immd(sq, wqe->write.u.immd_src, wr,
					 T4_MAX_WRITE_INLINE, &plen);
			if (ret)
				return ret;
543 544 545
			size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
			       plen;
		} else {
546 547 548 549 550 551
			ret = build_isgl((__be64 *)sq->queue,
					 (__be64 *)&sq->queue[sq->size],
					 wqe->write.u.isgl_src,
					 wr->sg_list, wr->num_sge, &plen);
			if (ret)
				return ret;
552 553 554 555 556 557 558 559 560
			size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
			       wr->num_sge * sizeof(struct fw_ri_sge);
		}
	} else {
		wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
		wqe->write.u.immd_src[0].r1 = 0;
		wqe->write.u.immd_src[0].r2 = 0;
		wqe->write.u.immd_src[0].immdlen = 0;
		size = sizeof wqe->write + sizeof(struct fw_ri_immd);
561
		plen = 0;
562 563 564 565 566 567 568 569 570 571
	}
	*len16 = DIV_ROUND_UP(size, 16);
	wqe->write.plen = cpu_to_be32(plen);
	return 0;
}

static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16)
{
	if (wr->num_sge > 1)
		return -EINVAL;
572
	if (wr->num_sge && wr->sg_list[0].length) {
C
Christoph Hellwig 已提交
573 574
		wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey);
		wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr
575
							>> 32));
C
Christoph Hellwig 已提交
576
		wqe->read.to_src_lo = cpu_to_be32((u32)rdma_wr(wr)->remote_addr);
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
		wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey);
		wqe->read.plen = cpu_to_be32(wr->sg_list[0].length);
		wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr
							 >> 32));
		wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr));
	} else {
		wqe->read.stag_src = cpu_to_be32(2);
		wqe->read.to_src_hi = 0;
		wqe->read.to_src_lo = 0;
		wqe->read.stag_sink = cpu_to_be32(2);
		wqe->read.plen = 0;
		wqe->read.to_sink_hi = 0;
		wqe->read.to_sink_lo = 0;
	}
	wqe->read.r2 = 0;
	wqe->read.r5 = 0;
	*len16 = DIV_ROUND_UP(sizeof wqe->read, 16);
	return 0;
}

static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
			   struct ib_recv_wr *wr, u8 *len16)
{
600
	int ret;
601

602 603 604 605 606
	ret = build_isgl((__be64 *)qhp->wq.rq.queue,
			 (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size],
			 &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL);
	if (ret)
		return ret;
607 608 609 610 611
	*len16 = DIV_ROUND_UP(sizeof wqe->recv +
			      wr->num_sge * sizeof(struct fw_ri_sge), 16);
	return 0;
}

612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr,
			      struct ib_reg_wr *wr, struct c4iw_mr *mhp,
			      u8 *len16)
{
	__be64 *p = (__be64 *)fr->pbl;

	fr->r2 = cpu_to_be32(0);
	fr->stag = cpu_to_be32(mhp->ibmr.rkey);

	fr->tpte.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
		FW_RI_TPTE_STAGKEY_V((mhp->ibmr.rkey & FW_RI_TPTE_STAGKEY_M)) |
		FW_RI_TPTE_STAGSTATE_V(1) |
		FW_RI_TPTE_STAGTYPE_V(FW_RI_STAG_NSMR) |
		FW_RI_TPTE_PDID_V(mhp->attr.pdid));
	fr->tpte.locread_to_qpid = cpu_to_be32(
		FW_RI_TPTE_PERM_V(c4iw_ib_to_tpt_access(wr->access)) |
		FW_RI_TPTE_ADDRTYPE_V(FW_RI_VA_BASED_TO) |
		FW_RI_TPTE_PS_V(ilog2(wr->mr->page_size) - 12));
	fr->tpte.nosnoop_pbladdr = cpu_to_be32(FW_RI_TPTE_PBLADDR_V(
		PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3));
	fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0);
	fr->tpte.len_hi = cpu_to_be32(0);
	fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length);
	fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
	fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);

	p[0] = cpu_to_be64((u64)mhp->mpl[0]);
	p[1] = cpu_to_be64((u64)mhp->mpl[1]);

	*len16 = DIV_ROUND_UP(sizeof(*fr), 16);
}

644
static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
645 646
			struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16,
			bool dsgl_supported)
647 648 649 650 651 652 653
{
	struct fw_ri_immd *imdp;
	__be64 *p;
	int i;
	int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32);
	int rem;

654
	if (mhp->mpl_len > t4_max_fr_depth(dsgl_supported && use_dsgl))
655 656 657 658 659 660 661 662 663 664 665 666 667
		return -EINVAL;

	wqe->fr.qpbinde_to_dcacpu = 0;
	wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12;
	wqe->fr.addr_type = FW_RI_VA_BASED_TO;
	wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access);
	wqe->fr.len_hi = 0;
	wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length);
	wqe->fr.stag = cpu_to_be32(wr->key);
	wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
	wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova &
					0xffffffff);

668
	if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) {
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
		struct fw_ri_dsgl *sglp;

		for (i = 0; i < mhp->mpl_len; i++)
			mhp->mpl[i] = (__force u64)cpu_to_be64((u64)mhp->mpl[i]);

		sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1);
		sglp->op = FW_RI_DATA_DSGL;
		sglp->r1 = 0;
		sglp->nsge = cpu_to_be16(1);
		sglp->addr0 = cpu_to_be64(mhp->mpl_addr);
		sglp->len0 = cpu_to_be32(pbllen);

		*len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16);
	} else {
		imdp = (struct fw_ri_immd *)(&wqe->fr + 1);
		imdp->op = FW_RI_DATA_IMMD;
		imdp->r1 = 0;
		imdp->r2 = 0;
		imdp->immdlen = cpu_to_be32(pbllen);
		p = (__be64 *)(imdp + 1);
		rem = pbllen;
		for (i = 0; i < mhp->mpl_len; i++) {
			*p = cpu_to_be64((u64)mhp->mpl[i]);
			rem -= sizeof(*p);
			if (++p == (__be64 *)&sq->queue[sq->size])
				p = (__be64 *)sq->queue;
		}
		BUG_ON(rem < 0);
		while (rem) {
			*p = 0;
			rem -= sizeof(*p);
			if (++p == (__be64 *)&sq->queue[sq->size])
				p = (__be64 *)sq->queue;
		}
		*len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp)
				      + pbllen, 16);
	}
	return 0;
}

709
static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16)
710 711 712 713 714 715 716
{
	wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
	wqe->inv.r2 = 0;
	*len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
	return 0;
}

717 718 719 720 721 722 723 724 725 726
static void free_qp_work(struct work_struct *work)
{
	struct c4iw_ucontext *ucontext;
	struct c4iw_qp *qhp;
	struct c4iw_dev *rhp;

	qhp = container_of(work, struct c4iw_qp, free_work);
	ucontext = qhp->ucontext;
	rhp = qhp->rhp;

727
	pr_debug("qhp %p ucontext %p\n", qhp, ucontext);
728 729 730 731 732 733 734 735 736
	destroy_qp(&rhp->rdev, &qhp->wq,
		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx);

	if (ucontext)
		c4iw_put_ucontext(ucontext);
	kfree(qhp);
}

static void queue_qp_free(struct kref *kref)
737 738 739 740
{
	struct c4iw_qp *qhp;

	qhp = container_of(kref, struct c4iw_qp, kref);
741
	pr_debug("qhp %p\n", qhp);
742
	queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work);
743 744
}

745 746
void c4iw_qp_add_ref(struct ib_qp *qp)
{
747
	pr_debug("ib_qp %p\n", qp);
748
	kref_get(&to_c4iw_qp(qp)->kref);
749 750 751 752
}

void c4iw_qp_rem_ref(struct ib_qp *qp)
{
753
	pr_debug("ib_qp %p\n", qp);
754
	kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free);
755 756
}

757 758 759 760 761 762 763 764 765 766 767 768
static void add_to_fc_list(struct list_head *head, struct list_head *entry)
{
	if (list_empty(entry))
		list_add_tail(entry, head);
}

static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
{
	unsigned long flags;

	spin_lock_irqsave(&qhp->rhp->lock, flags);
	spin_lock(&qhp->lock);
769
	if (qhp->rhp->db_state == NORMAL)
770
		t4_ring_sq_db(&qhp->wq, inc, NULL);
771
	else {
772 773 774 775 776 777 778 779 780 781 782 783 784 785
		add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
		qhp->wq.sq.wq_pidx_inc += inc;
	}
	spin_unlock(&qhp->lock);
	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
	return 0;
}

static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
{
	unsigned long flags;

	spin_lock_irqsave(&qhp->rhp->lock, flags);
	spin_lock(&qhp->lock);
786
	if (qhp->rhp->db_state == NORMAL)
787
		t4_ring_rq_db(&qhp->wq, inc, NULL);
788
	else {
789 790 791 792 793 794 795 796
		add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
		qhp->wq.rq.wq_pidx_inc += inc;
	}
	spin_unlock(&qhp->lock);
	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
	return 0;
}

797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
static void complete_sq_drain_wr(struct c4iw_qp *qhp, struct ib_send_wr *wr)
{
	struct t4_cqe cqe = {};
	struct c4iw_cq *schp;
	unsigned long flag;
	struct t4_cq *cq;

	schp = to_c4iw_cq(qhp->ibqp.send_cq);
	cq = &schp->cq;

	cqe.u.drain_cookie = wr->wr_id;
	cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
				 CQE_OPCODE_V(C4IW_DRAIN_OPCODE) |
				 CQE_TYPE_V(1) |
				 CQE_SWCQE_V(1) |
				 CQE_QPID_V(qhp->wq.sq.qid));

	spin_lock_irqsave(&schp->lock, flag);
	cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
	cq->sw_queue[cq->sw_pidx] = cqe;
	t4_swcq_produce(cq);
	spin_unlock_irqrestore(&schp->lock, flag);

	spin_lock_irqsave(&schp->comp_handler_lock, flag);
	(*schp->ibcq.comp_handler)(&schp->ibcq,
				   schp->ibcq.cq_context);
	spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
}

static void complete_rq_drain_wr(struct c4iw_qp *qhp, struct ib_recv_wr *wr)
{
	struct t4_cqe cqe = {};
	struct c4iw_cq *rchp;
	unsigned long flag;
	struct t4_cq *cq;

	rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
	cq = &rchp->cq;

	cqe.u.drain_cookie = wr->wr_id;
	cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
				 CQE_OPCODE_V(C4IW_DRAIN_OPCODE) |
				 CQE_TYPE_V(0) |
				 CQE_SWCQE_V(1) |
				 CQE_QPID_V(qhp->wq.sq.qid));

	spin_lock_irqsave(&rchp->lock, flag);
	cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
	cq->sw_queue[cq->sw_pidx] = cqe;
	t4_swcq_produce(cq);
	spin_unlock_irqrestore(&rchp->lock, flag);

	spin_lock_irqsave(&rchp->comp_handler_lock, flag);
	(*rchp->ibcq.comp_handler)(&rchp->ibcq,
				   rchp->ibcq.cq_context);
	spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
}

855 856 857 858 859 860 861 862
int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
		   struct ib_send_wr **bad_wr)
{
	int err = 0;
	u8 len16 = 0;
	enum fw_wr_opcodes fw_opcode = 0;
	enum fw_ri_wr_flags fw_flags;
	struct c4iw_qp *qhp;
863
	union t4_wr *wqe = NULL;
864 865 866 867 868 869 870 871 872
	u32 num_wrs;
	struct t4_swsqe *swsqe;
	unsigned long flag;
	u16 idx = 0;

	qhp = to_c4iw_qp(ibqp);
	spin_lock_irqsave(&qhp->lock, flag);
	if (t4_wq_in_error(&qhp->wq)) {
		spin_unlock_irqrestore(&qhp->lock, flag);
873 874
		complete_sq_drain_wr(qhp, wr);
		return err;
875 876 877 878
	}
	num_wrs = t4_sq_avail(&qhp->wq);
	if (num_wrs == 0) {
		spin_unlock_irqrestore(&qhp->lock, flag);
879
		*bad_wr = wr;
880 881 882 883 884 885 886 887
		return -ENOMEM;
	}
	while (wr) {
		if (num_wrs == 0) {
			err = -ENOMEM;
			*bad_wr = wr;
			break;
		}
888 889 890
		wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue +
		      qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE);

891 892 893
		fw_flags = 0;
		if (wr->send_flags & IB_SEND_SOLICITED)
			fw_flags |= FW_RI_SOLICITED_EVENT_FLAG;
894
		if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all)
895 896 897 898 899 900 901 902 903 904 905 906
			fw_flags |= FW_RI_COMPLETION_FLAG;
		swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx];
		switch (wr->opcode) {
		case IB_WR_SEND_WITH_INV:
		case IB_WR_SEND:
			if (wr->send_flags & IB_SEND_FENCE)
				fw_flags |= FW_RI_READ_FENCE_FLAG;
			fw_opcode = FW_RI_SEND_WR;
			if (wr->opcode == IB_WR_SEND)
				swsqe->opcode = FW_RI_SEND;
			else
				swsqe->opcode = FW_RI_SEND_WITH_INV;
907
			err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16);
908 909 910 911
			break;
		case IB_WR_RDMA_WRITE:
			fw_opcode = FW_RI_RDMA_WRITE_WR;
			swsqe->opcode = FW_RI_RDMA_WRITE;
912
			err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16);
913 914
			break;
		case IB_WR_RDMA_READ:
915
		case IB_WR_RDMA_READ_WITH_INV:
916 917
			fw_opcode = FW_RI_RDMA_READ_WR;
			swsqe->opcode = FW_RI_READ_REQ;
918 919 920
			if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) {
				c4iw_invalidate_mr(qhp->rhp,
						   wr->sg_list[0].lkey);
921
				fw_flags = FW_RI_RDMA_READ_INVALIDATE;
922
			} else {
923
				fw_flags = 0;
924
			}
925 926 927 928 929 930 931
			err = build_rdma_read(wqe, wr, &len16);
			if (err)
				break;
			swsqe->read_len = wr->sg_list[0].length;
			if (!qhp->wq.sq.oldest_read)
				qhp->wq.sq.oldest_read = swsqe;
			break;
932 933 934
		case IB_WR_REG_MR: {
			struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr);

935
			swsqe->opcode = FW_RI_FAST_REGISTER;
936 937 938 939 940 941 942 943 944 945 946 947 948 949
			if (qhp->rhp->rdev.lldi.fr_nsmr_tpte_wr_support &&
			    !mhp->attr.state && mhp->mpl_len <= 2) {
				fw_opcode = FW_RI_FR_NSMR_TPTE_WR;
				build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr),
						  mhp, &len16);
			} else {
				fw_opcode = FW_RI_FR_NSMR_WR;
				err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr),
				       mhp, &len16,
				       qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
				if (err)
					break;
			}
			mhp->attr.state = 1;
950
			break;
951
		}
952
		case IB_WR_LOCAL_INV:
953 954
			if (wr->send_flags & IB_SEND_FENCE)
				fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
955 956
			fw_opcode = FW_RI_INV_LSTAG_WR;
			swsqe->opcode = FW_RI_LOCAL_INV;
957 958
			err = build_inv_stag(wqe, wr, &len16);
			c4iw_invalidate_mr(qhp->rhp, wr->ex.invalidate_rkey);
959 960
			break;
		default:
J
Joe Perches 已提交
961 962
			pr_debug("%s post of type=%d TBD!\n", __func__,
				 wr->opcode);
963 964 965 966 967 968 969 970
			err = -EINVAL;
		}
		if (err) {
			*bad_wr = wr;
			break;
		}
		swsqe->idx = qhp->wq.sq.pidx;
		swsqe->complete = 0;
971 972
		swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) ||
				  qhp->sq_sig_all;
S
Steve Wise 已提交
973
		swsqe->flushed = 0;
974
		swsqe->wr_id = wr->wr_id;
975 976 977 978 979
		if (c4iw_wr_log) {
			swsqe->sge_ts = cxgb4_read_sge_timestamp(
					qhp->rhp->rdev.lldi.ports[0]);
			getnstimeofday(&swsqe->host_ts);
		}
980 981 982

		init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);

983
		pr_debug("cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u\n",
J
Joe Perches 已提交
984 985
			 (unsigned long long)wr->wr_id, qhp->wq.sq.pidx,
			 swsqe->opcode, swsqe->read_len);
986 987
		wr = wr->next;
		num_wrs--;
988 989
		t4_sq_produce(&qhp->wq, len16);
		idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
990
	}
991
	if (!qhp->rhp->rdev.status_page->db_off) {
992
		t4_ring_sq_db(&qhp->wq, idx, wqe);
993 994 995 996 997
		spin_unlock_irqrestore(&qhp->lock, flag);
	} else {
		spin_unlock_irqrestore(&qhp->lock, flag);
		ring_kernel_sq_db(qhp, idx);
	}
998 999 1000 1001 1002 1003 1004 1005
	return err;
}

int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
		      struct ib_recv_wr **bad_wr)
{
	int err = 0;
	struct c4iw_qp *qhp;
1006
	union t4_recv_wr *wqe = NULL;
1007 1008 1009 1010 1011 1012 1013 1014 1015
	u32 num_wrs;
	u8 len16 = 0;
	unsigned long flag;
	u16 idx = 0;

	qhp = to_c4iw_qp(ibqp);
	spin_lock_irqsave(&qhp->lock, flag);
	if (t4_wq_in_error(&qhp->wq)) {
		spin_unlock_irqrestore(&qhp->lock, flag);
1016 1017
		complete_rq_drain_wr(qhp, wr);
		return err;
1018 1019 1020 1021
	}
	num_wrs = t4_rq_avail(&qhp->wq);
	if (num_wrs == 0) {
		spin_unlock_irqrestore(&qhp->lock, flag);
1022
		*bad_wr = wr;
1023 1024 1025 1026 1027 1028 1029 1030
		return -ENOMEM;
	}
	while (wr) {
		if (wr->num_sge > T4_MAX_RECV_SGE) {
			err = -EINVAL;
			*bad_wr = wr;
			break;
		}
1031 1032 1033
		wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue +
					   qhp->wq.rq.wq_pidx *
					   T4_EQ_ENTRY_SIZE);
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
		if (num_wrs)
			err = build_rdma_recv(qhp, wqe, wr, &len16);
		else
			err = -ENOMEM;
		if (err) {
			*bad_wr = wr;
			break;
		}

		qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id;
1044 1045 1046 1047 1048 1049 1050
		if (c4iw_wr_log) {
			qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].sge_ts =
				cxgb4_read_sge_timestamp(
						qhp->rhp->rdev.lldi.ports[0]);
			getnstimeofday(
				&qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_ts);
		}
1051 1052 1053 1054 1055 1056 1057 1058

		wqe->recv.opcode = FW_RI_RECV_WR;
		wqe->recv.r1 = 0;
		wqe->recv.wrid = qhp->wq.rq.pidx;
		wqe->recv.r2[0] = 0;
		wqe->recv.r2[1] = 0;
		wqe->recv.r2[2] = 0;
		wqe->recv.len16 = len16;
1059
		pr_debug("cookie 0x%llx pidx %u\n",
J
Joe Perches 已提交
1060
			 (unsigned long long)wr->wr_id, qhp->wq.rq.pidx);
1061 1062
		t4_rq_produce(&qhp->wq, len16);
		idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
1063 1064 1065
		wr = wr->next;
		num_wrs--;
	}
1066
	if (!qhp->rhp->rdev.status_page->db_off) {
1067
		t4_ring_rq_db(&qhp->wq, idx, wqe);
1068 1069 1070 1071 1072
		spin_unlock_irqrestore(&qhp->lock, flag);
	} else {
		spin_unlock_irqrestore(&qhp->lock, flag);
		ring_kernel_rq_db(qhp, idx);
	}
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
	return err;
}

static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type,
				    u8 *ecode)
{
	int status;
	int tagged;
	int opcode;
	int rqtype;
	int send_inv;

	if (!err_cqe) {
		*layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
		*ecode = 0;
		return;
	}

	status = CQE_STATUS(err_cqe);
	opcode = CQE_OPCODE(err_cqe);
	rqtype = RQ_TYPE(err_cqe);
	send_inv = (opcode == FW_RI_SEND_WITH_INV) ||
		   (opcode == FW_RI_SEND_WITH_SE_INV);
	tagged = (opcode == FW_RI_RDMA_WRITE) ||
		 (rqtype && (opcode == FW_RI_READ_RESP));

	switch (status) {
	case T4_ERR_STAG:
		if (send_inv) {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
			*ecode = RDMAP_CANT_INV_STAG;
		} else {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
			*ecode = RDMAP_INV_STAG;
		}
		break;
	case T4_ERR_PDID:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		if ((opcode == FW_RI_SEND_WITH_INV) ||
		    (opcode == FW_RI_SEND_WITH_SE_INV))
			*ecode = RDMAP_CANT_INV_STAG;
		else
			*ecode = RDMAP_STAG_NOT_ASSOC;
		break;
	case T4_ERR_QPID:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_STAG_NOT_ASSOC;
		break;
	case T4_ERR_ACCESS:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_ACC_VIOL;
		break;
	case T4_ERR_WRAP:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
		*ecode = RDMAP_TO_WRAP;
		break;
	case T4_ERR_BOUND:
		if (tagged) {
			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
			*ecode = DDPT_BASE_BOUNDS;
		} else {
			*layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
			*ecode = RDMAP_BASE_BOUNDS;
		}
		break;
	case T4_ERR_INVALIDATE_SHARED_MR:
	case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_CANT_INV_STAG;
		break;
	case T4_ERR_ECC:
	case T4_ERR_ECC_PSTAG:
	case T4_ERR_INTERNAL_ERR:
		*layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA;
		*ecode = 0;
		break;
	case T4_ERR_OUT_OF_RQE:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MSN_NOBUF;
		break;
	case T4_ERR_PBL_ADDR_BOUND:
		*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
		*ecode = DDPT_BASE_BOUNDS;
		break;
	case T4_ERR_CRC:
		*layer_type = LAYER_MPA|DDP_LLP;
		*ecode = MPA_CRC_ERR;
		break;
	case T4_ERR_MARKER:
		*layer_type = LAYER_MPA|DDP_LLP;
		*ecode = MPA_MARKER_ERR;
		break;
	case T4_ERR_PDU_LEN_ERR:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_MSG_TOOBIG;
		break;
	case T4_ERR_DDP_VERSION:
		if (tagged) {
			*layer_type = LAYER_DDP|DDP_TAGGED_ERR;
			*ecode = DDPT_INV_VERS;
		} else {
			*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
			*ecode = DDPU_INV_VERS;
		}
		break;
	case T4_ERR_RDMA_VERSION:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_INV_VERS;
		break;
	case T4_ERR_OPCODE:
		*layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
		*ecode = RDMAP_INV_OPCODE;
		break;
	case T4_ERR_DDP_QUEUE_NUM:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_QN;
		break;
	case T4_ERR_MSN:
	case T4_ERR_MSN_GAP:
	case T4_ERR_MSN_RANGE:
	case T4_ERR_IRD_OVERFLOW:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MSN_RANGE;
		break;
	case T4_ERR_TBIT:
		*layer_type = LAYER_DDP|DDP_LOCAL_CATA;
		*ecode = 0;
		break;
	case T4_ERR_MO:
		*layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
		*ecode = DDPU_INV_MO;
		break;
	default:
		*layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
		*ecode = 0;
		break;
	}
}

1212 1213
static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
			   gfp_t gfp)
1214 1215 1216 1217 1218
{
	struct fw_ri_wr *wqe;
	struct sk_buff *skb;
	struct terminate_message *term;

1219
	pr_debug("qhp %p qid 0x%x tid %u\n", qhp, qhp->wq.sq.qid,
J
Joe Perches 已提交
1220
		 qhp->ep->hwtid);
1221

1222 1223
	skb = skb_dequeue(&qhp->ep->com.ep_skb_list);
	if (WARN_ON(!skb))
1224
		return;
1225

1226 1227
	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);

1228
	wqe = __skb_put(skb, sizeof(*wqe));
1229
	memset(wqe, 0, sizeof *wqe);
1230
	wqe->op_compl = cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR));
1231
	wqe->flowid_len16 = cpu_to_be32(
1232 1233
		FW_WR_FLOWID_V(qhp->ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
1234 1235 1236 1237

	wqe->u.terminate.type = FW_RI_TYPE_TERMINATE;
	wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term);
	term = (struct terminate_message *)wqe->u.terminate.termmsg;
1238 1239 1240 1241 1242
	if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) {
		term->layer_etype = qhp->attr.layer_etype;
		term->ecode = qhp->attr.ecode;
	} else
		build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
1243
	c4iw_ofld_send(&qhp->rhp->rdev, skb);
1244 1245 1246 1247 1248 1249
}

/*
 * Assumes qhp lock is held.
 */
static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
1250
		       struct c4iw_cq *schp)
1251 1252
{
	int count;
1253
	int rq_flushed, sq_flushed;
1254
	unsigned long flag;
1255

1256
	pr_debug("qhp %p rchp %p schp %p\n", qhp, rchp, schp);
1257

1258
	/* locking hierarchy: cq lock first, then qp lock. */
1259
	spin_lock_irqsave(&rchp->lock, flag);
1260
	spin_lock(&qhp->lock);
S
Steve Wise 已提交
1261 1262 1263 1264 1265 1266 1267 1268 1269

	if (qhp->wq.flushed) {
		spin_unlock(&qhp->lock);
		spin_unlock_irqrestore(&rchp->lock, flag);
		return;
	}
	qhp->wq.flushed = 1;

	c4iw_flush_hw_cq(rchp);
1270
	c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count);
1271
	rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count);
1272
	spin_unlock(&qhp->lock);
1273
	spin_unlock_irqrestore(&rchp->lock, flag);
1274

1275
	/* locking hierarchy: cq lock first, then qp lock. */
1276
	spin_lock_irqsave(&schp->lock, flag);
1277
	spin_lock(&qhp->lock);
S
Steve Wise 已提交
1278 1279
	if (schp != rchp)
		c4iw_flush_hw_cq(schp);
1280
	sq_flushed = c4iw_flush_sq(qhp);
1281
	spin_unlock(&qhp->lock);
1282
	spin_unlock_irqrestore(&schp->lock, flag);
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304

	if (schp == rchp) {
		if (t4_clear_cq_armed(&rchp->cq) &&
		    (rq_flushed || sq_flushed)) {
			spin_lock_irqsave(&rchp->comp_handler_lock, flag);
			(*rchp->ibcq.comp_handler)(&rchp->ibcq,
						   rchp->ibcq.cq_context);
			spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
		}
	} else {
		if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) {
			spin_lock_irqsave(&rchp->comp_handler_lock, flag);
			(*rchp->ibcq.comp_handler)(&rchp->ibcq,
						   rchp->ibcq.cq_context);
			spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
		}
		if (t4_clear_cq_armed(&schp->cq) && sq_flushed) {
			spin_lock_irqsave(&schp->comp_handler_lock, flag);
			(*schp->ibcq.comp_handler)(&schp->ibcq,
						   schp->ibcq.cq_context);
			spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
		}
1305
	}
1306 1307
}

1308
static void flush_qp(struct c4iw_qp *qhp)
1309 1310
{
	struct c4iw_cq *rchp, *schp;
1311
	unsigned long flag;
1312

S
Steve Wise 已提交
1313 1314
	rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
	schp = to_c4iw_cq(qhp->ibqp.send_cq);
1315

S
Steve Wise 已提交
1316
	t4_set_wq_in_error(&qhp->wq);
1317 1318
	if (qhp->ibqp.uobject) {
		t4_set_cq_in_error(&rchp->cq);
1319
		spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1320
		(*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
1321
		spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
1322
		if (schp != rchp) {
1323
			t4_set_cq_in_error(&schp->cq);
1324
			spin_lock_irqsave(&schp->comp_handler_lock, flag);
1325 1326
			(*schp->ibcq.comp_handler)(&schp->ibcq,
					schp->ibcq.cq_context);
1327
			spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
1328
		}
1329 1330
		return;
	}
1331
	__flush_qp(qhp, rchp, schp);
1332 1333
}

S
Steve Wise 已提交
1334 1335
static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
		     struct c4iw_ep *ep)
1336 1337 1338 1339 1340
{
	struct fw_ri_wr *wqe;
	int ret;
	struct sk_buff *skb;

1341
	pr_debug("qhp %p qid 0x%x tid %u\n", qhp, qhp->wq.sq.qid, ep->hwtid);
1342

1343 1344
	skb = skb_dequeue(&ep->com.ep_skb_list);
	if (WARN_ON(!skb))
1345
		return -ENOMEM;
1346

S
Steve Wise 已提交
1347
	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
1348

1349
	wqe = __skb_put(skb, sizeof(*wqe));
1350 1351
	memset(wqe, 0, sizeof *wqe);
	wqe->op_compl = cpu_to_be32(
1352 1353
		FW_WR_OP_V(FW_RI_INIT_WR) |
		FW_WR_COMPL_F);
1354
	wqe->flowid_len16 = cpu_to_be32(
1355 1356
		FW_WR_FLOWID_V(ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
H
Hariprasad S 已提交
1357
	wqe->cookie = (uintptr_t)&ep->com.wr_wait;
1358 1359 1360 1361 1362 1363

	wqe->u.fini.type = FW_RI_TYPE_FINI;
	ret = c4iw_ofld_send(&rhp->rdev, skb);
	if (ret)
		goto out;

1364
	ret = c4iw_wait_for_reply(&rhp->rdev, &ep->com.wr_wait, qhp->ep->hwtid,
1365
			     qhp->wq.sq.qid, __func__);
1366
out:
1367
	pr_debug("ret %d\n", ret);
1368 1369 1370 1371 1372
	return ret;
}

static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
{
1373
	pr_debug("p2p_type = %d\n", p2p_type);
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
	memset(&init->u, 0, sizeof init->u);
	switch (p2p_type) {
	case FW_RI_INIT_P2PTYPE_RDMA_WRITE:
		init->u.write.opcode = FW_RI_RDMA_WRITE_WR;
		init->u.write.stag_sink = cpu_to_be32(1);
		init->u.write.to_sink = cpu_to_be64(1);
		init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;
		init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write +
						   sizeof(struct fw_ri_immd),
						   16);
		break;
	case FW_RI_INIT_P2PTYPE_READ_REQ:
		init->u.write.opcode = FW_RI_RDMA_READ_WR;
		init->u.read.stag_src = cpu_to_be32(1);
		init->u.read.to_src_lo = cpu_to_be32(1);
		init->u.read.stag_sink = cpu_to_be32(1);
		init->u.read.to_sink_lo = cpu_to_be32(1);
		init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16);
		break;
	}
}

static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
{
	struct fw_ri_wr *wqe;
	int ret;
	struct sk_buff *skb;

1402
	pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp,
J
Joe Perches 已提交
1403
		 qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord);
1404

1405
	skb = alloc_skb(sizeof *wqe, GFP_KERNEL);
1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
	if (!skb) {
		ret = -ENOMEM;
		goto out;
	}
	ret = alloc_ird(rhp, qhp->attr.max_ird);
	if (ret) {
		qhp->attr.max_ird = 0;
		kfree_skb(skb);
		goto out;
	}
1416 1417
	set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);

1418
	wqe = __skb_put(skb, sizeof(*wqe));
1419 1420
	memset(wqe, 0, sizeof *wqe);
	wqe->op_compl = cpu_to_be32(
1421 1422
		FW_WR_OP_V(FW_RI_INIT_WR) |
		FW_WR_COMPL_F);
1423
	wqe->flowid_len16 = cpu_to_be32(
1424 1425
		FW_WR_FLOWID_V(qhp->ep->hwtid) |
		FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
1426

H
Hariprasad S 已提交
1427
	wqe->cookie = (uintptr_t)&qhp->ep->com.wr_wait;
1428 1429 1430

	wqe->u.init.type = FW_RI_TYPE_INIT;
	wqe->u.init.mpareqbit_p2ptype =
1431 1432
		FW_RI_WR_MPAREQBIT_V(qhp->attr.mpa_attr.initiator) |
		FW_RI_WR_P2PTYPE_V(qhp->attr.mpa_attr.p2p_type);
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
	wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE;
	if (qhp->attr.mpa_attr.recv_marker_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE;
	if (qhp->attr.mpa_attr.xmit_marker_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE;
	if (qhp->attr.mpa_attr.crc_enabled)
		wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE;

	wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE |
			    FW_RI_QP_RDMA_WRITE_ENABLE |
			    FW_RI_QP_BIND_ENABLE;
	if (!qhp->ibqp.uobject)
		wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE |
				     FW_RI_QP_STAG0_ENABLE;
	wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq));
	wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd);
	wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid);
	wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid);
	wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid);
	wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq);
	wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq);
	wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord);
	wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird);
	wqe->u.init.iss = cpu_to_be32(qhp->ep->snd_seq);
	wqe->u.init.irs = cpu_to_be32(qhp->ep->rcv_seq);
	wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size);
	wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr -
					 rhp->rdev.lldi.vr->rq.start);
	if (qhp->attr.mpa_attr.initiator)
		build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init);

	ret = c4iw_ofld_send(&rhp->rdev, skb);
	if (ret)
1466
		goto err1;
1467

1468 1469
	ret = c4iw_wait_for_reply(&rhp->rdev, &qhp->ep->com.wr_wait,
				  qhp->ep->hwtid, qhp->wq.sq.qid, __func__);
1470 1471 1472 1473
	if (!ret)
		goto out;
err1:
	free_ird(rhp, qhp->attr.max_ird);
1474
out:
1475
	pr_debug("ret %d\n", ret);
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
	return ret;
}

int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
		   enum c4iw_qp_attr_mask mask,
		   struct c4iw_qp_attributes *attrs,
		   int internal)
{
	int ret = 0;
	struct c4iw_qp_attributes newattr = qhp->attr;
	int disconnect = 0;
	int terminate = 0;
	int abort = 0;
	int free = 0;
	struct c4iw_ep *ep = NULL;

1492
	pr_debug("qhp %p sqid 0x%x rqid 0x%x ep %p state %d -> %d\n",
J
Joe Perches 已提交
1493 1494
		 qhp, qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep, qhp->attr.state,
		 (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1);
1495

1496
	mutex_lock(&qhp->mutex);
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510

	/* Process attr changes if in IDLE */
	if (mask & C4IW_QP_ATTR_VALID_MODIFY) {
		if (qhp->attr.state != C4IW_QP_STATE_IDLE) {
			ret = -EIO;
			goto out;
		}
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ)
			newattr.enable_rdma_read = attrs->enable_rdma_read;
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE)
			newattr.enable_rdma_write = attrs->enable_rdma_write;
		if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND)
			newattr.enable_bind = attrs->enable_bind;
		if (mask & C4IW_QP_ATTR_MAX_ORD) {
1511
			if (attrs->max_ord > c4iw_max_read_depth) {
1512 1513 1514 1515 1516 1517
				ret = -EINVAL;
				goto out;
			}
			newattr.max_ord = attrs->max_ord;
		}
		if (mask & C4IW_QP_ATTR_MAX_IRD) {
1518
			if (attrs->max_ird > cur_max_read_depth(rhp)) {
1519 1520 1521 1522 1523 1524 1525 1526
				ret = -EINVAL;
				goto out;
			}
			newattr.max_ird = attrs->max_ird;
		}
		qhp->attr = newattr;
	}

1527
	if (mask & C4IW_QP_ATTR_SQ_DB) {
1528
		ret = ring_kernel_sq_db(qhp, attrs->sq_db_inc);
1529 1530 1531
		goto out;
	}
	if (mask & C4IW_QP_ATTR_RQ_DB) {
1532
		ret = ring_kernel_rq_db(qhp, attrs->rq_db_inc);
1533 1534 1535
		goto out;
	}

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
	if (!(mask & C4IW_QP_ATTR_NEXT_STATE))
		goto out;
	if (qhp->attr.state == attrs->next_state)
		goto out;

	switch (qhp->attr.state) {
	case C4IW_QP_STATE_IDLE:
		switch (attrs->next_state) {
		case C4IW_QP_STATE_RTS:
			if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) {
				ret = -EINVAL;
				goto out;
			}
			if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) {
				ret = -EINVAL;
				goto out;
			}
			qhp->attr.mpa_attr = attrs->mpa_attr;
			qhp->attr.llp_stream_handle = attrs->llp_stream_handle;
			qhp->ep = qhp->attr.llp_stream_handle;
1556
			set_state(qhp, C4IW_QP_STATE_RTS);
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569

			/*
			 * Ref the endpoint here and deref when we
			 * disassociate the endpoint from the QP.  This
			 * happens in CLOSING->IDLE transition or *->ERROR
			 * transition.
			 */
			c4iw_get_ep(&qhp->ep->com);
			ret = rdma_init(rhp, qhp);
			if (ret)
				goto err;
			break;
		case C4IW_QP_STATE_ERROR:
1570 1571
			set_state(qhp, C4IW_QP_STATE_ERROR);
			flush_qp(qhp);
1572 1573 1574 1575 1576 1577 1578 1579 1580
			break;
		default:
			ret = -EINVAL;
			goto out;
		}
		break;
	case C4IW_QP_STATE_RTS:
		switch (attrs->next_state) {
		case C4IW_QP_STATE_CLOSING:
1581
			BUG_ON(kref_read(&qhp->ep->com.kref) < 2);
S
Steve Wise 已提交
1582
			t4_set_wq_in_error(&qhp->wq);
1583
			set_state(qhp, C4IW_QP_STATE_CLOSING);
S
Steve Wise 已提交
1584
			ep = qhp->ep;
1585 1586 1587
			if (!internal) {
				abort = 0;
				disconnect = 1;
1588
				c4iw_get_ep(&qhp->ep->com);
1589
			}
S
Steve Wise 已提交
1590
			ret = rdma_fini(rhp, qhp, ep);
S
Steve Wise 已提交
1591
			if (ret)
1592 1593 1594
				goto err;
			break;
		case C4IW_QP_STATE_TERMINATE:
S
Steve Wise 已提交
1595
			t4_set_wq_in_error(&qhp->wq);
1596
			set_state(qhp, C4IW_QP_STATE_TERMINATE);
1597 1598
			qhp->attr.layer_etype = attrs->layer_etype;
			qhp->attr.ecode = attrs->ecode;
1599
			ep = qhp->ep;
1600 1601
			if (!internal) {
				c4iw_get_ep(&qhp->ep->com);
1602
				terminate = 1;
1603 1604 1605
				disconnect = 1;
			} else {
				terminate = qhp->attr.send_term;
1606 1607 1608 1609
				ret = rdma_fini(rhp, qhp, ep);
				if (ret)
					goto err;
			}
1610 1611
			break;
		case C4IW_QP_STATE_ERROR:
S
Steve Wise 已提交
1612
			t4_set_wq_in_error(&qhp->wq);
S
Steve Wise 已提交
1613
			set_state(qhp, C4IW_QP_STATE_ERROR);
1614 1615 1616 1617
			if (!internal) {
				abort = 1;
				disconnect = 1;
				ep = qhp->ep;
1618
				c4iw_get_ep(&qhp->ep->com);
1619 1620 1621 1622 1623 1624 1625 1626 1627
			}
			goto err;
			break;
		default:
			ret = -EINVAL;
			goto out;
		}
		break;
	case C4IW_QP_STATE_CLOSING:
1628 1629 1630 1631 1632 1633

		/*
		 * Allow kernel users to move to ERROR for qp draining.
		 */
		if (!internal && (qhp->ibqp.uobject || attrs->next_state !=
				  C4IW_QP_STATE_ERROR)) {
1634 1635 1636 1637 1638
			ret = -EINVAL;
			goto out;
		}
		switch (attrs->next_state) {
		case C4IW_QP_STATE_IDLE:
1639 1640
			flush_qp(qhp);
			set_state(qhp, C4IW_QP_STATE_IDLE);
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
			qhp->attr.llp_stream_handle = NULL;
			c4iw_put_ep(&qhp->ep->com);
			qhp->ep = NULL;
			wake_up(&qhp->wait);
			break;
		case C4IW_QP_STATE_ERROR:
			goto err;
		default:
			ret = -EINVAL;
			goto err;
		}
		break;
	case C4IW_QP_STATE_ERROR:
		if (attrs->next_state != C4IW_QP_STATE_IDLE) {
			ret = -EINVAL;
			goto out;
		}
		if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) {
			ret = -EINVAL;
			goto out;
		}
1662
		set_state(qhp, C4IW_QP_STATE_IDLE);
1663 1664 1665 1666 1667 1668 1669 1670 1671
		break;
	case C4IW_QP_STATE_TERMINATE:
		if (!internal) {
			ret = -EINVAL;
			goto out;
		}
		goto err;
		break;
	default:
1672
		pr_err("%s in a bad state %d\n", __func__, qhp->attr.state);
1673 1674 1675 1676 1677 1678
		ret = -EINVAL;
		goto err;
		break;
	}
	goto out;
err:
1679
	pr_debug("disassociating ep %p qpid 0x%x\n", qhp->ep,
J
Joe Perches 已提交
1680
		 qhp->wq.sq.qid);
1681 1682 1683

	/* disassociate the LLP connection */
	qhp->attr.llp_stream_handle = NULL;
S
Steve Wise 已提交
1684 1685
	if (!ep)
		ep = qhp->ep;
1686
	qhp->ep = NULL;
1687
	set_state(qhp, C4IW_QP_STATE_ERROR);
1688
	free = 1;
1689
	abort = 1;
1690
	BUG_ON(!ep);
1691
	flush_qp(qhp);
1692
	wake_up(&qhp->wait);
1693
out:
1694
	mutex_unlock(&qhp->mutex);
1695 1696

	if (terminate)
1697
		post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL);
1698 1699 1700 1701 1702 1703 1704

	/*
	 * If disconnect is 1, then we need to initiate a disconnect
	 * on the EP.  This can be a normal close (RTS->CLOSING) or
	 * an abnormal close (RTS/CLOSING->ERROR).
	 */
	if (disconnect) {
1705 1706
		c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC :
							 GFP_KERNEL);
1707 1708 1709 1710 1711 1712 1713 1714 1715
		c4iw_put_ep(&ep->com);
	}

	/*
	 * If free is 1, then we've disassociated the EP from the QP
	 * and we need to dereference the EP.
	 */
	if (free)
		c4iw_put_ep(&ep->com);
1716
	pr_debug("exit state %d\n", qhp->attr.state);
1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
	return ret;
}

int c4iw_destroy_qp(struct ib_qp *ib_qp)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	struct c4iw_qp_attributes attrs;

	qhp = to_c4iw_qp(ib_qp);
	rhp = qhp->rhp;

	attrs.next_state = C4IW_QP_STATE_ERROR;
1730 1731 1732 1733
	if (qhp->attr.state == C4IW_QP_STATE_TERMINATE)
		c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
	else
		c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
1734 1735
	wait_event(qhp->wait, !qhp->ep);

1736
	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
1737

1738 1739 1740 1741
	spin_lock_irq(&rhp->lock);
	if (!list_empty(&qhp->db_fc_entry))
		list_del_init(&qhp->db_fc_entry);
	spin_unlock_irq(&rhp->lock);
1742
	free_ird(rhp, qhp->attr.max_ird);
1743

1744 1745
	c4iw_qp_rem_ref(ib_qp);

1746
	pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp, qhp->wq.sq.qid);
1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
	return 0;
}

struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
			     struct ib_udata *udata)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	struct c4iw_pd *php;
	struct c4iw_cq *schp;
	struct c4iw_cq *rchp;
	struct c4iw_create_qp_resp uresp;
1759
	unsigned int sqsize, rqsize;
1760 1761
	struct c4iw_ucontext *ucontext;
	int ret;
1762 1763
	struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;
	struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;
1764

1765
	pr_debug("ib_pd %p\n", pd);
1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779

	if (attrs->qp_type != IB_QPT_RC)
		return ERR_PTR(-EINVAL);

	php = to_c4iw_pd(pd);
	rhp = php->rhp;
	schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid);
	rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid);
	if (!schp || !rchp)
		return ERR_PTR(-EINVAL);

	if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
		return ERR_PTR(-EINVAL);

1780
	if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size)
1781
		return ERR_PTR(-E2BIG);
1782 1783 1784
	rqsize = attrs->cap.max_recv_wr + 1;
	if (rqsize < 8)
		rqsize = 8;
1785

1786
	if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size)
1787
		return ERR_PTR(-E2BIG);
1788 1789 1790
	sqsize = attrs->cap.max_send_wr + 1;
	if (sqsize < 8)
		sqsize = 8;
1791 1792 1793 1794 1795 1796 1797

	ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;

	qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
	if (!qhp)
		return ERR_PTR(-ENOMEM);
	qhp->wq.sq.size = sqsize;
1798 1799 1800
	qhp->wq.sq.memsize =
		(sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
		sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64);
S
Steve Wise 已提交
1801
	qhp->wq.sq.flush_cidx = -1;
1802
	qhp->wq.rq.size = rqsize;
1803 1804 1805
	qhp->wq.rq.memsize =
		(rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
		sizeof(*qhp->wq.rq.queue);
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834

	if (ucontext) {
		qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);
		qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE);
	}

	ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq,
			ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
	if (ret)
		goto err1;

	attrs->cap.max_recv_wr = rqsize - 1;
	attrs->cap.max_send_wr = sqsize - 1;
	attrs->cap.max_inline_data = T4_MAX_SEND_INLINE;

	qhp->rhp = rhp;
	qhp->attr.pd = php->pdid;
	qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid;
	qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid;
	qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
	qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
	qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
	qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
	qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
	qhp->attr.state = C4IW_QP_STATE_IDLE;
	qhp->attr.next_state = C4IW_QP_STATE_IDLE;
	qhp->attr.enable_rdma_read = 1;
	qhp->attr.enable_rdma_write = 1;
	qhp->attr.enable_bind = 1;
1835 1836
	qhp->attr.max_ord = 0;
	qhp->attr.max_ird = 0;
1837
	qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1838
	spin_lock_init(&qhp->lock);
1839
	mutex_init(&qhp->mutex);
1840
	init_waitqueue_head(&qhp->wait);
1841
	kref_init(&qhp->kref);
1842
	INIT_WORK(&qhp->free_work, free_qp_work);
1843

1844
	ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
1845 1846 1847 1848
	if (ret)
		goto err2;

	if (udata) {
1849 1850
		sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL);
		if (!sq_key_mm) {
1851
			ret = -ENOMEM;
1852
			goto err3;
1853
		}
1854 1855
		rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL);
		if (!rq_key_mm) {
1856
			ret = -ENOMEM;
1857
			goto err4;
1858
		}
1859 1860
		sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL);
		if (!sq_db_key_mm) {
1861
			ret = -ENOMEM;
1862
			goto err5;
1863
		}
1864 1865
		rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL);
		if (!rq_db_key_mm) {
1866
			ret = -ENOMEM;
1867
			goto err6;
1868
		}
S
Steve Wise 已提交
1869
		if (t4_sq_onchip(&qhp->wq.sq)) {
1870 1871 1872
			ma_sync_key_mm = kmalloc(sizeof(*ma_sync_key_mm),
						 GFP_KERNEL);
			if (!ma_sync_key_mm) {
S
Steve Wise 已提交
1873 1874 1875 1876 1877 1878
				ret = -ENOMEM;
				goto err7;
			}
			uresp.flags = C4IW_QPF_ONCHIP;
		} else
			uresp.flags = 0;
1879 1880 1881 1882 1883 1884 1885 1886
		uresp.qid_mask = rhp->rdev.qpmask;
		uresp.sqid = qhp->wq.sq.qid;
		uresp.sq_size = qhp->wq.sq.size;
		uresp.sq_memsize = qhp->wq.sq.memsize;
		uresp.rqid = qhp->wq.rq.qid;
		uresp.rq_size = qhp->wq.rq.size;
		uresp.rq_memsize = qhp->wq.rq.memsize;
		spin_lock(&ucontext->mmap_lock);
1887
		if (ma_sync_key_mm) {
S
Steve Wise 已提交
1888 1889
			uresp.ma_sync_key = ucontext->key;
			ucontext->key += PAGE_SIZE;
1890 1891
		} else {
			uresp.ma_sync_key =  0;
S
Steve Wise 已提交
1892
		}
1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
		uresp.sq_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.rq_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.sq_db_gts_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.rq_db_gts_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		spin_unlock(&ucontext->mmap_lock);
		ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
		if (ret)
S
Steve Wise 已提交
1904
			goto err8;
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
		sq_key_mm->key = uresp.sq_key;
		sq_key_mm->addr = qhp->wq.sq.phys_addr;
		sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize);
		insert_mmap(ucontext, sq_key_mm);
		rq_key_mm->key = uresp.rq_key;
		rq_key_mm->addr = virt_to_phys(qhp->wq.rq.queue);
		rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize);
		insert_mmap(ucontext, rq_key_mm);
		sq_db_key_mm->key = uresp.sq_db_gts_key;
		sq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.sq.bar2_pa;
		sq_db_key_mm->len = PAGE_SIZE;
		insert_mmap(ucontext, sq_db_key_mm);
		rq_db_key_mm->key = uresp.rq_db_gts_key;
		rq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.rq.bar2_pa;
		rq_db_key_mm->len = PAGE_SIZE;
		insert_mmap(ucontext, rq_db_key_mm);
		if (ma_sync_key_mm) {
			ma_sync_key_mm->key = uresp.ma_sync_key;
			ma_sync_key_mm->addr =
				(pci_resource_start(rhp->rdev.lldi.pdev, 0) +
				PCIE_MA_SYNC_A) & PAGE_MASK;
			ma_sync_key_mm->len = PAGE_SIZE;
			insert_mmap(ucontext, ma_sync_key_mm);
S
Steve Wise 已提交
1928
		}
1929 1930 1931

		c4iw_get_ucontext(ucontext);
		qhp->ucontext = ucontext;
1932 1933 1934
	}
	qhp->ibqp.qp_num = qhp->wq.sq.qid;
	init_timer(&(qhp->timer));
1935
	INIT_LIST_HEAD(&qhp->db_fc_entry);
1936
	pr_debug("sq id %u size %u memsize %zu num_entries %u rq id %u size %u memsize %zu num_entries %u\n",
J
Joe Perches 已提交
1937 1938 1939
		 qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize,
		 attrs->cap.max_send_wr, qhp->wq.rq.qid, qhp->wq.rq.size,
		 qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
1940
	return &qhp->ibqp;
S
Steve Wise 已提交
1941
err8:
1942
	kfree(ma_sync_key_mm);
1943
err7:
1944
	kfree(rq_db_key_mm);
1945
err6:
1946
	kfree(sq_db_key_mm);
1947
err5:
1948
	kfree(rq_key_mm);
1949
err4:
1950
	kfree(sq_key_mm);
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
err3:
	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
err2:
	destroy_qp(&rhp->rdev, &qhp->wq,
		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
err1:
	kfree(qhp);
	return ERR_PTR(ret);
}

int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		      int attr_mask, struct ib_udata *udata)
{
	struct c4iw_dev *rhp;
	struct c4iw_qp *qhp;
	enum c4iw_qp_attr_mask mask = 0;
	struct c4iw_qp_attributes attrs;

1969
	pr_debug("ib_qp %p\n", ibqp);
1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996

	/* iwarp does not support the RTR state */
	if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
		attr_mask &= ~IB_QP_STATE;

	/* Make sure we still have something left to do */
	if (!attr_mask)
		return 0;

	memset(&attrs, 0, sizeof attrs);
	qhp = to_c4iw_qp(ibqp);
	rhp = qhp->rhp;

	attrs.next_state = c4iw_convert_state(attr->qp_state);
	attrs.enable_rdma_read = (attr->qp_access_flags &
			       IB_ACCESS_REMOTE_READ) ?  1 : 0;
	attrs.enable_rdma_write = (attr->qp_access_flags &
				IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
	attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;


	mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0;
	mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
			(C4IW_QP_ATTR_ENABLE_RDMA_READ |
			 C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
			 C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0;

1997 1998 1999
	/*
	 * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for
	 * ringing the queue db when we're in DB_FULL mode.
2000
	 * Only allow this on T4 devices.
2001 2002 2003 2004 2005
	 */
	attrs.sq_db_inc = attr->sq_psn;
	attrs.rq_db_inc = attr->rq_psn;
	mask |= (attr_mask & IB_QP_SQ_PSN) ? C4IW_QP_ATTR_SQ_DB : 0;
	mask |= (attr_mask & IB_QP_RQ_PSN) ? C4IW_QP_ATTR_RQ_DB : 0;
2006
	if (!is_t4(to_c4iw_qp(ibqp)->rhp->rdev.lldi.adapter_type) &&
2007 2008
	    (mask & (C4IW_QP_ATTR_SQ_DB|C4IW_QP_ATTR_RQ_DB)))
		return -EINVAL;
2009

2010 2011 2012 2013 2014
	return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0);
}

struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn)
{
2015
	pr_debug("ib_dev %p qpn 0x%x\n", dev, qpn);
2016 2017
	return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn);
}
V
Vipul Pandya 已提交
2018 2019 2020 2021 2022 2023 2024 2025 2026

int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		     int attr_mask, struct ib_qp_init_attr *init_attr)
{
	struct c4iw_qp *qhp = to_c4iw_qp(ibqp);

	memset(attr, 0, sizeof *attr);
	memset(init_attr, 0, sizeof *init_attr);
	attr->qp_state = to_ib_qp_state(qhp->attr.state);
2027 2028 2029 2030 2031 2032
	init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
	init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
	init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
	init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges;
	init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE;
	init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0;
V
Vipul Pandya 已提交
2033 2034
	return 0;
}