cq.c 25.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *	  copyright notice, this list of conditions and the following
 *	  disclaimer in the documentation and/or other materials
 *	  provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "iw_cxgb4.h"

static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
36
		      struct c4iw_dev_ucontext *uctx, struct sk_buff *skb)
37 38 39 40 41 42 43 44 45 46
{
	struct fw_ri_res_wr *res_wr;
	struct fw_ri_res *res;
	int wr_len;
	struct c4iw_wr_wait wr_wait;
	int ret;

	wr_len = sizeof *res_wr + sizeof *res;
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);

47
	res_wr = __skb_put(skb, wr_len);
48 49
	memset(res_wr, 0, wr_len);
	res_wr->op_nres = cpu_to_be32(
50
			FW_WR_OP_V(FW_RI_RES_WR) |
51
			FW_RI_RES_WR_NRES_V(1) |
52
			FW_WR_COMPL_F);
53
	res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
H
Hariprasad S 已提交
54
	res_wr->cookie = (uintptr_t)&wr_wait;
55 56 57 58 59 60 61 62
	res = res_wr->res;
	res->u.cq.restype = FW_RI_RES_TYPE_CQ;
	res->u.cq.op = FW_RI_RES_OP_RESET;
	res->u.cq.iqid = cpu_to_be32(cq->cqid);

	c4iw_init_wr_wait(&wr_wait);
	ret = c4iw_ofld_send(rdev, skb);
	if (!ret) {
63
		ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
64 65 66 67 68
	}

	kfree(cq->sw_queue);
	dma_free_coherent(&(rdev->lldi.pdev->dev),
			  cq->memsize, cq->queue,
69
			  dma_unmap_addr(cq, mapping));
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
	c4iw_put_cqid(rdev, cq->cqid, uctx);
	return ret;
}

static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
		     struct c4iw_dev_ucontext *uctx)
{
	struct fw_ri_res_wr *res_wr;
	struct fw_ri_res *res;
	int wr_len;
	int user = (uctx != &rdev->uctx);
	struct c4iw_wr_wait wr_wait;
	int ret;
	struct sk_buff *skb;

	cq->cqid = c4iw_get_cqid(rdev, uctx);
	if (!cq->cqid) {
		ret = -ENOMEM;
		goto err1;
	}

	if (!user) {
		cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL);
		if (!cq->sw_queue) {
			ret = -ENOMEM;
			goto err2;
		}
	}
	cq->queue = dma_alloc_coherent(&rdev->lldi.pdev->dev, cq->memsize,
				       &cq->dma_addr, GFP_KERNEL);
	if (!cq->queue) {
		ret = -ENOMEM;
		goto err3;
	}
104
	dma_unmap_addr_set(cq, mapping, cq->dma_addr);
105 106 107 108 109
	memset(cq->queue, 0, cq->memsize);

	/* build fw_ri_res_wr */
	wr_len = sizeof *res_wr + sizeof *res;

110
	skb = alloc_skb(wr_len, GFP_KERNEL);
111 112 113 114 115 116
	if (!skb) {
		ret = -ENOMEM;
		goto err4;
	}
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);

117
	res_wr = __skb_put(skb, wr_len);
118 119
	memset(res_wr, 0, wr_len);
	res_wr->op_nres = cpu_to_be32(
120
			FW_WR_OP_V(FW_RI_RES_WR) |
121
			FW_RI_RES_WR_NRES_V(1) |
122
			FW_WR_COMPL_F);
123
	res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
H
Hariprasad S 已提交
124
	res_wr->cookie = (uintptr_t)&wr_wait;
125 126 127 128 129
	res = res_wr->res;
	res->u.cq.restype = FW_RI_RES_TYPE_CQ;
	res->u.cq.op = FW_RI_RES_OP_WRITE;
	res->u.cq.iqid = cpu_to_be32(cq->cqid);
	res->u.cq.iqandst_to_iqandstindex = cpu_to_be32(
130 131 132 133
			FW_RI_RES_WR_IQANUS_V(0) |
			FW_RI_RES_WR_IQANUD_V(1) |
			FW_RI_RES_WR_IQANDST_F |
			FW_RI_RES_WR_IQANDSTINDEX_V(
134
				rdev->lldi.ciq_ids[cq->vector]));
135
	res->u.cq.iqdroprss_to_iqesize = cpu_to_be16(
136 137 138 139 140
			FW_RI_RES_WR_IQDROPRSS_F |
			FW_RI_RES_WR_IQPCIECH_V(2) |
			FW_RI_RES_WR_IQINTCNTTHRESH_V(0) |
			FW_RI_RES_WR_IQO_F |
			FW_RI_RES_WR_IQESIZE_V(1));
141 142 143 144 145 146 147 148
	res->u.cq.iqsize = cpu_to_be16(cq->size);
	res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr);

	c4iw_init_wr_wait(&wr_wait);

	ret = c4iw_ofld_send(rdev, skb);
	if (ret)
		goto err4;
J
Joe Perches 已提交
149
	pr_debug("%s wait_event wr_wait %p\n", __func__, &wr_wait);
150
	ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
151 152 153 154
	if (ret)
		goto err4;

	cq->gen = 1;
155
	cq->gts = rdev->lldi.gts_reg;
156
	cq->rdev = rdev;
157

158 159 160
	cq->bar2_va = c4iw_bar2_addrs(rdev, cq->cqid, T4_BAR2_QTYPE_INGRESS,
				      &cq->bar2_qid,
				      user ? &cq->bar2_pa : NULL);
161
	if (user && !cq->bar2_pa) {
162
		pr_warn("%s: cqid %u not in BAR2 range\n",
163 164 165
			pci_name(rdev->lldi.pdev), cq->cqid);
		ret = -EINVAL;
		goto err4;
166 167 168 169
	}
	return 0;
err4:
	dma_free_coherent(&rdev->lldi.pdev->dev, cq->memsize, cq->queue,
170
			  dma_unmap_addr(cq, mapping));
171 172 173 174 175 176 177 178 179 180 181 182
err3:
	kfree(cq->sw_queue);
err2:
	c4iw_put_cqid(rdev, cq->cqid, uctx);
err1:
	return ret;
}

static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)
{
	struct t4_cqe cqe;

J
Joe Perches 已提交
183 184
	pr_debug("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__,
		 wq, cq, cq->sw_cidx, cq->sw_pidx);
185
	memset(&cqe, 0, sizeof(cqe));
186 187 188 189 190 191
	cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
				 CQE_OPCODE_V(FW_RI_SEND) |
				 CQE_TYPE_V(0) |
				 CQE_SWCQE_V(1) |
				 CQE_QPID_V(wq->sq.qid));
	cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
192 193 194 195 196 197 198 199 200 201
	cq->sw_queue[cq->sw_pidx] = cqe;
	t4_swcq_produce(cq);
}

int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)
{
	int flushed = 0;
	int in_use = wq->rq.in_use - count;

	BUG_ON(in_use < 0);
J
Joe Perches 已提交
202 203
	pr_debug("%s wq %p cq %p rq.in_use %u skip count %u\n", __func__,
		 wq, cq, wq->rq.in_use, count);
204 205 206 207 208 209 210 211 212 213 214 215
	while (in_use--) {
		insert_recv_cqe(wq, cq);
		flushed++;
	}
	return flushed;
}

static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
			  struct t4_swsqe *swcqe)
{
	struct t4_cqe cqe;

J
Joe Perches 已提交
216 217
	pr_debug("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__,
		 wq, cq, cq->sw_cidx, cq->sw_pidx);
218
	memset(&cqe, 0, sizeof(cqe));
219 220 221 222 223
	cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
				 CQE_OPCODE_V(swcqe->opcode) |
				 CQE_TYPE_V(1) |
				 CQE_SWCQE_V(1) |
				 CQE_QPID_V(wq->sq.qid));
224
	CQE_WRID_SQ_IDX(&cqe) = swcqe->idx;
225
	cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
226 227 228 229
	cq->sw_queue[cq->sw_pidx] = cqe;
	t4_swcq_produce(cq);
}

S
Steve Wise 已提交
230 231 232
static void advance_oldest_read(struct t4_wq *wq);

int c4iw_flush_sq(struct c4iw_qp *qhp)
233 234
{
	int flushed = 0;
S
Steve Wise 已提交
235 236 237 238 239 240 241 242 243 244 245
	struct t4_wq *wq = &qhp->wq;
	struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq);
	struct t4_cq *cq = &chp->cq;
	int idx;
	struct t4_swsqe *swsqe;

	if (wq->sq.flush_cidx == -1)
		wq->sq.flush_cidx = wq->sq.cidx;
	idx = wq->sq.flush_cidx;
	BUG_ON(idx >= wq->sq.size);
	while (idx != wq->sq.pidx) {
S
Steve Wise 已提交
246 247 248 249 250 251 252
		swsqe = &wq->sq.sw_sq[idx];
		BUG_ON(swsqe->flushed);
		swsqe->flushed = 1;
		insert_sq_cqe(wq, cq, swsqe);
		if (wq->sq.oldest_read == swsqe) {
			BUG_ON(swsqe->opcode != FW_RI_READ_REQ);
			advance_oldest_read(wq);
S
Steve Wise 已提交
253
		}
S
Steve Wise 已提交
254
		flushed++;
S
Steve Wise 已提交
255 256
		if (++idx == wq->sq.size)
			idx = 0;
257
	}
S
Steve Wise 已提交
258 259 260
	wq->sq.flush_cidx += flushed;
	if (wq->sq.flush_cidx >= wq->sq.size)
		wq->sq.flush_cidx -= wq->sq.size;
261 262 263
	return flushed;
}

S
Steve Wise 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
{
	struct t4_swsqe *swsqe;
	int cidx;

	if (wq->sq.flush_cidx == -1)
		wq->sq.flush_cidx = wq->sq.cidx;
	cidx = wq->sq.flush_cidx;
	BUG_ON(cidx > wq->sq.size);

	while (cidx != wq->sq.pidx) {
		swsqe = &wq->sq.sw_sq[cidx];
		if (!swsqe->signaled) {
			if (++cidx == wq->sq.size)
				cidx = 0;
		} else if (swsqe->complete) {

			BUG_ON(swsqe->flushed);

			/*
			 * Insert this completed cqe into the swcq.
			 */
J
Joe Perches 已提交
286 287
			pr_debug("%s moving cqe into swcq sq idx %u cq idx %u\n",
				 __func__, cidx, cq->sw_pidx);
288
			swsqe->cqe.header |= htonl(CQE_SWCQE_V(1));
S
Steve Wise 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
			cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
			t4_swcq_produce(cq);
			swsqe->flushed = 1;
			if (++cidx == wq->sq.size)
				cidx = 0;
			wq->sq.flush_cidx = cidx;
		} else
			break;
	}
}

static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
		struct t4_cqe *read_cqe)
{
	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
	read_cqe->len = htonl(wq->sq.oldest_read->read_len);
305 306 307 308
	read_cqe->header = htonl(CQE_QPID_V(CQE_QPID(hw_cqe)) |
			CQE_SWCQE_V(SW_CQE(hw_cqe)) |
			CQE_OPCODE_V(FW_RI_READ_REQ) |
			CQE_TYPE_V(1));
S
Steve Wise 已提交
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
	read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
}

static void advance_oldest_read(struct t4_wq *wq)
{

	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;

	if (rptr == wq->sq.size)
		rptr = 0;
	while (rptr != wq->sq.pidx) {
		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];

		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
			return;
		if (++rptr == wq->sq.size)
			rptr = 0;
	}
	wq->sq.oldest_read = NULL;
}

330 331
/*
 * Move all CQEs from the HWCQ into the SWCQ.
S
Steve Wise 已提交
332 333
 * Deal with out-of-order and/or completions that complete
 * prior unsignalled WRs.
334
 */
S
Steve Wise 已提交
335
void c4iw_flush_hw_cq(struct c4iw_cq *chp)
336
{
S
Steve Wise 已提交
337 338 339
	struct t4_cqe *hw_cqe, *swcqe, read_cqe;
	struct c4iw_qp *qhp;
	struct t4_swsqe *swsqe;
340 341
	int ret;

J
Joe Perches 已提交
342
	pr_debug("%s  cqid 0x%x\n", __func__, chp->cq.cqid);
S
Steve Wise 已提交
343 344 345 346 347 348 349
	ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);

	/*
	 * This logic is similar to poll_cq(), but not quite the same
	 * unfortunately.  Need to move pertinent HW CQEs to the SW CQ but
	 * also do any translation magic that poll_cq() normally does.
	 */
350
	while (!ret) {
S
Steve Wise 已提交
351 352 353 354 355 356 357 358 359 360 361 362 363
		qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe));

		/*
		 * drop CQEs with no associated QP
		 */
		if (qhp == NULL)
			goto next_cqe;

		if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE)
			goto next_cqe;

		if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) {

364 365 366 367 368 369 370 371
			/* If we have reached here because of async
			 * event or other error, and have egress error
			 * then drop
			 */
			if (CQE_TYPE(hw_cqe) == 1)
				goto next_cqe;

			/* drop peer2peer RTR reads.
S
Steve Wise 已提交
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
			 */
			if (CQE_WRID_STAG(hw_cqe) == 1)
				goto next_cqe;

			/*
			 * Eat completions for unsignaled read WRs.
			 */
			if (!qhp->wq.sq.oldest_read->signaled) {
				advance_oldest_read(&qhp->wq);
				goto next_cqe;
			}

			/*
			 * Don't write to the HWCQ, create a new read req CQE
			 * in local memory and move it into the swcq.
			 */
			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
			hw_cqe = &read_cqe;
			advance_oldest_read(&qhp->wq);
		}

		/* if its a SQ completion, then do the magic to move all the
		 * unsignaled and now in-order completions into the swcq.
		 */
		if (SQ_TYPE(hw_cqe)) {
			swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
			swsqe->cqe = *hw_cqe;
			swsqe->complete = 1;
			flush_completed_wrs(&qhp->wq, &chp->cq);
		} else {
			swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];
			*swcqe = *hw_cqe;
404
			swcqe->header |= cpu_to_be32(CQE_SWCQE_V(1));
S
Steve Wise 已提交
405 406 407 408 409
			t4_swcq_produce(&chp->cq);
		}
next_cqe:
		t4_hwcq_consume(&chp->cq);
		ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
	}
}

static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
{
	if (CQE_OPCODE(cqe) == FW_RI_TERMINATE)
		return 0;

	if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe))
		return 0;

	if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe))
		return 0;

	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))
		return 0;
	return 1;
}

void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
{
	struct t4_cqe *cqe;
	u32 ptr;

	*count = 0;
J
Joe Perches 已提交
435
	pr_debug("%s count zero %d\n", __func__, *count);
436 437 438 439
	ptr = cq->sw_cidx;
	while (ptr != cq->sw_pidx) {
		cqe = &cq->sw_queue[ptr];
		if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) &&
440
		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))
441 442 443 444
			(*count)++;
		if (++ptr == cq->size)
			ptr = 0;
	}
J
Joe Perches 已提交
445
	pr_debug("%s cq %p count %d\n", __func__, cq, *count);
446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
}

/*
 * poll_cq
 *
 * Caller must:
 *     check the validity of the first CQE,
 *     supply the wq assicated with the qpid.
 *
 * credit: cq credit to return to sge.
 * cqe_flushed: 1 iff the CQE is flushed.
 * cqe: copy of the polled CQE.
 *
 * return value:
 *    0		    CQE returned ok.
 *    -EAGAIN       CQE skipped, try again.
 *    -EOVERFLOW    CQ overflow detected.
 */
static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
		   u8 *cqe_flushed, u64 *cookie, u32 *credit)
{
	int ret = 0;
	struct t4_cqe *hw_cqe, read_cqe;

	*cqe_flushed = 0;
	*credit = 0;
	ret = t4_next_cqe(cq, &hw_cqe);
	if (ret)
		return ret;

J
Joe Perches 已提交
476 477 478 479 480
	pr_debug("%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
		 __func__, CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe),
		 CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe),
		 CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe),
		 CQE_WRID_LOW(hw_cqe));
481 482 483 484 485 486 487 488 489

	/*
	 * skip cqe's not affiliated with a QP.
	 */
	if (wq == NULL) {
		ret = -EAGAIN;
		goto skip_cqe;
	}

S
Steve Wise 已提交
490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
	/*
	* skip hw cqe's if the wq is flushed.
	*/
	if (wq->flushed && !SW_CQE(hw_cqe)) {
		ret = -EAGAIN;
		goto skip_cqe;
	}

	/*
	 * skip TERMINATE cqes...
	 */
	if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
		ret = -EAGAIN;
		goto skip_cqe;
	}

506 507 508 509 510 511 512 513 514
	/*
	 * Special cqe for drain WR completions...
	 */
	if (CQE_OPCODE(hw_cqe) == C4IW_DRAIN_OPCODE) {
		*cookie = CQE_DRAIN_COOKIE(hw_cqe);
		*cqe = *hw_cqe;
		goto skip_cqe;
	}

515 516 517 518 519 520 521 522 523
	/*
	 * Gotta tweak READ completions:
	 *	1) the cqe doesn't contain the sq_wptr from the wr.
	 *	2) opcode not reflected from the wr.
	 *	3) read_len not reflected from the wr.
	 *	4) cq_type is RQ_TYPE not SQ_TYPE.
	 */
	if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) {

524 525 526 527 528 529 530 531 532 533 534 535
		/* If we have reached here because of async
		 * event or other error, and have egress error
		 * then drop
		 */
		if (CQE_TYPE(hw_cqe) == 1) {
			if (CQE_STATUS(hw_cqe))
				t4_set_wq_in_error(wq);
			ret = -EAGAIN;
			goto skip_cqe;
		}

		/* If this is an unsolicited read response, then the read
536 537 538
		 * was generated by the kernel driver as part of peer-2-peer
		 * connection setup.  So ignore the completion.
		 */
S
Steve Wise 已提交
539
		if (CQE_WRID_STAG(hw_cqe) == 1) {
540 541 542 543 544 545
			if (CQE_STATUS(hw_cqe))
				t4_set_wq_in_error(wq);
			ret = -EAGAIN;
			goto skip_cqe;
		}

S
Steve Wise 已提交
546 547 548 549 550 551 552 553 554
		/*
		 * Eat completions for unsignaled read WRs.
		 */
		if (!wq->sq.oldest_read->signaled) {
			advance_oldest_read(wq);
			ret = -EAGAIN;
			goto skip_cqe;
		}

555 556 557 558 559 560 561 562 563 564
		/*
		 * Don't write to the HWCQ, so create a new read req CQE
		 * in local memory.
		 */
		create_read_req_cqe(wq, hw_cqe, &read_cqe);
		hw_cqe = &read_cqe;
		advance_oldest_read(wq);
	}

	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
S
Steve Wise 已提交
565
		*cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH);
566
		t4_set_wq_in_error(wq);
S
Steve Wise 已提交
567 568
	}

569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
	/*
	 * RECV completion.
	 */
	if (RQ_TYPE(hw_cqe)) {

		/*
		 * HW only validates 4 bits of MSN.  So we must validate that
		 * the MSN in the SEND is the next expected MSN.  If its not,
		 * then we complete this with T4_ERR_MSN and mark the wq in
		 * error.
		 */

		if (t4_rq_empty(wq)) {
			t4_set_wq_in_error(wq);
			ret = -EAGAIN;
			goto skip_cqe;
		}
		if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) {
			t4_set_wq_in_error(wq);
588
			hw_cqe->header |= htonl(CQE_STATUS_V(T4_ERR_MSN));
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
			goto proc_cqe;
		}
		goto proc_cqe;
	}

	/*
	 * If we get here its a send completion.
	 *
	 * Handle out of order completion. These get stuffed
	 * in the SW SQ. Then the SW SQ is walked to move any
	 * now in-order completions into the SW CQ.  This handles
	 * 2 cases:
	 *	1) reaping unsignaled WRs when the first subsequent
	 *	   signaled WR is completed.
	 *	2) out of order read completions.
	 */
	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {
		struct t4_swsqe *swsqe;

J
Joe Perches 已提交
608 609
		pr_debug("%s out of order completion going in sw_sq at idx %u\n",
			 __func__, CQE_WRID_SQ_IDX(hw_cqe));
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
		swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
		swsqe->cqe = *hw_cqe;
		swsqe->complete = 1;
		ret = -EAGAIN;
		goto flush_wq;
	}

proc_cqe:
	*cqe = *hw_cqe;

	/*
	 * Reap the associated WR(s) that are freed up with this
	 * completion.
	 */
	if (SQ_TYPE(hw_cqe)) {
S
Steve Wise 已提交
625
		int idx = CQE_WRID_SQ_IDX(hw_cqe);
626
		BUG_ON(idx >= wq->sq.size);
S
Steve Wise 已提交
627 628 629 630 631 632 633

		/*
		* Account for any unsignaled completions completed by
		* this signaled completion.  In this case, cidx points
		* to the first unsignaled one, and idx points to the
		* signaled one.  So adjust in_use based on this delta.
		* if this is not completing any unsigned wrs, then the
634
		* delta will be 0. Handle wrapping also!
S
Steve Wise 已提交
635
		*/
636 637 638 639
		if (idx < wq->sq.cidx)
			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
		else
			wq->sq.in_use -= idx - wq->sq.cidx;
640
		BUG_ON(wq->sq.in_use <= 0 && wq->sq.in_use >= wq->sq.size);
S
Steve Wise 已提交
641 642

		wq->sq.cidx = (uint16_t)idx;
J
Joe Perches 已提交
643
		pr_debug("%s completing sq idx %u\n", __func__, wq->sq.cidx);
644
		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
645 646
		if (c4iw_wr_log)
			c4iw_log_wr_stats(wq, hw_cqe);
647 648
		t4_sq_consume(wq);
	} else {
J
Joe Perches 已提交
649
		pr_debug("%s completing rq idx %u\n", __func__, wq->rq.cidx);
650 651
		*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
		BUG_ON(t4_rq_empty(wq));
652 653
		if (c4iw_wr_log)
			c4iw_log_wr_stats(wq, hw_cqe);
654
		t4_rq_consume(wq);
S
Steve Wise 已提交
655
		goto skip_cqe;
656 657 658 659 660 661 662 663 664 665
	}

flush_wq:
	/*
	 * Flush any completed cqes that are now in-order.
	 */
	flush_completed_wrs(wq, cq);

skip_cqe:
	if (SW_CQE(hw_cqe)) {
J
Joe Perches 已提交
666 667
		pr_debug("%s cq %p cqid 0x%x skip sw cqe cidx %u\n",
			 __func__, cq, cq->cqid, cq->sw_cidx);
668 669
		t4_swcq_consume(cq);
	} else {
J
Joe Perches 已提交
670 671
		pr_debug("%s cq %p cqid 0x%x skip hw cqe cidx %u\n",
			 __func__, cq, cq->cqid, cq->cidx);
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
		t4_hwcq_consume(cq);
	}
	return ret;
}

/*
 * Get one cq entry from c4iw and map it to openib.
 *
 * Returns:
 *	0			cqe returned
 *	-ENODATA		EMPTY;
 *	-EAGAIN			caller must try again
 *	any other -errno	fatal error
 */
static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
{
	struct c4iw_qp *qhp = NULL;
S
Steve Wise 已提交
689
	struct t4_cqe uninitialized_var(cqe), *rd_cqe;
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
	struct t4_wq *wq;
	u32 credit = 0;
	u8 cqe_flushed;
	u64 cookie = 0;
	int ret;

	ret = t4_next_cqe(&chp->cq, &rd_cqe);

	if (ret)
		return ret;

	qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe));
	if (!qhp)
		wq = NULL;
	else {
		spin_lock(&qhp->lock);
		wq = &(qhp->wq);
	}
	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);
	if (ret)
		goto out;

	wc->wr_id = cookie;
	wc->qp = &qhp->ibqp;
	wc->vendor_err = CQE_STATUS(&cqe);
	wc->wc_flags = 0;

J
Joe Perches 已提交
717 718 719 720 721 722
	pr_debug("%s qpid 0x%x type %d opcode %d status 0x%x len %u wrid hi 0x%x lo 0x%x cookie 0x%llx\n",
		 __func__, CQE_QPID(&cqe),
		 CQE_TYPE(&cqe), CQE_OPCODE(&cqe),
		 CQE_STATUS(&cqe), CQE_LEN(&cqe),
		 CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe),
		 (unsigned long long)cookie);
723 724 725 726 727 728 729 730 731 732 733

	if (CQE_TYPE(&cqe) == 0) {
		if (!CQE_STATUS(&cqe))
			wc->byte_len = CQE_LEN(&cqe);
		else
			wc->byte_len = 0;
		wc->opcode = IB_WC_RECV;
		if (CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_INV ||
		    CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) {
			wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
734
			c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758
		}
	} else {
		switch (CQE_OPCODE(&cqe)) {
		case FW_RI_RDMA_WRITE:
			wc->opcode = IB_WC_RDMA_WRITE;
			break;
		case FW_RI_READ_REQ:
			wc->opcode = IB_WC_RDMA_READ;
			wc->byte_len = CQE_LEN(&cqe);
			break;
		case FW_RI_SEND_WITH_INV:
		case FW_RI_SEND_WITH_SE_INV:
			wc->opcode = IB_WC_SEND;
			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
			break;
		case FW_RI_SEND:
		case FW_RI_SEND_WITH_SE:
			wc->opcode = IB_WC_SEND;
			break;

		case FW_RI_LOCAL_INV:
			wc->opcode = IB_WC_LOCAL_INV;
			break;
		case FW_RI_FAST_REGISTER:
S
Sagi Grimberg 已提交
759
			wc->opcode = IB_WC_REG_MR;
760 761 762

			/* Invalidate the MR if the fastreg failed */
			if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
763 764
				c4iw_invalidate_mr(qhp->rhp,
						   CQE_WRID_FR_STAG(&cqe));
765
			break;
766 767 768
		case C4IW_DRAIN_OPCODE:
			wc->opcode = IB_WC_SEND;
			break;
769
		default:
770
			pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
			       CQE_OPCODE(&cqe), CQE_QPID(&cqe));
			ret = -EINVAL;
			goto out;
		}
	}

	if (cqe_flushed)
		wc->status = IB_WC_WR_FLUSH_ERR;
	else {

		switch (CQE_STATUS(&cqe)) {
		case T4_ERR_SUCCESS:
			wc->status = IB_WC_SUCCESS;
			break;
		case T4_ERR_STAG:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_PDID:
			wc->status = IB_WC_LOC_PROT_ERR;
			break;
		case T4_ERR_QPID:
		case T4_ERR_ACCESS:
			wc->status = IB_WC_LOC_ACCESS_ERR;
			break;
		case T4_ERR_WRAP:
			wc->status = IB_WC_GENERAL_ERR;
			break;
		case T4_ERR_BOUND:
			wc->status = IB_WC_LOC_LEN_ERR;
			break;
		case T4_ERR_INVALIDATE_SHARED_MR:
		case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
			wc->status = IB_WC_MW_BIND_ERR;
			break;
		case T4_ERR_CRC:
		case T4_ERR_MARKER:
		case T4_ERR_PDU_LEN_ERR:
		case T4_ERR_OUT_OF_RQE:
		case T4_ERR_DDP_VERSION:
		case T4_ERR_RDMA_VERSION:
		case T4_ERR_DDP_QUEUE_NUM:
		case T4_ERR_MSN:
		case T4_ERR_TBIT:
		case T4_ERR_MO:
		case T4_ERR_MSN_RANGE:
		case T4_ERR_IRD_OVERFLOW:
		case T4_ERR_OPCODE:
S
Steve Wise 已提交
818
		case T4_ERR_INTERNAL_ERR:
819 820 821 822 823 824
			wc->status = IB_WC_FATAL_ERR;
			break;
		case T4_ERR_SWFLUSH:
			wc->status = IB_WC_WR_FLUSH_ERR;
			break;
		default:
825
			pr_err("Unexpected cqe_status 0x%x for QPID=0x%0x\n",
826
			       CQE_STATUS(&cqe), CQE_QPID(&cqe));
827
			wc->status = IB_WC_FATAL_ERR;
828 829 830
		}
	}
out:
831
	if (wq)
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
		spin_unlock(&qhp->lock);
	return ret;
}

int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
{
	struct c4iw_cq *chp;
	unsigned long flags;
	int npolled;
	int err = 0;

	chp = to_c4iw_cq(ibcq);

	spin_lock_irqsave(&chp->lock, flags);
	for (npolled = 0; npolled < num_entries; ++npolled) {
		do {
			err = c4iw_poll_cq_one(chp, wc + npolled);
		} while (err == -EAGAIN);
		if (err)
			break;
	}
	spin_unlock_irqrestore(&chp->lock, flags);
	return !err || err == -ENODATA ? npolled : err;
}

int c4iw_destroy_cq(struct ib_cq *ib_cq)
{
	struct c4iw_cq *chp;
	struct c4iw_ucontext *ucontext;

J
Joe Perches 已提交
862
	pr_debug("%s ib_cq %p\n", __func__, ib_cq);
863 864 865 866 867 868 869 870 871
	chp = to_c4iw_cq(ib_cq);

	remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
	atomic_dec(&chp->refcnt);
	wait_event(chp->wait, !atomic_read(&chp->refcnt));

	ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context)
				  : NULL;
	destroy_cq(&chp->rhp->rdev, &chp->cq,
872 873 874
		   ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx,
		   chp->destroy_skb);
	chp->destroy_skb = NULL;
875 876 877 878
	kfree(chp);
	return 0;
}

879 880 881
struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
			     const struct ib_cq_init_attr *attr,
			     struct ib_ucontext *ib_context,
882 883
			     struct ib_udata *udata)
{
884 885
	int entries = attr->cqe;
	int vector = attr->comp_vector;
886 887 888 889
	struct c4iw_dev *rhp;
	struct c4iw_cq *chp;
	struct c4iw_create_cq_resp uresp;
	struct c4iw_ucontext *ucontext = NULL;
890
	int ret, wr_len;
891
	size_t memsize, hwentries;
892 893
	struct c4iw_mm_entry *mm, *mm2;

J
Joe Perches 已提交
894
	pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
895 896
	if (attr->flags)
		return ERR_PTR(-EINVAL);
897 898 899

	rhp = to_c4iw_dev(ibdev);

900 901 902
	if (vector >= rhp->rdev.lldi.nciq)
		return ERR_PTR(-EINVAL);

903 904 905 906
	chp = kzalloc(sizeof(*chp), GFP_KERNEL);
	if (!chp)
		return ERR_PTR(-ENOMEM);

907 908 909 910 911 912 913
	wr_len = sizeof(struct fw_ri_res_wr) + sizeof(struct fw_ri_res);
	chp->destroy_skb = alloc_skb(wr_len, GFP_KERNEL);
	if (!chp->destroy_skb) {
		ret = -ENOMEM;
		goto err1;
	}

914 915 916 917 918 919
	if (ib_context)
		ucontext = to_c4iw_ucontext(ib_context);

	/* account for the status page. */
	entries++;

920 921 922
	/* IQ needs one extra entry to differentiate full vs empty. */
	entries++;

923 924 925 926
	/*
	 * entries must be multiple of 16 for HW.
	 */
	entries = roundup(entries, 16);
927 928 929 930

	/*
	 * Make actual HW queue 2x to avoid cdix_inc overflows.
	 */
931
	hwentries = min(entries * 2, rhp->rdev.hw_queue.t4_max_iq_size);
932 933 934 935 936 937 938 939 940

	/*
	 * Make HW queue at least 64 entries so GTS updates aren't too
	 * frequent.
	 */
	if (hwentries < 64)
		hwentries = 64;

	memsize = hwentries * sizeof *chp->cq.queue;
941 942 943 944

	/*
	 * memsize must be a multiple of the page size if its a user cq.
	 */
945
	if (ucontext)
946
		memsize = roundup(memsize, PAGE_SIZE);
947
	chp->cq.size = hwentries;
948
	chp->cq.memsize = memsize;
949
	chp->cq.vector = vector;
950 951 952 953

	ret = create_cq(&rhp->rdev, &chp->cq,
			ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
	if (ret)
954
		goto err2;
955 956 957

	chp->rhp = rhp;
	chp->cq.size--;				/* status page */
958
	chp->ibcq.cqe = entries - 2;
959
	spin_lock_init(&chp->lock);
960
	spin_lock_init(&chp->comp_handler_lock);
961 962 963 964
	atomic_set(&chp->refcnt, 1);
	init_waitqueue_head(&chp->wait);
	ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid);
	if (ret)
965
		goto err3;
966 967 968 969

	if (ucontext) {
		mm = kmalloc(sizeof *mm, GFP_KERNEL);
		if (!mm)
970
			goto err4;
971 972
		mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
		if (!mm2)
973
			goto err5;
974 975 976 977 978 979 980 981 982 983 984

		uresp.qid_mask = rhp->rdev.cqmask;
		uresp.cqid = chp->cq.cqid;
		uresp.size = chp->cq.size;
		uresp.memsize = chp->cq.memsize;
		spin_lock(&ucontext->mmap_lock);
		uresp.key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		uresp.gts_key = ucontext->key;
		ucontext->key += PAGE_SIZE;
		spin_unlock(&ucontext->mmap_lock);
985 986
		ret = ib_copy_to_udata(udata, &uresp,
				       sizeof(uresp) - sizeof(uresp.reserved));
987
		if (ret)
988
			goto err6;
989 990 991 992 993 994 995

		mm->key = uresp.key;
		mm->addr = virt_to_phys(chp->cq.queue);
		mm->len = chp->cq.memsize;
		insert_mmap(ucontext, mm);

		mm2->key = uresp.gts_key;
996
		mm2->addr = chp->cq.bar2_pa;
997 998 999
		mm2->len = PAGE_SIZE;
		insert_mmap(ucontext, mm2);
	}
J
Joe Perches 已提交
1000 1001 1002
	pr_debug("%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n",
		 __func__, chp->cq.cqid, chp, chp->cq.size,
		 chp->cq.memsize, (unsigned long long)chp->cq.dma_addr);
1003
	return &chp->ibcq;
1004
err6:
1005
	kfree(mm2);
1006
err5:
1007
	kfree(mm);
1008
err4:
1009
	remove_handle(rhp, &rhp->cqidr, chp->cq.cqid);
1010
err3:
1011
	destroy_cq(&chp->rhp->rdev, &chp->cq,
1012 1013 1014 1015
		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
		   chp->destroy_skb);
err2:
	kfree_skb(chp->destroy_skb);
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
err1:
	kfree(chp);
	return ERR_PTR(ret);
}

int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
{
	return -ENOSYS;
}

int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{
	struct c4iw_cq *chp;
1029
	int ret = 0;
1030 1031 1032 1033
	unsigned long flag;

	chp = to_c4iw_cq(ibcq);
	spin_lock_irqsave(&chp->lock, flag);
1034 1035 1036 1037
	t4_arm_cq(&chp->cq,
		  (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
		ret = t4_cq_notempty(&chp->cq);
1038 1039 1040
	spin_unlock_irqrestore(&chp->lock, flag);
	return ret;
}