nicvf_queues.c 49.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * Copyright (C) 2015 Cavium, Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License
 * as published by the Free Software Foundation.
 */

#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/ip.h>
#include <linux/etherdevice.h>
13
#include <linux/iommu.h>
14 15 16 17 18 19 20 21
#include <net/ip.h>
#include <net/tso.h>

#include "nic_reg.h"
#include "nic.h"
#include "q_struct.h"
#include "nicvf_queues.h"

22 23
static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
					       int size, u64 data);
24 25 26 27 28
static void nicvf_get_page(struct nicvf *nic)
{
	if (!nic->rb_pageref || !nic->rb_page)
		return;

29
	page_ref_add(nic->rb_page, nic->rb_pageref);
30 31 32
	nic->rb_pageref = 0;
}

33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
/* Poll a register for a specific value */
static int nicvf_poll_reg(struct nicvf *nic, int qidx,
			  u64 reg, int bit_pos, int bits, int val)
{
	u64 bit_mask;
	u64 reg_val;
	int timeout = 10;

	bit_mask = (1ULL << bits) - 1;
	bit_mask = (bit_mask << bit_pos);

	while (timeout) {
		reg_val = nicvf_queue_reg_read(nic, reg, qidx);
		if (((reg_val & bit_mask) >> bit_pos) == val)
			return 0;
		usleep_range(1000, 2000);
		timeout--;
	}
	netdev_err(nic->netdev, "Poll on reg 0x%llx failed\n", reg);
	return 1;
}

/* Allocate memory for a queue's descriptors */
static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem,
				  int q_len, int desc_size, int align_bytes)
{
	dmem->q_len = q_len;
	dmem->size = (desc_size * q_len) + align_bytes;
	/* Save address, need it while freeing */
	dmem->unalign_base = dma_zalloc_coherent(&nic->pdev->dev, dmem->size,
						&dmem->dma, GFP_KERNEL);
	if (!dmem->unalign_base)
		return -ENOMEM;

	/* Align memory address for 'align_bytes' */
	dmem->phys_base = NICVF_ALIGNED_ADDR((u64)dmem->dma, align_bytes);
69
	dmem->base = dmem->unalign_base + (dmem->phys_base - dmem->dma);
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
	return 0;
}

/* Free queue's descriptor memory */
static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem)
{
	if (!dmem)
		return;

	dma_free_coherent(&nic->pdev->dev, dmem->size,
			  dmem->unalign_base, dmem->dma);
	dmem->unalign_base = NULL;
	dmem->base = NULL;
}

85 86 87 88 89 90 91
/* Allocate a new page or recycle one if possible
 *
 * We cannot optimize dma mapping here, since
 * 1. It's only one RBDR ring for 8 Rx queues.
 * 2. CQE_RX gives address of the buffer where pkt has been DMA'ed
 *    and not idx into RBDR ring, so can't refer to saved info.
 * 3. There are multiple receive buffers per page
92
 */
93 94
static struct pgcache *nicvf_alloc_page(struct nicvf *nic,
					struct rbdr *rbdr, gfp_t gfp)
95
{
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
	struct page *page = NULL;
	struct pgcache *pgcache, *next;

	/* Check if page is already allocated */
	pgcache = &rbdr->pgcache[rbdr->pgidx];
	page = pgcache->page;
	/* Check if page can be recycled */
	if (page && (page_ref_count(page) != 1))
		page = NULL;

	if (!page) {
		page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, 0);
		if (!page)
			return NULL;

		this_cpu_inc(nic->pnicvf->drv_stats->page_alloc);

		/* Check for space */
		if (rbdr->pgalloc >= rbdr->pgcnt) {
			/* Page can still be used */
			nic->rb_page = page;
			return NULL;
		}

		/* Save the page in page cache */
		pgcache->page = page;
122
		pgcache->dma_addr = 0;
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
		rbdr->pgalloc++;
	}

	/* Take extra page reference for recycling */
	page_ref_add(page, 1);

	rbdr->pgidx++;
	rbdr->pgidx &= (rbdr->pgcnt - 1);

	/* Prefetch refcount of next page in page cache */
	next = &rbdr->pgcache[rbdr->pgidx];
	page = next->page;
	if (page)
		prefetch(&page->_refcount);

	return pgcache;
}

/* Allocate buffer for packet reception */
static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr,
143
					 gfp_t gfp, u32 buf_len, u64 *rbuf)
144 145
{
	struct pgcache *pgcache = NULL;
146

147 148 149
	/* Check if request can be accomodated in previous allocated page.
	 * But in XDP mode only one buffer per page is permitted.
	 */
150
	if (!rbdr->is_xdp && nic->rb_page &&
151
	    ((nic->rb_page_offset + buf_len) <= PAGE_SIZE)) {
152 153
		nic->rb_pageref++;
		goto ret;
154 155
	}

156
	nicvf_get_page(nic);
157
	nic->rb_page = NULL;
158

159 160 161
	/* Get new page, either recycled or new one */
	pgcache = nicvf_alloc_page(nic, rbdr, gfp);
	if (!pgcache && !nic->rb_page) {
162 163
		this_cpu_inc(nic->pnicvf->drv_stats->rcv_buffer_alloc_failures);
		return -ENOMEM;
164
	}
165

166
	nic->rb_page_offset = 0;
167 168 169 170 171

	/* Reserve space for header modifications by BPF program */
	if (rbdr->is_xdp)
		buf_len += XDP_PACKET_HEADROOM;

172 173 174
	/* Check if it's recycled */
	if (pgcache)
		nic->rb_page = pgcache->page;
175
ret:
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
	if (rbdr->is_xdp && pgcache && pgcache->dma_addr) {
		*rbuf = pgcache->dma_addr;
	} else {
		/* HW will ensure data coherency, CPU sync not required */
		*rbuf = (u64)dma_map_page_attrs(&nic->pdev->dev, nic->rb_page,
						nic->rb_page_offset, buf_len,
						DMA_FROM_DEVICE,
						DMA_ATTR_SKIP_CPU_SYNC);
		if (dma_mapping_error(&nic->pdev->dev, (dma_addr_t)*rbuf)) {
			if (!nic->rb_page_offset)
				__free_pages(nic->rb_page, 0);
			nic->rb_page = NULL;
			return -ENOMEM;
		}
		if (pgcache)
191
			pgcache->dma_addr = *rbuf + XDP_PACKET_HEADROOM;
192
		nic->rb_page_offset += buf_len;
193
	}
194 195 196 197

	return 0;
}

198
/* Build skb around receive buffer */
199 200 201
static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic,
					   u64 rb_ptr, int len)
{
202
	void *data;
203 204
	struct sk_buff *skb;

205
	data = phys_to_virt(rb_ptr);
206 207

	/* Now build an skb to give to stack */
208
	skb = build_skb(data, RCV_FRAG_LEN);
209
	if (!skb) {
210
		put_page(virt_to_page(data));
211 212 213
		return NULL;
	}

214
	prefetch(skb->data);
215 216 217 218 219 220 221 222
	return skb;
}

/* Allocate RBDR ring and populate receive buffers */
static int  nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr,
			    int ring_len, int buf_size)
{
	int idx;
223
	u64 rbuf;
224 225 226 227 228 229 230 231 232 233 234 235 236 237
	struct rbdr_entry_t *desc;
	int err;

	err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len,
				     sizeof(struct rbdr_entry_t),
				     NICVF_RCV_BUF_ALIGN_BYTES);
	if (err)
		return err;

	rbdr->desc = rbdr->dmem.base;
	/* Buffer size has to be in multiples of 128 bytes */
	rbdr->dma_size = buf_size;
	rbdr->enable = true;
	rbdr->thresh = RBDR_THRESH;
238 239
	rbdr->head = 0;
	rbdr->tail = 0;
240

241 242 243 244 245 246
	/* Initialize page recycling stuff.
	 *
	 * Can't use single buffer per page especially with 64K pages.
	 * On embedded platforms i.e 81xx/83xx available memory itself
	 * is low and minimum ring size of RBDR is 8K, that takes away
	 * lots of memory.
247 248
	 *
	 * But for XDP it has to be a single buffer per page.
249
	 */
250 251 252 253 254 255 256
	if (!nic->pnicvf->xdp_prog) {
		rbdr->pgcnt = ring_len / (PAGE_SIZE / buf_size);
		rbdr->is_xdp = false;
	} else {
		rbdr->pgcnt = ring_len;
		rbdr->is_xdp = true;
	}
257 258 259 260 261 262 263 264
	rbdr->pgcnt = roundup_pow_of_two(rbdr->pgcnt);
	rbdr->pgcache = kzalloc(sizeof(*rbdr->pgcache) *
				rbdr->pgcnt, GFP_KERNEL);
	if (!rbdr->pgcache)
		return -ENOMEM;
	rbdr->pgidx = 0;
	rbdr->pgalloc = 0;

265 266
	nic->rb_page = NULL;
	for (idx = 0; idx < ring_len; idx++) {
267 268
		err = nicvf_alloc_rcv_buffer(nic, rbdr, GFP_KERNEL,
					     RCV_FRAG_LEN, &rbuf);
269 270 271
		if (err) {
			/* To free already allocated and mapped ones */
			rbdr->tail = idx - 1;
272
			return err;
273
		}
274 275

		desc = GET_RBDR_DESC(rbdr, idx);
276
		desc->buf_addr = rbuf & ~(NICVF_RCV_BUF_ALIGN_BYTES - 1);
277
	}
278 279 280

	nicvf_get_page(nic);

281 282 283 284 285 286 287
	return 0;
}

/* Free RBDR ring and its receive buffers */
static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr)
{
	int head, tail;
288
	u64 buf_addr, phys_addr;
289
	struct pgcache *pgcache;
290 291 292 293 294 295 296 297 298 299 300 301
	struct rbdr_entry_t *desc;

	if (!rbdr)
		return;

	rbdr->enable = false;
	if (!rbdr->dmem.base)
		return;

	head = rbdr->head;
	tail = rbdr->tail;

302
	/* Release page references */
303 304
	while (head != tail) {
		desc = GET_RBDR_DESC(rbdr, head);
305
		buf_addr = desc->buf_addr;
306 307 308 309 310
		phys_addr = nicvf_iova_to_phys(nic, buf_addr);
		dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN,
				     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
		if (phys_addr)
			put_page(virt_to_page(phys_to_virt(phys_addr)));
311 312 313
		head++;
		head &= (rbdr->dmem.q_len - 1);
	}
314
	/* Release buffer of tail desc */
315
	desc = GET_RBDR_DESC(rbdr, tail);
316
	buf_addr = desc->buf_addr;
317 318 319 320 321
	phys_addr = nicvf_iova_to_phys(nic, buf_addr);
	dma_unmap_page_attrs(&nic->pdev->dev, buf_addr, RCV_FRAG_LEN,
			     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
	if (phys_addr)
		put_page(virt_to_page(phys_to_virt(phys_addr)));
322

323 324 325 326 327 328 329 330 331 332 333 334
	/* Sync page cache info */
	smp_rmb();

	/* Release additional page references held for recycling */
	head = 0;
	while (head < rbdr->pgcnt) {
		pgcache = &rbdr->pgcache[head];
		if (pgcache->page && page_ref_count(pgcache->page) != 0)
			put_page(pgcache->page);
		head++;
	}

335 336 337 338 339 340
	/* Free RBDR ring */
	nicvf_free_q_desc_mem(nic, &rbdr->dmem);
}

/* Refill receive buffer descriptors with new buffers.
 */
341
static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp)
342 343 344 345 346 347 348
{
	struct queue_set *qs = nic->qs;
	int rbdr_idx = qs->rbdr_cnt;
	int tail, qcount;
	int refill_rb_cnt;
	struct rbdr *rbdr;
	struct rbdr_entry_t *desc;
349
	u64 rbuf;
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
	int new_rb = 0;

refill:
	if (!rbdr_idx)
		return;
	rbdr_idx--;
	rbdr = &qs->rbdr[rbdr_idx];
	/* Check if it's enabled */
	if (!rbdr->enable)
		goto next_rbdr;

	/* Get no of desc's to be refilled */
	qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx);
	qcount &= 0x7FFFF;
	/* Doorbell can be ringed with a max of ring size minus 1 */
	if (qcount >= (qs->rbdr_len - 1))
		goto next_rbdr;
	else
		refill_rb_cnt = qs->rbdr_len - qcount - 1;

370 371 372
	/* Sync page cache info */
	smp_rmb();

373 374 375 376 377 378
	/* Start filling descs from tail */
	tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3;
	while (refill_rb_cnt) {
		tail++;
		tail &= (rbdr->dmem.q_len - 1);

379
		if (nicvf_alloc_rcv_buffer(nic, rbdr, gfp, RCV_FRAG_LEN, &rbuf))
380 381 382
			break;

		desc = GET_RBDR_DESC(rbdr, tail);
383
		desc->buf_addr = rbuf & ~(NICVF_RCV_BUF_ALIGN_BYTES - 1);
384 385 386 387
		refill_rb_cnt--;
		new_rb++;
	}

388 389
	nicvf_get_page(nic);

390 391 392 393 394 395 396 397 398 399 400 401 402 403
	/* make sure all memory stores are done before ringing doorbell */
	smp_wmb();

	/* Check if buffer allocation failed */
	if (refill_rb_cnt)
		nic->rb_alloc_fail = true;
	else
		nic->rb_alloc_fail = false;

	/* Notify HW */
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
			      rbdr_idx, new_rb);
next_rbdr:
	/* Re-enable RBDR interrupts only if buffer allocation is success */
404 405
	if (!nic->rb_alloc_fail && rbdr->enable &&
	    netif_running(nic->pnicvf->netdev))
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
		nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx);

	if (rbdr_idx)
		goto refill;
}

/* Alloc rcv buffers in non-atomic mode for better success */
void nicvf_rbdr_work(struct work_struct *work)
{
	struct nicvf *nic = container_of(work, struct nicvf, rbdr_work.work);

	nicvf_refill_rbdr(nic, GFP_KERNEL);
	if (nic->rb_alloc_fail)
		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
	else
		nic->rb_work_scheduled = false;
}

/* In Softirq context, alloc rcv buffers in atomic mode */
void nicvf_rbdr_task(unsigned long data)
{
	struct nicvf *nic = (struct nicvf *)data;

	nicvf_refill_rbdr(nic, GFP_ATOMIC);
	if (nic->rb_alloc_fail) {
		nic->rb_work_scheduled = true;
		schedule_delayed_work(&nic->rbdr_work, msecs_to_jiffies(10));
	}
}

/* Initialize completion queue */
static int nicvf_init_cmp_queue(struct nicvf *nic,
				struct cmp_queue *cq, int q_len)
{
	int err;

	err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE,
				     NICVF_CQ_BASE_ALIGN_BYTES);
	if (err)
		return err;

	cq->desc = cq->dmem.base;
448
	cq->thresh = pass1_silicon(nic->pdev) ? 0 : CMP_QUEUE_CQE_THRESH;
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
	nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1;

	return 0;
}

static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq)
{
	if (!cq)
		return;
	if (!cq->dmem.base)
		return;

	nicvf_free_q_desc_mem(nic, &cq->dmem);
}

/* Initialize transmit queue */
static int nicvf_init_snd_queue(struct nicvf *nic,
466
				struct snd_queue *sq, int q_len, int qidx)
467 468 469 470 471 472 473 474 475
{
	int err;

	err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE,
				     NICVF_SQ_BASE_ALIGN_BYTES);
	if (err)
		return err;

	sq->desc = sq->dmem.base;
476
	sq->skbuff = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
477 478
	if (!sq->skbuff)
		return -ENOMEM;
479

480 481 482 483
	sq->head = 0;
	sq->tail = 0;
	sq->thresh = SND_QUEUE_THRESH;

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
	/* Check if this SQ is a XDP TX queue */
	if (nic->sqs_mode)
		qidx += ((nic->sqs_id + 1) * MAX_SND_QUEUES_PER_QS);
	if (qidx < nic->pnicvf->xdp_tx_queues) {
		/* Alloc memory to save page pointers for XDP_TX */
		sq->xdp_page = kcalloc(q_len, sizeof(u64), GFP_KERNEL);
		if (!sq->xdp_page)
			return -ENOMEM;
		sq->xdp_desc_cnt = 0;
		sq->xdp_free_cnt = q_len - 1;
		sq->is_xdp = true;
	} else {
		sq->xdp_page = NULL;
		sq->xdp_desc_cnt = 0;
		sq->xdp_free_cnt = 0;
		sq->is_xdp = false;

		atomic_set(&sq->free_cnt, q_len - 1);

		/* Preallocate memory for TSO segment's header */
		sq->tso_hdrs = dma_alloc_coherent(&nic->pdev->dev,
						  q_len * TSO_HEADER_SIZE,
						  &sq->tso_hdrs_phys,
						  GFP_KERNEL);
		if (!sq->tso_hdrs)
			return -ENOMEM;
	}
511 512 513 514

	return 0;
}

515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq,
			      int hdr_sqe, u8 subdesc_cnt)
{
	u8 idx;
	struct sq_gather_subdesc *gather;

	/* Unmap DMA mapped skb data buffers */
	for (idx = 0; idx < subdesc_cnt; idx++) {
		hdr_sqe++;
		hdr_sqe &= (sq->dmem.q_len - 1);
		gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, hdr_sqe);
		/* HW will ensure data coherency, CPU sync not required */
		dma_unmap_page_attrs(&nic->pdev->dev, gather->addr,
				     gather->size, DMA_TO_DEVICE,
				     DMA_ATTR_SKIP_CPU_SYNC);
	}
}

533 534
static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq)
{
535
	struct sk_buff *skb;
536
	struct page *page;
537 538
	struct sq_hdr_subdesc *hdr;
	struct sq_hdr_subdesc *tso_sqe;
539

540 541 542 543 544 545
	if (!sq)
		return;
	if (!sq->dmem.base)
		return;

	if (sq->tso_hdrs)
546 547
		dma_free_coherent(&nic->pdev->dev,
				  sq->dmem.q_len * TSO_HEADER_SIZE,
548 549
				  sq->tso_hdrs, sq->tso_hdrs_phys);

550 551 552 553
	/* Free pending skbs in the queue */
	smp_rmb();
	while (sq->head != sq->tail) {
		skb = (struct sk_buff *)sq->skbuff[sq->head];
554
		if (!skb || !sq->xdp_page)
555
			goto next;
556 557 558 559 560 561 562

		page = (struct page *)sq->xdp_page[sq->head];
		if (!page)
			goto next;
		else
			put_page(page);

563 564 565 566 567 568 569 570 571 572 573 574
		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);
		/* Check for dummy descriptor used for HW TSO offload on 88xx */
		if (hdr->dont_send) {
			/* Get actual TSO descriptors and unmap them */
			tso_sqe =
			 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
			nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
						 tso_sqe->subdesc_cnt);
		} else {
			nicvf_unmap_sndq_buffers(nic, sq, sq->head,
						 hdr->subdesc_cnt);
		}
575 576
		if (skb)
			dev_kfree_skb_any(skb);
577
next:
578 579 580
		sq->head++;
		sq->head &= (sq->dmem.q_len - 1);
	}
581
	kfree(sq->skbuff);
582
	kfree(sq->xdp_page);
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
	nicvf_free_q_desc_mem(nic, &sq->dmem);
}

static void nicvf_reclaim_snd_queue(struct nicvf *nic,
				    struct queue_set *qs, int qidx)
{
	/* Disable send queue */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0);
	/* Check if SQ is stopped */
	if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01))
		return;
	/* Reset send queue */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);
}

static void nicvf_reclaim_rcv_queue(struct nicvf *nic,
				    struct queue_set *qs, int qidx)
{
	union nic_mbx mbx = {};

	/* Make sure all packets in the pipeline are written back into mem */
	mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC;
	nicvf_send_msg_to_pf(nic, &mbx);
}

static void nicvf_reclaim_cmp_queue(struct nicvf *nic,
				    struct queue_set *qs, int qidx)
{
	/* Disable timer threshold (doesn't get reset upon CQ reset */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0);
	/* Disable completion queue */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0);
	/* Reset completion queue */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);
}

static void nicvf_reclaim_rbdr(struct nicvf *nic,
			       struct rbdr *rbdr, int qidx)
{
	u64 tmp, fifo_state;
	int timeout = 10;

	/* Save head and tail pointers for feeing up buffers */
	rbdr->head = nicvf_queue_reg_read(nic,
					  NIC_QSET_RBDR_0_1_HEAD,
					  qidx) >> 3;
	rbdr->tail = nicvf_queue_reg_read(nic,
					  NIC_QSET_RBDR_0_1_TAIL,
					  qidx) >> 3;

	/* If RBDR FIFO is in 'FAIL' state then do a reset first
	 * before relaiming.
	 */
	fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx);
	if (((fifo_state >> 62) & 0x03) == 0x3)
		nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
				      qidx, NICVF_RBDR_RESET);

	/* Disable RBDR */
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0);
	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
		return;
	while (1) {
		tmp = nicvf_queue_reg_read(nic,
					   NIC_QSET_RBDR_0_1_PREFETCH_STATUS,
					   qidx);
		if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF))
			break;
		usleep_range(1000, 2000);
		timeout--;
		if (!timeout) {
			netdev_err(nic->netdev,
				   "Failed polling on prefetch status\n");
			return;
		}
	}
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
			      qidx, NICVF_RBDR_RESET);

	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02))
		return;
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00);
	if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00))
		return;
}

669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features)
{
	u64 rq_cfg;
	int sqs;

	rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0);

	/* Enable first VLAN stripping */
	if (features & NETIF_F_HW_VLAN_CTAG_RX)
		rq_cfg |= (1ULL << 25);
	else
		rq_cfg &= ~(1ULL << 25);
	nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);

	/* Configure Secondary Qsets, if any */
	for (sqs = 0; sqs < nic->sqs_count; sqs++)
		if (nic->snicvf[sqs])
			nicvf_queue_reg_write(nic->snicvf[sqs],
					      NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
}

690 691 692 693
static void nicvf_reset_rcv_queue_stats(struct nicvf *nic)
{
	union nic_mbx mbx = {};

694
	/* Reset all RQ/SQ and VF stats */
695
	mbx.reset_stat.msg = NIC_MBOX_MSG_RESET_STAT_COUNTER;
696 697
	mbx.reset_stat.rx_stat_mask = 0x3FFF;
	mbx.reset_stat.tx_stat_mask = 0x1F;
698
	mbx.reset_stat.rq_stat_mask = 0xFFFF;
699
	mbx.reset_stat.sq_stat_mask = 0xFFFF;
700 701 702
	nicvf_send_msg_to_pf(nic, &mbx);
}

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
/* Configures receive queue */
static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
				   int qidx, bool enable)
{
	union nic_mbx mbx = {};
	struct rcv_queue *rq;
	struct rq_cfg rq_cfg;

	rq = &qs->rq[qidx];
	rq->enable = enable;

	/* Disable receive queue */
	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0);

	if (!rq->enable) {
		nicvf_reclaim_rcv_queue(nic, qs, qidx);
		return;
	}

	rq->cq_qs = qs->vnic_id;
	rq->cq_idx = qidx;
	rq->start_rbdr_qs = qs->vnic_id;
	rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1;
	rq->cont_rbdr_qs = qs->vnic_id;
	rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1;
	/* all writes of RBDR data to be loaded into L2 Cache as well*/
	rq->caching = 1;

	/* Send a mailbox msg to PF to config RQ */
	mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG;
	mbx.rq.qs_num = qs->vnic_id;
	mbx.rq.rq_num = qidx;
	mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) |
			  (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) |
			  (rq->cont_qs_rbdr_idx << 8) |
			  (rq->start_rbdr_qs << 1) | (rq->start_qs_rbdr_idx);
	nicvf_send_msg_to_pf(nic, &mbx);

	mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG;
742 743 744
	mbx.rq.cfg = BIT_ULL(63) | BIT_ULL(62) |
		     (RQ_PASS_RBDR_LVL << 16) | (RQ_PASS_CQ_LVL << 8) |
		     (qs->vnic_id << 0);
745 746 747 748 749 750
	nicvf_send_msg_to_pf(nic, &mbx);

	/* RQ drop config
	 * Enable CQ drop to reserve sufficient CQEs for all tx packets
	 */
	mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG;
751 752 753
	mbx.rq.cfg = BIT_ULL(63) | BIT_ULL(62) |
		     (RQ_PASS_RBDR_LVL << 40) | (RQ_DROP_RBDR_LVL << 32) |
		     (RQ_PASS_CQ_LVL << 16) | (RQ_DROP_CQ_LVL << 8);
754 755
	nicvf_send_msg_to_pf(nic, &mbx);

756
	if (!nic->sqs_mode && (qidx == 0)) {
757 758 759
		/* Enable checking L3/L4 length and TCP/UDP checksums
		 * Also allow IPv6 pkts with zero UDP checksum.
		 */
760
		nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0,
761
				      (BIT(24) | BIT(23) | BIT(21) | BIT(20)));
762
		nicvf_config_vlan_stripping(nic, nic->netdev->features);
763
	}
764 765

	/* Enable Receive queue */
766
	memset(&rq_cfg, 0, sizeof(struct rq_cfg));
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
	rq_cfg.ena = 1;
	rq_cfg.tcp_ena = 0;
	nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, *(u64 *)&rq_cfg);
}

/* Configures completion queue */
void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs,
			    int qidx, bool enable)
{
	struct cmp_queue *cq;
	struct cq_cfg cq_cfg;

	cq = &qs->cq[qidx];
	cq->enable = enable;

	if (!cq->enable) {
		nicvf_reclaim_cmp_queue(nic, qs, qidx);
		return;
	}

	/* Reset completion queue */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET);

	if (!cq->enable)
		return;

	spin_lock_init(&cq->lock);
	/* Set completion queue base address */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE,
			      qidx, (u64)(cq->dmem.phys_base));

	/* Enable Completion queue */
799
	memset(&cq_cfg, 0, sizeof(struct cq_cfg));
800 801 802
	cq_cfg.ena = 1;
	cq_cfg.reset = 0;
	cq_cfg.caching = 0;
803
	cq_cfg.qsize = ilog2(qs->cq_len >> 10);
804 805 806 807 808 809
	cq_cfg.avg_con = 0;
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(u64 *)&cq_cfg);

	/* Set threshold value for interrupt generation */
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh);
	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2,
810
			      qidx, CMP_QUEUE_TIMER_THRESH);
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
}

/* Configures transmit queue */
static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs,
				   int qidx, bool enable)
{
	union nic_mbx mbx = {};
	struct snd_queue *sq;
	struct sq_cfg sq_cfg;

	sq = &qs->sq[qidx];
	sq->enable = enable;

	if (!sq->enable) {
		nicvf_reclaim_snd_queue(nic, qs, qidx);
		return;
	}

	/* Reset send queue */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET);

	sq->cq_qs = qs->vnic_id;
	sq->cq_idx = qidx;

	/* Send a mailbox msg to PF to config SQ */
	mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG;
	mbx.sq.qs_num = qs->vnic_id;
	mbx.sq.sq_num = qidx;
839
	mbx.sq.sqs_mode = nic->sqs_mode;
840 841 842 843 844 845 846 847
	mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx;
	nicvf_send_msg_to_pf(nic, &mbx);

	/* Set queue base address */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE,
			      qidx, (u64)(sq->dmem.phys_base));

	/* Enable send queue  & set queue size */
848
	memset(&sq_cfg, 0, sizeof(struct sq_cfg));
849 850 851
	sq_cfg.ena = 1;
	sq_cfg.reset = 0;
	sq_cfg.ldwb = 0;
852
	sq_cfg.qsize = ilog2(qs->sq_len >> 10);
853
	sq_cfg.tstmp_bgx_intf = 0;
854 855 856 857
	/* CQ's level at which HW will stop processing SQEs to avoid
	 * transmitting a pkt with no space in CQ to post CQE_TX.
	 */
	sq_cfg.cq_limit = (CMP_QUEUE_PIPELINE_RSVD * 256) / qs->cq_len;
858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(u64 *)&sq_cfg);

	/* Set threshold value for interrupt generation */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh);

	/* Set queue:cpu affinity for better load distribution */
	if (cpu_online(qidx)) {
		cpumask_set_cpu(qidx, &sq->affinity_mask);
		netif_set_xps_queue(nic->netdev,
				    &sq->affinity_mask, qidx);
	}
}

/* Configures receive buffer descriptor ring */
static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs,
			      int qidx, bool enable)
{
	struct rbdr *rbdr;
	struct rbdr_cfg rbdr_cfg;

	rbdr = &qs->rbdr[qidx];
	nicvf_reclaim_rbdr(nic, rbdr, qidx);
	if (!enable)
		return;

	/* Set descriptor base address */
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE,
			      qidx, (u64)(rbdr->dmem.phys_base));

	/* Enable RBDR  & set queue size */
	/* Buffer size should be in multiples of 128 bytes */
889
	memset(&rbdr_cfg, 0, sizeof(struct rbdr_cfg));
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
	rbdr_cfg.ena = 1;
	rbdr_cfg.reset = 0;
	rbdr_cfg.ldwb = 0;
	rbdr_cfg.qsize = RBDR_SIZE;
	rbdr_cfg.avg_con = 0;
	rbdr_cfg.lines = rbdr->dma_size / 128;
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG,
			      qidx, *(u64 *)&rbdr_cfg);

	/* Notify HW */
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR,
			      qidx, qs->rbdr_len - 1);

	/* Set threshold value for interrupt generation */
	nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH,
			      qidx, rbdr->thresh - 1);
}

/* Requests PF to assign and enable Qset */
void nicvf_qset_config(struct nicvf *nic, bool enable)
{
	union nic_mbx mbx = {};
	struct queue_set *qs = nic->qs;
	struct qs_cfg *qs_cfg;

	if (!qs) {
		netdev_warn(nic->netdev,
			    "Qset is still not allocated, don't init queues\n");
		return;
	}

	qs->enable = enable;
	qs->vnic_id = nic->vf_id;

	/* Send a mailbox msg to PF to config Qset */
	mbx.qs.msg = NIC_MBOX_MSG_QS_CFG;
	mbx.qs.num = qs->vnic_id;
927
	mbx.qs.sqs_count = nic->sqs_count;
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972

	mbx.qs.cfg = 0;
	qs_cfg = (struct qs_cfg *)&mbx.qs.cfg;
	if (qs->enable) {
		qs_cfg->ena = 1;
#ifdef __BIG_ENDIAN
		qs_cfg->be = 1;
#endif
		qs_cfg->vnic = qs->vnic_id;
	}
	nicvf_send_msg_to_pf(nic, &mbx);
}

static void nicvf_free_resources(struct nicvf *nic)
{
	int qidx;
	struct queue_set *qs = nic->qs;

	/* Free receive buffer descriptor ring */
	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
		nicvf_free_rbdr(nic, &qs->rbdr[qidx]);

	/* Free completion queue */
	for (qidx = 0; qidx < qs->cq_cnt; qidx++)
		nicvf_free_cmp_queue(nic, &qs->cq[qidx]);

	/* Free send queue */
	for (qidx = 0; qidx < qs->sq_cnt; qidx++)
		nicvf_free_snd_queue(nic, &qs->sq[qidx]);
}

static int nicvf_alloc_resources(struct nicvf *nic)
{
	int qidx;
	struct queue_set *qs = nic->qs;

	/* Alloc receive buffer descriptor ring */
	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
		if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len,
				    DMA_BUFFER_LEN))
			goto alloc_fail;
	}

	/* Alloc send queue */
	for (qidx = 0; qidx < qs->sq_cnt; qidx++) {
973
		if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx))
974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
			goto alloc_fail;
	}

	/* Alloc completion queue */
	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
		if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len))
			goto alloc_fail;
	}

	return 0;
alloc_fail:
	nicvf_free_resources(nic);
	return -ENOMEM;
}

int nicvf_set_qset_resources(struct nicvf *nic)
{
	struct queue_set *qs;

	qs = devm_kzalloc(&nic->pdev->dev, sizeof(*qs), GFP_KERNEL);
	if (!qs)
		return -ENOMEM;
	nic->qs = qs;

	/* Set count of each queue */
999 1000 1001 1002
	qs->rbdr_cnt = DEFAULT_RBDR_CNT;
	qs->rq_cnt = min_t(u8, MAX_RCV_QUEUES_PER_QS, num_online_cpus());
	qs->sq_cnt = min_t(u8, MAX_SND_QUEUES_PER_QS, num_online_cpus());
	qs->cq_cnt = max_t(u8, qs->rq_cnt, qs->sq_cnt);
1003 1004 1005 1006 1007

	/* Set queue lengths */
	qs->rbdr_len = RCV_BUF_COUNT;
	qs->sq_len = SND_QUEUE_LEN;
	qs->cq_len = CMP_QUEUE_LEN;
1008 1009 1010

	nic->rx_queues = qs->rq_cnt;
	nic->tx_queues = qs->sq_cnt;
1011
	nic->xdp_tx_queues = 0;
1012

1013 1014 1015 1016 1017 1018 1019
	return 0;
}

int nicvf_config_data_transfer(struct nicvf *nic, bool enable)
{
	bool disable = false;
	struct queue_set *qs = nic->qs;
1020
	struct queue_set *pqs = nic->pnicvf->qs;
1021 1022 1023 1024 1025
	int qidx;

	if (!qs)
		return 0;

1026 1027 1028 1029 1030 1031 1032 1033 1034
	/* Take primary VF's queue lengths.
	 * This is needed to take queue lengths set from ethtool
	 * into consideration.
	 */
	if (nic->sqs_mode && pqs) {
		qs->cq_len = pqs->cq_len;
		qs->sq_len = pqs->sq_len;
	}

1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	if (enable) {
		if (nicvf_alloc_resources(nic))
			return -ENOMEM;

		for (qidx = 0; qidx < qs->sq_cnt; qidx++)
			nicvf_snd_queue_config(nic, qs, qidx, enable);
		for (qidx = 0; qidx < qs->cq_cnt; qidx++)
			nicvf_cmp_queue_config(nic, qs, qidx, enable);
		for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
			nicvf_rbdr_config(nic, qs, qidx, enable);
		for (qidx = 0; qidx < qs->rq_cnt; qidx++)
			nicvf_rcv_queue_config(nic, qs, qidx, enable);
	} else {
		for (qidx = 0; qidx < qs->rq_cnt; qidx++)
			nicvf_rcv_queue_config(nic, qs, qidx, disable);
		for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
			nicvf_rbdr_config(nic, qs, qidx, disable);
		for (qidx = 0; qidx < qs->sq_cnt; qidx++)
			nicvf_snd_queue_config(nic, qs, qidx, disable);
		for (qidx = 0; qidx < qs->cq_cnt; qidx++)
			nicvf_cmp_queue_config(nic, qs, qidx, disable);

		nicvf_free_resources(nic);
	}

1060 1061 1062 1063 1064
	/* Reset RXQ's stats.
	 * SQ's stats will get reset automatically once SQ is reset.
	 */
	nicvf_reset_rcv_queue_stats(nic);

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
	return 0;
}

/* Get a free desc from SQ
 * returns descriptor ponter & descriptor number
 */
static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt)
{
	int qentry;

	qentry = sq->tail;
1076 1077 1078 1079
	if (!sq->is_xdp)
		atomic_sub(desc_cnt, &sq->free_cnt);
	else
		sq->xdp_free_cnt -= desc_cnt;
1080 1081 1082 1083 1084 1085
	sq->tail += desc_cnt;
	sq->tail &= (sq->dmem.q_len - 1);

	return qentry;
}

1086 1087 1088 1089 1090 1091 1092 1093
/* Rollback to previous tail pointer when descriptors not used */
static inline void nicvf_rollback_sq_desc(struct snd_queue *sq,
					  int qentry, int desc_cnt)
{
	sq->tail = qentry;
	atomic_add(desc_cnt, &sq->free_cnt);
}

1094 1095 1096
/* Free descriptor back to SQ for future use */
void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt)
{
1097 1098 1099 1100
	if (!sq->is_xdp)
		atomic_add(desc_cnt, &sq->free_cnt);
	else
		sq->xdp_free_cnt += desc_cnt;
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
	sq->head += desc_cnt;
	sq->head &= (sq->dmem.q_len - 1);
}

static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry)
{
	qentry++;
	qentry &= (sq->dmem.q_len - 1);
	return qentry;
}

void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx)
{
	u64 sq_cfg;

	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
	sq_cfg |= NICVF_SQ_EN;
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
	/* Ring doorbell so that H/W restarts processing SQEs */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0);
}

void nicvf_sq_disable(struct nicvf *nic, int qidx)
{
	u64 sq_cfg;

	sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx);
	sq_cfg &= ~NICVF_SQ_EN;
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg);
}

void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq,
			      int qidx)
{
	u64 head, tail;
	struct sk_buff *skb;
	struct nicvf *nic = netdev_priv(netdev);
	struct sq_hdr_subdesc *hdr;

	head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4;
	tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4;
	while (sq->head != head) {
		hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head);
		if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) {
			nicvf_put_sq_desc(sq, 1);
			continue;
		}
		skb = (struct sk_buff *)sq->skbuff[sq->head];
1149 1150
		if (skb)
			dev_kfree_skb_any(skb);
1151 1152 1153 1154 1155 1156 1157
		atomic64_add(1, (atomic64_t *)&netdev->stats.tx_packets);
		atomic64_add(hdr->tot_len,
			     (atomic64_t *)&netdev->stats.tx_bytes);
		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
	}
}

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
/* XDP Transmit APIs */
void nicvf_xdp_sq_doorbell(struct nicvf *nic,
			   struct snd_queue *sq, int sq_num)
{
	if (!sq->xdp_desc_cnt)
		return;

	/* make sure all memory stores are done before ringing doorbell */
	wmb();

	/* Inform HW to xmit all TSO segments */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
			      sq_num, sq->xdp_desc_cnt);
	sq->xdp_desc_cnt = 0;
}

static inline void
nicvf_xdp_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry,
			     int subdesc_cnt, u64 data, int len)
{
	struct sq_hdr_subdesc *hdr;

	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
	hdr->subdesc_cnt = subdesc_cnt;
	hdr->tot_len = len;
	hdr->post_cqe = 1;
	sq->xdp_page[qentry] = (u64)virt_to_page((void *)data);
}

int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq,
			    u64 bufaddr, u64 dma_addr, u16 len)
{
	int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;
	int qentry;

	if (subdesc_cnt > sq->xdp_free_cnt)
		return 0;

	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);

	nicvf_xdp_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, bufaddr, len);

	qentry = nicvf_get_nxt_sqentry(sq, qentry);
	nicvf_sq_add_gather_subdesc(sq, qentry, len, dma_addr);

	sq->xdp_desc_cnt += subdesc_cnt;

	return 1;
}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
/* Calculate no of SQ subdescriptors needed to transmit all
 * segments of this TSO packet.
 * Taken from 'Tilera network driver' with a minor modification.
 */
static int nicvf_tso_count_subdescs(struct sk_buff *skb)
{
	struct skb_shared_info *sh = skb_shinfo(skb);
	unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
	unsigned int data_len = skb->len - sh_len;
	unsigned int p_len = sh->gso_size;
	long f_id = -1;    /* id of the current fragment */
	long f_size = skb_headlen(skb) - sh_len;  /* current fragment size */
	long f_used = 0;  /* bytes used from the current fragment */
	long n;            /* size of the current piece of payload */
	int num_edescs = 0;
	int segment;

	for (segment = 0; segment < sh->gso_segs; segment++) {
		unsigned int p_used = 0;

		/* One edesc for header and for each piece of the payload. */
		for (num_edescs++; p_used < p_len; num_edescs++) {
			/* Advance as needed. */
			while (f_used >= f_size) {
				f_id++;
				f_size = skb_frag_size(&sh->frags[f_id]);
				f_used = 0;
			}

			/* Use bytes from the current fragment. */
			n = p_len - p_used;
			if (n > f_size - f_used)
				n = f_size - f_used;
			f_used += n;
			p_used += n;
		}

		/* The last segment may be less than gso_size. */
		data_len -= p_len;
		if (data_len < p_len)
			p_len = data_len;
	}

	/* '+ gso_segs' for SQ_HDR_SUDESCs for each segment */
	return num_edescs + sh->gso_segs;
}

1257 1258
#define POST_CQE_DESC_COUNT 2

1259 1260 1261 1262 1263
/* Get the number of SQ descriptors needed to xmit this skb */
static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb)
{
	int subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT;

1264
	if (skb_shinfo(skb)->gso_size && !nic->hw_tso) {
1265 1266 1267 1268
		subdesc_cnt = nicvf_tso_count_subdescs(skb);
		return subdesc_cnt;
	}

1269 1270 1271 1272
	/* Dummy descriptors to get TSO pkt completion notification */
	if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size)
		subdesc_cnt += POST_CQE_DESC_COUNT;

1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
	if (skb_shinfo(skb)->nr_frags)
		subdesc_cnt += skb_shinfo(skb)->nr_frags;

	return subdesc_cnt;
}

/* Add SQ HEADER subdescriptor.
 * First subdescriptor for every send descriptor.
 */
static inline void
1283
nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry,
1284 1285 1286 1287
			 int subdesc_cnt, struct sk_buff *skb, int len)
{
	int proto;
	struct sq_hdr_subdesc *hdr;
1288 1289 1290 1291 1292
	union {
		struct iphdr *v4;
		struct ipv6hdr *v6;
		unsigned char *hdr;
	} ip;
1293

1294
	ip.hdr = skb_network_header(skb);
1295 1296 1297
	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310

	if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) {
		/* post_cqe = 0, to avoid HW posting a CQE for every TSO
		 * segment transmitted on 88xx.
		 */
		hdr->subdesc_cnt = subdesc_cnt - POST_CQE_DESC_COUNT;
	} else {
		sq->skbuff[qentry] = (u64)skb;
		/* Enable notification via CQE after processing SQE */
		hdr->post_cqe = 1;
		/* No of subdescriptors following this */
		hdr->subdesc_cnt = subdesc_cnt;
	}
1311 1312 1313 1314 1315 1316 1317 1318
	hdr->tot_len = len;

	/* Offload checksum calculation to HW */
	if (skb->ip_summed == CHECKSUM_PARTIAL) {
		hdr->csum_l3 = 1; /* Enable IP csum calculation */
		hdr->l3_offset = skb_network_offset(skb);
		hdr->l4_offset = skb_transport_offset(skb);

1319 1320 1321
		proto = (ip.v4->version == 4) ? ip.v4->protocol :
			ip.v6->nexthdr;

1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
		switch (proto) {
		case IPPROTO_TCP:
			hdr->csum_l4 = SEND_L4_CSUM_TCP;
			break;
		case IPPROTO_UDP:
			hdr->csum_l4 = SEND_L4_CSUM_UDP;
			break;
		case IPPROTO_SCTP:
			hdr->csum_l4 = SEND_L4_CSUM_SCTP;
			break;
		}
	}
1334 1335 1336 1337 1338 1339 1340

	if (nic->hw_tso && skb_shinfo(skb)->gso_size) {
		hdr->tso = 1;
		hdr->tso_start = skb_transport_offset(skb) + tcp_hdrlen(skb);
		hdr->tso_max_paysize = skb_shinfo(skb)->gso_size;
		/* For non-tunneled pkts, point this to L2 ethertype */
		hdr->inner_l3_offset = skb_network_offset(skb) - 2;
1341
		this_cpu_inc(nic->pnicvf->drv_stats->tx_tso);
1342
	}
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
}

/* SQ GATHER subdescriptor
 * Must follow HDR descriptor
 */
static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
					       int size, u64 data)
{
	struct sq_gather_subdesc *gather;

	qentry &= (sq->dmem.q_len - 1);
	gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry);

	memset(gather, 0, SND_QUEUE_DESC_SIZE);
	gather->subdesc_type = SQ_DESC_TYPE_GATHER;
1358
	gather->ld_type = NIC_SEND_LD_TYPE_E_LDD;
1359 1360 1361 1362
	gather->size = size;
	gather->addr = data;
}

1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
/* Add HDR + IMMEDIATE subdescriptors right after descriptors of a TSO
 * packet so that a CQE is posted as a notifation for transmission of
 * TSO packet.
 */
static inline void nicvf_sq_add_cqe_subdesc(struct snd_queue *sq, int qentry,
					    int tso_sqe, struct sk_buff *skb)
{
	struct sq_imm_subdesc *imm;
	struct sq_hdr_subdesc *hdr;

	sq->skbuff[qentry] = (u64)skb;

	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
	/* Enable notification via CQE after processing SQE */
	hdr->post_cqe = 1;
	/* There is no packet to transmit here */
	hdr->dont_send = 1;
	hdr->subdesc_cnt = POST_CQE_DESC_COUNT - 1;
	hdr->tot_len = 1;
	/* Actual TSO header SQE index, needed for cleanup */
	hdr->rsvd2 = tso_sqe;

	qentry = nicvf_get_nxt_sqentry(sq, qentry);
	imm = (struct sq_imm_subdesc *)GET_SQ_DESC(sq, qentry);
	memset(imm, 0, SND_QUEUE_DESC_SIZE);
	imm->subdesc_type = SQ_DESC_TYPE_IMMEDIATE;
	imm->len = 1;
}

1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
static inline void nicvf_sq_doorbell(struct nicvf *nic, struct sk_buff *skb,
				     int sq_num, int desc_cnt)
{
	struct netdev_queue *txq;

	txq = netdev_get_tx_queue(nic->pnicvf->netdev,
				  skb_get_queue_mapping(skb));

	netdev_tx_sent_queue(txq, skb->len);

	/* make sure all memory stores are done before ringing doorbell */
	smp_wmb();

	/* Inform HW to xmit all TSO segments */
	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
			      sq_num, desc_cnt);
}

1412 1413 1414 1415
/* Segment a TSO packet into 'gso_size' segments and append
 * them to SQ for transfer
 */
static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq,
1416
			       int sq_num, int qentry, struct sk_buff *skb)
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
{
	struct tso_t tso;
	int seg_subdescs = 0, desc_cnt = 0;
	int seg_len, total_len, data_left;
	int hdr_qentry = qentry;
	int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);

	tso_start(skb, &tso);
	total_len = skb->len - hdr_len;
	while (total_len > 0) {
		char *hdr;

		/* Save Qentry for adding HDR_SUBDESC at the end */
		hdr_qentry = qentry;

		data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len);
		total_len -= data_left;

		/* Add segment's header */
		qentry = nicvf_get_nxt_sqentry(sq, qentry);
		hdr = sq->tso_hdrs + qentry * TSO_HEADER_SIZE;
		tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0);
		nicvf_sq_add_gather_subdesc(sq, qentry, hdr_len,
					    sq->tso_hdrs_phys +
					    qentry * TSO_HEADER_SIZE);
		/* HDR_SUDESC + GATHER */
		seg_subdescs = 2;
		seg_len = hdr_len;

		/* Add segment's payload fragments */
		while (data_left > 0) {
			int size;

			size = min_t(int, tso.size, data_left);

			qentry = nicvf_get_nxt_sqentry(sq, qentry);
			nicvf_sq_add_gather_subdesc(sq, qentry, size,
						    virt_to_phys(tso.data));
			seg_subdescs++;
			seg_len += size;

			data_left -= size;
			tso_build_data(skb, &tso, size);
		}
1461
		nicvf_sq_add_hdr_subdesc(nic, sq, hdr_qentry,
1462
					 seg_subdescs - 1, skb, seg_len);
1463
		sq->skbuff[hdr_qentry] = (u64)NULL;
1464 1465 1466 1467 1468 1469 1470
		qentry = nicvf_get_nxt_sqentry(sq, qentry);

		desc_cnt += seg_subdescs;
	}
	/* Save SKB in the last segment for freeing */
	sq->skbuff[hdr_qentry] = (u64)skb;

1471
	nicvf_sq_doorbell(nic, skb, sq_num, desc_cnt);
1472

1473
	this_cpu_inc(nic->pnicvf->drv_stats->tx_tso);
1474 1475 1476 1477
	return 1;
}

/* Append an skb to a SQ for packet transfer. */
1478 1479
int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq,
			struct sk_buff *skb, u8 sq_num)
1480 1481
{
	int i, size;
1482
	int subdesc_cnt, hdr_sqe = 0;
1483
	int qentry;
1484
	u64 dma_addr;
1485 1486 1487 1488 1489 1490 1491 1492

	subdesc_cnt = nicvf_sq_subdesc_required(nic, skb);
	if (subdesc_cnt > atomic_read(&sq->free_cnt))
		goto append_fail;

	qentry = nicvf_get_sq_desc(sq, subdesc_cnt);

	/* Check if its a TSO packet */
1493
	if (skb_shinfo(skb)->gso_size && !nic->hw_tso)
1494
		return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb);
1495 1496

	/* Add SQ header subdesc */
1497 1498
	nicvf_sq_add_hdr_subdesc(nic, sq, qentry, subdesc_cnt - 1,
				 skb, skb->len);
1499
	hdr_sqe = qentry;
1500 1501 1502 1503

	/* Add SQ gather subdescs */
	qentry = nicvf_get_nxt_sqentry(sq, qentry);
	size = skb_is_nonlinear(skb) ? skb_headlen(skb) : skb->len;
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
	/* HW will ensure data coherency, CPU sync not required */
	dma_addr = dma_map_page_attrs(&nic->pdev->dev, virt_to_page(skb->data),
				      offset_in_page(skb->data), size,
				      DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
	if (dma_mapping_error(&nic->pdev->dev, dma_addr)) {
		nicvf_rollback_sq_desc(sq, qentry, subdesc_cnt);
		return 0;
	}

	nicvf_sq_add_gather_subdesc(sq, qentry, size, dma_addr);
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525

	/* Check for scattered buffer */
	if (!skb_is_nonlinear(skb))
		goto doorbell;

	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		const struct skb_frag_struct *frag;

		frag = &skb_shinfo(skb)->frags[i];

		qentry = nicvf_get_nxt_sqentry(sq, qentry);
		size = skb_frag_size(frag);
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
		dma_addr = dma_map_page_attrs(&nic->pdev->dev,
					      skb_frag_page(frag),
					      frag->page_offset, size,
					      DMA_TO_DEVICE,
					      DMA_ATTR_SKIP_CPU_SYNC);
		if (dma_mapping_error(&nic->pdev->dev, dma_addr)) {
			/* Free entire chain of mapped buffers
			 * here 'i' = frags mapped + above mapped skb->data
			 */
			nicvf_unmap_sndq_buffers(nic, sq, hdr_sqe, i);
			nicvf_rollback_sq_desc(sq, qentry, subdesc_cnt);
			return 0;
		}
		nicvf_sq_add_gather_subdesc(sq, qentry, size, dma_addr);
1540 1541 1542
	}

doorbell:
1543 1544
	if (nic->t88 && skb_shinfo(skb)->gso_size) {
		qentry = nicvf_get_nxt_sqentry(sq, qentry);
1545
		nicvf_sq_add_cqe_subdesc(sq, qentry, hdr_sqe, skb);
1546 1547
	}

1548
	nicvf_sq_doorbell(nic, skb, sq_num, subdesc_cnt);
1549 1550 1551 1552

	return 1;

append_fail:
1553 1554
	/* Use original PCI dev for debug log */
	nic = nic->pnicvf;
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
	netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n");
	return 0;
}

static inline unsigned frag_num(unsigned i)
{
#ifdef __BIG_ENDIAN
	return (i & ~3) + 3 - (i & 3);
#else
	return i;
#endif
}

1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
static void nicvf_unmap_rcv_buffer(struct nicvf *nic, u64 dma_addr,
				   u64 buf_addr, bool xdp)
{
	struct page *page = NULL;
	int len = RCV_FRAG_LEN;

	if (xdp) {
		page = virt_to_page(phys_to_virt(buf_addr));
		/* Check if it's a recycled page, if not
		 * unmap the DMA mapping.
		 *
		 * Recycled page holds an extra reference.
		 */
		if (page_ref_count(page) != 1)
			return;
1583 1584

		len += XDP_PACKET_HEADROOM;
1585 1586 1587 1588 1589 1590 1591
		/* Receive buffers in XDP mode are mapped from page start */
		dma_addr &= PAGE_MASK;
	}
	dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, len,
			     DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
}

1592
/* Returns SKB for a received packet */
1593 1594
struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic,
				  struct cqe_rx_t *cqe_rx, bool xdp)
1595 1596 1597 1598
{
	int frag;
	int payload_len = 0;
	struct sk_buff *skb = NULL;
1599 1600
	struct page *page;
	int offset;
1601 1602
	u16 *rb_lens = NULL;
	u64 *rb_ptrs = NULL;
1603
	u64 phys_addr;
1604 1605

	rb_lens = (void *)cqe_rx + (3 * sizeof(u64));
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
	/* Except 88xx pass1 on all other chips CQE_RX2_S is added to
	 * CQE_RX at word6, hence buffer pointers move by word
	 *
	 * Use existing 'hw_tso' flag which will be set for all chips
	 * except 88xx pass1 instead of a additional cache line
	 * access (or miss) by using pci dev's revision.
	 */
	if (!nic->hw_tso)
		rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64));
	else
		rb_ptrs = (void *)cqe_rx + (7 * sizeof(u64));
1617 1618 1619

	for (frag = 0; frag < cqe_rx->rb_cnt; frag++) {
		payload_len = rb_lens[frag_num(frag)];
1620 1621 1622 1623 1624 1625 1626
		phys_addr = nicvf_iova_to_phys(nic, *rb_ptrs);
		if (!phys_addr) {
			if (skb)
				dev_kfree_skb_any(skb);
			return NULL;
		}

1627 1628
		if (!frag) {
			/* First fragment */
1629 1630 1631
			nicvf_unmap_rcv_buffer(nic,
					       *rb_ptrs - cqe_rx->align_pad,
					       phys_addr, xdp);
1632
			skb = nicvf_rb_ptr_to_skb(nic,
1633
						  phys_addr - cqe_rx->align_pad,
1634 1635 1636 1637 1638 1639 1640
						  payload_len);
			if (!skb)
				return NULL;
			skb_reserve(skb, cqe_rx->align_pad);
			skb_put(skb, payload_len);
		} else {
			/* Add fragments */
1641
			nicvf_unmap_rcv_buffer(nic, *rb_ptrs, phys_addr, xdp);
1642 1643
			page = virt_to_page(phys_to_virt(phys_addr));
			offset = phys_to_virt(phys_addr) - page_address(page);
1644 1645
			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
					offset, payload_len, RCV_FRAG_LEN);
1646 1647 1648 1649 1650 1651 1652
		}
		/* Next buffer pointer */
		rb_ptrs++;
	}
	return skb;
}

1653
static u64 nicvf_int_type_to_mask(int int_type, int q_idx)
1654 1655 1656 1657 1658
{
	u64 reg_val;

	switch (int_type) {
	case NICVF_INTR_CQ:
1659
		reg_val = ((1ULL << q_idx) << NICVF_INTR_CQ_SHIFT);
1660 1661
		break;
	case NICVF_INTR_SQ:
1662
		reg_val = ((1ULL << q_idx) << NICVF_INTR_SQ_SHIFT);
1663 1664
		break;
	case NICVF_INTR_RBDR:
1665
		reg_val = ((1ULL << q_idx) << NICVF_INTR_RBDR_SHIFT);
1666 1667
		break;
	case NICVF_INTR_PKT_DROP:
1668
		reg_val = (1ULL << NICVF_INTR_PKT_DROP_SHIFT);
1669 1670
		break;
	case NICVF_INTR_TCP_TIMER:
1671
		reg_val = (1ULL << NICVF_INTR_TCP_TIMER_SHIFT);
1672 1673
		break;
	case NICVF_INTR_MBOX:
1674
		reg_val = (1ULL << NICVF_INTR_MBOX_SHIFT);
1675 1676
		break;
	case NICVF_INTR_QS_ERR:
1677
		reg_val = (1ULL << NICVF_INTR_QS_ERR_SHIFT);
1678 1679
		break;
	default:
1680
		reg_val = 0;
1681 1682
	}

1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697
	return reg_val;
}

/* Enable interrupt */
void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx)
{
	u64 mask = nicvf_int_type_to_mask(int_type, q_idx);

	if (!mask) {
		netdev_dbg(nic->netdev,
			   "Failed to enable interrupt: unknown type\n");
		return;
	}
	nicvf_reg_write(nic, NIC_VF_ENA_W1S,
			nicvf_reg_read(nic, NIC_VF_ENA_W1S) | mask);
1698 1699 1700 1701 1702
}

/* Disable interrupt */
void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx)
{
1703
	u64 mask = nicvf_int_type_to_mask(int_type, q_idx);
1704

1705 1706
	if (!mask) {
		netdev_dbg(nic->netdev,
1707
			   "Failed to disable interrupt: unknown type\n");
1708
		return;
1709 1710
	}

1711
	nicvf_reg_write(nic, NIC_VF_ENA_W1C, mask);
1712 1713 1714 1715 1716
}

/* Clear interrupt */
void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx)
{
1717
	u64 mask = nicvf_int_type_to_mask(int_type, q_idx);
1718

1719 1720
	if (!mask) {
		netdev_dbg(nic->netdev,
1721
			   "Failed to clear interrupt: unknown type\n");
1722
		return;
1723 1724
	}

1725
	nicvf_reg_write(nic, NIC_VF_INT, mask);
1726 1727 1728 1729 1730
}

/* Check if interrupt is enabled */
int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx)
{
1731 1732 1733 1734
	u64 mask = nicvf_int_type_to_mask(int_type, q_idx);
	/* If interrupt type is unknown, we treat it disabled. */
	if (!mask) {
		netdev_dbg(nic->netdev,
1735
			   "Failed to check interrupt enable: unknown type\n");
1736
		return 0;
1737 1738
	}

1739
	return mask & nicvf_reg_read(nic, NIC_VF_ENA_W1S);
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768
}

void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx)
{
	struct rcv_queue *rq;

#define GET_RQ_STATS(reg) \
	nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\
			    (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))

	rq = &nic->qs->rq[rq_idx];
	rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS);
	rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS);
}

void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx)
{
	struct snd_queue *sq;

#define GET_SQ_STATS(reg) \
	nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\
			    (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3))

	sq = &nic->qs->sq[sq_idx];
	sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS);
	sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS);
}

/* Check for errors in the receive cmp.queue entry */
1769
int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
1770 1771 1772 1773 1774 1775 1776 1777 1778
{
	if (netif_msg_rx_err(nic))
		netdev_err(nic->netdev,
			   "%s: RX error CQE err_level 0x%x err_opcode 0x%x\n",
			   nic->netdev->name,
			   cqe_rx->err_level, cqe_rx->err_opcode);

	switch (cqe_rx->err_opcode) {
	case CQ_RX_ERROP_RE_PARTIAL:
1779
		this_cpu_inc(nic->drv_stats->rx_bgx_truncated_pkts);
1780 1781
		break;
	case CQ_RX_ERROP_RE_JABBER:
1782
		this_cpu_inc(nic->drv_stats->rx_jabber_errs);
1783 1784
		break;
	case CQ_RX_ERROP_RE_FCS:
1785
		this_cpu_inc(nic->drv_stats->rx_fcs_errs);
1786 1787
		break;
	case CQ_RX_ERROP_RE_RX_CTL:
1788
		this_cpu_inc(nic->drv_stats->rx_bgx_errs);
1789 1790
		break;
	case CQ_RX_ERROP_PREL2_ERR:
1791
		this_cpu_inc(nic->drv_stats->rx_prel2_errs);
1792 1793
		break;
	case CQ_RX_ERROP_L2_MAL:
1794
		this_cpu_inc(nic->drv_stats->rx_l2_hdr_malformed);
1795 1796
		break;
	case CQ_RX_ERROP_L2_OVERSIZE:
1797
		this_cpu_inc(nic->drv_stats->rx_oversize);
1798 1799
		break;
	case CQ_RX_ERROP_L2_UNDERSIZE:
1800
		this_cpu_inc(nic->drv_stats->rx_undersize);
1801 1802
		break;
	case CQ_RX_ERROP_L2_LENMISM:
1803
		this_cpu_inc(nic->drv_stats->rx_l2_len_mismatch);
1804 1805
		break;
	case CQ_RX_ERROP_L2_PCLP:
1806
		this_cpu_inc(nic->drv_stats->rx_l2_pclp);
1807 1808
		break;
	case CQ_RX_ERROP_IP_NOT:
1809
		this_cpu_inc(nic->drv_stats->rx_ip_ver_errs);
1810 1811
		break;
	case CQ_RX_ERROP_IP_CSUM_ERR:
1812
		this_cpu_inc(nic->drv_stats->rx_ip_csum_errs);
1813 1814
		break;
	case CQ_RX_ERROP_IP_MAL:
1815
		this_cpu_inc(nic->drv_stats->rx_ip_hdr_malformed);
1816 1817
		break;
	case CQ_RX_ERROP_IP_MALD:
1818
		this_cpu_inc(nic->drv_stats->rx_ip_payload_malformed);
1819 1820
		break;
	case CQ_RX_ERROP_IP_HOP:
1821
		this_cpu_inc(nic->drv_stats->rx_ip_ttl_errs);
1822 1823
		break;
	case CQ_RX_ERROP_L3_PCLP:
1824
		this_cpu_inc(nic->drv_stats->rx_l3_pclp);
1825 1826
		break;
	case CQ_RX_ERROP_L4_MAL:
1827
		this_cpu_inc(nic->drv_stats->rx_l4_malformed);
1828 1829
		break;
	case CQ_RX_ERROP_L4_CHK:
1830
		this_cpu_inc(nic->drv_stats->rx_l4_csum_errs);
1831 1832
		break;
	case CQ_RX_ERROP_UDP_LEN:
1833
		this_cpu_inc(nic->drv_stats->rx_udp_len_errs);
1834 1835
		break;
	case CQ_RX_ERROP_L4_PORT:
1836
		this_cpu_inc(nic->drv_stats->rx_l4_port_errs);
1837 1838
		break;
	case CQ_RX_ERROP_TCP_FLAG:
1839
		this_cpu_inc(nic->drv_stats->rx_tcp_flag_errs);
1840 1841
		break;
	case CQ_RX_ERROP_TCP_OFFSET:
1842
		this_cpu_inc(nic->drv_stats->rx_tcp_offset_errs);
1843 1844
		break;
	case CQ_RX_ERROP_L4_PCLP:
1845
		this_cpu_inc(nic->drv_stats->rx_l4_pclp);
1846 1847
		break;
	case CQ_RX_ERROP_RBDR_TRUNC:
1848
		this_cpu_inc(nic->drv_stats->rx_truncated_pkts);
1849 1850 1851 1852 1853 1854 1855
		break;
	}

	return 1;
}

/* Check for errors in the send cmp.queue entry */
1856
int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx)
1857 1858 1859
{
	switch (cqe_tx->send_status) {
	case CQ_TX_ERROP_DESC_FAULT:
1860
		this_cpu_inc(nic->drv_stats->tx_desc_fault);
1861 1862
		break;
	case CQ_TX_ERROP_HDR_CONS_ERR:
1863
		this_cpu_inc(nic->drv_stats->tx_hdr_cons_err);
1864 1865
		break;
	case CQ_TX_ERROP_SUBDC_ERR:
1866
		this_cpu_inc(nic->drv_stats->tx_subdesc_err);
1867
		break;
1868
	case CQ_TX_ERROP_MAX_SIZE_VIOL:
1869
		this_cpu_inc(nic->drv_stats->tx_max_size_exceeded);
1870
		break;
1871
	case CQ_TX_ERROP_IMM_SIZE_OFLOW:
1872
		this_cpu_inc(nic->drv_stats->tx_imm_size_oflow);
1873 1874
		break;
	case CQ_TX_ERROP_DATA_SEQUENCE_ERR:
1875
		this_cpu_inc(nic->drv_stats->tx_data_seq_err);
1876 1877
		break;
	case CQ_TX_ERROP_MEM_SEQUENCE_ERR:
1878
		this_cpu_inc(nic->drv_stats->tx_mem_seq_err);
1879 1880
		break;
	case CQ_TX_ERROP_LOCK_VIOL:
1881
		this_cpu_inc(nic->drv_stats->tx_lock_viol);
1882 1883
		break;
	case CQ_TX_ERROP_DATA_FAULT:
1884
		this_cpu_inc(nic->drv_stats->tx_data_fault);
1885 1886
		break;
	case CQ_TX_ERROP_TSTMP_CONFLICT:
1887
		this_cpu_inc(nic->drv_stats->tx_tstmp_conflict);
1888 1889
		break;
	case CQ_TX_ERROP_TSTMP_TIMEOUT:
1890
		this_cpu_inc(nic->drv_stats->tx_tstmp_timeout);
1891 1892
		break;
	case CQ_TX_ERROP_MEM_FAULT:
1893
		this_cpu_inc(nic->drv_stats->tx_mem_fault);
1894 1895
		break;
	case CQ_TX_ERROP_CK_OVERLAP:
1896
		this_cpu_inc(nic->drv_stats->tx_csum_overlap);
1897 1898
		break;
	case CQ_TX_ERROP_CK_OFLOW:
1899
		this_cpu_inc(nic->drv_stats->tx_csum_overflow);
1900 1901 1902 1903 1904
		break;
	}

	return 1;
}
新手
引导
客服 返回
顶部