nhi.c 25.1 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Thunderbolt Cactus Ridge driver - NHI driver
 *
 * The NHI (native host interface) is the pci device that allows us to send and
 * receive frames from the thunderbolt bus.
 *
 * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
 */

10
#include <linux/pm_runtime.h>
11 12 13 14 15
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/module.h>
16
#include <linux/delay.h>
17 18 19

#include "nhi.h"
#include "nhi_regs.h"
20
#include "tb.h"
21 22 23

#define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring")

24 25 26 27 28 29
/*
 * Used to enable end-to-end workaround for missing RX packets. Do not
 * use this ring for anything else.
 */
#define RING_E2E_UNUSED_HOPID	2

M
Mika Westerberg 已提交
30 31 32 33 34 35
/*
 * Minimal number of vectors when we use MSI-X. Two for control channel
 * Rx/Tx and the rest four are for cross domain DMA paths.
 */
#define MSIX_MIN_VECS		6
#define MSIX_MAX_VECS		16
36

37 38
#define NHI_MAILBOX_TIMEOUT	500 /* ms */

39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
static int ring_interrupt_index(struct tb_ring *ring)
{
	int bit = ring->hop;
	if (!ring->is_tx)
		bit += ring->nhi->hop_count;
	return bit;
}

/**
 * ring_interrupt_active() - activate/deactivate interrupts for a single ring
 *
 * ring->nhi->lock must be held.
 */
static void ring_interrupt_active(struct tb_ring *ring, bool active)
{
54 55
	int reg = REG_RING_INTERRUPT_BASE +
		  ring_interrupt_index(ring) / 32 * 4;
56 57 58
	int bit = ring_interrupt_index(ring) & 31;
	int mask = 1 << bit;
	u32 old, new;
M
Mika Westerberg 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89

	if (ring->irq > 0) {
		u32 step, shift, ivr, misc;
		void __iomem *ivr_base;
		int index;

		if (ring->is_tx)
			index = ring->hop;
		else
			index = ring->hop + ring->nhi->hop_count;

		/*
		 * Ask the hardware to clear interrupt status bits automatically
		 * since we already know which interrupt was triggered.
		 */
		misc = ioread32(ring->nhi->iobase + REG_DMA_MISC);
		if (!(misc & REG_DMA_MISC_INT_AUTO_CLEAR)) {
			misc |= REG_DMA_MISC_INT_AUTO_CLEAR;
			iowrite32(misc, ring->nhi->iobase + REG_DMA_MISC);
		}

		ivr_base = ring->nhi->iobase + REG_INT_VEC_ALLOC_BASE;
		step = index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
		shift = index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
		ivr = ioread32(ivr_base + step);
		ivr &= ~(REG_INT_VEC_ALLOC_MASK << shift);
		if (active)
			ivr |= ring->vector << shift;
		iowrite32(ivr, ivr_base + step);
	}

90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
	old = ioread32(ring->nhi->iobase + reg);
	if (active)
		new = old | mask;
	else
		new = old & ~mask;

	dev_info(&ring->nhi->pdev->dev,
		 "%s interrupt at register %#x bit %d (%#x -> %#x)\n",
		 active ? "enabling" : "disabling", reg, bit, old, new);

	if (new == old)
		dev_WARN(&ring->nhi->pdev->dev,
					 "interrupt for %s %d is already %s\n",
					 RING_TYPE(ring), ring->hop,
					 active ? "enabled" : "disabled");
	iowrite32(new, ring->nhi->iobase + reg);
}

/**
 * nhi_disable_interrupts() - disable interrupts for all rings
 *
 * Use only during init and shutdown.
 */
static void nhi_disable_interrupts(struct tb_nhi *nhi)
{
	int i = 0;
	/* disable interrupts */
	for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++)
		iowrite32(0, nhi->iobase + REG_RING_INTERRUPT_BASE + 4 * i);

	/* clear interrupt status bits */
	for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++)
		ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + 4 * i);
}

/* ring helper methods */

static void __iomem *ring_desc_base(struct tb_ring *ring)
{
	void __iomem *io = ring->nhi->iobase;
	io += ring->is_tx ? REG_TX_RING_BASE : REG_RX_RING_BASE;
	io += ring->hop * 16;
	return io;
}

static void __iomem *ring_options_base(struct tb_ring *ring)
{
	void __iomem *io = ring->nhi->iobase;
	io += ring->is_tx ? REG_TX_OPTIONS_BASE : REG_RX_OPTIONS_BASE;
	io += ring->hop * 32;
	return io;
}

static void ring_iowrite16desc(struct tb_ring *ring, u32 value, u32 offset)
{
	iowrite16(value, ring_desc_base(ring) + offset);
}

static void ring_iowrite32desc(struct tb_ring *ring, u32 value, u32 offset)
{
	iowrite32(value, ring_desc_base(ring) + offset);
}

static void ring_iowrite64desc(struct tb_ring *ring, u64 value, u32 offset)
{
	iowrite32(value, ring_desc_base(ring) + offset);
	iowrite32(value >> 32, ring_desc_base(ring) + offset + 4);
}

static void ring_iowrite32options(struct tb_ring *ring, u32 value, u32 offset)
{
	iowrite32(value, ring_options_base(ring) + offset);
}

static bool ring_full(struct tb_ring *ring)
{
	return ((ring->head + 1) % ring->size) == ring->tail;
}

static bool ring_empty(struct tb_ring *ring)
{
	return ring->head == ring->tail;
}

/**
 * ring_write_descriptors() - post frames from ring->queue to the controller
 *
 * ring->lock is held.
 */
static void ring_write_descriptors(struct tb_ring *ring)
{
	struct ring_frame *frame, *n;
	struct ring_desc *descriptor;
	list_for_each_entry_safe(frame, n, &ring->queue, list) {
		if (ring_full(ring))
			break;
		list_move_tail(&frame->list, &ring->in_flight);
		descriptor = &ring->descriptors[ring->head];
		descriptor->phys = frame->buffer_phy;
		descriptor->time = 0;
		descriptor->flags = RING_DESC_POSTED | RING_DESC_INTERRUPT;
		if (ring->is_tx) {
			descriptor->length = frame->size;
			descriptor->eof = frame->eof;
			descriptor->sof = frame->sof;
		}
		ring->head = (ring->head + 1) % ring->size;
		ring_iowrite16desc(ring, ring->head, ring->is_tx ? 10 : 8);
	}
}

/**
 * ring_work() - progress completed frames
 *
 * If the ring is shutting down then all frames are marked as canceled and
 * their callbacks are invoked.
 *
 * Otherwise we collect all completed frame from the ring buffer, write new
 * frame to the ring buffer and invoke the callbacks for the completed frames.
 */
static void ring_work(struct work_struct *work)
{
	struct tb_ring *ring = container_of(work, typeof(*ring), work);
	struct ring_frame *frame;
	bool canceled = false;
215
	unsigned long flags;
216
	LIST_HEAD(done);
217 218

	spin_lock_irqsave(&ring->lock, flags);
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245

	if (!ring->running) {
		/*  Move all frames to done and mark them as canceled. */
		list_splice_tail_init(&ring->in_flight, &done);
		list_splice_tail_init(&ring->queue, &done);
		canceled = true;
		goto invoke_callback;
	}

	while (!ring_empty(ring)) {
		if (!(ring->descriptors[ring->tail].flags
				& RING_DESC_COMPLETED))
			break;
		frame = list_first_entry(&ring->in_flight, typeof(*frame),
					 list);
		list_move_tail(&frame->list, &done);
		if (!ring->is_tx) {
			frame->size = ring->descriptors[ring->tail].length;
			frame->eof = ring->descriptors[ring->tail].eof;
			frame->sof = ring->descriptors[ring->tail].sof;
			frame->flags = ring->descriptors[ring->tail].flags;
		}
		ring->tail = (ring->tail + 1) % ring->size;
	}
	ring_write_descriptors(ring);

invoke_callback:
246 247
	/* allow callbacks to schedule new work */
	spin_unlock_irqrestore(&ring->lock, flags);
248 249 250 251 252 253 254 255 256 257 258
	while (!list_empty(&done)) {
		frame = list_first_entry(&done, typeof(*frame), list);
		/*
		 * The callback may reenqueue or delete frame.
		 * Do not hold on to it.
		 */
		list_del_init(&frame->list);
		frame->callback(ring, frame, canceled);
	}
}

259
int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
260
{
261
	unsigned long flags;
262
	int ret = 0;
263 264

	spin_lock_irqsave(&ring->lock, flags);
265 266 267 268 269 270
	if (ring->running) {
		list_add_tail(&frame->list, &ring->queue);
		ring_write_descriptors(ring);
	} else {
		ret = -ESHUTDOWN;
	}
271
	spin_unlock_irqrestore(&ring->lock, flags);
272 273
	return ret;
}
274
EXPORT_SYMBOL_GPL(__tb_ring_enqueue);
275

M
Mika Westerberg 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
static irqreturn_t ring_msix(int irq, void *data)
{
	struct tb_ring *ring = data;

	schedule_work(&ring->work);
	return IRQ_HANDLED;
}

static int ring_request_msix(struct tb_ring *ring, bool no_suspend)
{
	struct tb_nhi *nhi = ring->nhi;
	unsigned long irqflags;
	int ret;

	if (!nhi->pdev->msix_enabled)
		return 0;

	ret = ida_simple_get(&nhi->msix_ida, 0, MSIX_MAX_VECS, GFP_KERNEL);
	if (ret < 0)
		return ret;

	ring->vector = ret;

	ring->irq = pci_irq_vector(ring->nhi->pdev, ring->vector);
	if (ring->irq < 0)
		return ring->irq;

	irqflags = no_suspend ? IRQF_NO_SUSPEND : 0;
	return request_irq(ring->irq, ring_msix, irqflags, "thunderbolt", ring);
}

static void ring_release_msix(struct tb_ring *ring)
{
	if (ring->irq <= 0)
		return;

	free_irq(ring->irq, ring);
	ida_simple_remove(&ring->nhi->msix_ida, ring->vector);
	ring->vector = 0;
	ring->irq = 0;
}

318 319 320
static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size,
				     bool transmit, unsigned int flags,
				     u16 sof_mask, u16 eof_mask)
321 322 323 324 325
{
	struct tb_ring *ring = NULL;
	dev_info(&nhi->pdev->dev, "allocating %s ring %d of size %d\n",
		 transmit ? "TX" : "RX", hop, size);

326 327 328 329
	/* Tx Ring 2 is reserved for E2E workaround */
	if (transmit && hop == RING_E2E_UNUSED_HOPID)
		return NULL;

330 331
	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
	if (!ring)
332
		return NULL;
333

334
	spin_lock_init(&ring->lock);
335 336 337 338 339 340 341 342
	INIT_LIST_HEAD(&ring->queue);
	INIT_LIST_HEAD(&ring->in_flight);
	INIT_WORK(&ring->work, ring_work);

	ring->nhi = nhi;
	ring->hop = hop;
	ring->is_tx = transmit;
	ring->size = size;
M
Mika Westerberg 已提交
343
	ring->flags = flags;
344 345
	ring->sof_mask = sof_mask;
	ring->eof_mask = eof_mask;
346 347 348
	ring->head = 0;
	ring->tail = 0;
	ring->running = false;
M
Mika Westerberg 已提交
349

350 351 352 353
	ring->descriptors = dma_alloc_coherent(&ring->nhi->pdev->dev,
			size * sizeof(*ring->descriptors),
			&ring->descriptors_dma, GFP_KERNEL | __GFP_ZERO);
	if (!ring->descriptors)
354
		goto err_free_ring;
355

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
	if (ring_request_msix(ring, flags & RING_FLAG_NO_SUSPEND))
		goto err_free_descs;

	spin_lock_irq(&nhi->lock);
	if (hop >= nhi->hop_count) {
		dev_WARN(&nhi->pdev->dev, "invalid hop: %d\n", hop);
		goto err_release_msix;
	}
	if (transmit && nhi->tx_rings[hop]) {
		dev_WARN(&nhi->pdev->dev, "TX hop %d already allocated\n", hop);
		goto err_release_msix;
	} else if (!transmit && nhi->rx_rings[hop]) {
		dev_WARN(&nhi->pdev->dev, "RX hop %d already allocated\n", hop);
		goto err_release_msix;
	}
371 372 373 374
	if (transmit)
		nhi->tx_rings[hop] = ring;
	else
		nhi->rx_rings[hop] = ring;
375 376
	spin_unlock_irq(&nhi->lock);

377 378
	return ring;

379 380 381 382 383 384 385 386
err_release_msix:
	spin_unlock_irq(&nhi->lock);
	ring_release_msix(ring);
err_free_descs:
	dma_free_coherent(&ring->nhi->pdev->dev,
			  ring->size * sizeof(*ring->descriptors),
			  ring->descriptors, ring->descriptors_dma);
err_free_ring:
387
	kfree(ring);
388

389 390 391
	return NULL;
}

392 393 394 395 396 397 398 399 400
/**
 * tb_ring_alloc_tx() - Allocate DMA ring for transmit
 * @nhi: Pointer to the NHI the ring is to be allocated
 * @hop: HopID (ring) to allocate
 * @size: Number of entries in the ring
 * @flags: Flags for the ring
 */
struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size,
				 unsigned int flags)
401
{
402
	return tb_ring_alloc(nhi, hop, size, true, flags, 0, 0);
403
}
404
EXPORT_SYMBOL_GPL(tb_ring_alloc_tx);
405

406 407 408 409 410 411 412 413 414 415 416
/**
 * tb_ring_alloc_rx() - Allocate DMA ring for receive
 * @nhi: Pointer to the NHI the ring is to be allocated
 * @hop: HopID (ring) to allocate
 * @size: Number of entries in the ring
 * @flags: Flags for the ring
 * @sof_mask: Mask of PDF values that start a frame
 * @eof_mask: Mask of PDF values that end a frame
 */
struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size,
				 unsigned int flags, u16 sof_mask, u16 eof_mask)
417
{
418
	return tb_ring_alloc(nhi, hop, size, false, flags, sof_mask, eof_mask);
419
}
420
EXPORT_SYMBOL_GPL(tb_ring_alloc_rx);
421 422

/**
423
 * tb_ring_start() - enable a ring
424
 *
425
 * Must not be invoked in parallel with tb_ring_stop().
426
 */
427
void tb_ring_start(struct tb_ring *ring)
428
{
429 430 431
	u16 frame_size;
	u32 flags;

432 433
	spin_lock_irq(&ring->nhi->lock);
	spin_lock(&ring->lock);
434 435
	if (ring->nhi->going_away)
		goto err;
436 437 438 439 440 441 442
	if (ring->running) {
		dev_WARN(&ring->nhi->pdev->dev, "ring already started\n");
		goto err;
	}
	dev_info(&ring->nhi->pdev->dev, "starting %s %d\n",
		 RING_TYPE(ring), ring->hop);

443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
	if (ring->flags & RING_FLAG_FRAME) {
		/* Means 4096 */
		frame_size = 0;
		flags = RING_FLAG_ENABLE;
	} else {
		frame_size = TB_FRAME_SIZE;
		flags = RING_FLAG_ENABLE | RING_FLAG_RAW;
	}

	if (ring->flags & RING_FLAG_E2E && !ring->is_tx) {
		u32 hop;

		/*
		 * In order not to lose Rx packets we enable end-to-end
		 * workaround which transfers Rx credits to an unused Tx
		 * HopID.
		 */
		hop = RING_E2E_UNUSED_HOPID << REG_RX_OPTIONS_E2E_HOP_SHIFT;
		hop &= REG_RX_OPTIONS_E2E_HOP_MASK;
		flags |= hop | RING_FLAG_E2E_FLOW_CONTROL;
	}

465 466 467 468
	ring_iowrite64desc(ring, ring->descriptors_dma, 0);
	if (ring->is_tx) {
		ring_iowrite32desc(ring, ring->size, 12);
		ring_iowrite32options(ring, 0, 4); /* time releated ? */
469
		ring_iowrite32options(ring, flags, 0);
470
	} else {
471 472 473 474 475
		u32 sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask;

		ring_iowrite32desc(ring, (frame_size << 16) | ring->size, 12);
		ring_iowrite32options(ring, sof_eof_mask, 4);
		ring_iowrite32options(ring, flags, 0);
476 477 478 479
	}
	ring_interrupt_active(ring, true);
	ring->running = true;
err:
480 481
	spin_unlock(&ring->lock);
	spin_unlock_irq(&ring->nhi->lock);
482
}
483
EXPORT_SYMBOL_GPL(tb_ring_start);
484 485

/**
486
 * tb_ring_stop() - shutdown a ring
487 488 489
 *
 * Must not be invoked from a callback.
 *
490 491 492
 * This method will disable the ring. Further calls to
 * tb_ring_tx/tb_ring_rx will return -ESHUTDOWN until ring_stop has been
 * called.
493 494 495 496 497
 *
 * All enqueued frames will be canceled and their callbacks will be executed
 * with frame->canceled set to true (on the callback thread). This method
 * returns only after all callback invocations have finished.
 */
498
void tb_ring_stop(struct tb_ring *ring)
499
{
500 501
	spin_lock_irq(&ring->nhi->lock);
	spin_lock(&ring->lock);
502 503
	dev_info(&ring->nhi->pdev->dev, "stopping %s %d\n",
		 RING_TYPE(ring), ring->hop);
504 505
	if (ring->nhi->going_away)
		goto err;
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
	if (!ring->running) {
		dev_WARN(&ring->nhi->pdev->dev, "%s %d already stopped\n",
			 RING_TYPE(ring), ring->hop);
		goto err;
	}
	ring_interrupt_active(ring, false);

	ring_iowrite32options(ring, 0, 0);
	ring_iowrite64desc(ring, 0, 0);
	ring_iowrite16desc(ring, 0, ring->is_tx ? 10 : 8);
	ring_iowrite32desc(ring, 0, 12);
	ring->head = 0;
	ring->tail = 0;
	ring->running = false;

err:
522 523
	spin_unlock(&ring->lock);
	spin_unlock_irq(&ring->nhi->lock);
524 525 526 527 528 529 530

	/*
	 * schedule ring->work to invoke callbacks on all remaining frames.
	 */
	schedule_work(&ring->work);
	flush_work(&ring->work);
}
531
EXPORT_SYMBOL_GPL(tb_ring_stop);
532 533

/*
534
 * tb_ring_free() - free ring
535 536 537 538 539 540 541 542
 *
 * When this method returns all invocations of ring->callback will have
 * finished.
 *
 * Ring must be stopped.
 *
 * Must NOT be called from ring_frame->callback!
 */
543
void tb_ring_free(struct tb_ring *ring)
544
{
545
	spin_lock_irq(&ring->nhi->lock);
546 547 548 549 550 551 552 553 554 555 556 557 558 559
	/*
	 * Dissociate the ring from the NHI. This also ensures that
	 * nhi_interrupt_work cannot reschedule ring->work.
	 */
	if (ring->is_tx)
		ring->nhi->tx_rings[ring->hop] = NULL;
	else
		ring->nhi->rx_rings[ring->hop] = NULL;

	if (ring->running) {
		dev_WARN(&ring->nhi->pdev->dev, "%s %d still running\n",
			 RING_TYPE(ring), ring->hop);
	}

M
Mika Westerberg 已提交
560 561
	ring_release_msix(ring);

562 563 564 565
	dma_free_coherent(&ring->nhi->pdev->dev,
			  ring->size * sizeof(*ring->descriptors),
			  ring->descriptors, ring->descriptors_dma);

566
	ring->descriptors = NULL;
567 568 569 570 571 572 573 574
	ring->descriptors_dma = 0;


	dev_info(&ring->nhi->pdev->dev,
		 "freeing %s %d\n",
		 RING_TYPE(ring),
		 ring->hop);

575
	spin_unlock_irq(&ring->nhi->lock);
576
	/**
M
Mika Westerberg 已提交
577 578 579
	 * ring->work can no longer be scheduled (it is scheduled only
	 * by nhi_interrupt_work, ring_stop and ring_msix). Wait for it
	 * to finish before freeing the ring.
580 581 582 583
	 */
	flush_work(&ring->work);
	kfree(ring);
}
584
EXPORT_SYMBOL_GPL(tb_ring_free);
585

586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
/**
 * nhi_mailbox_cmd() - Send a command through NHI mailbox
 * @nhi: Pointer to the NHI structure
 * @cmd: Command to send
 * @data: Data to be send with the command
 *
 * Sends mailbox command to the firmware running on NHI. Returns %0 in
 * case of success and negative errno in case of failure.
 */
int nhi_mailbox_cmd(struct tb_nhi *nhi, enum nhi_mailbox_cmd cmd, u32 data)
{
	ktime_t timeout;
	u32 val;

	iowrite32(data, nhi->iobase + REG_INMAIL_DATA);

	val = ioread32(nhi->iobase + REG_INMAIL_CMD);
	val &= ~(REG_INMAIL_CMD_MASK | REG_INMAIL_ERROR);
	val |= REG_INMAIL_OP_REQUEST | cmd;
	iowrite32(val, nhi->iobase + REG_INMAIL_CMD);

	timeout = ktime_add_ms(ktime_get(), NHI_MAILBOX_TIMEOUT);
	do {
		val = ioread32(nhi->iobase + REG_INMAIL_CMD);
		if (!(val & REG_INMAIL_OP_REQUEST))
			break;
		usleep_range(10, 20);
	} while (ktime_before(ktime_get(), timeout));

	if (val & REG_INMAIL_OP_REQUEST)
		return -ETIMEDOUT;
	if (val & REG_INMAIL_ERROR)
		return -EIO;

	return 0;
}

/**
 * nhi_mailbox_mode() - Return current firmware operation mode
 * @nhi: Pointer to the NHI structure
 *
 * The function reads current firmware operation mode using NHI mailbox
 * registers and returns it to the caller.
 */
enum nhi_fw_mode nhi_mailbox_mode(struct tb_nhi *nhi)
{
	u32 val;

	val = ioread32(nhi->iobase + REG_OUTMAIL_CMD);
	val &= REG_OUTMAIL_CMD_OPMODE_MASK;
	val >>= REG_OUTMAIL_CMD_OPMODE_SHIFT;

	return (enum nhi_fw_mode)val;
}

641 642 643 644 645 646 647 648 649
static void nhi_interrupt_work(struct work_struct *work)
{
	struct tb_nhi *nhi = container_of(work, typeof(*nhi), interrupt_work);
	int value = 0; /* Suppress uninitialized usage warning. */
	int bit;
	int hop = -1;
	int type = 0; /* current interrupt type 0: TX, 1: RX, 2: RX overflow */
	struct tb_ring *ring;

650
	spin_lock_irq(&nhi->lock);
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687

	/*
	 * Starting at REG_RING_NOTIFY_BASE there are three status bitfields
	 * (TX, RX, RX overflow). We iterate over the bits and read a new
	 * dwords as required. The registers are cleared on read.
	 */
	for (bit = 0; bit < 3 * nhi->hop_count; bit++) {
		if (bit % 32 == 0)
			value = ioread32(nhi->iobase
					 + REG_RING_NOTIFY_BASE
					 + 4 * (bit / 32));
		if (++hop == nhi->hop_count) {
			hop = 0;
			type++;
		}
		if ((value & (1 << (bit % 32))) == 0)
			continue;
		if (type == 2) {
			dev_warn(&nhi->pdev->dev,
				 "RX overflow for ring %d\n",
				 hop);
			continue;
		}
		if (type == 0)
			ring = nhi->tx_rings[hop];
		else
			ring = nhi->rx_rings[hop];
		if (ring == NULL) {
			dev_warn(&nhi->pdev->dev,
				 "got interrupt for inactive %s ring %d\n",
				 type ? "RX" : "TX",
				 hop);
			continue;
		}
		/* we do not check ring->running, this is done in ring->work */
		schedule_work(&ring->work);
	}
688
	spin_unlock_irq(&nhi->lock);
689 690 691 692 693 694 695 696 697
}

static irqreturn_t nhi_msi(int irq, void *data)
{
	struct tb_nhi *nhi = data;
	schedule_work(&nhi->interrupt_work);
	return IRQ_HANDLED;
}

698 699 700 701
static int nhi_suspend_noirq(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct tb *tb = pci_get_drvdata(pdev);
702 703

	return tb_domain_suspend_noirq(tb);
704 705
}

706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
static void nhi_enable_int_throttling(struct tb_nhi *nhi)
{
	/* Throttling is specified in 256ns increments */
	u32 throttle = DIV_ROUND_UP(128 * NSEC_PER_USEC, 256);
	unsigned int i;

	/*
	 * Configure interrupt throttling for all vectors even if we
	 * only use few.
	 */
	for (i = 0; i < MSIX_MAX_VECS; i++) {
		u32 reg = REG_INT_THROTTLING_RATE + i * 4;
		iowrite32(throttle, nhi->iobase + reg);
	}
}

722 723 724 725
static int nhi_resume_noirq(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct tb *tb = pci_get_drvdata(pdev);
726

727 728 729 730 731 732 733
	/*
	 * Check that the device is still there. It may be that the user
	 * unplugged last device which causes the host controller to go
	 * away on PCs.
	 */
	if (!pci_device_is_present(pdev))
		tb->nhi->going_away = true;
734 735
	else
		nhi_enable_int_throttling(tb->nhi);
736

737
	return tb_domain_resume_noirq(tb);
738 739
}

740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
static int nhi_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct tb *tb = pci_get_drvdata(pdev);

	return tb_domain_suspend(tb);
}

static void nhi_complete(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct tb *tb = pci_get_drvdata(pdev);

	tb_domain_complete(tb);
}

756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
static void nhi_shutdown(struct tb_nhi *nhi)
{
	int i;
	dev_info(&nhi->pdev->dev, "shutdown\n");

	for (i = 0; i < nhi->hop_count; i++) {
		if (nhi->tx_rings[i])
			dev_WARN(&nhi->pdev->dev,
				 "TX ring %d is still active\n", i);
		if (nhi->rx_rings[i])
			dev_WARN(&nhi->pdev->dev,
				 "RX ring %d is still active\n", i);
	}
	nhi_disable_interrupts(nhi);
	/*
	 * We have to release the irq before calling flush_work. Otherwise an
	 * already executing IRQ handler could call schedule_work again.
	 */
M
Mika Westerberg 已提交
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
	if (!nhi->pdev->msix_enabled) {
		devm_free_irq(&nhi->pdev->dev, nhi->pdev->irq, nhi);
		flush_work(&nhi->interrupt_work);
	}
	ida_destroy(&nhi->msix_ida);
}

static int nhi_init_msi(struct tb_nhi *nhi)
{
	struct pci_dev *pdev = nhi->pdev;
	int res, irq, nvec;

	/* In case someone left them on. */
	nhi_disable_interrupts(nhi);

789 790
	nhi_enable_int_throttling(nhi);

M
Mika Westerberg 已提交
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
	ida_init(&nhi->msix_ida);

	/*
	 * The NHI has 16 MSI-X vectors or a single MSI. We first try to
	 * get all MSI-X vectors and if we succeed, each ring will have
	 * one MSI-X. If for some reason that does not work out, we
	 * fallback to a single MSI.
	 */
	nvec = pci_alloc_irq_vectors(pdev, MSIX_MIN_VECS, MSIX_MAX_VECS,
				     PCI_IRQ_MSIX);
	if (nvec < 0) {
		nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
		if (nvec < 0)
			return nvec;

		INIT_WORK(&nhi->interrupt_work, nhi_interrupt_work);

		irq = pci_irq_vector(nhi->pdev, 0);
		if (irq < 0)
			return irq;

		res = devm_request_irq(&pdev->dev, irq, nhi_msi,
				       IRQF_NO_SUSPEND, "thunderbolt", nhi);
		if (res) {
			dev_err(&pdev->dev, "request_irq failed, aborting\n");
			return res;
		}
	}

	return 0;
821 822 823 824 825
}

static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct tb_nhi *nhi;
826
	struct tb *tb;
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
	int res;

	res = pcim_enable_device(pdev);
	if (res) {
		dev_err(&pdev->dev, "cannot enable PCI device, aborting\n");
		return res;
	}

	res = pcim_iomap_regions(pdev, 1 << 0, "thunderbolt");
	if (res) {
		dev_err(&pdev->dev, "cannot obtain PCI resources, aborting\n");
		return res;
	}

	nhi = devm_kzalloc(&pdev->dev, sizeof(*nhi), GFP_KERNEL);
	if (!nhi)
		return -ENOMEM;

	nhi->pdev = pdev;
	/* cannot fail - table is allocated bin pcim_iomap_regions */
	nhi->iobase = pcim_iomap_table(pdev)[0];
	nhi->hop_count = ioread32(nhi->iobase + REG_HOP_COUNT) & 0x3ff;
849
	if (nhi->hop_count != 12 && nhi->hop_count != 32)
850 851 852
		dev_warn(&pdev->dev, "unexpected hop count: %d\n",
			 nhi->hop_count);

H
Himangi Saraogi 已提交
853 854 855 856
	nhi->tx_rings = devm_kcalloc(&pdev->dev, nhi->hop_count,
				     sizeof(*nhi->tx_rings), GFP_KERNEL);
	nhi->rx_rings = devm_kcalloc(&pdev->dev, nhi->hop_count,
				     sizeof(*nhi->rx_rings), GFP_KERNEL);
857 858 859
	if (!nhi->tx_rings || !nhi->rx_rings)
		return -ENOMEM;

M
Mika Westerberg 已提交
860
	res = nhi_init_msi(nhi);
861
	if (res) {
M
Mika Westerberg 已提交
862
		dev_err(&pdev->dev, "cannot enable MSI, aborting\n");
863 864 865
		return res;
	}

866
	spin_lock_init(&nhi->lock);
867 868 869

	pci_set_master(pdev);

870
	tb = icm_probe(nhi);
871
	if (!tb)
872 873 874 875
		tb = tb_probe(nhi);
	if (!tb) {
		dev_err(&nhi->pdev->dev,
			"failed to determine connection manager, aborting\n");
876
		return -ENODEV;
877 878 879
	}

	dev_info(&nhi->pdev->dev, "NHI initialized, starting thunderbolt\n");
880 881 882

	res = tb_domain_add(tb);
	if (res) {
883 884 885 886
		/*
		 * At this point the RX/TX rings might already have been
		 * activated. Do a proper shutdown.
		 */
887
		tb_domain_put(tb);
888 889 890 891
		nhi_shutdown(nhi);
		return -EIO;
	}
	pci_set_drvdata(pdev, tb);
892 893 894 895 896 897

	return 0;
}

static void nhi_remove(struct pci_dev *pdev)
{
898 899
	struct tb *tb = pci_get_drvdata(pdev);
	struct tb_nhi *nhi = tb->nhi;
900 901

	tb_domain_remove(tb);
902 903 904
	nhi_shutdown(nhi);
}

905 906 907 908 909 910 911 912 913 914 915 916 917
/*
 * The tunneled pci bridges are siblings of us. Use resume_noirq to reenable
 * the tunnels asap. A corresponding pci quirk blocks the downstream bridges
 * resume_noirq until we are done.
 */
static const struct dev_pm_ops nhi_pm_ops = {
	.suspend_noirq = nhi_suspend_noirq,
	.resume_noirq = nhi_resume_noirq,
	.freeze_noirq = nhi_suspend_noirq, /*
					    * we just disable hotplug, the
					    * pci-tunnels stay alive.
					    */
	.restore_noirq = nhi_resume_noirq,
918 919 920 921
	.suspend = nhi_suspend,
	.freeze = nhi_suspend,
	.poweroff = nhi_suspend,
	.complete = nhi_complete,
922 923
};

S
Sachin Kamat 已提交
924
static struct pci_device_id nhi_ids[] = {
925 926
	/*
	 * We have to specify class, the TB bridges use the same device and
927
	 * vendor (sub)id on gen 1 and gen 2 controllers.
928
	 */
929 930 931 932 933 934
	{
		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
		.vendor = PCI_VENDOR_ID_INTEL,
		.device = PCI_DEVICE_ID_INTEL_LIGHT_RIDGE,
		.subvendor = 0x2222, .subdevice = 0x1111,
	},
935 936
	{
		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
937 938
		.vendor = PCI_VENDOR_ID_INTEL,
		.device = PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C,
939 940
		.subvendor = 0x2222, .subdevice = 0x1111,
	},
941 942 943 944 945 946
	{
		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
		.vendor = PCI_VENDOR_ID_INTEL,
		.device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI,
		.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID,
	},
947 948
	{
		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
949 950
		.vendor = PCI_VENDOR_ID_INTEL,
		.device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI,
951
		.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID,
952
	},
953 954 955 956 957 958 959 960 961 962 963

	/* Thunderbolt 3 */
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_USBONLY_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_USBONLY_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI) },
	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_USBONLY_NHI) },

964 965 966 967 968 969 970 971 972 973 974
	{ 0,}
};

MODULE_DEVICE_TABLE(pci, nhi_ids);
MODULE_LICENSE("GPL");

static struct pci_driver nhi_driver = {
	.name = "thunderbolt",
	.id_table = nhi_ids,
	.probe = nhi_probe,
	.remove = nhi_remove,
975
	.driver.pm = &nhi_pm_ops,
976 977 978 979
};

static int __init nhi_init(void)
{
980 981 982 983 984 985 986 987 988
	int ret;

	ret = tb_domain_init();
	if (ret)
		return ret;
	ret = pci_register_driver(&nhi_driver);
	if (ret)
		tb_domain_exit();
	return ret;
989 990 991 992 993
}

static void __exit nhi_unload(void)
{
	pci_unregister_driver(&nhi_driver);
994
	tb_domain_exit();
995 996 997 998
}

module_init(nhi_init);
module_exit(nhi_unload);