ntb_hw_intel.c 58.2 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Intel PCIe NTB Linux driver
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
50

51
#include <linux/debugfs.h>
J
Jon Mason 已提交
52
#include <linux/delay.h>
53 54 55 56
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
J
Jon Mason 已提交
57
#include <linux/random.h>
58
#include <linux/slab.h>
59 60
#include <linux/ntb.h>

61
#include "ntb_hw_intel.h"
62

63 64 65
#define NTB_NAME	"ntb_hw_intel"
#define NTB_DESC	"Intel(R) PCI-E Non-Transparent Bridge Driver"
#define NTB_VER		"2.0"
66

67
MODULE_DESCRIPTION(NTB_DESC);
68 69 70 71
MODULE_VERSION(NTB_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

72 73 74
#define bar0_off(base, bar) ((base) + ((bar) << 2))
#define bar2_off(base, bar) bar0_off(base, (bar) - 2)

75 76 77 78 79 80 81 82 83 84 85 86 87 88
static const struct intel_ntb_reg atom_reg;
static const struct intel_ntb_alt_reg atom_pri_reg;
static const struct intel_ntb_alt_reg atom_sec_reg;
static const struct intel_ntb_alt_reg atom_b2b_reg;
static const struct intel_ntb_xlat_reg atom_pri_xlat;
static const struct intel_ntb_xlat_reg atom_sec_xlat;
static const struct intel_ntb_reg xeon_reg;
static const struct intel_ntb_alt_reg xeon_pri_reg;
static const struct intel_ntb_alt_reg xeon_sec_reg;
static const struct intel_ntb_alt_reg xeon_b2b_reg;
static const struct intel_ntb_xlat_reg xeon_pri_xlat;
static const struct intel_ntb_xlat_reg xeon_sec_xlat;
static struct intel_b2b_addr xeon_b2b_usd_addr;
static struct intel_b2b_addr xeon_b2b_dsd_addr;
89 90 91
static const struct ntb_dev_ops intel_ntb_ops;

static const struct file_operations intel_ntb_debugfs_info;
92 93
static struct dentry *debugfs_dir;

94 95 96 97 98 99 100 101 102 103 104 105 106 107
static int b2b_mw_idx = -1;
module_param(b2b_mw_idx, int, 0644);
MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
		 "value of zero or positive starts from first mw idx, and a "
		 "negative value starts from last mw idx.  Both sides MUST "
		 "set the same value here!");

static unsigned int b2b_mw_share;
module_param(b2b_mw_share, uint, 0644);
MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
		 "ntb so that the peer ntb only occupies the first half of "
		 "the mw, so the second half can still be used as a mw.  Both "
		 "sides MUST set the same value here!");

108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
module_param_named(xeon_b2b_usd_bar2_addr64,
		   xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
		 "XEON B2B USD BAR 2 64-bit address");

module_param_named(xeon_b2b_usd_bar4_addr64,
		   xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
		 "XEON B2B USD BAR 4 64-bit address");

module_param_named(xeon_b2b_usd_bar4_addr32,
		   xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
		 "XEON B2B USD split-BAR 4 32-bit address");

module_param_named(xeon_b2b_usd_bar5_addr32,
		   xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
		 "XEON B2B USD split-BAR 5 32-bit address");

module_param_named(xeon_b2b_dsd_bar2_addr64,
		   xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
		 "XEON B2B DSD BAR 2 64-bit address");

module_param_named(xeon_b2b_dsd_bar4_addr64,
		   xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
		 "XEON B2B DSD BAR 4 64-bit address");

module_param_named(xeon_b2b_dsd_bar4_addr32,
		   xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
		 "XEON B2B DSD split-BAR 4 32-bit address");

module_param_named(xeon_b2b_dsd_bar5_addr32,
		   xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
		 "XEON B2B DSD split-BAR 5 32-bit address");
147

148 149 150 151 152 153
#ifndef ioread64
#ifdef readq
#define ioread64 readq
#else
#define ioread64 _ioread64
static inline u64 _ioread64(void __iomem *mmio)
154
{
155
	u64 low, high;
156

157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
	low = ioread32(mmio);
	high = ioread32(mmio + sizeof(u32));
	return low | (high << 32);
}
#endif
#endif

#ifndef iowrite64
#ifdef writeq
#define iowrite64 writeq
#else
#define iowrite64 _iowrite64
static inline void _iowrite64(u64 val, void __iomem *mmio)
{
	iowrite32(val, mmio);
	iowrite32(val >> 32, mmio + sizeof(u32));
173
}
174 175
#endif
#endif
176

177
static inline int pdev_is_atom(struct pci_dev *pdev)
178
{
179
	switch (pdev->device) {
180 181 182 183 184 185
	case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
		return 1;
	}
	return 0;
}

186
static inline int pdev_is_xeon(struct pci_dev *pdev)
187
{
188
	switch (pdev->device) {
189 190 191 192 193 194 195 196 197 198 199 200
	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
201
		return 1;
202
	}
203
	return 0;
204 205
}

206
static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
207
{
208 209 210 211 212 213 214 215 216 217 218 219 220
	ndev->unsafe_flags = 0;
	ndev->unsafe_flags_ignore = 0;

	/* Only B2B has a workaround to avoid SDOORBELL */
	if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
		if (!ntb_topo_is_b2b(ndev->ntb.topo))
			ndev->unsafe_flags |= NTB_UNSAFE_DB;

	/* No low level workaround to avoid SB01BASE */
	if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
		ndev->unsafe_flags |= NTB_UNSAFE_DB;
		ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
	}
221 222
}

223 224
static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
				 unsigned long flag)
225
{
226
	return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
227 228
}

229 230
static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
				     unsigned long flag)
231
{
232 233
	flag &= ndev->unsafe_flags;
	ndev->unsafe_flags_ignore |= flag;
234

235
	return !!flag;
236 237
}

238
static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
239
{
240
	if (idx < 0 || idx > ndev->mw_count)
241
		return -EINVAL;
242 243
	return ndev->reg->mw_bar[idx];
}
244

245 246 247 248 249
static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
			       phys_addr_t *db_addr, resource_size_t *db_size,
			       phys_addr_t reg_addr, unsigned long reg)
{
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
250

251 252 253 254
	if (db_addr) {
		*db_addr = reg_addr + reg;
		dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
	}
255

256 257 258 259
	if (db_size) {
		*db_size = ndev->reg->db_size;
		dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
	}
260 261 262 263

	return 0;
}

264 265
static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
			       void __iomem *mmio)
266
{
267
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
268

269 270 271 272 273 274 275
	return ndev->reg->db_ioread(mmio);
}

static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
				void __iomem *mmio)
{
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
276

277 278
	if (db_bits & ~ndev->db_valid_mask)
		return -EINVAL;
279

280
	ndev->reg->db_iowrite(db_bits, mmio);
281

282
	return 0;
283 284
}

285 286
static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
				   void __iomem *mmio)
287
{
288
	unsigned long irqflags;
289

290 291 292 293
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));

	if (db_bits & ~ndev->db_valid_mask)
		return -EINVAL;
294

295 296 297 298 299 300
	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
	{
		ndev->db_mask |= db_bits;
		ndev->reg->db_iowrite(ndev->db_mask, mmio);
	}
	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
301

302
	return 0;
303 304
}

305 306
static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
				     void __iomem *mmio)
307
{
308
	unsigned long irqflags;
309

310 311 312 313
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));

	if (db_bits & ~ndev->db_valid_mask)
		return -EINVAL;
314

315 316 317 318 319 320
	spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
	{
		ndev->db_mask &= ~db_bits;
		ndev->reg->db_iowrite(ndev->db_mask, mmio);
	}
	spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
321

322
	return 0;
323 324
}

325
static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
326
{
327
	u64 shift, mask;
328

329 330
	shift = ndev->db_vec_shift;
	mask = BIT_ULL(shift) - 1;
331

332
	return mask << (shift * db_vector);
333 334
}

335 336 337
static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
				 phys_addr_t *spad_addr, phys_addr_t reg_addr,
				 unsigned long reg)
338
{
339 340 341
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));

	if (idx < 0 || idx >= ndev->spad_count)
342 343
		return -EINVAL;

344 345 346 347
	if (spad_addr) {
		*spad_addr = reg_addr + reg + (idx << 2);
		dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
	}
348 349 350 351

	return 0;
}

352 353
static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
				 void __iomem *mmio)
354
{
355
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
356

357 358
	if (idx < 0 || idx >= ndev->spad_count)
		return 0;
359

360
	return ioread32(mmio + (idx << 2));
361 362
}

363 364
static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
				  void __iomem *mmio)
365
{
366 367 368
	WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));

	if (idx < 0 || idx >= ndev->spad_count)
369 370
		return -EINVAL;

371
	iowrite32(val, mmio + (idx << 2));
372 373 374 375

	return 0;
}

376
static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
377
{
378 379 380 381 382
	u64 vec_mask;

	vec_mask = ndev_vec_mask(ndev, vec);

	dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
383

384 385 386 387 388 389 390 391 392 393 394
	ndev->last_ts = jiffies;

	if (vec_mask & ndev->db_link_mask) {
		if (ndev->reg->poll_link(ndev))
			ntb_link_event(&ndev->ntb);
	}

	if (vec_mask & ndev->db_valid_mask)
		ntb_db_event(&ndev->ntb, vec);

	return IRQ_HANDLED;
395 396
}

397
static irqreturn_t ndev_vec_isr(int irq, void *dev)
398
{
399
	struct intel_ntb_vec *nvec = dev;
400

401
	return ndev_interrupt(nvec->ndev, nvec->num);
402 403
}

404
static irqreturn_t ndev_irq_isr(int irq, void *dev)
405
{
406
	struct intel_ntb_dev *ndev = dev;
407

408
	return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
409 410
}

411 412 413
static int ndev_init_isr(struct intel_ntb_dev *ndev,
			 int msix_min, int msix_max,
			 int msix_shift, int total_shift)
414
{
415
	struct pci_dev *pdev;
416
	int rc, i, msix_count, node;
417

418
	pdev = ndev_pdev(ndev);
419

420 421
	node = dev_to_node(&pdev->dev);

422 423 424 425 426
	/* Mask all doorbell interrupts */
	ndev->db_mask = ndev->db_valid_mask;
	ndev->reg->db_iowrite(ndev->db_mask,
			      ndev->self_mmio +
			      ndev->self_reg->db_mask);
427

428 429
	/* Try to set up msix irq */

430 431
	ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
				 GFP_KERNEL, node);
432 433 434
	if (!ndev->vec)
		goto err_msix_vec_alloc;

435 436
	ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
				  GFP_KERNEL, node);
437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
	if (!ndev->msix)
		goto err_msix_alloc;

	for (i = 0; i < msix_max; ++i)
		ndev->msix[i].entry = i;

	msix_count = pci_enable_msix_range(pdev, ndev->msix,
					   msix_min, msix_max);
	if (msix_count < 0)
		goto err_msix_enable;

	for (i = 0; i < msix_count; ++i) {
		ndev->vec[i].ndev = ndev;
		ndev->vec[i].num = i;
		rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
				 "ndev_vec_isr", &ndev->vec[i]);
		if (rc)
			goto err_msix_request;
455 456
	}

457 458 459 460
	dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
	ndev->db_vec_count = msix_count;
	ndev->db_vec_shift = msix_shift;
	return 0;
461

462 463 464 465 466 467 468 469 470 471 472
err_msix_request:
	while (i-- > 0)
		free_irq(ndev->msix[i].vector, ndev);
	pci_disable_msix(pdev);
err_msix_enable:
	kfree(ndev->msix);
err_msix_alloc:
	kfree(ndev->vec);
err_msix_vec_alloc:
	ndev->msix = NULL;
	ndev->vec = NULL;
473

474
	/* Try to set up msi irq */
J
Jon Mason 已提交
475

476 477 478
	rc = pci_enable_msi(pdev);
	if (rc)
		goto err_msi_enable;
J
Jon Mason 已提交
479

480 481 482 483
	rc = request_irq(pdev->irq, ndev_irq_isr, 0,
			 "ndev_irq_isr", ndev);
	if (rc)
		goto err_msi_request;
J
Jon Mason 已提交
484

485 486 487 488
	dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
	ndev->db_vec_count = 1;
	ndev->db_vec_shift = total_shift;
	return 0;
J
Jon Mason 已提交
489

490 491 492
err_msi_request:
	pci_disable_msi(pdev);
err_msi_enable:
J
Jon Mason 已提交
493

494
	/* Try to set up intx irq */
J
Jon Mason 已提交
495

496
	pci_intx(pdev, 1);
J
Jon Mason 已提交
497

498 499 500 501 502 503 504 505 506 507 508 509
	rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
			 "ndev_irq_isr", ndev);
	if (rc)
		goto err_intx_request;

	dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
	ndev->db_vec_count = 1;
	ndev->db_vec_shift = total_shift;
	return 0;

err_intx_request:
	return rc;
J
Jon Mason 已提交
510 511
}

512
static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
513
{
514 515
	struct pci_dev *pdev;
	int i;
516

517
	pdev = ndev_pdev(ndev);
518

519 520 521 522 523
	/* Mask all doorbell interrupts */
	ndev->db_mask = ndev->db_valid_mask;
	ndev->reg->db_iowrite(ndev->db_mask,
			      ndev->self_mmio +
			      ndev->self_reg->db_mask);
J
Jon Mason 已提交
524

525 526 527 528 529 530 531
	if (ndev->msix) {
		i = ndev->db_vec_count;
		while (i--)
			free_irq(ndev->msix[i].vector, &ndev->vec[i]);
		pci_disable_msix(pdev);
		kfree(ndev->msix);
		kfree(ndev->vec);
532
	} else {
533 534 535
		free_irq(pdev->irq, ndev);
		if (pci_dev_msi_enabled(pdev))
			pci_disable_msi(pdev);
536 537 538
	}
}

539 540
static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
				 size_t count, loff_t *offp)
541
{
542 543 544 545 546 547
	struct intel_ntb_dev *ndev;
	void __iomem *mmio;
	char *buf;
	size_t buf_size;
	ssize_t ret, off;
	union { u64 v64; u32 v32; u16 v16; } u;
548

549 550
	ndev = filp->private_data;
	mmio = ndev->self_mmio;
551

552
	buf_size = min(count, 0x800ul);
553

554 555 556
	buf = kmalloc(buf_size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
557

558
	off = 0;
559

560 561
	off += scnprintf(buf + off, buf_size - off,
			 "NTB Device Information:\n");
562

563 564 565
	off += scnprintf(buf + off, buf_size - off,
			 "Connection Topology -\t%s\n",
			 ntb_topo_string(ndev->ntb.topo));
566

567 568 569 570 571 572 573
	off += scnprintf(buf + off, buf_size - off,
			 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
	off += scnprintf(buf + off, buf_size - off,
			 "B2B MW Idx -\t\t%d\n", ndev->b2b_idx);
	off += scnprintf(buf + off, buf_size - off,
			 "BAR4 Split -\t\t%s\n",
			 ndev->bar4_split ? "yes" : "no");
J
Jon Mason 已提交
574

575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
	off += scnprintf(buf + off, buf_size - off,
			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
	off += scnprintf(buf + off, buf_size - off,
			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);

	if (!ndev->reg->link_is_up(ndev)) {
		off += scnprintf(buf + off, buf_size - off,
				 "Link Status -\t\tDown\n");
	} else {
		off += scnprintf(buf + off, buf_size - off,
				 "Link Status -\t\tUp\n");
		off += scnprintf(buf + off, buf_size - off,
				 "Link Speed -\t\tPCI-E Gen %u\n",
				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
		off += scnprintf(buf + off, buf_size - off,
				 "Link Width -\t\tx%u\n",
				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
J
Jon Mason 已提交
592 593
	}

594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
	off += scnprintf(buf + off, buf_size - off,
			 "Memory Window Count -\t%u\n", ndev->mw_count);
	off += scnprintf(buf + off, buf_size - off,
			 "Scratchpad Count -\t%u\n", ndev->spad_count);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Count -\t%u\n", ndev->db_count);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);

	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);

	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Mask -\t\t%#llx\n", u.v64);

	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
	off += scnprintf(buf + off, buf_size - off,
			 "Doorbell Bell -\t\t%#llx\n", u.v64);

	off += scnprintf(buf + off, buf_size - off,
			 "\nNTB Incoming XLAT:\n");

	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
	off += scnprintf(buf + off, buf_size - off,
			 "XLAT23 -\t\t%#018llx\n", u.v64);

	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
	off += scnprintf(buf + off, buf_size - off,
			 "XLAT45 -\t\t%#018llx\n", u.v64);

	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
	off += scnprintf(buf + off, buf_size - off,
			 "LMT23 -\t\t\t%#018llx\n", u.v64);

	u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
	off += scnprintf(buf + off, buf_size - off,
			 "LMT45 -\t\t\t%#018llx\n", u.v64);

639
	if (pdev_is_xeon(ndev->ntb.pdev)) {
640 641 642 643
		if (ntb_topo_is_b2b(ndev->ntb.topo)) {
			off += scnprintf(buf + off, buf_size - off,
					 "\nNTB Outgoing B2B XLAT:\n");

644
			u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
645 646 647
			off += scnprintf(buf + off, buf_size - off,
					 "B2B XLAT23 -\t\t%#018llx\n", u.v64);

648
			u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
649 650 651
			off += scnprintf(buf + off, buf_size - off,
					 "B2B XLAT45 -\t\t%#018llx\n", u.v64);

652
			u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
653 654 655
			off += scnprintf(buf + off, buf_size - off,
					 "B2B LMT23 -\t\t%#018llx\n", u.v64);

656
			u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
657 658 659 660 661 662
			off += scnprintf(buf + off, buf_size - off,
					 "B2B LMT45 -\t\t%#018llx\n", u.v64);

			off += scnprintf(buf + off, buf_size - off,
					 "\nNTB Secondary BAR:\n");

663
			u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
664 665 666
			off += scnprintf(buf + off, buf_size - off,
					 "SBAR01 -\t\t%#018llx\n", u.v64);

667
			u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
668 669 670
			off += scnprintf(buf + off, buf_size - off,
					 "SBAR23 -\t\t%#018llx\n", u.v64);

671
			u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
672 673 674 675 676
			off += scnprintf(buf + off, buf_size - off,
					 "SBAR45 -\t\t%#018llx\n", u.v64);
		}

		off += scnprintf(buf + off, buf_size - off,
677
				 "\nXEON NTB Statistics:\n");
678

679
		u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
680 681 682 683
		off += scnprintf(buf + off, buf_size - off,
				 "Upstream Memory Miss -\t%u\n", u.v16);

		off += scnprintf(buf + off, buf_size - off,
684
				 "\nXEON NTB Hardware Errors:\n");
685 686

		if (!pci_read_config_word(ndev->ntb.pdev,
687
					  XEON_DEVSTS_OFFSET, &u.v16))
688 689 690 691
			off += scnprintf(buf + off, buf_size - off,
					 "DEVSTS -\t\t%#06x\n", u.v16);

		if (!pci_read_config_word(ndev->ntb.pdev,
692
					  XEON_LINK_STATUS_OFFSET, &u.v16))
693 694
			off += scnprintf(buf + off, buf_size - off,
					 "LNKSTS -\t\t%#06x\n", u.v16);
J
Jon Mason 已提交
695

696
		if (!pci_read_config_dword(ndev->ntb.pdev,
697
					   XEON_UNCERRSTS_OFFSET, &u.v32))
698 699 700 701
			off += scnprintf(buf + off, buf_size - off,
					 "UNCERRSTS -\t\t%#06x\n", u.v32);

		if (!pci_read_config_dword(ndev->ntb.pdev,
702
					   XEON_CORERRSTS_OFFSET, &u.v32))
703 704 705 706 707 708 709
			off += scnprintf(buf + off, buf_size - off,
					 "CORERRSTS -\t\t%#06x\n", u.v32);
	}

	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
	kfree(buf);
	return ret;
J
Jon Mason 已提交
710 711
}

712
static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
713
{
714 715 716 717 718 719 720 721 722 723 724 725 726
	if (!debugfs_dir) {
		ndev->debugfs_dir = NULL;
		ndev->debugfs_info = NULL;
	} else {
		ndev->debugfs_dir =
			debugfs_create_dir(ndev_name(ndev), debugfs_dir);
		if (!ndev->debugfs_dir)
			ndev->debugfs_info = NULL;
		else
			ndev->debugfs_info =
				debugfs_create_file("info", S_IRUSR,
						    ndev->debugfs_dir, ndev,
						    &intel_ntb_debugfs_info);
727
	}
728
}
729

730 731 732
static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
{
	debugfs_remove_recursive(ndev->debugfs_dir);
733 734
}

735
static int intel_ntb_mw_count(struct ntb_dev *ntb)
736
{
737 738
	return ntb_ndev(ntb)->mw_count;
}
J
Jon Mason 已提交
739

740 741 742 743 744 745 746 747
static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
				  phys_addr_t *base,
				  resource_size_t *size,
				  resource_size_t *align,
				  resource_size_t *align_size)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
	int bar;
J
Jon Mason 已提交
748

749 750
	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
		idx += 1;
751

752 753 754
	bar = ndev_mw_to_bar(ndev, idx);
	if (bar < 0)
		return bar;
J
Jon Mason 已提交
755

756 757 758
	if (base)
		*base = pci_resource_start(ndev->ntb.pdev, bar) +
			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
759

760 761 762
	if (size)
		*size = pci_resource_len(ndev->ntb.pdev, bar) -
			(idx == ndev->b2b_idx ? ndev->b2b_off : 0);
763

764 765
	if (align)
		*align = pci_resource_len(ndev->ntb.pdev, bar);
J
Jon Mason 已提交
766

767 768
	if (align_size)
		*align_size = 1;
769 770 771 772

	return 0;
}

773 774
static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
				  dma_addr_t addr, resource_size_t size)
775
{
776 777 778 779 780 781
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
	unsigned long base_reg, xlat_reg, limit_reg;
	resource_size_t bar_size, mw_size;
	void __iomem *mmio;
	u64 base, limit, reg_val;
	int bar;
782

783 784
	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
		idx += 1;
785

786 787 788
	bar = ndev_mw_to_bar(ndev, idx);
	if (bar < 0)
		return bar;
789

790 791 792 793 794 795 796 797 798
	bar_size = pci_resource_len(ndev->ntb.pdev, bar);

	if (idx == ndev->b2b_idx)
		mw_size = bar_size - ndev->b2b_off;
	else
		mw_size = bar_size;

	/* hardware requires that addr is aligned to bar size */
	if (addr & (bar_size - 1))
799
		return -EINVAL;
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865

	/* make sure the range fits in the usable mw size */
	if (size > mw_size)
		return -EINVAL;

	mmio = ndev->self_mmio;
	base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
	xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
	limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);

	if (bar < 4 || !ndev->bar4_split) {
		base = ioread64(mmio + base_reg);

		/* Set the limit if supported, if size is not mw_size */
		if (limit_reg && size != mw_size)
			limit = base + size;
		else
			limit = 0;

		/* set and verify setting the translation address */
		iowrite64(addr, mmio + xlat_reg);
		reg_val = ioread64(mmio + xlat_reg);
		if (reg_val != addr) {
			iowrite64(0, mmio + xlat_reg);
			return -EIO;
		}

		/* set and verify setting the limit */
		iowrite64(limit, mmio + limit_reg);
		reg_val = ioread64(mmio + limit_reg);
		if (reg_val != limit) {
			iowrite64(base, mmio + limit_reg);
			iowrite64(0, mmio + xlat_reg);
			return -EIO;
		}
	} else {
		/* split bar addr range must all be 32 bit */
		if (addr & (~0ull << 32))
			return -EINVAL;
		if ((addr + size) & (~0ull << 32))
			return -EINVAL;

		base = ioread32(mmio + base_reg);

		/* Set the limit if supported, if size is not mw_size */
		if (limit_reg && size != mw_size)
			limit = base + size;
		else
			limit = 0;

		/* set and verify setting the translation address */
		iowrite32(addr, mmio + xlat_reg);
		reg_val = ioread32(mmio + xlat_reg);
		if (reg_val != addr) {
			iowrite32(0, mmio + xlat_reg);
			return -EIO;
		}

		/* set and verify setting the limit */
		iowrite32(limit, mmio + limit_reg);
		reg_val = ioread32(mmio + limit_reg);
		if (reg_val != limit) {
			iowrite32(base, mmio + limit_reg);
			iowrite32(0, mmio + xlat_reg);
			return -EIO;
		}
866 867
	}

868 869
	return 0;
}
870

871 872 873 874 875
static int intel_ntb_link_is_up(struct ntb_dev *ntb,
				enum ntb_speed *speed,
				enum ntb_width *width)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
876

877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	if (ndev->reg->link_is_up(ndev)) {
		if (speed)
			*speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
		if (width)
			*width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
		return 1;
	} else {
		/* TODO MAYBE: is it possible to observe the link speed and
		 * width while link is training? */
		if (speed)
			*speed = NTB_SPEED_NONE;
		if (width)
			*width = NTB_WIDTH_NONE;
		return 0;
	}
}

static int intel_ntb_link_enable(struct ntb_dev *ntb,
				 enum ntb_speed max_speed,
				 enum ntb_width max_width)
{
	struct intel_ntb_dev *ndev;
	u32 ntb_ctl;

	ndev = container_of(ntb, struct intel_ntb_dev, ntb);

	if (ndev->ntb.topo == NTB_TOPO_SEC)
		return -EINVAL;

	dev_dbg(ndev_dev(ndev),
		"Enabling link with max_speed %d max_width %d\n",
		max_speed, max_width);
	if (max_speed != NTB_SPEED_AUTO)
		dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
	if (max_width != NTB_WIDTH_AUTO)
		dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);

	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
	ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
	ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
	ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
	if (ndev->bar4_split)
		ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
921 922 923 924

	return 0;
}

925
static int intel_ntb_link_disable(struct ntb_dev *ntb)
926
{
927 928
	struct intel_ntb_dev *ndev;
	u32 ntb_cntl;
929

930
	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
931

932 933
	if (ndev->ntb.topo == NTB_TOPO_SEC)
		return -EINVAL;
934

935 936 937 938 939 940 941 942 943 944
	dev_dbg(ndev_dev(ndev), "Disabling link\n");

	/* Bring NTB link down */
	ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
	ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
	ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
	if (ndev->bar4_split)
		ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
	ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
	iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
945

946
	return 0;
947 948
}

949
static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
950
{
951
	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
952 953
}

954
static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
955
{
956 957
	return ntb_ndev(ntb)->db_valid_mask;
}
958

959 960 961
static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
{
	struct intel_ntb_dev *ndev;
962

963
	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
964

965 966
	return ndev->db_vec_count;
}
967

968 969 970
static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
971

972 973
	if (db_vector < 0 || db_vector > ndev->db_vec_count)
		return 0;
974

975
	return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
976 977
}

978
static u64 intel_ntb_db_read(struct ntb_dev *ntb)
979
{
980
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
981

982 983 984 985
	return ndev_db_read(ndev,
			    ndev->self_mmio +
			    ndev->self_reg->db_bell);
}
986

987 988 989
static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
990

991 992 993 994
	return ndev_db_write(ndev, db_bits,
			     ndev->self_mmio +
			     ndev->self_reg->db_bell);
}
995

996 997 998
static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
999

1000 1001 1002
	return ndev_db_set_mask(ndev, db_bits,
				ndev->self_mmio +
				ndev->self_reg->db_mask);
1003 1004
}

1005
static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1006
{
1007
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1008

1009 1010 1011 1012
	return ndev_db_clear_mask(ndev, db_bits,
				  ndev->self_mmio +
				  ndev->self_reg->db_mask);
}
1013

1014 1015 1016 1017 1018
static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
				  phys_addr_t *db_addr,
				  resource_size_t *db_size)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1019

1020 1021
	return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
			    ndev->peer_reg->db_bell);
1022 1023
}

1024
static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1025
{
1026
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1027

1028 1029 1030 1031
	return ndev_db_write(ndev, db_bits,
			     ndev->peer_mmio +
			     ndev->peer_reg->db_bell);
}
1032

1033 1034 1035 1036
static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
{
	return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
}
1037

1038 1039 1040
static int intel_ntb_spad_count(struct ntb_dev *ntb)
{
	struct intel_ntb_dev *ndev;
1041

1042
	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1043

1044 1045
	return ndev->spad_count;
}
1046

1047 1048 1049
static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1050

1051 1052 1053
	return ndev_spad_read(ndev, idx,
			      ndev->self_mmio +
			      ndev->self_reg->spad);
1054 1055
}

1056 1057
static int intel_ntb_spad_write(struct ntb_dev *ntb,
				int idx, u32 val)
1058
{
1059
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1060

1061 1062 1063 1064
	return ndev_spad_write(ndev, idx, val,
			       ndev->self_mmio +
			       ndev->self_reg->spad);
}
1065

1066 1067 1068 1069
static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
				    phys_addr_t *spad_addr)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1070

1071 1072 1073
	return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
			      ndev->peer_reg->spad);
}
1074

1075 1076 1077
static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1078

1079 1080 1081 1082
	return ndev_spad_read(ndev, idx,
			      ndev->peer_mmio +
			      ndev->peer_reg->spad);
}
1083

1084 1085 1086 1087
static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
				     int idx, u32 val)
{
	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1088

1089 1090 1091 1092
	return ndev_spad_write(ndev, idx, val,
			       ndev->peer_mmio +
			       ndev->peer_reg->spad);
}
1093

1094
/* ATOM */
1095

1096
static u64 atom_db_ioread(void __iomem *mmio)
1097 1098 1099 1100
{
	return ioread64(mmio);
}

1101
static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1102 1103
{
	iowrite64(bits, mmio);
1104 1105
}

1106
static int atom_poll_link(struct intel_ntb_dev *ndev)
1107
{
1108
	u32 ntb_ctl;
1109

1110
	ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1111

1112 1113
	if (ntb_ctl == ndev->ntb_ctl)
		return 0;
1114

1115
	ndev->ntb_ctl = ntb_ctl;
1116

1117
	ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1118

1119 1120
	return 1;
}
1121

1122
static int atom_link_is_up(struct intel_ntb_dev *ndev)
1123
{
1124
	return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1125
}
1126

1127
static int atom_link_is_err(struct intel_ntb_dev *ndev)
1128
{
1129 1130
	if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
	    & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1131
		return 1;
1132

1133 1134
	if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
	    & ATOM_IBIST_ERR_OFLOW)
1135 1136 1137
		return 1;

	return 0;
1138 1139
}

1140
static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1141
{
1142 1143
	switch (ppd & ATOM_PPD_TOPO_MASK) {
	case ATOM_PPD_TOPO_B2B_USD:
1144 1145 1146
		dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
		return NTB_TOPO_B2B_USD;

1147
	case ATOM_PPD_TOPO_B2B_DSD:
1148 1149 1150
		dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
		return NTB_TOPO_B2B_DSD;

1151 1152 1153 1154
	case ATOM_PPD_TOPO_PRI_USD:
	case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
	case ATOM_PPD_TOPO_SEC_USD:
	case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1155 1156 1157
		dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
		return NTB_TOPO_NONE;
	}
1158

1159 1160 1161 1162
	dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
	return NTB_TOPO_NONE;
}

1163
static void atom_link_hb(struct work_struct *work)
1164 1165 1166 1167 1168 1169
{
	struct intel_ntb_dev *ndev = hb_ndev(work);
	unsigned long poll_ts;
	void __iomem *mmio;
	u32 status32;

1170
	poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1171 1172 1173 1174

	/* Delay polling the link status if an interrupt was received,
	 * unless the cached link status says the link is down.
	 */
1175
	if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1176 1177
		schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
		return;
1178 1179
	}

1180
	if (atom_poll_link(ndev))
1181 1182
		ntb_link_event(&ndev->ntb);

1183 1184
	if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
		schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1185
		return;
1186 1187
	}

1188
	/* Link is down with error: recover the link! */
1189

1190
	mmio = ndev->self_mmio;
1191

1192
	/* Driver resets the NTB ModPhy lanes - magic! */
1193 1194 1195 1196
	iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
	iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
	iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
	iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1197

1198 1199 1200 1201
	/* Driver waits 100ms to allow the NTB ModPhy to settle */
	msleep(100);

	/* Clear AER Errors, write to clear */
1202
	status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1203 1204
	dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
	status32 &= PCI_ERR_COR_REP_ROLL;
1205
	iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1206 1207

	/* Clear unexpected electrical idle event in LTSSM, write to clear */
1208
	status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1209
	dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1210 1211
	status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
	iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1212 1213

	/* Clear DeSkew Buffer error, write to clear */
1214
	status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1215
	dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1216 1217
	status32 |= ATOM_DESKEWSTS_DBERR;
	iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1218

1219
	status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1220
	dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1221 1222
	status32 &= ATOM_IBIST_ERR_OFLOW;
	iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1223 1224

	/* Releases the NTB state machine to allow the link to retrain */
1225
	status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1226
	dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1227 1228
	status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
	iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1229 1230 1231 1232 1233 1234

	/* There is a potential race between the 2 NTB devices recovering at the
	 * same time.  If the times are the same, the link will not recover and
	 * the driver will be stuck in this loop forever.  Add a random interval
	 * to the recovery time to prevent this race.
	 */
1235 1236
	schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
			      + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1237 1238
}

1239
static int atom_init_isr(struct intel_ntb_dev *ndev)
1240 1241 1242
{
	int rc;

1243 1244
	rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
			   ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1245 1246 1247
	if (rc)
		return rc;

1248
	/* ATOM doesn't have link status interrupt, poll on that platform */
1249
	ndev->last_ts = jiffies;
1250 1251
	INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
	schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1252 1253 1254 1255

	return 0;
}

1256
static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1257
{
1258 1259 1260
	cancel_delayed_work_sync(&ndev->hb_timer);
	ndev_deinit_isr(ndev);
}
1261

1262
static int atom_init_ntb(struct intel_ntb_dev *ndev)
1263
{
1264 1265 1266
	ndev->mw_count = ATOM_MW_COUNT;
	ndev->spad_count = ATOM_SPAD_COUNT;
	ndev->db_count = ATOM_DB_COUNT;
1267

1268 1269 1270
	switch (ndev->ntb.topo) {
	case NTB_TOPO_B2B_USD:
	case NTB_TOPO_B2B_DSD:
1271 1272 1273
		ndev->self_reg = &atom_pri_reg;
		ndev->peer_reg = &atom_b2b_reg;
		ndev->xlat_reg = &atom_sec_xlat;
1274 1275 1276

		/* Enable Bus Master and Memory Space on the secondary side */
		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1277
			  ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1278 1279 1280 1281 1282 1283 1284 1285

		break;

	default:
		return -EINVAL;
	}

	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1286 1287 1288 1289

	return 0;
}

1290
static int atom_init_dev(struct intel_ntb_dev *ndev)
1291
{
1292
	u32 ppd;
1293 1294
	int rc;

1295
	rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1296 1297
	if (rc)
		return -EIO;
1298

1299
	ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1300 1301
	if (ndev->ntb.topo == NTB_TOPO_NONE)
		return -EINVAL;
1302

1303
	rc = atom_init_ntb(ndev);
1304 1305
	if (rc)
		return rc;
1306

1307
	rc = atom_init_isr(ndev);
1308
	if (rc)
1309
		return rc;
1310 1311 1312

	if (ndev->ntb.topo != NTB_TOPO_SEC) {
		/* Initiate PCI-E link training */
1313 1314
		rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
					    ppd | ATOM_PPD_INIT_LINK);
1315 1316
		if (rc)
			return rc;
1317 1318 1319 1320 1321
	}

	return 0;
}

1322
static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1323
{
1324
	atom_deinit_isr(ndev);
1325
}
1326

1327
/* XEON */
1328

1329
static u64 xeon_db_ioread(void __iomem *mmio)
1330 1331 1332
{
	return (u64)ioread16(mmio);
}
1333

1334
static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1335 1336 1337
{
	iowrite16((u16)bits, mmio);
}
1338

1339
static int xeon_poll_link(struct intel_ntb_dev *ndev)
1340 1341 1342 1343 1344 1345 1346 1347 1348
{
	u16 reg_val;
	int rc;

	ndev->reg->db_iowrite(ndev->db_link_mask,
			      ndev->self_mmio +
			      ndev->self_reg->db_bell);

	rc = pci_read_config_word(ndev->ntb.pdev,
1349
				  XEON_LINK_STATUS_OFFSET, &reg_val);
1350 1351 1352 1353 1354 1355 1356 1357 1358
	if (rc)
		return 0;

	if (reg_val == ndev->lnk_sta)
		return 0;

	ndev->lnk_sta = reg_val;

	return 1;
1359 1360
}

1361
static int xeon_link_is_up(struct intel_ntb_dev *ndev)
1362
{
1363 1364 1365
	if (ndev->ntb.topo == NTB_TOPO_SEC)
		return 1;

1366 1367
	return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
}
1368

1369
static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1370
{
1371 1372
	switch (ppd & XEON_PPD_TOPO_MASK) {
	case XEON_PPD_TOPO_B2B_USD:
1373 1374
		return NTB_TOPO_B2B_USD;

1375
	case XEON_PPD_TOPO_B2B_DSD:
1376 1377
		return NTB_TOPO_B2B_DSD;

1378 1379
	case XEON_PPD_TOPO_PRI_USD:
	case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1380
		return NTB_TOPO_PRI;
1381

1382 1383
	case XEON_PPD_TOPO_SEC_USD:
	case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1384
		return NTB_TOPO_SEC;
1385 1386
	}

1387
	return NTB_TOPO_NONE;
1388 1389
}

1390
static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1391
{
1392
	if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1393 1394 1395 1396 1397
		dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
		return 1;
	}
	return 0;
}
1398

1399
static int xeon_init_isr(struct intel_ntb_dev *ndev)
1400
{
1401 1402 1403 1404
	return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
			     XEON_DB_MSIX_VECTOR_COUNT,
			     XEON_DB_MSIX_VECTOR_SHIFT,
			     XEON_DB_TOTAL_SHIFT);
1405
}
1406

1407
static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1408 1409
{
	ndev_deinit_isr(ndev);
1410 1411
}

1412 1413 1414
static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
			     const struct intel_b2b_addr *addr,
			     const struct intel_b2b_addr *peer_addr)
J
Jon Mason 已提交
1415
{
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
	struct pci_dev *pdev;
	void __iomem *mmio;
	resource_size_t bar_size;
	phys_addr_t bar_addr;
	int b2b_bar;
	u8 bar_sz;

	pdev = ndev_pdev(ndev);
	mmio = ndev->self_mmio;

	if (ndev->b2b_idx >= ndev->mw_count) {
		dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
		b2b_bar = 0;
		ndev->b2b_off = 0;
	} else {
		b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
		if (b2b_bar < 0)
			return -EIO;
J
Jon Mason 已提交
1434

1435
		dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
J
Jon Mason 已提交
1436

1437
		bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
J
Jon Mason 已提交
1438

1439
		dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
J
Jon Mason 已提交
1440

1441
		if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1442 1443 1444
			dev_dbg(ndev_dev(ndev),
				"b2b using first half of bar\n");
			ndev->b2b_off = bar_size >> 1;
1445
		} else if (XEON_B2B_MIN_SIZE <= bar_size) {
1446 1447 1448 1449 1450 1451 1452 1453 1454
			dev_dbg(ndev_dev(ndev),
				"b2b using whole bar\n");
			ndev->b2b_off = 0;
			--ndev->mw_count;
		} else {
			dev_dbg(ndev_dev(ndev),
				"b2b bar size is too small\n");
			return -EIO;
		}
J
Jon Mason 已提交
1455 1456
	}

1457 1458 1459 1460 1461 1462
	/* Reset the secondary bar sizes to match the primary bar sizes,
	 * except disable or halve the size of the b2b secondary bar.
	 *
	 * Note: code for each specific bar size register, because the register
	 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
	 */
1463
	pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1464 1465 1466 1467 1468 1469 1470
	dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
	if (b2b_bar == 2) {
		if (ndev->b2b_off)
			bar_sz -= 1;
		else
			bar_sz = 0;
	}
1471 1472
	pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
	pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1473 1474 1475
	dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);

	if (!ndev->bar4_split) {
1476
		pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1477 1478 1479 1480 1481 1482 1483
		dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
		if (b2b_bar == 4) {
			if (ndev->b2b_off)
				bar_sz -= 1;
			else
				bar_sz = 0;
		}
1484 1485
		pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
		pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1486 1487
		dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
	} else {
1488
		pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1489 1490 1491 1492 1493 1494 1495
		dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
		if (b2b_bar == 4) {
			if (ndev->b2b_off)
				bar_sz -= 1;
			else
				bar_sz = 0;
		}
1496 1497
		pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
		pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1498 1499
		dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);

1500
		pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1501 1502 1503 1504 1505 1506 1507
		dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
		if (b2b_bar == 5) {
			if (ndev->b2b_off)
				bar_sz -= 1;
			else
				bar_sz = 0;
		}
1508 1509
		pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
		pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1510 1511
		dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
	}
J
Jon Mason 已提交
1512

1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
	/* SBAR01 hit by first part of the b2b bar */
	if (b2b_bar == 0)
		bar_addr = addr->bar0_addr;
	else if (b2b_bar == 2)
		bar_addr = addr->bar2_addr64;
	else if (b2b_bar == 4 && !ndev->bar4_split)
		bar_addr = addr->bar4_addr64;
	else if (b2b_bar == 4)
		bar_addr = addr->bar4_addr32;
	else if (b2b_bar == 5)
		bar_addr = addr->bar5_addr32;
	else
		return -EIO;
J
Jon Mason 已提交
1526

1527
	dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
1528
	iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
J
Jon Mason 已提交
1529

1530 1531 1532 1533
	/* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
	 * The b2b bar is either disabled above, or configured half-size, and
	 * it starts at the PBAR xlat + offset.
	 */
1534

1535
	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1536 1537
	iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
	bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1538 1539 1540 1541 1542
	dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);

	if (!ndev->bar4_split) {
		bar_addr = addr->bar4_addr64 +
			(b2b_bar == 4 ? ndev->b2b_off : 0);
1543 1544
		iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
		bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1545 1546 1547 1548
		dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
	} else {
		bar_addr = addr->bar4_addr32 +
			(b2b_bar == 4 ? ndev->b2b_off : 0);
1549 1550
		iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
		bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1551 1552 1553 1554
		dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);

		bar_addr = addr->bar5_addr32 +
			(b2b_bar == 5 ? ndev->b2b_off : 0);
1555 1556
		iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
		bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1557 1558
		dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
	}
1559

1560
	/* setup incoming bar limits == base addrs (zero length windows) */
1561

1562
	bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1563 1564
	iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
	bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1565
	dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
1566

1567 1568 1569
	if (!ndev->bar4_split) {
		bar_addr = addr->bar4_addr64 +
			(b2b_bar == 4 ? ndev->b2b_off : 0);
1570 1571
		iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
		bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1572 1573 1574 1575
		dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
	} else {
		bar_addr = addr->bar4_addr32 +
			(b2b_bar == 4 ? ndev->b2b_off : 0);
1576 1577
		iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
		bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1578 1579 1580 1581
		dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);

		bar_addr = addr->bar5_addr32 +
			(b2b_bar == 5 ? ndev->b2b_off : 0);
1582 1583
		iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
		bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1584
		dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
1585 1586
	}

1587
	/* zero incoming translation addrs */
1588
	iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
J
Jon Mason 已提交
1589

1590
	if (!ndev->bar4_split) {
1591
		iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1592
	} else {
1593 1594
		iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
		iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1595
	}
1596

1597
	/* zero outgoing translation limits (whole bar size windows) */
1598
	iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1599
	if (!ndev->bar4_split) {
1600
		iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1601
	} else {
1602 1603
		iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
		iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
J
Jon Mason 已提交
1604
	}
J
Jon Mason 已提交
1605

1606 1607
	/* set outgoing translation offsets */
	bar_addr = peer_addr->bar2_addr64;
1608 1609
	iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
	bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1610 1611 1612 1613
	dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);

	if (!ndev->bar4_split) {
		bar_addr = peer_addr->bar4_addr64;
1614 1615
		iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
		bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1616 1617 1618
		dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
	} else {
		bar_addr = peer_addr->bar4_addr32;
1619 1620
		iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
		bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1621 1622 1623
		dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);

		bar_addr = peer_addr->bar5_addr32;
1624 1625
		iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
		bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1626 1627
		dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
	}
J
Jon Mason 已提交
1628

1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
	/* set the translation offset for b2b registers */
	if (b2b_bar == 0)
		bar_addr = peer_addr->bar0_addr;
	else if (b2b_bar == 2)
		bar_addr = peer_addr->bar2_addr64;
	else if (b2b_bar == 4 && !ndev->bar4_split)
		bar_addr = peer_addr->bar4_addr64;
	else if (b2b_bar == 4)
		bar_addr = peer_addr->bar4_addr32;
	else if (b2b_bar == 5)
		bar_addr = peer_addr->bar5_addr32;
	else
		return -EIO;

	/* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
	dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
1645 1646
	iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
	iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1647 1648 1649 1650

	if (b2b_bar) {
		/* map peer ntb mmio config space registers */
		ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1651
					    XEON_B2B_MIN_SIZE);
1652 1653
		if (!ndev->peer_mmio)
			return -EIO;
J
Jon Mason 已提交
1654 1655
	}

1656
	return 0;
J
Jon Mason 已提交
1657 1658
}

1659
static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1660
{
1661
	int rc;
1662
	u32 ntb_ctl;
1663 1664 1665

	if (ndev->bar4_split)
		ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1666
	else
1667
		ndev->mw_count = XEON_MW_COUNT;
1668

1669 1670 1671
	ndev->spad_count = XEON_SPAD_COUNT;
	ndev->db_count = XEON_DB_COUNT;
	ndev->db_link_mask = XEON_DB_LINK_BIT;
1672

1673 1674 1675 1676 1677 1678
	switch (ndev->ntb.topo) {
	case NTB_TOPO_PRI:
		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
			dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
			return -EINVAL;
		}
1679 1680 1681 1682 1683 1684

		/* enable link to allow secondary side device to appear */
		ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
		ntb_ctl &= ~NTB_CTL_DISABLE;
		iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);

1685 1686
		/* use half the spads for the peer */
		ndev->spad_count >>= 1;
1687 1688 1689
		ndev->self_reg = &xeon_pri_reg;
		ndev->peer_reg = &xeon_sec_reg;
		ndev->xlat_reg = &xeon_sec_xlat;
1690
		break;
1691

1692 1693 1694 1695 1696 1697 1698
	case NTB_TOPO_SEC:
		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
			dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
			return -EINVAL;
		}
		/* use half the spads for the peer */
		ndev->spad_count >>= 1;
1699 1700 1701
		ndev->self_reg = &xeon_sec_reg;
		ndev->peer_reg = &xeon_pri_reg;
		ndev->xlat_reg = &xeon_pri_xlat;
1702
		break;
1703

1704 1705
	case NTB_TOPO_B2B_USD:
	case NTB_TOPO_B2B_DSD:
1706 1707 1708
		ndev->self_reg = &xeon_pri_reg;
		ndev->peer_reg = &xeon_b2b_reg;
		ndev->xlat_reg = &xeon_sec_xlat;
1709

1710
		if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1711
			ndev->peer_reg = &xeon_pri_reg;
1712

1713 1714 1715 1716
			if (b2b_mw_idx < 0)
				ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
			else
				ndev->b2b_idx = b2b_mw_idx;
1717

1718 1719 1720 1721 1722 1723 1724 1725 1726 1727
			dev_dbg(ndev_dev(ndev),
				"setting up b2b mw idx %d means %d\n",
				b2b_mw_idx, ndev->b2b_idx);

		} else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
			dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
			ndev->db_count -= 1;
		}

		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1728 1729 1730
			rc = xeon_setup_b2b_mw(ndev,
					       &xeon_b2b_dsd_addr,
					       &xeon_b2b_usd_addr);
1731
		} else {
1732 1733 1734
			rc = xeon_setup_b2b_mw(ndev,
					       &xeon_b2b_usd_addr,
					       &xeon_b2b_dsd_addr);
1735 1736 1737 1738 1739 1740
		}
		if (rc)
			return rc;

		/* Enable Bus Master and Memory Space on the secondary side */
		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1741
			  ndev->self_mmio + XEON_SPCICMD_OFFSET);
1742

1743
		break;
1744

1745
	default:
1746
		return -EINVAL;
1747 1748
	}

1749 1750 1751 1752 1753
	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;

	ndev->reg->db_iowrite(ndev->db_valid_mask,
			      ndev->self_mmio +
			      ndev->self_reg->db_mask);
1754

1755 1756 1757
	return 0;
}

1758
static int xeon_init_dev(struct intel_ntb_dev *ndev)
1759
{
1760 1761 1762 1763
	struct pci_dev *pdev;
	u8 ppd;
	int rc, mem;

1764 1765 1766
	pdev = ndev_pdev(ndev);

	switch (pdev->device) {
1767 1768 1769 1770 1771 1772
	/* There is a Xeon hardware errata related to writes to SDOORBELL or
	 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
	 * which may hang the system.  To workaround this use the second memory
	 * window to access the interrupt and scratch pad registers on the
	 * remote system.
	 */
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
		ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
		break;
	}
1788

1789
	switch (pdev->device) {
1790 1791 1792
	/* There is a hardware errata related to accessing any register in
	 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
	 */
1793 1794 1795 1796 1797 1798 1799 1800 1801
	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
		ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
		break;
	}
1802

1803
	switch (pdev->device) {
1804 1805 1806 1807
	/* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
	 * mirrored to the remote system.  Shrink the number of bits by one,
	 * since bit 14 is the last bit.
	 */
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
	case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
	case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
	case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
	case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
	case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
		ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
		break;
	}
1823

1824
	ndev->reg = &xeon_reg;
1825

1826
	rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1827
	if (rc)
1828
		return -EIO;
1829

1830
	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1831 1832 1833
	dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
		ntb_topo_string(ndev->ntb.topo));
	if (ndev->ntb.topo == NTB_TOPO_NONE)
1834
		return -EINVAL;
1835 1836

	if (ndev->ntb.topo != NTB_TOPO_SEC) {
1837
		ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
		dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
			ppd, ndev->bar4_split);
	} else {
		/* This is a way for transparent BAR to figure out if we are
		 * doing split BAR or not. There is no way for the hw on the
		 * transparent side to know and set the PPD.
		 */
		mem = pci_select_bars(pdev, IORESOURCE_MEM);
		ndev->bar4_split = hweight32(mem) ==
			HSX_SPLIT_BAR_MW_COUNT + 1;
		dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
			mem, ndev->bar4_split);
1850 1851
	}

1852
	rc = xeon_init_ntb(ndev);
1853 1854
	if (rc)
		return rc;
1855

1856
	return xeon_init_isr(ndev);
1857 1858
}

1859
static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1860
{
1861
	xeon_deinit_isr(ndev);
1862 1863
}

1864
static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1865 1866 1867
{
	int rc;

1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
	pci_set_drvdata(pdev, ndev);

	rc = pci_enable_device(pdev);
	if (rc)
		goto err_pci_enable;

	rc = pci_request_regions(pdev, NTB_NAME);
	if (rc)
		goto err_pci_regions;

	pci_set_master(pdev);

	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
	if (rc) {
		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (rc)
			goto err_dma_mask;
		dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
	}
1887

1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	if (rc) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
		if (rc)
			goto err_dma_mask;
		dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
	}

	ndev->self_mmio = pci_iomap(pdev, 0, 0);
	if (!ndev->self_mmio) {
		rc = -EIO;
		goto err_mmio;
	}
	ndev->peer_mmio = ndev->self_mmio;
1902 1903

	return 0;
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913

err_mmio:
err_dma_mask:
	pci_clear_master(pdev);
	pci_release_regions(pdev);
err_pci_regions:
	pci_disable_device(pdev);
err_pci_enable:
	pci_set_drvdata(pdev, NULL);
	return rc;
1914 1915
}

1916
static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1917
{
1918
	struct pci_dev *pdev = ndev_pdev(ndev);
1919

1920 1921 1922
	if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
		pci_iounmap(pdev, ndev->peer_mmio);
	pci_iounmap(pdev, ndev->self_mmio);
1923

1924 1925 1926 1927 1928
	pci_clear_master(pdev);
	pci_release_regions(pdev);
	pci_disable_device(pdev);
	pci_set_drvdata(pdev, NULL);
}
1929

1930 1931 1932 1933 1934 1935
static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
				    struct pci_dev *pdev)
{
	ndev->ntb.pdev = pdev;
	ndev->ntb.topo = NTB_TOPO_NONE;
	ndev->ntb.ops = &intel_ntb_ops;
1936

1937 1938
	ndev->b2b_off = 0;
	ndev->b2b_idx = INT_MAX;
1939

1940
	ndev->bar4_split = 0;
1941

1942 1943 1944 1945 1946
	ndev->mw_count = 0;
	ndev->spad_count = 0;
	ndev->db_count = 0;
	ndev->db_vec_count = 0;
	ndev->db_vec_shift = 0;
1947

1948 1949
	ndev->ntb_ctl = 0;
	ndev->lnk_sta = 0;
1950

1951 1952 1953
	ndev->db_valid_mask = 0;
	ndev->db_link_mask = 0;
	ndev->db_mask = 0;
1954

1955 1956
	spin_lock_init(&ndev->db_mask_lock);
}
1957

1958 1959 1960 1961
static int intel_ntb_pci_probe(struct pci_dev *pdev,
			       const struct pci_device_id *id)
{
	struct intel_ntb_dev *ndev;
1962 1963 1964
	int rc, node;

	node = dev_to_node(&pdev->dev);
1965

1966
	if (pdev_is_atom(pdev)) {
1967
		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1968 1969 1970 1971
		if (!ndev) {
			rc = -ENOMEM;
			goto err_ndev;
		}
1972

1973
		ndev_init_struct(ndev, pdev);
1974

1975 1976 1977 1978
		rc = intel_ntb_init_pci(ndev, pdev);
		if (rc)
			goto err_init_pci;

1979
		rc = atom_init_dev(ndev);
1980 1981
		if (rc)
			goto err_init_dev;
1982

1983
	} else if (pdev_is_xeon(pdev)) {
1984
		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1985 1986 1987
		if (!ndev) {
			rc = -ENOMEM;
			goto err_ndev;
1988 1989
		}

1990
		ndev_init_struct(ndev, pdev);
1991

1992 1993 1994
		rc = intel_ntb_init_pci(ndev, pdev);
		if (rc)
			goto err_init_pci;
1995

1996
		rc = xeon_init_dev(ndev);
1997
		if (rc)
1998
			goto err_init_dev;
1999

2000 2001 2002
	} else {
		rc = -EINVAL;
		goto err_ndev;
2003 2004
	}

2005
	ndev_reset_unsafe_flags(ndev);
2006

2007
	ndev->reg->poll_link(ndev);
2008

2009
	ndev_init_debugfs(ndev);
2010

2011
	rc = ntb_register_device(&ndev->ntb);
2012
	if (rc)
2013
		goto err_register;
2014 2015 2016

	return 0;

2017 2018
err_register:
	ndev_deinit_debugfs(ndev);
2019 2020 2021 2022
	if (pdev_is_atom(pdev))
		atom_deinit_dev(ndev);
	else if (pdev_is_xeon(pdev))
		xeon_deinit_dev(ndev);
2023 2024 2025
err_init_dev:
	intel_ntb_deinit_pci(ndev);
err_init_pci:
2026
	kfree(ndev);
2027
err_ndev:
2028 2029 2030
	return rc;
}

2031
static void intel_ntb_pci_remove(struct pci_dev *pdev)
2032
{
2033 2034 2035 2036
	struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);

	ntb_unregister_device(&ndev->ntb);
	ndev_deinit_debugfs(ndev);
2037 2038 2039 2040
	if (pdev_is_atom(pdev))
		atom_deinit_dev(ndev);
	else if (pdev_is_xeon(pdev))
		xeon_deinit_dev(ndev);
2041 2042 2043
	intel_ntb_deinit_pci(ndev);
	kfree(ndev);
}
2044

2045 2046 2047 2048 2049
static const struct intel_ntb_reg atom_reg = {
	.poll_link		= atom_poll_link,
	.link_is_up		= atom_link_is_up,
	.db_ioread		= atom_db_ioread,
	.db_iowrite		= atom_db_iowrite,
2050
	.db_size		= sizeof(u64),
2051
	.ntb_ctl		= ATOM_NTBCNTL_OFFSET,
2052 2053
	.mw_bar			= {2, 4},
};
2054

2055 2056 2057 2058
static const struct intel_ntb_alt_reg atom_pri_reg = {
	.db_bell		= ATOM_PDOORBELL_OFFSET,
	.db_mask		= ATOM_PDBMSK_OFFSET,
	.spad			= ATOM_SPAD_OFFSET,
2059
};
2060

2061 2062 2063
static const struct intel_ntb_alt_reg atom_b2b_reg = {
	.db_bell		= ATOM_B2B_DOORBELL_OFFSET,
	.spad			= ATOM_B2B_SPAD_OFFSET,
2064
};
2065

2066 2067 2068 2069
static const struct intel_ntb_xlat_reg atom_sec_xlat = {
	/* FIXME : .bar0_base	= ATOM_SBAR0BASE_OFFSET, */
	/* FIXME : .bar2_limit	= ATOM_SBAR2LMT_OFFSET, */
	.bar2_xlat		= ATOM_SBAR2XLAT_OFFSET,
2070
};
2071

2072 2073 2074 2075 2076
static const struct intel_ntb_reg xeon_reg = {
	.poll_link		= xeon_poll_link,
	.link_is_up		= xeon_link_is_up,
	.db_ioread		= xeon_db_ioread,
	.db_iowrite		= xeon_db_iowrite,
2077
	.db_size		= sizeof(u32),
2078
	.ntb_ctl		= XEON_NTBCNTL_OFFSET,
2079 2080
	.mw_bar			= {2, 4, 5},
};
2081

2082 2083 2084 2085
static const struct intel_ntb_alt_reg xeon_pri_reg = {
	.db_bell		= XEON_PDOORBELL_OFFSET,
	.db_mask		= XEON_PDBMSK_OFFSET,
	.spad			= XEON_SPAD_OFFSET,
2086 2087
};

2088 2089 2090
static const struct intel_ntb_alt_reg xeon_sec_reg = {
	.db_bell		= XEON_SDOORBELL_OFFSET,
	.db_mask		= XEON_SDBMSK_OFFSET,
2091
	/* second half of the scratchpads */
2092
	.spad			= XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2093
};
2094

2095 2096 2097
static const struct intel_ntb_alt_reg xeon_b2b_reg = {
	.db_bell		= XEON_B2B_DOORBELL_OFFSET,
	.spad			= XEON_B2B_SPAD_OFFSET,
2098 2099
};

2100
static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
	/* Note: no primary .bar0_base visible to the secondary side.
	 *
	 * The secondary side cannot get the base address stored in primary
	 * bars.  The base address is necessary to set the limit register to
	 * any value other than zero, or unlimited.
	 *
	 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
	 * window by setting the limit equal to base, nor can it limit the size
	 * of the memory window by setting the limit to base + size.
	 */
2111 2112
	.bar2_limit		= XEON_PBAR23LMT_OFFSET,
	.bar2_xlat		= XEON_PBAR23XLAT_OFFSET,
2113 2114
};

2115 2116 2117 2118
static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
	.bar0_base		= XEON_SBAR0BASE_OFFSET,
	.bar2_limit		= XEON_SBAR23LMT_OFFSET,
	.bar2_xlat		= XEON_SBAR23XLAT_OFFSET,
2119 2120
};

2121 2122 2123 2124 2125
static struct intel_b2b_addr xeon_b2b_usd_addr = {
	.bar2_addr64		= XEON_B2B_BAR2_USD_ADDR64,
	.bar4_addr64		= XEON_B2B_BAR4_USD_ADDR64,
	.bar4_addr32		= XEON_B2B_BAR4_USD_ADDR32,
	.bar5_addr32		= XEON_B2B_BAR5_USD_ADDR32,
2126 2127
};

2128 2129 2130 2131 2132
static struct intel_b2b_addr xeon_b2b_dsd_addr = {
	.bar2_addr64		= XEON_B2B_BAR2_DSD_ADDR64,
	.bar4_addr64		= XEON_B2B_BAR4_DSD_ADDR64,
	.bar4_addr32		= XEON_B2B_BAR4_DSD_ADDR32,
	.bar5_addr32		= XEON_B2B_BAR5_DSD_ADDR32,
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
};

/* operations for primary side of local ntb */
static const struct ntb_dev_ops intel_ntb_ops = {
	.mw_count		= intel_ntb_mw_count,
	.mw_get_range		= intel_ntb_mw_get_range,
	.mw_set_trans		= intel_ntb_mw_set_trans,
	.link_is_up		= intel_ntb_link_is_up,
	.link_enable		= intel_ntb_link_enable,
	.link_disable		= intel_ntb_link_disable,
	.db_is_unsafe		= intel_ntb_db_is_unsafe,
	.db_valid_mask		= intel_ntb_db_valid_mask,
	.db_vector_count	= intel_ntb_db_vector_count,
	.db_vector_mask		= intel_ntb_db_vector_mask,
	.db_read		= intel_ntb_db_read,
	.db_clear		= intel_ntb_db_clear,
	.db_set_mask		= intel_ntb_db_set_mask,
	.db_clear_mask		= intel_ntb_db_clear_mask,
	.peer_db_addr		= intel_ntb_peer_db_addr,
	.peer_db_set		= intel_ntb_peer_db_set,
	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
	.spad_count		= intel_ntb_spad_count,
	.spad_read		= intel_ntb_spad_read,
	.spad_write		= intel_ntb_spad_write,
	.peer_spad_addr		= intel_ntb_peer_spad_addr,
	.peer_spad_read		= intel_ntb_peer_spad_read,
	.peer_spad_write	= intel_ntb_peer_spad_write,
};

static const struct file_operations intel_ntb_debugfs_info = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.read = ndev_debugfs_read,
};

static const struct pci_device_id intel_ntb_pci_tbl[] = {
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
	{0}
};
MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);

static struct pci_driver intel_ntb_pci_driver = {
2187
	.name = KBUILD_MODNAME,
2188 2189 2190
	.id_table = intel_ntb_pci_tbl,
	.probe = intel_ntb_pci_probe,
	.remove = intel_ntb_pci_remove,
2191
};
J
Jon Mason 已提交
2192

2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
static int __init intel_ntb_pci_driver_init(void)
{
	if (debugfs_initialized())
		debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);

	return pci_register_driver(&intel_ntb_pci_driver);
}
module_init(intel_ntb_pci_driver_init);

static void __exit intel_ntb_pci_driver_exit(void)
{
	pci_unregister_driver(&intel_ntb_pci_driver);

	debugfs_remove_recursive(debugfs_dir);
}
module_exit(intel_ntb_pci_driver_exit);