mthca_main.c 34.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4
 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
40
#include <linux/gfp.h>
L
Linus Torvalds 已提交
41 42 43 44 45 46

#include "mthca_dev.h"
#include "mthca_config_reg.h"
#include "mthca_cmd.h"
#include "mthca_profile.h"
#include "mthca_memfree.h"
47
#include "mthca_wqe.h"
L
Linus Torvalds 已提交
48 49 50 51 52

MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
MODULE_LICENSE("Dual BSD/GPL");

53 54 55 56 57 58 59 60
#ifdef CONFIG_INFINIBAND_MTHCA_DEBUG

int mthca_debug_level = 0;
module_param_named(debug_level, mthca_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");

#endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */

L
Linus Torvalds 已提交
61 62
#ifdef CONFIG_PCI_MSI

63
static int msi_x = 1;
L
Linus Torvalds 已提交
64 65 66 67 68 69 70 71 72
module_param(msi_x, int, 0444);
MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");

#else /* CONFIG_PCI_MSI */

#define msi_x (0)

#endif /* CONFIG_PCI_MSI */

73 74 75 76
static int tune_pci = 0;
module_param(tune_pci, int, 0444);
MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero");

77
DEFINE_MUTEX(mthca_device_mutex);
78

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
#define MTHCA_DEFAULT_NUM_QP            (1 << 16)
#define MTHCA_DEFAULT_RDB_PER_QP        (1 << 2)
#define MTHCA_DEFAULT_NUM_CQ            (1 << 16)
#define MTHCA_DEFAULT_NUM_MCG           (1 << 13)
#define MTHCA_DEFAULT_NUM_MPT           (1 << 17)
#define MTHCA_DEFAULT_NUM_MTT           (1 << 20)
#define MTHCA_DEFAULT_NUM_UDAV          (1 << 15)
#define MTHCA_DEFAULT_NUM_RESERVED_MTTS (1 << 18)
#define MTHCA_DEFAULT_NUM_UARC_SIZE     (1 << 18)

static struct mthca_profile hca_profile = {
	.num_qp             = MTHCA_DEFAULT_NUM_QP,
	.rdb_per_qp         = MTHCA_DEFAULT_RDB_PER_QP,
	.num_cq             = MTHCA_DEFAULT_NUM_CQ,
	.num_mcg            = MTHCA_DEFAULT_NUM_MCG,
	.num_mpt            = MTHCA_DEFAULT_NUM_MPT,
	.num_mtt            = MTHCA_DEFAULT_NUM_MTT,
	.num_udav           = MTHCA_DEFAULT_NUM_UDAV,          /* Tavor only */
	.fmr_reserved_mtts  = MTHCA_DEFAULT_NUM_RESERVED_MTTS, /* Tavor only */
	.uarc_size          = MTHCA_DEFAULT_NUM_UARC_SIZE,     /* Arbel only */
};

module_param_named(num_qp, hca_profile.num_qp, int, 0444);
MODULE_PARM_DESC(num_qp, "maximum number of QPs per HCA");

module_param_named(rdb_per_qp, hca_profile.rdb_per_qp, int, 0444);
MODULE_PARM_DESC(rdb_per_qp, "number of RDB buffers per QP");

module_param_named(num_cq, hca_profile.num_cq, int, 0444);
MODULE_PARM_DESC(num_cq, "maximum number of CQs per HCA");

module_param_named(num_mcg, hca_profile.num_mcg, int, 0444);
MODULE_PARM_DESC(num_mcg, "maximum number of multicast groups per HCA");

module_param_named(num_mpt, hca_profile.num_mpt, int, 0444);
MODULE_PARM_DESC(num_mpt,
		"maximum number of memory protection table entries per HCA");

module_param_named(num_mtt, hca_profile.num_mtt, int, 0444);
MODULE_PARM_DESC(num_mtt,
		 "maximum number of memory translation table segments per HCA");

module_param_named(num_udav, hca_profile.num_udav, int, 0444);
MODULE_PARM_DESC(num_udav, "maximum number of UD address vectors per HCA");

module_param_named(fmr_reserved_mtts, hca_profile.fmr_reserved_mtts, int, 0444);
MODULE_PARM_DESC(fmr_reserved_mtts,
		 "number of memory translation table segments reserved for FMR");

128 129 130 131
static int log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8);
module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444);
MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-5)");

132
static char mthca_version[] =
133
	DRV_NAME ": Mellanox InfiniBand HCA driver v"
L
Linus Torvalds 已提交
134 135
	DRV_VERSION " (" DRV_RELDATE ")\n";

R
Roland Dreier 已提交
136
static int mthca_tune_pci(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
137
{
138 139 140
	if (!tune_pci)
		return 0;

L
Linus Torvalds 已提交
141
	/* First try to max out Read Byte Count */
142 143 144 145
	if (pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX)) {
		if (pcix_set_mmrbc(mdev->pdev, pcix_get_max_mmrbc(mdev->pdev))) {
			mthca_err(mdev, "Couldn't set PCI-X max read count, "
				"aborting.\n");
L
Linus Torvalds 已提交
146 147
			return -ENODEV;
		}
148
	} else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
L
Linus Torvalds 已提交
149 150
		mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");

151
	if (pci_is_pcie(mdev->pdev)) {
152 153 154
		if (pcie_set_readrq(mdev->pdev, 4096)) {
			mthca_err(mdev, "Couldn't write PCI Express read request, "
				"aborting.\n");
L
Linus Torvalds 已提交
155 156
			return -ENODEV;
		}
157
	} else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
L
Linus Torvalds 已提交
158 159 160 161 162 163
		mthca_info(mdev, "No PCI Express capability, "
			   "not setting Max Read Request Size.\n");

	return 0;
}

R
Roland Dreier 已提交
164
static int mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
L
Linus Torvalds 已提交
165 166 167
{
	int err;

168
	mdev->limits.mtt_seg_size = (1 << log_mtts_per_seg) * 8;
169
	err = mthca_QUERY_DEV_LIM(mdev, dev_lim);
L
Linus Torvalds 已提交
170
	if (err) {
171 172
		mthca_err(mdev, "QUERY_DEV_LIM command returned %d"
				", aborting.\n", err);
L
Linus Torvalds 已提交
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
		return err;
	}
	if (dev_lim->min_page_sz > PAGE_SIZE) {
		mthca_err(mdev, "HCA minimum page size of %d bigger than "
			  "kernel PAGE_SIZE of %ld, aborting.\n",
			  dev_lim->min_page_sz, PAGE_SIZE);
		return -ENODEV;
	}
	if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
		mthca_err(mdev, "HCA has %d ports, but we only support %d, "
			  "aborting.\n",
			  dev_lim->num_ports, MTHCA_MAX_PORTS);
		return -ENODEV;
	}

188 189
	if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) {
		mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than "
190 191 192
			  "PCI resource 2 size of 0x%llx, aborting.\n",
			  dev_lim->uar_size,
			  (unsigned long long)pci_resource_len(mdev->pdev, 2));
193 194 195
		return -ENODEV;
	}

L
Linus Torvalds 已提交
196 197 198 199 200 201
	mdev->limits.num_ports      	= dev_lim->num_ports;
	mdev->limits.vl_cap             = dev_lim->max_vl;
	mdev->limits.mtu_cap            = dev_lim->max_mtu;
	mdev->limits.gid_table_len  	= dev_lim->max_gids;
	mdev->limits.pkey_table_len 	= dev_lim->max_pkeys;
	mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
202 203 204 205 206 207 208 209 210 211 212 213
	/*
	 * Need to allow for worst case send WQE overhead and check
	 * whether max_desc_sz imposes a lower limit than max_sg; UD
	 * send has the biggest overhead.
	 */
	mdev->limits.max_sg		= min_t(int, dev_lim->max_sg,
					      (dev_lim->max_desc_sz -
					       sizeof (struct mthca_next_seg) -
					       (mthca_is_memfree(mdev) ?
						sizeof (struct mthca_arbel_ud_seg) :
						sizeof (struct mthca_tavor_ud_seg))) /
						sizeof (struct mthca_data_seg));
214 215
	mdev->limits.max_wqes           = dev_lim->max_qp_sz;
	mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
L
Linus Torvalds 已提交
216
	mdev->limits.reserved_qps       = dev_lim->reserved_qps;
217
	mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
L
Linus Torvalds 已提交
218 219
	mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
	mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
220
	mdev->limits.max_desc_sz        = dev_lim->max_desc_sz;
221
	mdev->limits.max_srq_sge	= mthca_max_srq_sge(mdev);
222 223 224 225 226 227
	/*
	 * Subtract 1 from the limit because we need to allocate a
	 * spare CQE so the HCA HW can tell the difference between an
	 * empty CQ and a full CQ.
	 */
	mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
L
Linus Torvalds 已提交
228 229 230 231 232 233
	mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
	mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
	mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;
	mdev->limits.reserved_mrws      = dev_lim->reserved_mrws;
	mdev->limits.reserved_uars      = dev_lim->reserved_uars;
	mdev->limits.reserved_pds       = dev_lim->reserved_pds;
234
	mdev->limits.port_width_cap     = dev_lim->max_port_width;
235
	mdev->limits.page_size_cap      = ~(u32) (dev_lim->min_page_sz - 1);
236
	mdev->limits.flags              = dev_lim->flags;
237 238 239 240 241 242 243 244 245 246 247 248
	/*
	 * For old FW that doesn't return static rate support, use a
	 * value of 0x3 (only static rate values of 0 or 1 are handled),
	 * except on Sinai, where even old FW can handle static rate
	 * values of 2 and 3.
	 */
	if (dev_lim->stat_rate_support)
		mdev->limits.stat_rate_support = dev_lim->stat_rate_support;
	else if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
		mdev->limits.stat_rate_support = 0xf;
	else
		mdev->limits.stat_rate_support = 0x3;
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279

	/* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
	   May be doable since hardware supports it for SRQ.

	   IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.

	   IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
	   supported by driver. */
	mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
		IB_DEVICE_PORT_ACTIVE_EVENT |
		IB_DEVICE_SYS_IMAGE_GUID |
		IB_DEVICE_RC_RNR_NAK_GEN;

	if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
		mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;

	if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
		mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;

	if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
		mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;

	if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
		mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;

	if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
		mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;

	if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
		mdev->mthca_flags |= MTHCA_FLAG_SRQ;

280 281 282 283
	if (mthca_is_memfree(mdev))
		if (dev_lim->flags & DEV_LIM_FLAG_IPOIB_CSUM)
			mdev->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;

L
Linus Torvalds 已提交
284 285 286
	return 0;
}

R
Roland Dreier 已提交
287
static int mthca_init_tavor(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
288
{
289
	s64 size;
L
Linus Torvalds 已提交
290 291 292 293 294
	int err;
	struct mthca_dev_lim        dev_lim;
	struct mthca_profile        profile;
	struct mthca_init_hca_param init_hca;

295
	err = mthca_SYS_EN(mdev);
L
Linus Torvalds 已提交
296
	if (err) {
297
		mthca_err(mdev, "SYS_EN command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
298 299 300
		return err;
	}

301
	err = mthca_QUERY_FW(mdev);
L
Linus Torvalds 已提交
302
	if (err) {
303 304
		mthca_err(mdev, "QUERY_FW command returned %d,"
				" aborting.\n", err);
L
Linus Torvalds 已提交
305 306
		goto err_disable;
	}
307
	err = mthca_QUERY_DDR(mdev);
L
Linus Torvalds 已提交
308
	if (err) {
309
		mthca_err(mdev, "QUERY_DDR command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
310 311 312 313
		goto err_disable;
	}

	err = mthca_dev_lim(mdev, &dev_lim);
314
	if (err) {
315
		mthca_err(mdev, "QUERY_DEV_LIM command returned %d, aborting.\n", err);
316 317
		goto err_disable;
	}
L
Linus Torvalds 已提交
318

319
	profile = hca_profile;
L
Linus Torvalds 已提交
320 321
	profile.num_uar   = dev_lim.uar_size / PAGE_SIZE;
	profile.uarc_size = 0;
322 323
	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
		profile.num_srq = dev_lim.max_srqs;
L
Linus Torvalds 已提交
324

325 326 327
	size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
	if (size < 0) {
		err = size;
L
Linus Torvalds 已提交
328
		goto err_disable;
329
	}
L
Linus Torvalds 已提交
330

331
	err = mthca_INIT_HCA(mdev, &init_hca);
L
Linus Torvalds 已提交
332
	if (err) {
333
		mthca_err(mdev, "INIT_HCA command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
334 335 336 337 338 339
		goto err_disable;
	}

	return 0;

err_disable:
340
	mthca_SYS_DIS(mdev);
L
Linus Torvalds 已提交
341 342 343 344

	return err;
}

R
Roland Dreier 已提交
345
static int mthca_load_fw(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
346 347 348 349 350 351 352
{
	int err;

	/* FIXME: use HCA-attached memory for FW if present */

	mdev->fw.arbel.fw_icm =
		mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
353
				GFP_HIGHUSER | __GFP_NOWARN, 0);
L
Linus Torvalds 已提交
354 355 356 357 358
	if (!mdev->fw.arbel.fw_icm) {
		mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
		return -ENOMEM;
	}

359
	err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm);
L
Linus Torvalds 已提交
360
	if (err) {
361
		mthca_err(mdev, "MAP_FA command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
362 363
		goto err_free;
	}
364
	err = mthca_RUN_FW(mdev);
L
Linus Torvalds 已提交
365
	if (err) {
366
		mthca_err(mdev, "RUN_FW command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
367 368 369 370 371 372
		goto err_unmap_fa;
	}

	return 0;

err_unmap_fa:
373
	mthca_UNMAP_FA(mdev);
L
Linus Torvalds 已提交
374 375

err_free:
376
	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
L
Linus Torvalds 已提交
377 378 379
	return err;
}

R
Roland Dreier 已提交
380 381 382 383
static int mthca_init_icm(struct mthca_dev *mdev,
			  struct mthca_dev_lim *dev_lim,
			  struct mthca_init_hca_param *init_hca,
			  u64 icm_size)
L
Linus Torvalds 已提交
384 385 386 387
{
	u64 aux_pages;
	int err;

388
	err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages);
L
Linus Torvalds 已提交
389
	if (err) {
390
		mthca_err(mdev, "SET_ICM_SIZE command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
391 392 393 394 395 396 397 398
		return err;
	}

	mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
		  (unsigned long long) icm_size >> 10,
		  (unsigned long long) aux_pages << 2);

	mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
399
						 GFP_HIGHUSER | __GFP_NOWARN, 0);
L
Linus Torvalds 已提交
400 401 402 403 404
	if (!mdev->fw.arbel.aux_icm) {
		mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
		return -ENOMEM;
	}

405
	err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm);
L
Linus Torvalds 已提交
406
	if (err) {
407
		mthca_err(mdev, "MAP_ICM_AUX returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
408 409 410 411 412 413 414 415 416
		goto err_free_aux;
	}

	err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
	if (err) {
		mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
		goto err_unmap_aux;
	}

417
	/* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */
418 419
	mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * mdev->limits.mtt_seg_size,
					   dma_get_cache_alignment()) / mdev->limits.mtt_seg_size;
420

L
Linus Torvalds 已提交
421
	mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
422
							 mdev->limits.mtt_seg_size,
L
Linus Torvalds 已提交
423
							 mdev->limits.num_mtt_segs,
424 425
							 mdev->limits.reserved_mtts,
							 1, 0);
L
Linus Torvalds 已提交
426 427 428 429 430 431 432 433 434
	if (!mdev->mr_table.mtt_table) {
		mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
		err = -ENOMEM;
		goto err_unmap_eq;
	}

	mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
							 dev_lim->mpt_entry_sz,
							 mdev->limits.num_mpts,
435 436
							 mdev->limits.reserved_mrws,
							 1, 1);
L
Linus Torvalds 已提交
437 438 439 440 441 442 443 444 445
	if (!mdev->mr_table.mpt_table) {
		mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
		err = -ENOMEM;
		goto err_unmap_mtt;
	}

	mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
							dev_lim->qpc_entry_sz,
							mdev->limits.num_qps,
446 447
							mdev->limits.reserved_qps,
							0, 0);
L
Linus Torvalds 已提交
448 449 450 451 452 453 454 455 456
	if (!mdev->qp_table.qp_table) {
		mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
		err = -ENOMEM;
		goto err_unmap_mpt;
	}

	mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
							 dev_lim->eqpc_entry_sz,
							 mdev->limits.num_qps,
457 458
							 mdev->limits.reserved_qps,
							 0, 0);
L
Linus Torvalds 已提交
459 460 461 462 463 464
	if (!mdev->qp_table.eqp_table) {
		mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
		err = -ENOMEM;
		goto err_unmap_qp;
	}

465 466 467
	mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
							 MTHCA_RDB_ENTRY_SIZE,
							 mdev->limits.num_qps <<
468
							 mdev->qp_table.rdb_shift, 0,
469 470 471 472
							 0, 0);
	if (!mdev->qp_table.rdb_table) {
		mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
		err = -ENOMEM;
473
		goto err_unmap_eqp;
474 475 476
	}

       mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
477 478
						    dev_lim->cqc_entry_sz,
						    mdev->limits.num_cqs,
479 480
						    mdev->limits.reserved_cqs,
						    0, 0);
L
Linus Torvalds 已提交
481 482 483
	if (!mdev->cq_table.table) {
		mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
		err = -ENOMEM;
484
		goto err_unmap_rdb;
L
Linus Torvalds 已提交
485 486
	}

487 488 489 490 491
	if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
		mdev->srq_table.table =
			mthca_alloc_icm_table(mdev, init_hca->srqc_base,
					      dev_lim->srq_entry_sz,
					      mdev->limits.num_srqs,
492 493
					      mdev->limits.reserved_srqs,
					      0, 0);
494 495 496 497 498 499 500 501
		if (!mdev->srq_table.table) {
			mthca_err(mdev, "Failed to map SRQ context memory, "
				  "aborting.\n");
			err = -ENOMEM;
			goto err_unmap_cq;
		}
	}

L
Linus Torvalds 已提交
502 503 504 505 506 507 508 509 510 511 512
	/*
	 * It's not strictly required, but for simplicity just map the
	 * whole multicast group table now.  The table isn't very big
	 * and it's a lot easier than trying to track ref counts.
	 */
	mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
						      MTHCA_MGM_ENTRY_SIZE,
						      mdev->limits.num_mgms +
						      mdev->limits.num_amgms,
						      mdev->limits.num_mgms +
						      mdev->limits.num_amgms,
513
						      0, 0);
L
Linus Torvalds 已提交
514 515 516
	if (!mdev->mcg_table.table) {
		mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
		err = -ENOMEM;
517
		goto err_unmap_srq;
L
Linus Torvalds 已提交
518 519 520 521
	}

	return 0;

522 523 524 525
err_unmap_srq:
	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
		mthca_free_icm_table(mdev, mdev->srq_table.table);

L
Linus Torvalds 已提交
526 527 528
err_unmap_cq:
	mthca_free_icm_table(mdev, mdev->cq_table.table);

529 530 531
err_unmap_rdb:
	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);

L
Linus Torvalds 已提交
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
err_unmap_eqp:
	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);

err_unmap_qp:
	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);

err_unmap_mpt:
	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);

err_unmap_mtt:
	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);

err_unmap_eq:
	mthca_unmap_eq_icm(mdev);

err_unmap_aux:
548
	mthca_UNMAP_ICM_AUX(mdev);
L
Linus Torvalds 已提交
549 550

err_free_aux:
551
	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0);
L
Linus Torvalds 已提交
552 553 554 555

	return err;
}

556 557 558 559 560 561 562 563 564 565 566 567 568 569
static void mthca_free_icms(struct mthca_dev *mdev)
{

	mthca_free_icm_table(mdev, mdev->mcg_table.table);
	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
		mthca_free_icm_table(mdev, mdev->srq_table.table);
	mthca_free_icm_table(mdev, mdev->cq_table.table);
	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
	mthca_unmap_eq_icm(mdev);

570
	mthca_UNMAP_ICM_AUX(mdev);
571
	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0);
572 573
}

R
Roland Dreier 已提交
574
static int mthca_init_arbel(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
575 576 577 578
{
	struct mthca_dev_lim        dev_lim;
	struct mthca_profile        profile;
	struct mthca_init_hca_param init_hca;
579
	s64 icm_size;
L
Linus Torvalds 已提交
580 581
	int err;

582
	err = mthca_QUERY_FW(mdev);
L
Linus Torvalds 已提交
583
	if (err) {
584
		mthca_err(mdev, "QUERY_FW command failed %d, aborting.\n", err);
L
Linus Torvalds 已提交
585 586 587
		return err;
	}

588 589
	err = mthca_ENABLE_LAM(mdev);
	if (err == -EAGAIN) {
L
Linus Torvalds 已提交
590 591
		mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
		mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
592 593 594
	} else if (err) {
		mthca_err(mdev, "ENABLE_LAM returned %d, aborting.\n", err);
		return err;
L
Linus Torvalds 已提交
595 596 597 598
	}

	err = mthca_load_fw(mdev);
	if (err) {
599
		mthca_err(mdev, "Loading FW returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
600 601 602 603 604
		goto err_disable;
	}

	err = mthca_dev_lim(mdev, &dev_lim);
	if (err) {
605
		mthca_err(mdev, "QUERY_DEV_LIM returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
606 607 608
		goto err_stop_fw;
	}

609
	profile = hca_profile;
L
Linus Torvalds 已提交
610 611
	profile.num_uar  = dev_lim.uar_size / PAGE_SIZE;
	profile.num_udav = 0;
612 613
	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
		profile.num_srq = dev_lim.max_srqs;
L
Linus Torvalds 已提交
614 615

	icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
616
	if (icm_size < 0) {
L
Linus Torvalds 已提交
617 618 619 620 621 622 623 624
		err = icm_size;
		goto err_stop_fw;
	}

	err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
	if (err)
		goto err_stop_fw;

625
	err = mthca_INIT_HCA(mdev, &init_hca);
L
Linus Torvalds 已提交
626
	if (err) {
627
		mthca_err(mdev, "INIT_HCA command returned %d, aborting.\n", err);
L
Linus Torvalds 已提交
628 629 630 631 632 633
		goto err_free_icm;
	}

	return 0;

err_free_icm:
634
	mthca_free_icms(mdev);
L
Linus Torvalds 已提交
635 636

err_stop_fw:
637
	mthca_UNMAP_FA(mdev);
638
	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
L
Linus Torvalds 已提交
639 640 641

err_disable:
	if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
642
		mthca_DISABLE_LAM(mdev);
L
Linus Torvalds 已提交
643 644 645 646

	return err;
}

647 648
static void mthca_close_hca(struct mthca_dev *mdev)
{
649
	mthca_CLOSE_HCA(mdev, 0);
650 651

	if (mthca_is_memfree(mdev)) {
652
		mthca_free_icms(mdev);
653

654
		mthca_UNMAP_FA(mdev);
655
		mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0);
656 657

		if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
658
			mthca_DISABLE_LAM(mdev);
659
	} else
660
		mthca_SYS_DIS(mdev);
661 662
}

R
Roland Dreier 已提交
663
static int mthca_init_hca(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
664
{
665 666 667
	int err;
	struct mthca_adapter adapter;

668
	if (mthca_is_memfree(mdev))
669
		err = mthca_init_arbel(mdev);
L
Linus Torvalds 已提交
670
	else
671 672 673 674 675
		err = mthca_init_tavor(mdev);

	if (err)
		return err;

676
	err = mthca_QUERY_ADAPTER(mdev, &adapter);
677
	if (err) {
678
		mthca_err(mdev, "QUERY_ADAPTER command returned %d, aborting.\n", err);
679 680 681 682
		goto err_close;
	}

	mdev->eq_table.inta_pin = adapter.inta_pin;
683 684
	if (!mthca_is_memfree(mdev))
		mdev->rev_id = adapter.revision_id;
685 686 687 688 689 690 691
	memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id);

	return 0;

err_close:
	mthca_close_hca(mdev);
	return err;
L
Linus Torvalds 已提交
692 693
}

R
Roland Dreier 已提交
694
static int mthca_setup_hca(struct mthca_dev *dev)
L
Linus Torvalds 已提交
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
{
	int err;

	MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);

	err = mthca_init_uar_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "user access region table, aborting.\n");
		return err;
	}

	err = mthca_uar_alloc(dev, &dev->driver_uar);
	if (err) {
		mthca_err(dev, "Failed to allocate driver access region, "
			  "aborting.\n");
		goto err_uar_table_free;
	}

714
	dev->kar = ioremap((phys_addr_t) dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
L
Linus Torvalds 已提交
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
	if (!dev->kar) {
		mthca_err(dev, "Couldn't map kernel access region, "
			  "aborting.\n");
		err = -ENOMEM;
		goto err_uar_free;
	}

	err = mthca_init_pd_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "protection domain table, aborting.\n");
		goto err_kar_unmap;
	}

	err = mthca_init_mr_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "memory region table, aborting.\n");
		goto err_pd_table_free;
	}

736
	err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
L
Linus Torvalds 已提交
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
	if (err) {
		mthca_err(dev, "Failed to create driver PD, "
			  "aborting.\n");
		goto err_mr_table_free;
	}

	err = mthca_init_eq_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "event queue table, aborting.\n");
		goto err_pd_free;
	}

	err = mthca_cmd_use_events(dev);
	if (err) {
		mthca_err(dev, "Failed to switch to event-driven "
			  "firmware commands, aborting.\n");
		goto err_eq_table_free;
	}

757 758
	err = mthca_NOP(dev);
	if (err) {
759
		if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {
760 761
			mthca_warn(dev, "NOP command failed to generate interrupt "
				   "(IRQ %d).\n",
762 763
				   dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector);
			mthca_warn(dev, "Trying again with MSI-X disabled.\n");
764 765 766 767
		} else {
			mthca_err(dev, "NOP command failed to generate interrupt "
				  "(IRQ %d), aborting.\n",
				  dev->pdev->irq);
L
Linus Torvalds 已提交
768
			mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
769
		}
L
Linus Torvalds 已提交
770 771 772 773 774 775 776 777 778 779 780 781 782

		goto err_cmd_poll;
	}

	mthca_dbg(dev, "NOP command IRQ test passed\n");

	err = mthca_init_cq_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "completion queue table, aborting.\n");
		goto err_cmd_poll;
	}

783 784 785 786 787 788 789
	err = mthca_init_srq_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "shared receive queue table, aborting.\n");
		goto err_cq_table_free;
	}

L
Linus Torvalds 已提交
790 791 792 793
	err = mthca_init_qp_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "queue pair table, aborting.\n");
794
		goto err_srq_table_free;
L
Linus Torvalds 已提交
795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
	}

	err = mthca_init_av_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "address vector table, aborting.\n");
		goto err_qp_table_free;
	}

	err = mthca_init_mcg_table(dev);
	if (err) {
		mthca_err(dev, "Failed to initialize "
			  "multicast group table, aborting.\n");
		goto err_av_table_free;
	}

	return 0;

err_av_table_free:
	mthca_cleanup_av_table(dev);

err_qp_table_free:
	mthca_cleanup_qp_table(dev);

819 820 821
err_srq_table_free:
	mthca_cleanup_srq_table(dev);

L
Linus Torvalds 已提交
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
err_cq_table_free:
	mthca_cleanup_cq_table(dev);

err_cmd_poll:
	mthca_cmd_use_polling(dev);

err_eq_table_free:
	mthca_cleanup_eq_table(dev);

err_pd_free:
	mthca_pd_free(dev, &dev->driver_pd);

err_mr_table_free:
	mthca_cleanup_mr_table(dev);

err_pd_table_free:
	mthca_cleanup_pd_table(dev);

err_kar_unmap:
	iounmap(dev->kar);

err_uar_free:
	mthca_uar_free(dev, &dev->driver_uar);

err_uar_table_free:
	mthca_cleanup_uar_table(dev);
	return err;
}

R
Roland Dreier 已提交
851
static int mthca_enable_msi_x(struct mthca_dev *mdev)
L
Linus Torvalds 已提交
852 853 854
{
	int err;

855 856
	err = pci_alloc_irq_vectors(mdev->pdev, 3, 3, PCI_IRQ_MSIX);
	if (err < 0)
L
Linus Torvalds 已提交
857 858
		return err;

859 860 861 862 863 864
	mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector =
			pci_irq_vector(mdev->pdev, 0);
	mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector =
			pci_irq_vector(mdev->pdev, 1);
	mdev->eq_table.eq[MTHCA_EQ_CMD  ].msi_x_vector =
			pci_irq_vector(mdev->pdev, 2);
L
Linus Torvalds 已提交
865 866 867 868

	return 0;
}

869 870 871 872 873 874 875 876 877 878 879 880 881
/* Types of supported HCA */
enum {
	TAVOR,			/* MT23108                        */
	ARBEL_COMPAT,		/* MT25208 in Tavor compat mode   */
	ARBEL_NATIVE,		/* MT25208 with extended features */
	SINAI			/* MT25204 */
};

#define MTHCA_FW_VER(major, minor, subminor) \
	(((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor))

static struct {
	u64 latest_fw;
882
	u32 flags;
883
} mthca_hca_table[] = {
884
	[TAVOR]        = { .latest_fw = MTHCA_FW_VER(3, 5, 0),
885
			   .flags     = 0 },
886
	[ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 8, 200),
887
			   .flags     = MTHCA_FLAG_PCIE },
888
	[ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 3, 0),
889 890
			   .flags     = MTHCA_FLAG_MEMFREE |
					MTHCA_FLAG_PCIE },
891
	[SINAI]        = { .latest_fw = MTHCA_FW_VER(1, 2, 0),
892 893 894
			   .flags     = MTHCA_FLAG_MEMFREE |
					MTHCA_FLAG_PCIE    |
					MTHCA_FLAG_SINAI_OPT }
895 896
};

897
static int __mthca_init_one(struct pci_dev *pdev, int hca_type)
L
Linus Torvalds 已提交
898 899 900 901 902
{
	int ddr_hidden = 0;
	int err;
	struct mthca_dev *mdev;

903 904
	printk(KERN_INFO PFX "Initializing %s\n",
	       pci_name(pdev));
L
Linus Torvalds 已提交
905 906 907 908 909 910 911 912 913 914 915 916 917 918

	err = pci_enable_device(pdev);
	if (err) {
		dev_err(&pdev->dev, "Cannot enable PCI device, "
			"aborting.\n");
		return err;
	}

	/*
	 * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
	 * be present)
	 */
	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
	    pci_resource_len(pdev, 0) != 1 << 20) {
919
		dev_err(&pdev->dev, "Missing DCS, aborting.\n");
L
Linus Torvalds 已提交
920 921 922
		err = -ENODEV;
		goto err_disable_pdev;
	}
923
	if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
924
		dev_err(&pdev->dev, "Missing UAR, aborting.\n");
L
Linus Torvalds 已提交
925 926 927 928 929 930
		err = -ENODEV;
		goto err_disable_pdev;
	}
	if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
		ddr_hidden = 1;

931
	err = pci_request_regions(pdev, DRV_NAME);
L
Linus Torvalds 已提交
932 933 934 935 936 937 938 939
	if (err) {
		dev_err(&pdev->dev, "Cannot obtain PCI resources, "
			"aborting.\n");
		goto err_disable_pdev;
	}

	pci_set_master(pdev);

940
	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
L
Linus Torvalds 已提交
941 942
	if (err) {
		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
943
		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
L
Linus Torvalds 已提交
944 945 946 947 948
		if (err) {
			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
			goto err_free_res;
		}
	}
949
	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
L
Linus Torvalds 已提交
950 951 952
	if (err) {
		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
			 "consistent PCI DMA mask.\n");
953
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
L
Linus Torvalds 已提交
954 955 956 957 958 959 960
		if (err) {
			dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
				"aborting.\n");
			goto err_free_res;
		}
	}

961 962 963
	/* We can handle large RDMA requests, so allow larger segments. */
	dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024);

L
Linus Torvalds 已提交
964 965 966 967 968 969 970 971
	mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
	if (!mdev) {
		dev_err(&pdev->dev, "Device struct alloc failed, "
			"aborting.\n");
		err = -ENOMEM;
		goto err_free_res;
	}

972
	mdev->pdev = pdev;
L
Linus Torvalds 已提交
973

974
	mdev->mthca_flags = mthca_hca_table[hca_type].flags;
L
Linus Torvalds 已提交
975 976 977 978 979 980 981 982 983 984 985 986 987 988
	if (ddr_hidden)
		mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;

	/*
	 * Now reset the HCA before we touch the PCI capabilities or
	 * attempt a firmware command, since a boot ROM may have left
	 * the HCA in an undefined state.
	 */
	err = mthca_reset(mdev);
	if (err) {
		mthca_err(mdev, "Failed to reset HCA, aborting.\n");
		goto err_free_dev;
	}

989 990
	if (mthca_cmd_init(mdev)) {
		mthca_err(mdev, "Failed to init command interface, aborting.\n");
L
Linus Torvalds 已提交
991 992 993 994 995
		goto err_free_dev;
	}

	err = mthca_tune_pci(mdev);
	if (err)
996
		goto err_cmd;
L
Linus Torvalds 已提交
997 998 999

	err = mthca_init_hca(mdev);
	if (err)
1000
		goto err_cmd;
L
Linus Torvalds 已提交
1001

1002
	if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) {
1003
		mthca_warn(mdev, "HCA FW version %d.%d.%03d is old (%d.%d.%03d is current).\n",
1004 1005
			   (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
			   (int) (mdev->fw_ver & 0xffff),
1006 1007 1008
			   (int) (mthca_hca_table[hca_type].latest_fw >> 32),
			   (int) (mthca_hca_table[hca_type].latest_fw >> 16) & 0xffff,
			   (int) (mthca_hca_table[hca_type].latest_fw & 0xffff));
1009 1010 1011
		mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
	}

1012 1013 1014
	if (msi_x && !mthca_enable_msi_x(mdev))
		mdev->mthca_flags |= MTHCA_FLAG_MSI_X;

L
Linus Torvalds 已提交
1015
	err = mthca_setup_hca(mdev);
1016
	if (err == -EBUSY && (mdev->mthca_flags & MTHCA_FLAG_MSI_X)) {
1017
		if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1018
			pci_free_irq_vectors(pdev);
1019
		mdev->mthca_flags &= ~MTHCA_FLAG_MSI_X;
1020 1021 1022 1023

		err = mthca_setup_hca(mdev);
	}

L
Linus Torvalds 已提交
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
	if (err)
		goto err_close;

	err = mthca_register_device(mdev);
	if (err)
		goto err_cleanup;

	err = mthca_create_agents(mdev);
	if (err)
		goto err_unregister;

	pci_set_drvdata(pdev, mdev);
1036
	mdev->hca_type = hca_type;
L
Linus Torvalds 已提交
1037

1038 1039
	mdev->active = true;

L
Linus Torvalds 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048
	return 0;

err_unregister:
	mthca_unregister_device(mdev);

err_cleanup:
	mthca_cleanup_mcg_table(mdev);
	mthca_cleanup_av_table(mdev);
	mthca_cleanup_qp_table(mdev);
1049
	mthca_cleanup_srq_table(mdev);
L
Linus Torvalds 已提交
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	mthca_cleanup_cq_table(mdev);
	mthca_cmd_use_polling(mdev);
	mthca_cleanup_eq_table(mdev);

	mthca_pd_free(mdev, &mdev->driver_pd);

	mthca_cleanup_mr_table(mdev);
	mthca_cleanup_pd_table(mdev);
	mthca_cleanup_uar_table(mdev);

err_close:
1061
	if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1062
		pci_free_irq_vectors(pdev);
1063

L
Linus Torvalds 已提交
1064 1065
	mthca_close_hca(mdev);

1066 1067
err_cmd:
	mthca_cmd_cleanup(mdev);
L
Linus Torvalds 已提交
1068 1069 1070 1071 1072

err_free_dev:
	ib_dealloc_device(&mdev->ib_dev);

err_free_res:
1073
	pci_release_regions(pdev);
L
Linus Torvalds 已提交
1074 1075 1076 1077 1078 1079 1080

err_disable_pdev:
	pci_disable_device(pdev);
	pci_set_drvdata(pdev, NULL);
	return err;
}

1081
static void __mthca_remove_one(struct pci_dev *pdev)
L
Linus Torvalds 已提交
1082 1083 1084 1085 1086 1087 1088 1089 1090
{
	struct mthca_dev *mdev = pci_get_drvdata(pdev);
	int p;

	if (mdev) {
		mthca_free_agents(mdev);
		mthca_unregister_device(mdev);

		for (p = 1; p <= mdev->limits.num_ports; ++p)
1091
			mthca_CLOSE_IB(mdev, p);
L
Linus Torvalds 已提交
1092 1093 1094 1095

		mthca_cleanup_mcg_table(mdev);
		mthca_cleanup_av_table(mdev);
		mthca_cleanup_qp_table(mdev);
1096
		mthca_cleanup_srq_table(mdev);
L
Linus Torvalds 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
		mthca_cleanup_cq_table(mdev);
		mthca_cmd_use_polling(mdev);
		mthca_cleanup_eq_table(mdev);

		mthca_pd_free(mdev, &mdev->driver_pd);

		mthca_cleanup_mr_table(mdev);
		mthca_cleanup_pd_table(mdev);

		iounmap(mdev->kar);
		mthca_uar_free(mdev, &mdev->driver_uar);
		mthca_cleanup_uar_table(mdev);
		mthca_close_hca(mdev);
1110
		mthca_cmd_cleanup(mdev);
L
Linus Torvalds 已提交
1111 1112

		if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1113
			pci_free_irq_vectors(pdev);
L
Linus Torvalds 已提交
1114 1115

		ib_dealloc_device(&mdev->ib_dev);
1116
		pci_release_regions(pdev);
L
Linus Torvalds 已提交
1117 1118 1119 1120 1121
		pci_disable_device(pdev);
		pci_set_drvdata(pdev, NULL);
	}
}

1122 1123 1124
int __mthca_restart_one(struct pci_dev *pdev)
{
	struct mthca_dev *mdev;
1125
	int hca_type;
1126 1127 1128 1129

	mdev = pci_get_drvdata(pdev);
	if (!mdev)
		return -ENODEV;
1130
	hca_type = mdev->hca_type;
1131
	__mthca_remove_one(pdev);
1132
	return __mthca_init_one(pdev, hca_type);
1133 1134
}

1135
static int mthca_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
1136 1137 1138 1139 1140
{
	int ret;

	mutex_lock(&mthca_device_mutex);

1141
	printk_once(KERN_INFO "%s", mthca_version);
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156

	if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
		printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
		       pci_name(pdev), id->driver_data);
		mutex_unlock(&mthca_device_mutex);
		return -ENODEV;
	}

	ret = __mthca_init_one(pdev, id->driver_data);

	mutex_unlock(&mthca_device_mutex);

	return ret;
}

1157
static void mthca_remove_one(struct pci_dev *pdev)
1158 1159 1160 1161 1162 1163
{
	mutex_lock(&mthca_device_mutex);
	__mthca_remove_one(pdev);
	mutex_unlock(&mthca_device_mutex);
}

L
Linus Torvalds 已提交
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
static struct pci_device_id mthca_pci_table[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
	  .driver_data = TAVOR },
	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
	  .driver_data = TAVOR },
	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
	  .driver_data = ARBEL_COMPAT },
	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
	  .driver_data = ARBEL_COMPAT },
	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
	  .driver_data = ARBEL_NATIVE },
	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
	  .driver_data = ARBEL_NATIVE },
1177 1178 1179 1180 1181 1182 1183 1184
	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI),
	  .driver_data = SINAI },
	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI),
	  .driver_data = SINAI },
	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
	  .driver_data = SINAI },
	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
	  .driver_data = SINAI },
L
Linus Torvalds 已提交
1185 1186 1187 1188 1189 1190
	{ 0, }
};

MODULE_DEVICE_TABLE(pci, mthca_pci_table);

static struct pci_driver mthca_driver = {
1191
	.name		= DRV_NAME,
L
Linus Torvalds 已提交
1192 1193
	.id_table	= mthca_pci_table,
	.probe		= mthca_init_one,
1194
	.remove		= mthca_remove_one,
L
Linus Torvalds 已提交
1195 1196
};

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
static void __init __mthca_check_profile_val(const char *name, int *pval,
					     int pval_default)
{
	/* value must be positive and power of 2 */
	int old_pval = *pval;

	if (old_pval <= 0)
		*pval = pval_default;
	else
		*pval = roundup_pow_of_two(old_pval);

	if (old_pval != *pval) {
		printk(KERN_WARNING PFX "Invalid value %d for %s in module parameter.\n",
		       old_pval, name);
		printk(KERN_WARNING PFX "Corrected %s to %d.\n", name, *pval);
	}
}

#define mthca_check_profile_val(name, default)				\
	__mthca_check_profile_val(#name, &hca_profile.name, default)

static void __init mthca_validate_profile(void)
{
	mthca_check_profile_val(num_qp,            MTHCA_DEFAULT_NUM_QP);
	mthca_check_profile_val(rdb_per_qp,        MTHCA_DEFAULT_RDB_PER_QP);
	mthca_check_profile_val(num_cq,            MTHCA_DEFAULT_NUM_CQ);
	mthca_check_profile_val(num_mcg, 	   MTHCA_DEFAULT_NUM_MCG);
	mthca_check_profile_val(num_mpt, 	   MTHCA_DEFAULT_NUM_MPT);
	mthca_check_profile_val(num_mtt, 	   MTHCA_DEFAULT_NUM_MTT);
	mthca_check_profile_val(num_udav,          MTHCA_DEFAULT_NUM_UDAV);
	mthca_check_profile_val(fmr_reserved_mtts, MTHCA_DEFAULT_NUM_RESERVED_MTTS);

	if (hca_profile.fmr_reserved_mtts >= hca_profile.num_mtt) {
		printk(KERN_WARNING PFX "Invalid fmr_reserved_mtts module parameter %d.\n",
		       hca_profile.fmr_reserved_mtts);
		printk(KERN_WARNING PFX "(Must be smaller than num_mtt %d)\n",
		       hca_profile.num_mtt);
		hca_profile.fmr_reserved_mtts = hca_profile.num_mtt / 2;
		printk(KERN_WARNING PFX "Corrected fmr_reserved_mtts to %d.\n",
		       hca_profile.fmr_reserved_mtts);
	}
1238 1239 1240 1241 1242 1243

	if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 5)) {
		printk(KERN_WARNING PFX "bad log_mtts_per_seg (%d). Using default - %d\n",
		       log_mtts_per_seg, ilog2(MTHCA_MTT_SEG_SIZE / 8));
		log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8);
	}
1244 1245
}

L
Linus Torvalds 已提交
1246 1247 1248 1249
static int __init mthca_init(void)
{
	int ret;

1250 1251
	mthca_validate_profile();

1252 1253 1254 1255
	ret = mthca_catas_init();
	if (ret)
		return ret;

L
Linus Torvalds 已提交
1256
	ret = pci_register_driver(&mthca_driver);
1257 1258 1259 1260 1261 1262
	if (ret < 0) {
		mthca_catas_cleanup();
		return ret;
	}

	return 0;
L
Linus Torvalds 已提交
1263 1264 1265 1266 1267
}

static void __exit mthca_cleanup(void)
{
	pci_unregister_driver(&mthca_driver);
1268
	mthca_catas_cleanup();
L
Linus Torvalds 已提交
1269 1270 1271 1272
}

module_init(mthca_init);
module_exit(mthca_cleanup);