trans.c 61.1 KB
Newer Older
1 2 3 4 5 6 7
/******************************************************************************
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
W
Wey-Yi Guy 已提交
8
 * Copyright(c) 2007 - 2012 Intel Corporation. All rights reserved.
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
 * USA
 *
 * The full GNU General Public License is included in this distribution
 * in the file called LICENSE.GPL.
 *
 * Contact Information:
 *  Intel Linux Wireless <ilw@linux.intel.com>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 * BSD LICENSE
 *
W
Wey-Yi Guy 已提交
33
 * Copyright(c) 2005 - 2012 Intel Corporation. All rights reserved.
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/
63 64
#include <linux/pci.h>
#include <linux/pci-aspm.h>
65
#include <linux/interrupt.h>
66
#include <linux/debugfs.h>
67
#include <linux/sched.h>
68 69
#include <linux/bitops.h>
#include <linux/gfp.h>
70

71
#include "iwl-drv.h"
72
#include "iwl-trans.h"
73 74
#include "iwl-csr.h"
#include "iwl-prph.h"
75
#include "iwl-agn-hw.h"
76
#include "internal.h"
77
/* FIXME: need to abstract out TX command (once we know what it looks like) */
78
#include "dvm/commands.h"
79

80
#define SCD_QUEUECHAIN_SEL_ALL(trans, trans_pcie)	\
81
	(((1<<trans->cfg->base_params->num_of_queues) - 1) &\
82 83
	(~(1<<(trans_pcie)->cmd_queue)))

84
static int iwl_trans_rx_alloc(struct iwl_trans *trans)
85
{
86
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
87
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
88
	struct device *dev = trans->dev;
89

90
	memset(&trans_pcie->rxq, 0, sizeof(trans_pcie->rxq));
91 92 93 94 95 96 97

	spin_lock_init(&rxq->lock);

	if (WARN_ON(rxq->bd || rxq->rb_stts))
		return -EINVAL;

	/* Allocate the circular buffer of Read Buffer Descriptors (RBDs) */
98 99
	rxq->bd = dma_zalloc_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
				      &rxq->bd_dma, GFP_KERNEL);
100 101 102 103
	if (!rxq->bd)
		goto err_bd;

	/*Allocate the driver's pointer to receive buffer status */
104 105
	rxq->rb_stts = dma_zalloc_coherent(dev, sizeof(*rxq->rb_stts),
					   &rxq->rb_stts_dma, GFP_KERNEL);
106 107 108 109 110 111
	if (!rxq->rb_stts)
		goto err_rb_stts;

	return 0;

err_rb_stts:
112
	dma_free_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
113
			  rxq->bd, rxq->bd_dma);
114 115 116 117 118 119
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;
err_bd:
	return -ENOMEM;
}

120
static void iwl_trans_rxq_free_rx_bufs(struct iwl_trans *trans)
121
{
122
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
123
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
124
	int i;
125 126 127 128 129 130

	/* Fill the rx_used queue with _all_ of the Rx buffers */
	for (i = 0; i < RX_FREE_BUFFERS + RX_QUEUE_SIZE; i++) {
		/* In the reset function, these buffers may have been allocated
		 * to an SKB, so we need to unmap and free potential storage */
		if (rxq->pool[i].page != NULL) {
131
			dma_unmap_page(trans->dev, rxq->pool[i].page_dma,
132 133
				       PAGE_SIZE << trans_pcie->rx_page_order,
				       DMA_FROM_DEVICE);
134
			__free_pages(rxq->pool[i].page,
135
				     trans_pcie->rx_page_order);
136 137 138 139
			rxq->pool[i].page = NULL;
		}
		list_add_tail(&rxq->pool[i].list, &rxq->rx_used);
	}
140 141
}

142
static void iwl_trans_rx_hw_init(struct iwl_trans *trans,
143 144
				 struct iwl_rx_queue *rxq)
{
145
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
146 147
	u32 rb_size;
	const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */
148
	u32 rb_timeout = RX_RB_TIMEOUT; /* FIXME: RX_RB_TIMEOUT for all devices? */
149

150
	if (trans_pcie->rx_buf_size_8k)
151 152 153 154 155
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K;
	else
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;

	/* Stop Rx DMA */
156
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
157 158

	/* Reset driver's Rx queue write index */
159
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
160 161

	/* Tell device where to find RBD circular buffer in DRAM */
162
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG,
163 164 165
			   (u32)(rxq->bd_dma >> 8));

	/* Tell device where in DRAM to update its Rx status */
166
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG,
167 168 169 170 171 172 173 174 175 176
			   rxq->rb_stts_dma >> 4);

	/* Enable Rx DMA
	 * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
	 *      the credit mechanism in 5000 HW RX FIFO
	 * Direct rx interrupts to hosts
	 * Rx buffer size 4 or 8k
	 * RB timeout 0x10
	 * 256 RBDs
	 */
177
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG,
178 179 180 181 182 183 184 185
			   FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL |
			   FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY |
			   FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL |
			   rb_size|
			   (rb_timeout << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS)|
			   (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS));

	/* Set interrupt coalescing timer to default (2048 usecs) */
186
	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
187 188
}

189
static int iwl_rx_init(struct iwl_trans *trans)
190
{
191
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
192 193
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;

194 195 196 197
	int i, err;
	unsigned long flags;

	if (!rxq->bd) {
198
		err = iwl_trans_rx_alloc(trans);
199 200 201 202 203 204 205 206
		if (err)
			return err;
	}

	spin_lock_irqsave(&rxq->lock, flags);
	INIT_LIST_HEAD(&rxq->rx_free);
	INIT_LIST_HEAD(&rxq->rx_used);

207
	iwl_trans_rxq_free_rx_bufs(trans);
208 209 210 211 212 213 214 215 216 217 218

	for (i = 0; i < RX_QUEUE_SIZE; i++)
		rxq->queue[i] = NULL;

	/* Set us so that we have processed and used all buffers, but have
	 * not restocked the Rx queue with fresh buffers */
	rxq->read = rxq->write = 0;
	rxq->write_actual = 0;
	rxq->free_count = 0;
	spin_unlock_irqrestore(&rxq->lock, flags);

219
	iwlagn_rx_replenish(trans);
220

221
	iwl_trans_rx_hw_init(trans, rxq);
222

J
Johannes Berg 已提交
223
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
224
	rxq->need_update = 1;
225
	iwl_rx_queue_update_write_ptr(trans, rxq);
J
Johannes Berg 已提交
226
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
227

228 229 230
	return 0;
}

231
static void iwl_trans_pcie_rx_free(struct iwl_trans *trans)
232
{
233
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
234
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
235 236 237 238 239
	unsigned long flags;

	/*if rxq->bd is NULL, it means that nothing has been allocated,
	 * exit now */
	if (!rxq->bd) {
240
		IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
241 242 243 244
		return;
	}

	spin_lock_irqsave(&rxq->lock, flags);
245
	iwl_trans_rxq_free_rx_bufs(trans);
246 247
	spin_unlock_irqrestore(&rxq->lock, flags);

248
	dma_free_coherent(trans->dev, sizeof(__le32) * RX_QUEUE_SIZE,
249 250 251 252 253
			  rxq->bd, rxq->bd_dma);
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;

	if (rxq->rb_stts)
254
		dma_free_coherent(trans->dev,
255 256 257
				  sizeof(struct iwl_rb_status),
				  rxq->rb_stts, rxq->rb_stts_dma);
	else
258
		IWL_DEBUG_INFO(trans, "Free rxq->rb_stts which is NULL\n");
259 260 261 262
	memset(&rxq->rb_stts_dma, 0, sizeof(rxq->rb_stts_dma));
	rxq->rb_stts = NULL;
}

263
static int iwl_trans_rx_stop(struct iwl_trans *trans)
264 265 266
{

	/* stop Rx DMA */
267 268
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
	return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG,
269
				   FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, 1000);
270 271
}

272 273
static int iwlagn_alloc_dma_ptr(struct iwl_trans *trans,
				struct iwl_dma_ptr *ptr, size_t size)
274 275 276 277
{
	if (WARN_ON(ptr->addr))
		return -EINVAL;

278
	ptr->addr = dma_alloc_coherent(trans->dev, size,
279 280 281 282 283 284 285
				       &ptr->dma, GFP_KERNEL);
	if (!ptr->addr)
		return -ENOMEM;
	ptr->size = size;
	return 0;
}

286 287
static void iwlagn_free_dma_ptr(struct iwl_trans *trans,
				struct iwl_dma_ptr *ptr)
288 289 290 291
{
	if (unlikely(!ptr->addr))
		return;

292
	dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma);
293 294 295
	memset(ptr, 0, sizeof(*ptr));
}

296 297 298
static void iwl_trans_pcie_queue_stuck_timer(unsigned long data)
{
	struct iwl_tx_queue *txq = (void *)data;
299
	struct iwl_queue *q = &txq->q;
300 301
	struct iwl_trans_pcie *trans_pcie = txq->trans_pcie;
	struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie);
302 303 304 305
	u32 scd_sram_addr = trans_pcie->scd_base_addr +
		SCD_TX_STTS_MEM_LOWER_BOUND + (16 * txq->q.id);
	u8 buf[16];
	int i;
306 307 308 309 310 311 312 313 314 315 316 317 318 319

	spin_lock(&txq->lock);
	/* check if triggered erroneously */
	if (txq->q.read_ptr == txq->q.write_ptr) {
		spin_unlock(&txq->lock);
		return;
	}
	spin_unlock(&txq->lock);

	IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->q.id,
		jiffies_to_msecs(trans_pcie->wd_timeout));
	IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n",
		txq->q.read_ptr, txq->q.write_ptr);

320 321 322 323 324 325 326 327
	iwl_read_targ_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf));

	iwl_print_hex_error(trans, buf, sizeof(buf));

	for (i = 0; i < FH_TCSR_CHNL_NUM; i++)
		IWL_ERR(trans, "FH TRBs(%d) = 0x%08x\n", i,
			iwl_read_direct32(trans, FH_TX_TRB_REG(i)));

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
	for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) {
		u32 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(i));
		u8 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7;
		bool active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE));
		u32 tbl_dw =
			iwl_read_targ_mem(trans,
					  trans_pcie->scd_base_addr +
					  SCD_TRANS_TBL_OFFSET_QUEUE(i));

		if (i & 0x1)
			tbl_dw = (tbl_dw & 0xFFFF0000) >> 16;
		else
			tbl_dw = tbl_dw & 0x0000FFFF;

		IWL_ERR(trans,
			"Q %d is %sactive and mapped to fifo %d ra_tid 0x%04x [%d,%d]\n",
			i, active ? "" : "in", fifo, tbl_dw,
			iwl_read_prph(trans,
				      SCD_QUEUE_RDPTR(i)) & (txq->q.n_bd - 1),
			iwl_read_prph(trans, SCD_QUEUE_WRPTR(i)));
	}
349

350 351 352 353 354 355 356 357
	for (i = q->read_ptr; i != q->write_ptr;
	     i = iwl_queue_inc_wrap(i, q->n_bd)) {
		struct iwl_tx_cmd *tx_cmd =
			(struct iwl_tx_cmd *)txq->entries[i].cmd->payload;
		IWL_ERR(trans, "scratch %d = 0x%08x\n", i,
			get_unaligned_le32(&tx_cmd->scratch));
	}

358 359 360
	iwl_op_mode_nic_error(trans->op_mode);
}

361
static int iwl_trans_txq_alloc(struct iwl_trans *trans,
362 363
			       struct iwl_tx_queue *txq, int slots_num,
			       u32 txq_id)
364
{
365
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
366
	size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX;
367 368
	int i;

369
	if (WARN_ON(txq->entries || txq->tfds))
370 371
		return -EINVAL;

372 373 374 375
	setup_timer(&txq->stuck_timer, iwl_trans_pcie_queue_stuck_timer,
		    (unsigned long)txq);
	txq->trans_pcie = trans_pcie;

376 377
	txq->q.n_window = slots_num;

378 379 380
	txq->entries = kcalloc(slots_num,
			       sizeof(struct iwl_pcie_tx_queue_entry),
			       GFP_KERNEL);
381

382
	if (!txq->entries)
383 384
		goto error;

385
	if (txq_id == trans_pcie->cmd_queue)
386
		for (i = 0; i < slots_num; i++) {
387 388 389 390
			txq->entries[i].cmd =
				kmalloc(sizeof(struct iwl_device_cmd),
					GFP_KERNEL);
			if (!txq->entries[i].cmd)
391 392
				goto error;
		}
393 394 395

	/* Circular buffer of transmit frame descriptors (TFDs),
	 * shared with device */
396
	txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,
397
				       &txq->q.dma_addr, GFP_KERNEL);
398
	if (!txq->tfds) {
399
		IWL_ERR(trans, "dma_alloc_coherent(%zd) failed\n", tfd_sz);
400 401 402 403 404 405
		goto error;
	}
	txq->q.id = txq_id;

	return 0;
error:
406
	if (txq->entries && txq_id == trans_pcie->cmd_queue)
407
		for (i = 0; i < slots_num; i++)
408 409 410
			kfree(txq->entries[i].cmd);
	kfree(txq->entries);
	txq->entries = NULL;
411 412 413 414 415

	return -ENOMEM;

}

416
static int iwl_trans_txq_init(struct iwl_trans *trans, struct iwl_tx_queue *txq,
417
			      int slots_num, u32 txq_id)
418 419 420 421 422 423 424 425 426 427
{
	int ret;

	txq->need_update = 0;

	/* TFD_QUEUE_SIZE_MAX must be power-of-two size, otherwise
	 * iwl_queue_inc_wrap and iwl_queue_dec_wrap are broken. */
	BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1));

	/* Initialize queue's high/low-water marks, and head/tail indexes */
428
	ret = iwl_queue_init(&txq->q, TFD_QUEUE_SIZE_MAX, slots_num,
429 430 431 432
			txq_id);
	if (ret)
		return ret;

433 434
	spin_lock_init(&txq->lock);

435 436 437 438
	/*
	 * Tell nic where to find circular buffer of Tx Frame Descriptors for
	 * given Tx queue, and enable the DMA channel used for that queue.
	 * Circular buffer (TFD queue in DRAM) physical base address */
439
	iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(txq_id),
440 441 442 443 444
			     txq->q.dma_addr >> 8);

	return 0;
}

445 446 447
/**
 * iwl_tx_queue_unmap -  Unmap any remaining DMA mappings and free skb's
 */
448
static void iwl_tx_queue_unmap(struct iwl_trans *trans, int txq_id)
449
{
450 451
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
452
	struct iwl_queue *q = &txq->q;
453
	enum dma_data_direction dma_dir;
454 455 456 457

	if (!q->n_bd)
		return;

458 459 460
	/* In the command queue, all the TBs are mapped as BIDI
	 * so unmap them as such.
	 */
461
	if (txq_id == trans_pcie->cmd_queue)
462
		dma_dir = DMA_BIDIRECTIONAL;
463
	else
464 465
		dma_dir = DMA_TO_DEVICE;

466
	spin_lock_bh(&txq->lock);
467
	while (q->write_ptr != q->read_ptr) {
468
		iwl_txq_free_tfd(trans, txq, dma_dir);
469 470
		q->read_ptr = iwl_queue_inc_wrap(q->read_ptr, q->n_bd);
	}
471
	spin_unlock_bh(&txq->lock);
472 473
}

474 475 476 477 478 479 480 481
/**
 * iwl_tx_queue_free - Deallocate DMA queue.
 * @txq: Transmit queue to deallocate.
 *
 * Empty queue by removing and destroying all BD's.
 * Free all buffers.
 * 0-fill, but do not free "txq" descriptor structure.
 */
482
static void iwl_tx_queue_free(struct iwl_trans *trans, int txq_id)
483
{
484 485
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
486
	struct device *dev = trans->dev;
487
	int i;
488

489 490 491
	if (WARN_ON(!txq))
		return;

492
	iwl_tx_queue_unmap(trans, txq_id);
493 494

	/* De-alloc array of command/tx buffers */
495

496
	if (txq_id == trans_pcie->cmd_queue)
497
		for (i = 0; i < txq->q.n_window; i++)
498
			kfree(txq->entries[i].cmd);
499 500 501

	/* De-alloc circular buffer of TFDs */
	if (txq->q.n_bd) {
502
		dma_free_coherent(dev, sizeof(struct iwl_tfd) *
503 504 505 506
				  txq->q.n_bd, txq->tfds, txq->q.dma_addr);
		memset(&txq->q.dma_addr, 0, sizeof(txq->q.dma_addr));
	}

507 508
	kfree(txq->entries);
	txq->entries = NULL;
509

510 511
	del_timer_sync(&txq->stuck_timer);

512 513 514 515 516 517 518 519 520
	/* 0-fill queue descriptor structure */
	memset(txq, 0, sizeof(*txq));
}

/**
 * iwl_trans_tx_free - Free TXQ Context
 *
 * Destroy all TX DMA queues and structures
 */
521
static void iwl_trans_pcie_tx_free(struct iwl_trans *trans)
522 523
{
	int txq_id;
524
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
525 526

	/* Tx queues */
527
	if (trans_pcie->txq) {
528
		for (txq_id = 0;
529
		     txq_id < trans->cfg->base_params->num_of_queues; txq_id++)
530
			iwl_tx_queue_free(trans, txq_id);
531 532
	}

533 534
	kfree(trans_pcie->txq);
	trans_pcie->txq = NULL;
535

536
	iwlagn_free_dma_ptr(trans, &trans_pcie->kw);
537

538
	iwlagn_free_dma_ptr(trans, &trans_pcie->scd_bc_tbls);
539 540
}

541 542 543 544 545 546 547
/**
 * iwl_trans_tx_alloc - allocate TX context
 * Allocate all Tx DMA structures and initialize them
 *
 * @param priv
 * @return error code
 */
548
static int iwl_trans_tx_alloc(struct iwl_trans *trans)
549 550 551
{
	int ret;
	int txq_id, slots_num;
552
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
553

554
	u16 scd_bc_tbls_size = trans->cfg->base_params->num_of_queues *
555 556
			sizeof(struct iwlagn_scd_bc_tbl);

557 558
	/*It is not allowed to alloc twice, so warn when this happens.
	 * We cannot rely on the previous allocation, so free and fail */
559
	if (WARN_ON(trans_pcie->txq)) {
560 561 562 563
		ret = -EINVAL;
		goto error;
	}

564
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->scd_bc_tbls,
565
				   scd_bc_tbls_size);
566
	if (ret) {
567
		IWL_ERR(trans, "Scheduler BC Table allocation failed\n");
568 569 570 571
		goto error;
	}

	/* Alloc keep-warm buffer */
572
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE);
573
	if (ret) {
574
		IWL_ERR(trans, "Keep Warm allocation failed\n");
575 576 577
		goto error;
	}

578
	trans_pcie->txq = kcalloc(trans->cfg->base_params->num_of_queues,
579
				  sizeof(struct iwl_tx_queue), GFP_KERNEL);
580
	if (!trans_pcie->txq) {
581
		IWL_ERR(trans, "Not enough memory for txq\n");
582 583 584 585 586
		ret = ENOMEM;
		goto error;
	}

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
587
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
588
	     txq_id++) {
W
Wey-Yi Guy 已提交
589
		slots_num = (txq_id == trans_pcie->cmd_queue) ?
590
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
591 592
		ret = iwl_trans_txq_alloc(trans, &trans_pcie->txq[txq_id],
					  slots_num, txq_id);
593
		if (ret) {
594
			IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
595 596 597 598 599 600 601
			goto error;
		}
	}

	return 0;

error:
602
	iwl_trans_pcie_tx_free(trans);
603 604 605

	return ret;
}
606
static int iwl_tx_init(struct iwl_trans *trans)
607
{
608
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
609 610 611 612 613
	int ret;
	int txq_id, slots_num;
	unsigned long flags;
	bool alloc = false;

614
	if (!trans_pcie->txq) {
615
		ret = iwl_trans_tx_alloc(trans);
616 617 618 619 620
		if (ret)
			goto error;
		alloc = true;
	}

J
Johannes Berg 已提交
621
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
622 623

	/* Turn off all Tx DMA fifos */
624
	iwl_write_prph(trans, SCD_TXFACT, 0);
625 626

	/* Tell NIC where to find the "keep warm" buffer */
627
	iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
628
			   trans_pcie->kw.dma >> 4);
629

J
Johannes Berg 已提交
630
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
631 632

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
633
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
634
	     txq_id++) {
W
Wey-Yi Guy 已提交
635
		slots_num = (txq_id == trans_pcie->cmd_queue) ?
636
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
637 638
		ret = iwl_trans_txq_init(trans, &trans_pcie->txq[txq_id],
					 slots_num, txq_id);
639
		if (ret) {
640
			IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
641 642 643 644 645 646 647 648
			goto error;
		}
	}

	return 0;
error:
	/*Upon error, free only if we allocated something */
	if (alloc)
649
		iwl_trans_pcie_tx_free(trans);
650 651 652
	return ret;
}

653
static void iwl_set_pwr_vmain(struct iwl_trans *trans)
654 655 656 657 658 659
{
/*
 * (for documentation purposes)
 * to set power to V_AUX, do:

		if (pci_pme_capable(priv->pci_dev, PCI_D3cold))
660
			iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
661 662 663 664
					       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
					       ~APMG_PS_CTRL_MSK_PWR_SRC);
 */

665
	iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
666 667 668 669
			       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
			       ~APMG_PS_CTRL_MSK_PWR_SRC);
}

E
Emmanuel Grumbach 已提交
670 671 672 673 674 675 676
/* PCI registers */
#define PCI_CFG_RETRY_TIMEOUT	0x041
#define PCI_CFG_LINK_CTRL_VAL_L0S_EN	0x01
#define PCI_CFG_LINK_CTRL_VAL_L1_EN	0x02

static u16 iwl_pciexp_link_ctrl(struct iwl_trans *trans)
{
677
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
E
Emmanuel Grumbach 已提交
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
	int pos;
	u16 pci_lnk_ctl;

	struct pci_dev *pci_dev = trans_pcie->pci_dev;

	pos = pci_pcie_cap(pci_dev);
	pci_read_config_word(pci_dev, pos + PCI_EXP_LNKCTL, &pci_lnk_ctl);
	return pci_lnk_ctl;
}

static void iwl_apm_config(struct iwl_trans *trans)
{
	/*
	 * HW bug W/A for instability in PCIe bus L0S->L1 transition.
	 * Check if BIOS (or OS) enabled L1-ASPM on this device.
	 * If so (likely), disable L0S, so device moves directly L0->L1;
	 *    costs negligible amount of power savings.
	 * If not (unlikely), enable L0S, so there is at least some
	 *    power savings, even without L1.
	 */
	u16 lctl = iwl_pciexp_link_ctrl(trans);

	if ((lctl & PCI_CFG_LINK_CTRL_VAL_L1_EN) ==
				PCI_CFG_LINK_CTRL_VAL_L1_EN) {
		/* L1-ASPM enabled; disable(!) L0S */
		iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
		dev_printk(KERN_INFO, trans->dev,
			   "L1 Enabled; Disabling L0S\n");
	} else {
		/* L1-ASPM disabled; enable(!) L0S */
		iwl_clear_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
		dev_printk(KERN_INFO, trans->dev,
			   "L1 Disabled; Enabling L0S\n");
	}
712
	trans->pm_support = !(lctl & PCI_CFG_LINK_CTRL_VAL_L0S_EN);
E
Emmanuel Grumbach 已提交
713 714
}

715 716 717 718 719 720 721
/*
 * Start up NIC's basic functionality after it has been reset
 * (e.g. after platform boot, or shutdown via iwl_apm_stop())
 * NOTE:  This does not load uCode nor start the embedded processor
 */
static int iwl_apm_init(struct iwl_trans *trans)
{
D
Don Fry 已提交
722
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
723 724 725 726 727 728 729 730 731 732
	int ret = 0;
	IWL_DEBUG_INFO(trans, "Init card's basic functions\n");

	/*
	 * Use "set_bit" below rather than "write", to preserve any hardware
	 * bits already set by default after reset.
	 */

	/* Disable L0S exit timer (platform NMI Work/Around) */
	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
733
		    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
734 735 736 737 738 739

	/*
	 * Disable L0s without affecting L1;
	 *  don't wait for ICH L0s (ICH bug W/A)
	 */
	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
740
		    CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
741 742 743 744 745 746 747 748 749

	/* Set FH wait threshold to maximum (HW error during stress W/A) */
	iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);

	/*
	 * Enable HAP INTA (interrupt from management bus) to
	 * wake device's PCI Express link L1a -> L0s
	 */
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
750
		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
751

E
Emmanuel Grumbach 已提交
752
	iwl_apm_config(trans);
753 754

	/* Configure analog phase-lock-loop before activating to D0A */
755
	if (trans->cfg->base_params->pll_cfg_val)
756
		iwl_set_bit(trans, CSR_ANA_PLL_CFG,
757
			    trans->cfg->base_params->pll_cfg_val);
758 759 760 761 762 763 764 765 766 767 768 769 770

	/*
	 * Set "initialization complete" bit to move adapter from
	 * D0U* --> D0A* (powered-up active) state.
	 */
	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

	/*
	 * Wait for clock stabilization; once stabilized, access to
	 * device-internal resources is supported, e.g. iwl_write_prph()
	 * and accesses to uCode SRAM.
	 */
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
771 772
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000);
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
	if (ret < 0) {
		IWL_DEBUG_INFO(trans, "Failed to init the card\n");
		goto out;
	}

	/*
	 * Enable DMA clock and wait for it to stabilize.
	 *
	 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0" bits
	 * do not disable clocks.  This preserves any hardware bits already
	 * set by default in "CLK_CTRL_REG" after reset.
	 */
	iwl_write_prph(trans, APMG_CLK_EN_REG, APMG_CLK_VAL_DMA_CLK_RQT);
	udelay(20);

	/* Disable L1-Active */
	iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
			  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);

D
Don Fry 已提交
792
	set_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
793 794 795 796 797

out:
	return ret;
}

798 799 800 801 802 803 804 805
static int iwl_apm_stop_master(struct iwl_trans *trans)
{
	int ret = 0;

	/* stop device's busmaster DMA activity */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);

	ret = iwl_poll_bit(trans, CSR_RESET,
806 807
			   CSR_RESET_REG_FLAG_MASTER_DISABLED,
			   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
808 809 810 811 812 813 814 815 816 817
	if (ret)
		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");

	IWL_DEBUG_INFO(trans, "stop master\n");

	return ret;
}

static void iwl_apm_stop(struct iwl_trans *trans)
{
D
Don Fry 已提交
818
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
819 820
	IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");

D
Don Fry 已提交
821
	clear_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838

	/* Stop device's DMA activity */
	iwl_apm_stop_master(trans);

	/* Reset the entire device */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);

	udelay(10);

	/*
	 * Clear "initialization complete" bit to move adapter from
	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
	 */
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
}

839
static int iwl_nic_init(struct iwl_trans *trans)
840
{
J
Johannes Berg 已提交
841
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
842 843 844
	unsigned long flags;

	/* nic_init */
J
Johannes Berg 已提交
845
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
846
	iwl_apm_init(trans);
847 848

	/* Set interrupt coalescing calibration timer to default (512 usecs) */
849
	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_CALIB_TIMEOUT_DEF);
850

J
Johannes Berg 已提交
851
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
852

853
	iwl_set_pwr_vmain(trans);
854

J
Johannes Berg 已提交
855
	iwl_op_mode_nic_config(trans->op_mode);
856

857
#ifndef CONFIG_IWLWIFI_IDI
858
	/* Allocate the RX queue, or reset if it is already allocated */
859
	iwl_rx_init(trans);
860
#endif
861 862

	/* Allocate or reset and init all Tx and Command queues */
863
	if (iwl_tx_init(trans))
864 865
		return -ENOMEM;

866
	if (trans->cfg->base_params->shadow_reg_enable) {
867
		/* enable shadow regs in HW */
868
		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
869
		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
870 871 872 873 874 875 876 877
	}

	return 0;
}

#define HW_READY_TIMEOUT (50)

/* Note: returns poll_bit return value, which is >= 0 if success */
878
static int iwl_set_hw_ready(struct iwl_trans *trans)
879 880 881
{
	int ret;

882
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
883
		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
884 885

	/* See if we got it */
886
	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
887 888 889
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   HW_READY_TIMEOUT);
890

891
	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
892 893 894 895
	return ret;
}

/* Note: returns standard 0/-ERROR code */
896
static int iwl_prepare_card_hw(struct iwl_trans *trans)
897 898 899
{
	int ret;

900
	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
901

902
	ret = iwl_set_hw_ready(trans);
903
	/* If the card is ready, exit 0 */
904 905 906 907
	if (ret >= 0)
		return 0;

	/* If HW is not ready, prepare the conditions to check again */
908
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
909
		    CSR_HW_IF_CONFIG_REG_PREPARE);
910

911
	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
912 913
			   ~CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE,
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE, 150000);
914 915 916 917 918

	if (ret < 0)
		return ret;

	/* HW should be ready by now, check again. */
919
	ret = iwl_set_hw_ready(trans);
920 921 922 923 924
	if (ret >= 0)
		return 0;
	return ret;
}

925 926 927
/*
 * ucode
 */
D
David Spinadel 已提交
928 929
static int iwl_load_section(struct iwl_trans *trans, u8 section_num,
			    const struct fw_desc *section)
930
{
931
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
D
David Spinadel 已提交
932 933 934
	dma_addr_t phy_addr = section->p_addr;
	u32 byte_cnt = section->len;
	u32 dst_addr = section->offset;
935 936
	int ret;

937
	trans_pcie->ucode_write_complete = false;
938 939

	iwl_write_direct32(trans,
940 941
			   FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
942 943

	iwl_write_direct32(trans,
944 945
			   FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
			   dst_addr);
946 947 948 949 950 951

	iwl_write_direct32(trans,
		FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
		phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);

	iwl_write_direct32(trans,
952 953 954
			   FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
			   (iwl_get_dma_hi_addr(phy_addr)
				<< FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
955 956

	iwl_write_direct32(trans,
957 958 959 960
			   FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
			   1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
			   1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
			   FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
961 962

	iwl_write_direct32(trans,
963 964 965 966
			   FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE	|
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE	|
			   FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
967

D
David Spinadel 已提交
968 969
	IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
		     section_num);
970 971
	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
				 trans_pcie->ucode_write_complete, 5 * HZ);
972
	if (!ret) {
D
David Spinadel 已提交
973 974
		IWL_ERR(trans, "Could not load the [%d] uCode section\n",
			section_num);
975 976 977 978 979 980
		return -ETIMEDOUT;
	}

	return 0;
}

981 982
static int iwl_load_given_ucode(struct iwl_trans *trans,
				const struct fw_img *image)
983 984
{
	int ret = 0;
D
David Spinadel 已提交
985
		int i;
986

D
David Spinadel 已提交
987 988 989
		for (i = 0; i < IWL_UCODE_SECTION_MAX; i++) {
			if (!image->sec[i].p_addr)
				break;
990

D
David Spinadel 已提交
991 992 993 994
			ret = iwl_load_section(trans, i, &image->sec[i]);
			if (ret)
				return ret;
		}
995 996 997 998 999 1000 1001

	/* Remove all resets to allow NIC to operate */
	iwl_write32(trans, CSR_RESET, 0);

	return 0;
}

1002 1003
static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
				   const struct fw_img *fw)
1004 1005
{
	int ret;
1006
	bool hw_rfkill;
1007

1008 1009
	/* This may fail if AMT took ownership of the device */
	if (iwl_prepare_card_hw(trans)) {
1010
		IWL_WARN(trans, "Exit HW not ready\n");
1011 1012 1013
		return -EIO;
	}

1014 1015
	iwl_enable_rfkill_int(trans);

1016
	/* If platform's RF_KILL switch is NOT set to KILL */
1017
	hw_rfkill = iwl_is_rfkill_set(trans);
1018
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1019
	if (hw_rfkill)
1020 1021
		return -ERFKILL;

1022
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1023

1024
	ret = iwl_nic_init(trans);
1025
	if (ret) {
1026
		IWL_ERR(trans, "Unable to init nic\n");
1027 1028 1029 1030
		return ret;
	}

	/* make sure rfkill handshake bits are cleared */
1031 1032
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1033 1034 1035
		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);

	/* clear (again), then enable host interrupts */
1036
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1037
	iwl_enable_interrupts(trans);
1038 1039

	/* really make sure rfkill handshake bits are cleared */
1040 1041
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1042

1043
	/* Load the given image to the HW */
1044
	return iwl_load_given_ucode(trans, fw);
1045 1046
}

1047 1048 1049
/*
 * Activate/Deactivate Tx DMA/FIFO channels according tx fifos mask
 */
1050
static void iwl_trans_txq_set_sched(struct iwl_trans *trans, u32 mask)
1051
{
J
Johannes Berg 已提交
1052 1053 1054
	struct iwl_trans_pcie __maybe_unused *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);

1055
	iwl_write_prph(trans, SCD_TXFACT, mask);
1056 1057
}

1058
static void iwl_tx_start(struct iwl_trans *trans)
1059
{
1060
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1061
	u32 a;
1062
	int chan;
1063 1064
	u32 reg_val;

1065 1066 1067 1068
	/* make sure all queue are not stopped/used */
	memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped));
	memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used));

1069
	trans_pcie->scd_base_addr =
1070
		iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
1071
	a = trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_LOWER_BOUND;
1072
	/* reset conext data memory */
1073
	for (; a < trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_UPPER_BOUND;
1074
		a += 4)
1075
		iwl_write_targ_mem(trans, a, 0);
1076
	/* reset tx status memory */
1077
	for (; a < trans_pcie->scd_base_addr + SCD_TX_STTS_MEM_UPPER_BOUND;
1078
		a += 4)
1079
		iwl_write_targ_mem(trans, a, 0);
1080
	for (; a < trans_pcie->scd_base_addr +
1081
	       SCD_TRANS_TBL_OFFSET_QUEUE(
1082
				trans->cfg->base_params->num_of_queues);
1083
	       a += 4)
1084
		iwl_write_targ_mem(trans, a, 0);
1085

1086
	iwl_write_prph(trans, SCD_DRAM_BASE_ADDR,
1087
		       trans_pcie->scd_bc_tbls.dma >> 10);
1088

1089 1090 1091 1092 1093
	/* The chain extension of the SCD doesn't work well. This feature is
	 * enabled by default by the HW, so we need to disable it manually.
	 */
	iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);

1094 1095
	iwl_trans_ac_txq_enable(trans, trans_pcie->cmd_queue,
				trans_pcie->cmd_fifo);
1096

1097 1098 1099
	/* Activate all Tx DMA/FIFO channels */
	iwl_trans_txq_set_sched(trans, IWL_MASK(0, 7));

1100 1101
	/* Enable DMA channel */
	for (chan = 0; chan < FH_TCSR_CHNL_NUM ; chan++)
1102
		iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan),
1103 1104
				   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
				   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1105 1106

	/* Update FH chicken bits */
1107 1108
	reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG);
	iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG,
1109 1110 1111
			   reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);

	/* Enable L1-Active */
1112
	iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
1113
			    APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1114 1115
}

1116 1117 1118 1119 1120 1121
static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans)
{
	iwl_reset_ict(trans);
	iwl_tx_start(trans);
}

1122 1123 1124
/**
 * iwlagn_txq_ctx_stop - Stop all Tx DMA channels
 */
1125
static int iwl_trans_tx_stop(struct iwl_trans *trans)
1126
{
1127
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1128
	int ch, txq_id, ret;
1129 1130 1131
	unsigned long flags;

	/* Turn off all Tx DMA fifos */
J
Johannes Berg 已提交
1132
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1133

1134
	iwl_trans_txq_set_sched(trans, 0);
1135 1136

	/* Stop each Tx DMA channel, and wait for it to be idle */
1137
	for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) {
1138
		iwl_write_direct32(trans,
1139
				   FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0);
1140
		ret = iwl_poll_direct_bit(trans, FH_TSSR_TX_STATUS_REG,
1141
			FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch), 1000);
1142
		if (ret < 0)
1143
			IWL_ERR(trans,
1144
				"Failing on timeout while stopping DMA channel %d [0x%08x]\n",
1145 1146 1147
				ch,
				iwl_read_direct32(trans,
						  FH_TSSR_TX_STATUS_REG));
1148
	}
J
Johannes Berg 已提交
1149
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1150

1151
	if (!trans_pcie->txq) {
1152 1153
		IWL_WARN(trans,
			 "Stopping tx queues that aren't allocated...\n");
1154 1155 1156 1157
		return 0;
	}

	/* Unmap DMA from host system and free skb's */
1158
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
1159
	     txq_id++)
1160
		iwl_tx_queue_unmap(trans, txq_id);
1161 1162 1163 1164

	return 0;
}

1165
static void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1166
{
1167
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1168
	unsigned long flags;
1169

1170
	/* tell the device to stop sending interrupts */
J
Johannes Berg 已提交
1171
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1172
	iwl_disable_interrupts(trans);
J
Johannes Berg 已提交
1173
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1174

1175
	/* device going down, Stop using ICT table */
1176
	iwl_disable_ict(trans);
1177 1178 1179 1180 1181 1182 1183 1184

	/*
	 * If a HW restart happens during firmware loading,
	 * then the firmware loading might call this function
	 * and later it might be called again due to the
	 * restart. So don't process again if the device is
	 * already dead.
	 */
D
Don Fry 已提交
1185
	if (test_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status)) {
1186
		iwl_trans_tx_stop(trans);
1187
#ifndef CONFIG_IWLWIFI_IDI
1188
		iwl_trans_rx_stop(trans);
1189
#endif
1190
		/* Power-down device's busmaster DMA clocks */
1191
		iwl_write_prph(trans, APMG_CLK_DIS_REG,
1192 1193 1194 1195 1196
			       APMG_CLK_VAL_DMA_CLK_RQT);
		udelay(5);
	}

	/* Make sure (redundant) we've released our request to stay awake */
1197
	iwl_clear_bit(trans, CSR_GP_CNTRL,
1198
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1199 1200

	/* Stop the device, and put it in low power state */
1201
	iwl_apm_stop(trans);
1202 1203 1204 1205

	/* Upon stop, the APM issues an interrupt if HW RF kill is set.
	 * Clean again the interrupt here
	 */
J
Johannes Berg 已提交
1206
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1207
	iwl_disable_interrupts(trans);
J
Johannes Berg 已提交
1208
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1209

1210 1211
	iwl_enable_rfkill_int(trans);

1212
	/* wait to make sure we flush pending tasklet*/
J
Johannes Berg 已提交
1213
	synchronize_irq(trans_pcie->irq);
1214 1215
	tasklet_kill(&trans_pcie->irq_tasklet);

J
Johannes Berg 已提交
1216 1217
	cancel_work_sync(&trans_pcie->rx_replenish);

1218
	/* stop and reset the on-board processor */
1219
	iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_NEVO_RESET);
D
Don Fry 已提交
1220 1221 1222 1223 1224

	/* clear all status bits */
	clear_bit(STATUS_HCMD_ACTIVE, &trans_pcie->status);
	clear_bit(STATUS_INT_ENABLED, &trans_pcie->status);
	clear_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
1225
	clear_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
1226 1227
}

1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
static void iwl_trans_pcie_wowlan_suspend(struct iwl_trans *trans)
{
	/* let the ucode operate on its own */
	iwl_write32(trans, CSR_UCODE_DRV_GP1_SET,
		    CSR_UCODE_DRV_GP1_BIT_D3_CFG_COMPLETE);

	iwl_disable_interrupts(trans);
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
}

1239
static int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
1240
			     struct iwl_device_cmd *dev_cmd, int txq_id)
1241
{
1242 1243
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
1244
	struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *) dev_cmd->payload;
1245
	struct iwl_cmd_meta *out_meta;
1246 1247
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
1248 1249 1250 1251 1252
	dma_addr_t phys_addr = 0;
	dma_addr_t txcmd_phys;
	dma_addr_t scratch_phys;
	u16 len, firstlen, secondlen;
	u8 wait_write_ptr = 0;
1253
	__le16 fc = hdr->frame_control;
1254
	u8 hdr_len = ieee80211_hdrlen(fc);
1255
	u16 __maybe_unused wifi_seq;
1256

1257
	txq = &trans_pcie->txq[txq_id];
1258 1259
	q = &txq->q;

1260 1261 1262 1263
	if (unlikely(!test_bit(txq_id, trans_pcie->queue_used))) {
		WARN_ON_ONCE(1);
		return -EINVAL;
	}
1264

1265
	spin_lock(&txq->lock);
1266

1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
	/* In AGG mode, the index in the ring must correspond to the WiFi
	 * sequence number. This is a HW requirements to help the SCD to parse
	 * the BA.
	 * Check here that the packets are in the right place on the ring.
	 */
#ifdef CONFIG_IWLWIFI_DEBUG
	wifi_seq = SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
	WARN_ONCE((iwl_read_prph(trans, SCD_AGGR_SEL) & BIT(txq_id)) &&
		  ((wifi_seq & 0xff) != q->write_ptr),
		  "Q: %d WiFi Seq %d tfdNum %d",
		  txq_id, wifi_seq, q->write_ptr);
#endif

1280
	/* Set up driver data for this TFD */
1281 1282
	txq->entries[q->write_ptr].skb = skb;
	txq->entries[q->write_ptr].cmd = dev_cmd;
1283 1284

	dev_cmd->hdr.cmd = REPLY_TX;
1285 1286 1287
	dev_cmd->hdr.sequence =
		cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
			    INDEX_TO_SEQ(q->write_ptr)));
1288 1289

	/* Set up first empty entry in queue's array of Tx/cmd buffers */
1290
	out_meta = &txq->entries[q->write_ptr].meta;
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310

	/*
	 * Use the first empty entry in this queue's command buffer array
	 * to contain the Tx command and MAC header concatenated together
	 * (payload data will be in another buffer).
	 * Size of this varies, due to varying MAC header length.
	 * If end is not dword aligned, we'll have 2 extra bytes at the end
	 * of the MAC header (device reads on dword boundaries).
	 * We'll tell device about this padding later.
	 */
	len = sizeof(struct iwl_tx_cmd) +
		sizeof(struct iwl_cmd_header) + hdr_len;
	firstlen = (len + 3) & ~3;

	/* Tell NIC about any 2-byte padding after MAC header */
	if (firstlen != len)
		tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK;

	/* Physical address of this Tx command's header (not MAC header!),
	 * within command buffer array. */
1311
	txcmd_phys = dma_map_single(trans->dev,
1312 1313
				    &dev_cmd->hdr, firstlen,
				    DMA_BIDIRECTIONAL);
1314
	if (unlikely(dma_mapping_error(trans->dev, txcmd_phys)))
1315
		goto out_err;
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
	dma_unmap_addr_set(out_meta, mapping, txcmd_phys);
	dma_unmap_len_set(out_meta, len, firstlen);

	if (!ieee80211_has_morefrags(fc)) {
		txq->need_update = 1;
	} else {
		wait_write_ptr = 1;
		txq->need_update = 0;
	}

	/* Set up TFD's 2nd entry to point directly to remainder of skb,
	 * if any (802.11 null frames have no payload). */
	secondlen = skb->len - hdr_len;
	if (secondlen > 0) {
1330
		phys_addr = dma_map_single(trans->dev, skb->data + hdr_len,
1331
					   secondlen, DMA_TO_DEVICE);
1332 1333
		if (unlikely(dma_mapping_error(trans->dev, phys_addr))) {
			dma_unmap_single(trans->dev,
1334 1335 1336
					 dma_unmap_addr(out_meta, mapping),
					 dma_unmap_len(out_meta, len),
					 DMA_BIDIRECTIONAL);
1337
			goto out_err;
1338 1339 1340 1341
		}
	}

	/* Attach buffers to TFD */
1342
	iwlagn_txq_attach_buf_to_tfd(trans, txq, txcmd_phys, firstlen, 1);
1343
	if (secondlen > 0)
1344
		iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr,
1345 1346 1347 1348 1349 1350
					     secondlen, 0);

	scratch_phys = txcmd_phys + sizeof(struct iwl_cmd_header) +
				offsetof(struct iwl_tx_cmd, scratch);

	/* take back ownership of DMA buffer to enable update */
1351
	dma_sync_single_for_cpu(trans->dev, txcmd_phys, firstlen,
1352
				DMA_BIDIRECTIONAL);
1353 1354 1355
	tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
	tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);

1356
	IWL_DEBUG_TX(trans, "sequence nr = 0X%x\n",
1357
		     le16_to_cpu(dev_cmd->hdr.sequence));
1358
	IWL_DEBUG_TX(trans, "tx_flags = 0X%x\n", le32_to_cpu(tx_cmd->tx_flags));
1359 1360

	/* Set up entry for this TFD in Tx byte-count array */
1361
	iwl_trans_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len));
1362

1363
	dma_sync_single_for_device(trans->dev, txcmd_phys, firstlen,
1364
				   DMA_BIDIRECTIONAL);
1365

1366
	trace_iwlwifi_dev_tx(trans->dev,
1367
			     &txq->tfds[txq->q.write_ptr],
1368 1369 1370 1371
			     sizeof(struct iwl_tfd),
			     &dev_cmd->hdr, firstlen,
			     skb->data + hdr_len, secondlen);

1372
	/* start timer if queue currently empty */
1373 1374
	if (txq->need_update && q->read_ptr == q->write_ptr &&
	    trans_pcie->wd_timeout)
1375 1376
		mod_timer(&txq->stuck_timer, jiffies + trans_pcie->wd_timeout);

1377 1378
	/* Tell device the write index *just past* this latest filled TFD */
	q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd);
1379 1380
	iwl_txq_update_write_ptr(trans, txq);

1381 1382 1383 1384 1385 1386
	/*
	 * At this point the frame is "transmitted" successfully
	 * and we will get a TX status notification eventually,
	 * regardless of the value of ret. "ret" only indicates
	 * whether or not we should update the write pointer.
	 */
1387
	if (iwl_queue_space(q) < q->high_mark) {
1388 1389
		if (wait_write_ptr) {
			txq->need_update = 1;
1390
			iwl_txq_update_write_ptr(trans, txq);
1391
		} else {
1392
			iwl_stop_queue(trans, txq);
1393 1394
		}
	}
1395
	spin_unlock(&txq->lock);
1396
	return 0;
1397 1398 1399
 out_err:
	spin_unlock(&txq->lock);
	return -1;
1400 1401
}

1402
static int iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1403
{
1404
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1405
	int err;
1406
	bool hw_rfkill;
1407

1408 1409
	trans_pcie->inta_mask = CSR_INI_SET_MASK;

1410 1411 1412
	if (!trans_pcie->irq_requested) {
		tasklet_init(&trans_pcie->irq_tasklet, (void (*)(unsigned long))
			iwl_irq_tasklet, (unsigned long)trans);
1413

1414
		iwl_alloc_isr_ict(trans);
1415

J
Johannes Berg 已提交
1416
		err = request_irq(trans_pcie->irq, iwl_isr_ict, IRQF_SHARED,
1417
				  DRV_NAME, trans);
1418 1419
		if (err) {
			IWL_ERR(trans, "Error allocating IRQ %d\n",
J
Johannes Berg 已提交
1420
				trans_pcie->irq);
1421
			goto error;
1422 1423 1424 1425
		}

		INIT_WORK(&trans_pcie->rx_replenish, iwl_bg_rx_replenish);
		trans_pcie->irq_requested = true;
1426 1427
	}

1428 1429
	err = iwl_prepare_card_hw(trans);
	if (err) {
1430
		IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1431
		goto err_free_irq;
1432
	}
1433 1434 1435

	iwl_apm_init(trans);

1436 1437 1438
	/* From now on, the op_mode will be kept updated about RF kill state */
	iwl_enable_rfkill_int(trans);

1439
	hw_rfkill = iwl_is_rfkill_set(trans);
1440
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1441

1442 1443
	return err;

1444
err_free_irq:
1445
	trans_pcie->irq_requested = false;
J
Johannes Berg 已提交
1446
	free_irq(trans_pcie->irq, trans);
1447 1448 1449 1450
error:
	iwl_free_isr_ict(trans);
	tasklet_kill(&trans_pcie->irq_tasklet);
	return err;
1451 1452
}

1453 1454
static void iwl_trans_pcie_stop_hw(struct iwl_trans *trans,
				   bool op_mode_leaving)
1455
{
1456
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1457
	bool hw_rfkill;
1458
	unsigned long flags;
1459

1460 1461
	iwl_apm_stop(trans);

1462 1463 1464
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
	iwl_disable_interrupts(trans);
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1465

1466
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1467

1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
	if (!op_mode_leaving) {
		/*
		 * Even if we stop the HW, we still want the RF kill
		 * interrupt
		 */
		iwl_enable_rfkill_int(trans);

		/*
		 * Check again since the RF kill state may have changed while
		 * all the interrupts were disabled, in this case we couldn't
		 * receive the RF kill interrupt and update the state in the
		 * op_mode.
		 */
		hw_rfkill = iwl_is_rfkill_set(trans);
		iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
	}
1484 1485
}

1486 1487
static void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
				   struct sk_buff_head *skbs)
1488
{
1489 1490
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
1491 1492
	/* n_bd is usually 256 => n_bd - 1 = 0xff */
	int tfd_num = ssn & (txq->q.n_bd - 1);
1493
	int freed = 0;
1494

1495 1496
	spin_lock(&txq->lock);

1497
	if (txq->q.read_ptr != tfd_num) {
1498 1499
		IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n",
				   txq_id, txq->q.read_ptr, tfd_num, ssn);
1500
		freed = iwl_tx_queue_reclaim(trans, txq_id, tfd_num, skbs);
1501
		if (iwl_queue_space(&txq->q) > txq->q.low_mark)
1502
			iwl_wake_queue(trans, txq);
1503
	}
1504 1505

	spin_unlock(&txq->lock);
1506 1507
}

1508 1509
static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
{
1510
	writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1511 1512 1513 1514
}

static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
{
1515
	writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1516 1517 1518 1519
}

static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
{
1520
	return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1521 1522
}

1523
static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1524
				     const struct iwl_trans_config *trans_cfg)
1525 1526 1527 1528
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans_pcie->cmd_queue = trans_cfg->cmd_queue;
1529
	trans_pcie->cmd_fifo = trans_cfg->cmd_fifo;
1530 1531 1532 1533 1534 1535 1536
	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
		trans_pcie->n_no_reclaim_cmds = 0;
	else
		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
	if (trans_pcie->n_no_reclaim_cmds)
		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1537

1538 1539 1540 1541 1542
	trans_pcie->rx_buf_size_8k = trans_cfg->rx_buf_size_8k;
	if (trans_pcie->rx_buf_size_8k)
		trans_pcie->rx_page_order = get_order(8 * 1024);
	else
		trans_pcie->rx_page_order = get_order(4 * 1024);
1543 1544 1545

	trans_pcie->wd_timeout =
		msecs_to_jiffies(trans_cfg->queue_watchdog_timeout);
J
Johannes Berg 已提交
1546 1547

	trans_pcie->command_names = trans_cfg->command_names;
1548 1549
}

1550
void iwl_trans_pcie_free(struct iwl_trans *trans)
1551
{
1552
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1553

1554
	iwl_trans_pcie_tx_free(trans);
1555
#ifndef CONFIG_IWLWIFI_IDI
1556
	iwl_trans_pcie_rx_free(trans);
1557
#endif
1558
	if (trans_pcie->irq_requested == true) {
J
Johannes Berg 已提交
1559
		free_irq(trans_pcie->irq, trans);
1560 1561
		iwl_free_isr_ict(trans);
	}
1562 1563

	pci_disable_msi(trans_pcie->pci_dev);
1564
	iounmap(trans_pcie->hw_base);
1565 1566
	pci_release_regions(trans_pcie->pci_dev);
	pci_disable_device(trans_pcie->pci_dev);
1567
	kmem_cache_destroy(trans->dev_cmd_pool);
1568

1569
	kfree(trans);
1570 1571
}

D
Don Fry 已提交
1572 1573 1574 1575 1576
static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (state)
1577
		set_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
D
Don Fry 已提交
1578
	else
1579
		clear_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
D
Don Fry 已提交
1580 1581
}

J
Johannes Berg 已提交
1582
#ifdef CONFIG_PM_SLEEP
1583 1584 1585 1586 1587 1588 1589
static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
{
	return 0;
}

static int iwl_trans_pcie_resume(struct iwl_trans *trans)
{
1590
	bool hw_rfkill;
1591

1592 1593
	iwl_enable_rfkill_int(trans);

1594
	hw_rfkill = iwl_is_rfkill_set(trans);
1595
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1596

1597
	if (!hw_rfkill)
1598 1599
		iwl_enable_interrupts(trans);

1600 1601
	return 0;
}
J
Johannes Berg 已提交
1602
#endif /* CONFIG_PM_SLEEP */
1603

1604 1605 1606 1607
#define IWL_FLUSH_WAIT_MS	2000

static int iwl_trans_pcie_wait_tx_queue_empty(struct iwl_trans *trans)
{
1608
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1609 1610 1611 1612 1613 1614 1615
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	int cnt;
	unsigned long now = jiffies;
	int ret = 0;

	/* waiting for all the tx frames complete might take a while */
1616
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
W
Wey-Yi Guy 已提交
1617
		if (cnt == trans_pcie->cmd_queue)
1618
			continue;
1619
		txq = &trans_pcie->txq[cnt];
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
		q = &txq->q;
		while (q->read_ptr != q->write_ptr && !time_after(jiffies,
		       now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS)))
			msleep(1);

		if (q->read_ptr != q->write_ptr) {
			IWL_ERR(trans, "fail to flush all tx fifo queues\n");
			ret = -ETIMEDOUT;
			break;
		}
	}
	return ret;
}

1634 1635
static const char *get_fh_string(int cmd)
{
J
Johannes Berg 已提交
1636
#define IWL_CMD(x) case x: return #x
1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
	switch (cmd) {
	IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
	IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
	IWL_CMD(FH_RSCSR_CHNL0_WPTR);
	IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
	IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
	IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
	IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
	IWL_CMD(FH_TSSR_TX_STATUS_REG);
	IWL_CMD(FH_TSSR_TX_ERROR_REG);
	default:
		return "UNKNOWN";
	}
J
Johannes Berg 已提交
1650
#undef IWL_CMD
1651 1652
}

1653
int iwl_dump_fh(struct iwl_trans *trans, char **buf)
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
{
	int i;
	static const u32 fh_tbl[] = {
		FH_RSCSR_CHNL0_STTS_WPTR_REG,
		FH_RSCSR_CHNL0_RBDCB_BASE_REG,
		FH_RSCSR_CHNL0_WPTR,
		FH_MEM_RCSR_CHNL0_CONFIG_REG,
		FH_MEM_RSSR_SHARED_CTRL_REG,
		FH_MEM_RSSR_RX_STATUS_REG,
		FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
		FH_TSSR_TX_STATUS_REG,
		FH_TSSR_TX_ERROR_REG
	};
1667 1668 1669 1670 1671 1672

#ifdef CONFIG_IWLWIFI_DEBUGFS
	if (buf) {
		int pos = 0;
		size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;

1673 1674 1675
		*buf = kmalloc(bufsz, GFP_KERNEL);
		if (!*buf)
			return -ENOMEM;
1676

1677 1678
		pos += scnprintf(*buf + pos, bufsz - pos,
				"FH register values:\n");
1679 1680

		for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
1681 1682 1683
			pos += scnprintf(*buf + pos, bufsz - pos,
				"  %34s: 0X%08x\n",
				get_fh_string(fh_tbl[i]),
1684
				iwl_read_direct32(trans, fh_tbl[i]));
1685

1686 1687 1688
		return pos;
	}
#endif
1689

1690
	IWL_ERR(trans, "FH register values:\n");
1691
	for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++)
1692 1693
		IWL_ERR(trans, "  %34s: 0X%08x\n",
			get_fh_string(fh_tbl[i]),
1694
			iwl_read_direct32(trans, fh_tbl[i]));
1695

1696 1697 1698 1699 1700
	return 0;
}

static const char *get_csr_string(int cmd)
{
J
Johannes Berg 已提交
1701
#define IWL_CMD(x) case x: return #x
1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
	switch (cmd) {
	IWL_CMD(CSR_HW_IF_CONFIG_REG);
	IWL_CMD(CSR_INT_COALESCING);
	IWL_CMD(CSR_INT);
	IWL_CMD(CSR_INT_MASK);
	IWL_CMD(CSR_FH_INT_STATUS);
	IWL_CMD(CSR_GPIO_IN);
	IWL_CMD(CSR_RESET);
	IWL_CMD(CSR_GP_CNTRL);
	IWL_CMD(CSR_HW_REV);
	IWL_CMD(CSR_EEPROM_REG);
	IWL_CMD(CSR_EEPROM_GP);
	IWL_CMD(CSR_OTP_GP_REG);
	IWL_CMD(CSR_GIO_REG);
	IWL_CMD(CSR_GP_UCODE_REG);
	IWL_CMD(CSR_GP_DRIVER_REG);
	IWL_CMD(CSR_UCODE_DRV_GP1);
	IWL_CMD(CSR_UCODE_DRV_GP2);
	IWL_CMD(CSR_LED_REG);
	IWL_CMD(CSR_DRAM_INT_TBL_REG);
	IWL_CMD(CSR_GIO_CHICKEN_BITS);
	IWL_CMD(CSR_ANA_PLL_CFG);
	IWL_CMD(CSR_HW_REV_WA_REG);
	IWL_CMD(CSR_DBG_HPET_MEM_REG);
	default:
		return "UNKNOWN";
	}
J
Johannes Berg 已提交
1729
#undef IWL_CMD
1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765
}

void iwl_dump_csr(struct iwl_trans *trans)
{
	int i;
	static const u32 csr_tbl[] = {
		CSR_HW_IF_CONFIG_REG,
		CSR_INT_COALESCING,
		CSR_INT,
		CSR_INT_MASK,
		CSR_FH_INT_STATUS,
		CSR_GPIO_IN,
		CSR_RESET,
		CSR_GP_CNTRL,
		CSR_HW_REV,
		CSR_EEPROM_REG,
		CSR_EEPROM_GP,
		CSR_OTP_GP_REG,
		CSR_GIO_REG,
		CSR_GP_UCODE_REG,
		CSR_GP_DRIVER_REG,
		CSR_UCODE_DRV_GP1,
		CSR_UCODE_DRV_GP2,
		CSR_LED_REG,
		CSR_DRAM_INT_TBL_REG,
		CSR_GIO_CHICKEN_BITS,
		CSR_ANA_PLL_CFG,
		CSR_HW_REV_WA_REG,
		CSR_DBG_HPET_MEM_REG
	};
	IWL_ERR(trans, "CSR values:\n");
	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
		"CSR_INT_PERIODIC_REG)\n");
	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
		IWL_ERR(trans, "  %25s: 0X%08x\n",
			get_csr_string(csr_tbl[i]),
1766
			iwl_read32(trans, csr_tbl[i]));
1767 1768 1769
	}
}

1770 1771 1772
#ifdef CONFIG_IWLWIFI_DEBUGFS
/* create and remove of files */
#define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
1773
	if (!debugfs_create_file(#name, mode, parent, trans,		\
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
				 &iwl_dbgfs_##name##_ops))		\
		return -ENOMEM;						\
} while (0)

/* file operation */
#define DEBUGFS_READ_FUNC(name)                                         \
static ssize_t iwl_dbgfs_##name##_read(struct file *file,               \
					char __user *user_buf,          \
					size_t count, loff_t *ppos);

#define DEBUGFS_WRITE_FUNC(name)                                        \
static ssize_t iwl_dbgfs_##name##_write(struct file *file,              \
					const char __user *user_buf,    \
					size_t count, loff_t *ppos);


#define DEBUGFS_READ_FILE_OPS(name)					\
	DEBUGFS_READ_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.read = iwl_dbgfs_##name##_read,				\
1794
	.open = simple_open,						\
1795 1796 1797
	.llseek = generic_file_llseek,					\
};

1798 1799 1800 1801
#define DEBUGFS_WRITE_FILE_OPS(name)                                    \
	DEBUGFS_WRITE_FUNC(name);                                       \
static const struct file_operations iwl_dbgfs_##name##_ops = {          \
	.write = iwl_dbgfs_##name##_write,                              \
1802
	.open = simple_open,						\
1803 1804 1805
	.llseek = generic_file_llseek,					\
};

1806 1807 1808 1809 1810 1811
#define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
	DEBUGFS_READ_FUNC(name);					\
	DEBUGFS_WRITE_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.write = iwl_dbgfs_##name##_write,				\
	.read = iwl_dbgfs_##name##_read,				\
1812
	.open = simple_open,						\
1813 1814 1815 1816
	.llseek = generic_file_llseek,					\
};

static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
1817 1818
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
1819
{
1820
	struct iwl_trans *trans = file->private_data;
1821
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1822 1823 1824 1825 1826 1827
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	char *buf;
	int pos = 0;
	int cnt;
	int ret;
1828 1829
	size_t bufsz;

1830
	bufsz = sizeof(char) * 64 * trans->cfg->base_params->num_of_queues;
1831

J
Johannes Berg 已提交
1832
	if (!trans_pcie->txq)
1833
		return -EAGAIN;
J
Johannes Berg 已提交
1834

1835 1836 1837 1838
	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

1839
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
1840
		txq = &trans_pcie->txq[cnt];
1841 1842
		q = &txq->q;
		pos += scnprintf(buf + pos, bufsz - pos,
1843
				"hwq %.2d: read=%u write=%u use=%d stop=%d\n",
1844
				cnt, q->read_ptr, q->write_ptr,
1845 1846
				!!test_bit(cnt, trans_pcie->queue_used),
				!!test_bit(cnt, trans_pcie->queue_stopped));
1847 1848 1849 1850 1851 1852 1853
	}
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
1854 1855 1856
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
{
1857
	struct iwl_trans *trans = file->private_data;
1858
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1859
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
	char buf[256];
	int pos = 0;
	const size_t bufsz = sizeof(buf);

	pos += scnprintf(buf + pos, bufsz - pos, "read: %u\n",
						rxq->read);
	pos += scnprintf(buf + pos, bufsz - pos, "write: %u\n",
						rxq->write);
	pos += scnprintf(buf + pos, bufsz - pos, "free_count: %u\n",
						rxq->free_count);
	if (rxq->rb_stts) {
		pos += scnprintf(buf + pos, bufsz - pos, "closed_rb_num: %u\n",
			 le16_to_cpu(rxq->rb_stts->closed_rb_num) &  0x0FFF);
	} else {
		pos += scnprintf(buf + pos, bufsz - pos,
					"closed_rb_num: Not Allocated\n");
	}
	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
}

1880 1881
static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
					char __user *user_buf,
1882 1883
					size_t count, loff_t *ppos)
{
1884
	struct iwl_trans *trans = file->private_data;
1885
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1886 1887 1888 1889 1890 1891 1892 1893
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	int pos = 0;
	char *buf;
	int bufsz = 24 * 64; /* 24 items * 64 char per item */
	ssize_t ret;

	buf = kzalloc(bufsz, GFP_KERNEL);
J
Johannes Berg 已提交
1894
	if (!buf)
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
		return -ENOMEM;

	pos += scnprintf(buf + pos, bufsz - pos,
			"Interrupt Statistics Report:\n");

	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
		isr_stats->hw);
	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
		isr_stats->sw);
	if (isr_stats->sw || isr_stats->hw) {
		pos += scnprintf(buf + pos, bufsz - pos,
			"\tLast Restarting Code:  0x%X\n",
			isr_stats->err_code);
	}
#ifdef CONFIG_IWLWIFI_DEBUG
	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
		isr_stats->sch);
	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
		isr_stats->alive);
#endif
	pos += scnprintf(buf + pos, bufsz - pos,
		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);

	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
		isr_stats->ctkill);

	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
		isr_stats->wakeup);

	pos += scnprintf(buf + pos, bufsz - pos,
		"Rx command responses:\t\t %u\n", isr_stats->rx);

	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
		isr_stats->tx);

	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
		isr_stats->unhandled);

	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
					 const char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
1943
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	char buf[8];
	int buf_size;
	u32 reset_flag;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%x", &reset_flag) != 1)
		return -EFAULT;
	if (reset_flag == 0)
		memset(isr_stats, 0, sizeof(*isr_stats));

	return count;
}

1962
static ssize_t iwl_dbgfs_csr_write(struct file *file,
1963 1964
				   const char __user *user_buf,
				   size_t count, loff_t *ppos)
1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
{
	struct iwl_trans *trans = file->private_data;
	char buf[8];
	int buf_size;
	int csr;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%d", &csr) != 1)
		return -EFAULT;

	iwl_dump_csr(trans);

	return count;
}

static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
1984 1985
				     char __user *user_buf,
				     size_t count, loff_t *ppos)
1986 1987
{
	struct iwl_trans *trans = file->private_data;
1988
	char *buf = NULL;
1989 1990 1991
	int pos = 0;
	ssize_t ret = -EFAULT;

1992
	ret = pos = iwl_dump_fh(trans, &buf);
1993 1994 1995 1996 1997 1998 1999 2000 2001
	if (buf) {
		ret = simple_read_from_buffer(user_buf,
					      count, ppos, buf, pos);
		kfree(buf);
	}

	return ret;
}

2002 2003 2004 2005 2006 2007 2008 2009 2010
static ssize_t iwl_dbgfs_fw_restart_write(struct file *file,
					  const char __user *user_buf,
					  size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;

	if (!trans->op_mode)
		return -EAGAIN;

2011
	local_bh_disable();
2012
	iwl_op_mode_nic_error(trans->op_mode);
2013
	local_bh_enable();
2014 2015 2016 2017

	return count;
}

2018
DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2019
DEBUGFS_READ_FILE_OPS(fh_reg);
2020 2021
DEBUGFS_READ_FILE_OPS(rx_queue);
DEBUGFS_READ_FILE_OPS(tx_queue);
2022
DEBUGFS_WRITE_FILE_OPS(csr);
2023
DEBUGFS_WRITE_FILE_OPS(fw_restart);
2024 2025 2026 2027 2028 2029

/*
 * Create the debugfs files and directories
 *
 */
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
2030
					 struct dentry *dir)
2031 2032 2033
{
	DEBUGFS_ADD_FILE(rx_queue, dir, S_IRUSR);
	DEBUGFS_ADD_FILE(tx_queue, dir, S_IRUSR);
2034
	DEBUGFS_ADD_FILE(interrupt, dir, S_IWUSR | S_IRUSR);
2035 2036
	DEBUGFS_ADD_FILE(csr, dir, S_IWUSR);
	DEBUGFS_ADD_FILE(fh_reg, dir, S_IRUSR);
2037
	DEBUGFS_ADD_FILE(fw_restart, dir, S_IWUSR);
2038 2039 2040 2041
	return 0;
}
#else
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
2042 2043 2044 2045
					 struct dentry *dir)
{
	return 0;
}
2046 2047
#endif /*CONFIG_IWLWIFI_DEBUGFS */

2048
static const struct iwl_trans_ops trans_ops_pcie = {
2049
	.start_hw = iwl_trans_pcie_start_hw,
2050
	.stop_hw = iwl_trans_pcie_stop_hw,
2051
	.fw_alive = iwl_trans_pcie_fw_alive,
2052
	.start_fw = iwl_trans_pcie_start_fw,
2053
	.stop_device = iwl_trans_pcie_stop_device,
2054

2055 2056
	.wowlan_suspend = iwl_trans_pcie_wowlan_suspend,

2057
	.send_cmd = iwl_trans_pcie_send_cmd,
2058

2059
	.tx = iwl_trans_pcie_tx,
2060
	.reclaim = iwl_trans_pcie_reclaim,
2061

2062
	.txq_disable = iwl_trans_pcie_txq_disable,
2063
	.txq_enable = iwl_trans_pcie_txq_enable,
2064

2065
	.dbgfs_register = iwl_trans_pcie_dbgfs_register,
2066 2067 2068

	.wait_tx_queue_empty = iwl_trans_pcie_wait_tx_queue_empty,

J
Johannes Berg 已提交
2069
#ifdef CONFIG_PM_SLEEP
2070 2071
	.suspend = iwl_trans_pcie_suspend,
	.resume = iwl_trans_pcie_resume,
J
Johannes Berg 已提交
2072
#endif
2073 2074 2075
	.write8 = iwl_trans_pcie_write8,
	.write32 = iwl_trans_pcie_write32,
	.read32 = iwl_trans_pcie_read32,
2076
	.configure = iwl_trans_pcie_configure,
D
Don Fry 已提交
2077
	.set_pmi = iwl_trans_pcie_set_pmi,
2078
};
2079

2080
struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
2081 2082
				       const struct pci_device_id *ent,
				       const struct iwl_cfg *cfg)
2083 2084 2085 2086 2087 2088 2089
{
	struct iwl_trans_pcie *trans_pcie;
	struct iwl_trans *trans;
	u16 pci_cmd;
	int err;

	trans = kzalloc(sizeof(struct iwl_trans) +
2090
			sizeof(struct iwl_trans_pcie), GFP_KERNEL);
2091 2092 2093 2094 2095 2096 2097

	if (WARN_ON(!trans))
		return NULL;

	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans->ops = &trans_ops_pcie;
2098
	trans->cfg = cfg;
2099
	trans_pcie->trans = trans;
J
Johannes Berg 已提交
2100
	spin_lock_init(&trans_pcie->irq_lock);
2101
	init_waitqueue_head(&trans_pcie->ucode_write_waitq);
2102 2103 2104 2105

	/* W/A - seems to solve weird behavior. We need to remove this if we
	 * don't want to stay in L1 all the time. This wastes a lot of power */
	pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 |
2106
			       PCIE_LINK_STATE_CLKPM);
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121

	if (pci_enable_device(pdev)) {
		err = -ENODEV;
		goto out_no_pci;
	}

	pci_set_master(pdev);

	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(36));
	if (!err)
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(36));
	if (err) {
		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (!err)
			err = pci_set_consistent_dma_mask(pdev,
2122
							  DMA_BIT_MASK(32));
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
		/* both attempts failed: */
		if (err) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "No suitable DMA available.\n");
			goto out_pci_disable_device;
		}
	}

	err = pci_request_regions(pdev, DRV_NAME);
	if (err) {
2133 2134
		dev_printk(KERN_ERR, &pdev->dev,
			   "pci_request_regions failed\n");
2135 2136 2137
		goto out_pci_disable_device;
	}

2138
	trans_pcie->hw_base = pci_ioremap_bar(pdev, 0);
2139
	if (!trans_pcie->hw_base) {
2140
		dev_printk(KERN_ERR, &pdev->dev, "pci_ioremap_bar failed\n");
2141 2142 2143 2144 2145
		err = -ENODEV;
		goto out_pci_release_regions;
	}

	dev_printk(KERN_INFO, &pdev->dev,
2146 2147
		   "pci_resource_len = 0x%08llx\n",
		   (unsigned long long) pci_resource_len(pdev, 0));
2148
	dev_printk(KERN_INFO, &pdev->dev,
2149
		   "pci_resource_base = %p\n", trans_pcie->hw_base);
2150 2151

	dev_printk(KERN_INFO, &pdev->dev,
2152
		   "HW Revision ID = 0x%X\n", pdev->revision);
2153 2154 2155 2156 2157 2158 2159 2160

	/* We disable the RETRY_TIMEOUT register (0x41) to keep
	 * PCI Tx retries from interfering with C3 CPU state */
	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);

	err = pci_enable_msi(pdev);
	if (err)
		dev_printk(KERN_ERR, &pdev->dev,
2161
			   "pci_enable_msi failed(0X%x)\n", err);
2162 2163

	trans->dev = &pdev->dev;
J
Johannes Berg 已提交
2164
	trans_pcie->irq = pdev->irq;
2165
	trans_pcie->pci_dev = pdev;
2166
	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
E
Emmanuel Grumbach 已提交
2167
	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
2168 2169
	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
2170 2171 2172 2173 2174 2175 2176 2177 2178

	/* TODO: Move this away, not needed if not MSI */
	/* enable rfkill interrupt: hw bug w/a */
	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
	if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
		pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
		pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
	}

2179 2180
	/* Initialize the wait queue for commands */
	init_waitqueue_head(&trans->wait_command_queue);
2181
	spin_lock_init(&trans->reg_lock);
2182

2183 2184
	snprintf(trans->dev_cmd_pool_name, sizeof(trans->dev_cmd_pool_name),
		 "iwl_cmd_pool:%s", dev_name(trans->dev));
2185 2186 2187

	trans->dev_cmd_headroom = 0;
	trans->dev_cmd_pool =
2188
		kmem_cache_create(trans->dev_cmd_pool_name,
2189 2190 2191 2192 2193 2194 2195 2196 2197
				  sizeof(struct iwl_device_cmd)
				  + trans->dev_cmd_headroom,
				  sizeof(void *),
				  SLAB_HWCACHE_ALIGN,
				  NULL);

	if (!trans->dev_cmd_pool)
		goto out_pci_disable_msi;

2198 2199
	return trans;

2200 2201
out_pci_disable_msi:
	pci_disable_msi(pdev);
2202 2203 2204 2205 2206 2207 2208 2209
out_pci_release_regions:
	pci_release_regions(pdev);
out_pci_disable_device:
	pci_disable_device(pdev);
out_no_pci:
	kfree(trans);
	return NULL;
}