trans.c 61.1 KB
Newer Older
1 2 3 4 5 6 7
/******************************************************************************
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
W
Wey-Yi Guy 已提交
8
 * Copyright(c) 2007 - 2012 Intel Corporation. All rights reserved.
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
 * USA
 *
 * The full GNU General Public License is included in this distribution
 * in the file called LICENSE.GPL.
 *
 * Contact Information:
 *  Intel Linux Wireless <ilw@linux.intel.com>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 * BSD LICENSE
 *
W
Wey-Yi Guy 已提交
33
 * Copyright(c) 2005 - 2012 Intel Corporation. All rights reserved.
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/
63 64
#include <linux/pci.h>
#include <linux/pci-aspm.h>
65
#include <linux/interrupt.h>
66
#include <linux/debugfs.h>
67
#include <linux/sched.h>
68 69
#include <linux/bitops.h>
#include <linux/gfp.h>
70

71
#include "iwl-drv.h"
72
#include "iwl-trans.h"
73 74
#include "iwl-csr.h"
#include "iwl-prph.h"
75
#include "iwl-agn-hw.h"
76
#include "internal.h"
77
/* FIXME: need to abstract out TX command (once we know what it looks like) */
78
#include "dvm/commands.h"
79

80
#define SCD_QUEUECHAIN_SEL_ALL(trans, trans_pcie)	\
81
	(((1<<trans->cfg->base_params->num_of_queues) - 1) &\
82 83
	(~(1<<(trans_pcie)->cmd_queue)))

84
static int iwl_trans_rx_alloc(struct iwl_trans *trans)
85
{
86
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
87
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
88
	struct device *dev = trans->dev;
89

90
	memset(&trans_pcie->rxq, 0, sizeof(trans_pcie->rxq));
91 92 93 94 95 96 97

	spin_lock_init(&rxq->lock);

	if (WARN_ON(rxq->bd || rxq->rb_stts))
		return -EINVAL;

	/* Allocate the circular buffer of Read Buffer Descriptors (RBDs) */
98 99
	rxq->bd = dma_zalloc_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
				      &rxq->bd_dma, GFP_KERNEL);
100 101 102 103
	if (!rxq->bd)
		goto err_bd;

	/*Allocate the driver's pointer to receive buffer status */
104 105
	rxq->rb_stts = dma_zalloc_coherent(dev, sizeof(*rxq->rb_stts),
					   &rxq->rb_stts_dma, GFP_KERNEL);
106 107 108 109 110 111
	if (!rxq->rb_stts)
		goto err_rb_stts;

	return 0;

err_rb_stts:
112
	dma_free_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
113
			  rxq->bd, rxq->bd_dma);
114 115 116 117 118 119
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;
err_bd:
	return -ENOMEM;
}

120
static void iwl_trans_rxq_free_rx_bufs(struct iwl_trans *trans)
121
{
122
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
123
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
124
	int i;
125 126 127 128 129 130

	/* Fill the rx_used queue with _all_ of the Rx buffers */
	for (i = 0; i < RX_FREE_BUFFERS + RX_QUEUE_SIZE; i++) {
		/* In the reset function, these buffers may have been allocated
		 * to an SKB, so we need to unmap and free potential storage */
		if (rxq->pool[i].page != NULL) {
131
			dma_unmap_page(trans->dev, rxq->pool[i].page_dma,
132 133
				       PAGE_SIZE << trans_pcie->rx_page_order,
				       DMA_FROM_DEVICE);
134
			__free_pages(rxq->pool[i].page,
135
				     trans_pcie->rx_page_order);
136 137 138 139
			rxq->pool[i].page = NULL;
		}
		list_add_tail(&rxq->pool[i].list, &rxq->rx_used);
	}
140 141
}

142
static void iwl_trans_rx_hw_init(struct iwl_trans *trans,
143 144
				 struct iwl_rx_queue *rxq)
{
145
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
146 147
	u32 rb_size;
	const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */
148
	u32 rb_timeout = RX_RB_TIMEOUT; /* FIXME: RX_RB_TIMEOUT for all devices? */
149

150
	if (trans_pcie->rx_buf_size_8k)
151 152 153 154 155
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K;
	else
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;

	/* Stop Rx DMA */
156
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
157 158

	/* Reset driver's Rx queue write index */
159
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
160 161

	/* Tell device where to find RBD circular buffer in DRAM */
162
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG,
163 164 165
			   (u32)(rxq->bd_dma >> 8));

	/* Tell device where in DRAM to update its Rx status */
166
	iwl_write_direct32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG,
167 168 169 170 171 172 173 174 175 176
			   rxq->rb_stts_dma >> 4);

	/* Enable Rx DMA
	 * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
	 *      the credit mechanism in 5000 HW RX FIFO
	 * Direct rx interrupts to hosts
	 * Rx buffer size 4 or 8k
	 * RB timeout 0x10
	 * 256 RBDs
	 */
177
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG,
178 179 180 181 182 183 184 185
			   FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL |
			   FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY |
			   FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL |
			   rb_size|
			   (rb_timeout << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS)|
			   (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS));

	/* Set interrupt coalescing timer to default (2048 usecs) */
186
	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
187 188
}

189
static int iwl_rx_init(struct iwl_trans *trans)
190
{
191
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
192 193
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;

194 195 196 197
	int i, err;
	unsigned long flags;

	if (!rxq->bd) {
198
		err = iwl_trans_rx_alloc(trans);
199 200 201 202 203 204 205 206
		if (err)
			return err;
	}

	spin_lock_irqsave(&rxq->lock, flags);
	INIT_LIST_HEAD(&rxq->rx_free);
	INIT_LIST_HEAD(&rxq->rx_used);

207
	iwl_trans_rxq_free_rx_bufs(trans);
208 209 210 211 212 213 214 215 216 217 218

	for (i = 0; i < RX_QUEUE_SIZE; i++)
		rxq->queue[i] = NULL;

	/* Set us so that we have processed and used all buffers, but have
	 * not restocked the Rx queue with fresh buffers */
	rxq->read = rxq->write = 0;
	rxq->write_actual = 0;
	rxq->free_count = 0;
	spin_unlock_irqrestore(&rxq->lock, flags);

219
	iwlagn_rx_replenish(trans);
220

221
	iwl_trans_rx_hw_init(trans, rxq);
222

J
Johannes Berg 已提交
223
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
224
	rxq->need_update = 1;
225
	iwl_rx_queue_update_write_ptr(trans, rxq);
J
Johannes Berg 已提交
226
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
227

228 229 230
	return 0;
}

231
static void iwl_trans_pcie_rx_free(struct iwl_trans *trans)
232
{
233
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
234
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
235 236 237 238 239
	unsigned long flags;

	/*if rxq->bd is NULL, it means that nothing has been allocated,
	 * exit now */
	if (!rxq->bd) {
240
		IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
241 242 243 244
		return;
	}

	spin_lock_irqsave(&rxq->lock, flags);
245
	iwl_trans_rxq_free_rx_bufs(trans);
246 247
	spin_unlock_irqrestore(&rxq->lock, flags);

248
	dma_free_coherent(trans->dev, sizeof(__le32) * RX_QUEUE_SIZE,
249 250 251 252 253
			  rxq->bd, rxq->bd_dma);
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;

	if (rxq->rb_stts)
254
		dma_free_coherent(trans->dev,
255 256 257
				  sizeof(struct iwl_rb_status),
				  rxq->rb_stts, rxq->rb_stts_dma);
	else
258
		IWL_DEBUG_INFO(trans, "Free rxq->rb_stts which is NULL\n");
259 260 261 262
	memset(&rxq->rb_stts_dma, 0, sizeof(rxq->rb_stts_dma));
	rxq->rb_stts = NULL;
}

263
static int iwl_trans_rx_stop(struct iwl_trans *trans)
264 265 266
{

	/* stop Rx DMA */
267 268
	iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
	return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG,
269
				   FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, 1000);
270 271
}

272 273
static int iwlagn_alloc_dma_ptr(struct iwl_trans *trans,
				struct iwl_dma_ptr *ptr, size_t size)
274 275 276 277
{
	if (WARN_ON(ptr->addr))
		return -EINVAL;

278
	ptr->addr = dma_alloc_coherent(trans->dev, size,
279 280 281 282 283 284 285
				       &ptr->dma, GFP_KERNEL);
	if (!ptr->addr)
		return -ENOMEM;
	ptr->size = size;
	return 0;
}

286 287
static void iwlagn_free_dma_ptr(struct iwl_trans *trans,
				struct iwl_dma_ptr *ptr)
288 289 290 291
{
	if (unlikely(!ptr->addr))
		return;

292
	dma_free_coherent(trans->dev, ptr->size, ptr->addr, ptr->dma);
293 294 295
	memset(ptr, 0, sizeof(*ptr));
}

296 297 298
static void iwl_trans_pcie_queue_stuck_timer(unsigned long data)
{
	struct iwl_tx_queue *txq = (void *)data;
299
	struct iwl_queue *q = &txq->q;
300 301
	struct iwl_trans_pcie *trans_pcie = txq->trans_pcie;
	struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie);
302 303 304 305
	u32 scd_sram_addr = trans_pcie->scd_base_addr +
		SCD_TX_STTS_MEM_LOWER_BOUND + (16 * txq->q.id);
	u8 buf[16];
	int i;
306 307 308 309 310 311 312 313 314 315 316 317 318 319

	spin_lock(&txq->lock);
	/* check if triggered erroneously */
	if (txq->q.read_ptr == txq->q.write_ptr) {
		spin_unlock(&txq->lock);
		return;
	}
	spin_unlock(&txq->lock);

	IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->q.id,
		jiffies_to_msecs(trans_pcie->wd_timeout));
	IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n",
		txq->q.read_ptr, txq->q.write_ptr);

320 321 322 323 324 325 326 327
	iwl_read_targ_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf));

	iwl_print_hex_error(trans, buf, sizeof(buf));

	for (i = 0; i < FH_TCSR_CHNL_NUM; i++)
		IWL_ERR(trans, "FH TRBs(%d) = 0x%08x\n", i,
			iwl_read_direct32(trans, FH_TX_TRB_REG(i)));

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
	for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) {
		u32 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(i));
		u8 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7;
		bool active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE));
		u32 tbl_dw =
			iwl_read_targ_mem(trans,
					  trans_pcie->scd_base_addr +
					  SCD_TRANS_TBL_OFFSET_QUEUE(i));

		if (i & 0x1)
			tbl_dw = (tbl_dw & 0xFFFF0000) >> 16;
		else
			tbl_dw = tbl_dw & 0x0000FFFF;

		IWL_ERR(trans,
			"Q %d is %sactive and mapped to fifo %d ra_tid 0x%04x [%d,%d]\n",
			i, active ? "" : "in", fifo, tbl_dw,
			iwl_read_prph(trans,
				      SCD_QUEUE_RDPTR(i)) & (txq->q.n_bd - 1),
			iwl_read_prph(trans, SCD_QUEUE_WRPTR(i)));
	}

350 351 352 353 354 355 356 357
	for (i = q->read_ptr; i != q->write_ptr;
	     i = iwl_queue_inc_wrap(i, q->n_bd)) {
		struct iwl_tx_cmd *tx_cmd =
			(struct iwl_tx_cmd *)txq->entries[i].cmd->payload;
		IWL_ERR(trans, "scratch %d = 0x%08x\n", i,
			get_unaligned_le32(&tx_cmd->scratch));
	}

358 359 360
	iwl_op_mode_nic_error(trans->op_mode);
}

361
static int iwl_trans_txq_alloc(struct iwl_trans *trans,
362 363
			       struct iwl_tx_queue *txq, int slots_num,
			       u32 txq_id)
364
{
365
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
366
	size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX;
367 368
	int i;

369
	if (WARN_ON(txq->entries || txq->tfds))
370 371
		return -EINVAL;

372 373 374 375
	setup_timer(&txq->stuck_timer, iwl_trans_pcie_queue_stuck_timer,
		    (unsigned long)txq);
	txq->trans_pcie = trans_pcie;

376 377
	txq->q.n_window = slots_num;

378 379 380
	txq->entries = kcalloc(slots_num,
			       sizeof(struct iwl_pcie_tx_queue_entry),
			       GFP_KERNEL);
381

382
	if (!txq->entries)
383 384
		goto error;

385
	if (txq_id == trans_pcie->cmd_queue)
386
		for (i = 0; i < slots_num; i++) {
387 388 389 390
			txq->entries[i].cmd =
				kmalloc(sizeof(struct iwl_device_cmd),
					GFP_KERNEL);
			if (!txq->entries[i].cmd)
391 392
				goto error;
		}
393 394 395

	/* Circular buffer of transmit frame descriptors (TFDs),
	 * shared with device */
396
	txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,
397
				       &txq->q.dma_addr, GFP_KERNEL);
398
	if (!txq->tfds) {
399
		IWL_ERR(trans, "dma_alloc_coherent(%zd) failed\n", tfd_sz);
400 401 402 403 404 405
		goto error;
	}
	txq->q.id = txq_id;

	return 0;
error:
406
	if (txq->entries && txq_id == trans_pcie->cmd_queue)
407
		for (i = 0; i < slots_num; i++)
408 409 410
			kfree(txq->entries[i].cmd);
	kfree(txq->entries);
	txq->entries = NULL;
411 412 413 414 415

	return -ENOMEM;

}

416
static int iwl_trans_txq_init(struct iwl_trans *trans, struct iwl_tx_queue *txq,
417
			      int slots_num, u32 txq_id)
418 419 420 421 422 423 424 425 426 427
{
	int ret;

	txq->need_update = 0;

	/* TFD_QUEUE_SIZE_MAX must be power-of-two size, otherwise
	 * iwl_queue_inc_wrap and iwl_queue_dec_wrap are broken. */
	BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1));

	/* Initialize queue's high/low-water marks, and head/tail indexes */
428
	ret = iwl_queue_init(&txq->q, TFD_QUEUE_SIZE_MAX, slots_num,
429 430 431 432
			txq_id);
	if (ret)
		return ret;

433 434
	spin_lock_init(&txq->lock);

435 436 437 438
	/*
	 * Tell nic where to find circular buffer of Tx Frame Descriptors for
	 * given Tx queue, and enable the DMA channel used for that queue.
	 * Circular buffer (TFD queue in DRAM) physical base address */
439
	iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(txq_id),
440 441 442 443 444
			     txq->q.dma_addr >> 8);

	return 0;
}

445 446 447
/**
 * iwl_tx_queue_unmap -  Unmap any remaining DMA mappings and free skb's
 */
448
static void iwl_tx_queue_unmap(struct iwl_trans *trans, int txq_id)
449
{
450 451
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
452
	struct iwl_queue *q = &txq->q;
453
	enum dma_data_direction dma_dir;
454 455 456 457

	if (!q->n_bd)
		return;

458 459 460
	/* In the command queue, all the TBs are mapped as BIDI
	 * so unmap them as such.
	 */
461
	if (txq_id == trans_pcie->cmd_queue)
462
		dma_dir = DMA_BIDIRECTIONAL;
463
	else
464 465
		dma_dir = DMA_TO_DEVICE;

466
	spin_lock_bh(&txq->lock);
467
	while (q->write_ptr != q->read_ptr) {
468
		iwl_txq_free_tfd(trans, txq, dma_dir);
469 470
		q->read_ptr = iwl_queue_inc_wrap(q->read_ptr, q->n_bd);
	}
471
	spin_unlock_bh(&txq->lock);
472 473
}

474 475 476 477 478 479 480 481
/**
 * iwl_tx_queue_free - Deallocate DMA queue.
 * @txq: Transmit queue to deallocate.
 *
 * Empty queue by removing and destroying all BD's.
 * Free all buffers.
 * 0-fill, but do not free "txq" descriptor structure.
 */
482
static void iwl_tx_queue_free(struct iwl_trans *trans, int txq_id)
483
{
484 485
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
486
	struct device *dev = trans->dev;
487
	int i;
488

489 490 491
	if (WARN_ON(!txq))
		return;

492
	iwl_tx_queue_unmap(trans, txq_id);
493 494

	/* De-alloc array of command/tx buffers */
495

496
	if (txq_id == trans_pcie->cmd_queue)
497
		for (i = 0; i < txq->q.n_window; i++)
498
			kfree(txq->entries[i].cmd);
499 500 501

	/* De-alloc circular buffer of TFDs */
	if (txq->q.n_bd) {
502
		dma_free_coherent(dev, sizeof(struct iwl_tfd) *
503 504 505 506
				  txq->q.n_bd, txq->tfds, txq->q.dma_addr);
		memset(&txq->q.dma_addr, 0, sizeof(txq->q.dma_addr));
	}

507 508
	kfree(txq->entries);
	txq->entries = NULL;
509

510 511
	del_timer_sync(&txq->stuck_timer);

512 513 514 515 516 517 518 519 520
	/* 0-fill queue descriptor structure */
	memset(txq, 0, sizeof(*txq));
}

/**
 * iwl_trans_tx_free - Free TXQ Context
 *
 * Destroy all TX DMA queues and structures
 */
521
static void iwl_trans_pcie_tx_free(struct iwl_trans *trans)
522 523
{
	int txq_id;
524
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
525 526

	/* Tx queues */
527
	if (trans_pcie->txq) {
528
		for (txq_id = 0;
529
		     txq_id < trans->cfg->base_params->num_of_queues; txq_id++)
530
			iwl_tx_queue_free(trans, txq_id);
531 532
	}

533 534
	kfree(trans_pcie->txq);
	trans_pcie->txq = NULL;
535

536
	iwlagn_free_dma_ptr(trans, &trans_pcie->kw);
537

538
	iwlagn_free_dma_ptr(trans, &trans_pcie->scd_bc_tbls);
539 540
}

541 542 543 544 545 546 547
/**
 * iwl_trans_tx_alloc - allocate TX context
 * Allocate all Tx DMA structures and initialize them
 *
 * @param priv
 * @return error code
 */
548
static int iwl_trans_tx_alloc(struct iwl_trans *trans)
549 550 551
{
	int ret;
	int txq_id, slots_num;
552
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
553

554
	u16 scd_bc_tbls_size = trans->cfg->base_params->num_of_queues *
555 556
			sizeof(struct iwlagn_scd_bc_tbl);

557 558
	/*It is not allowed to alloc twice, so warn when this happens.
	 * We cannot rely on the previous allocation, so free and fail */
559
	if (WARN_ON(trans_pcie->txq)) {
560 561 562 563
		ret = -EINVAL;
		goto error;
	}

564
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->scd_bc_tbls,
565
				   scd_bc_tbls_size);
566
	if (ret) {
567
		IWL_ERR(trans, "Scheduler BC Table allocation failed\n");
568 569 570 571
		goto error;
	}

	/* Alloc keep-warm buffer */
572
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE);
573
	if (ret) {
574
		IWL_ERR(trans, "Keep Warm allocation failed\n");
575 576 577
		goto error;
	}

578
	trans_pcie->txq = kcalloc(trans->cfg->base_params->num_of_queues,
579
				  sizeof(struct iwl_tx_queue), GFP_KERNEL);
580
	if (!trans_pcie->txq) {
581
		IWL_ERR(trans, "Not enough memory for txq\n");
582 583 584 585 586
		ret = ENOMEM;
		goto error;
	}

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
587
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
588
	     txq_id++) {
W
Wey-Yi Guy 已提交
589
		slots_num = (txq_id == trans_pcie->cmd_queue) ?
590
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
591 592
		ret = iwl_trans_txq_alloc(trans, &trans_pcie->txq[txq_id],
					  slots_num, txq_id);
593
		if (ret) {
594
			IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
595 596 597 598 599 600 601
			goto error;
		}
	}

	return 0;

error:
602
	iwl_trans_pcie_tx_free(trans);
603 604 605

	return ret;
}
606
static int iwl_tx_init(struct iwl_trans *trans)
607
{
608
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
609 610 611 612 613
	int ret;
	int txq_id, slots_num;
	unsigned long flags;
	bool alloc = false;

614
	if (!trans_pcie->txq) {
615
		ret = iwl_trans_tx_alloc(trans);
616 617 618 619 620
		if (ret)
			goto error;
		alloc = true;
	}

J
Johannes Berg 已提交
621
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
622 623

	/* Turn off all Tx DMA fifos */
624
	iwl_write_prph(trans, SCD_TXFACT, 0);
625 626

	/* Tell NIC where to find the "keep warm" buffer */
627
	iwl_write_direct32(trans, FH_KW_MEM_ADDR_REG,
628
			   trans_pcie->kw.dma >> 4);
629

J
Johannes Berg 已提交
630
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
631 632

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
633
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
634
	     txq_id++) {
W
Wey-Yi Guy 已提交
635
		slots_num = (txq_id == trans_pcie->cmd_queue) ?
636
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
637 638
		ret = iwl_trans_txq_init(trans, &trans_pcie->txq[txq_id],
					 slots_num, txq_id);
639
		if (ret) {
640
			IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
641 642 643 644 645 646 647 648
			goto error;
		}
	}

	return 0;
error:
	/*Upon error, free only if we allocated something */
	if (alloc)
649
		iwl_trans_pcie_tx_free(trans);
650 651 652
	return ret;
}

653
static void iwl_set_pwr_vmain(struct iwl_trans *trans)
654 655 656 657 658 659
{
/*
 * (for documentation purposes)
 * to set power to V_AUX, do:

		if (pci_pme_capable(priv->pci_dev, PCI_D3cold))
660
			iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
661 662 663 664
					       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
					       ~APMG_PS_CTRL_MSK_PWR_SRC);
 */

665
	iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
666 667 668 669
			       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
			       ~APMG_PS_CTRL_MSK_PWR_SRC);
}

E
Emmanuel Grumbach 已提交
670 671 672 673 674 675 676
/* PCI registers */
#define PCI_CFG_RETRY_TIMEOUT	0x041
#define PCI_CFG_LINK_CTRL_VAL_L0S_EN	0x01
#define PCI_CFG_LINK_CTRL_VAL_L1_EN	0x02

static u16 iwl_pciexp_link_ctrl(struct iwl_trans *trans)
{
677
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
E
Emmanuel Grumbach 已提交
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
	int pos;
	u16 pci_lnk_ctl;

	struct pci_dev *pci_dev = trans_pcie->pci_dev;

	pos = pci_pcie_cap(pci_dev);
	pci_read_config_word(pci_dev, pos + PCI_EXP_LNKCTL, &pci_lnk_ctl);
	return pci_lnk_ctl;
}

static void iwl_apm_config(struct iwl_trans *trans)
{
	/*
	 * HW bug W/A for instability in PCIe bus L0S->L1 transition.
	 * Check if BIOS (or OS) enabled L1-ASPM on this device.
	 * If so (likely), disable L0S, so device moves directly L0->L1;
	 *    costs negligible amount of power savings.
	 * If not (unlikely), enable L0S, so there is at least some
	 *    power savings, even without L1.
	 */
	u16 lctl = iwl_pciexp_link_ctrl(trans);

	if ((lctl & PCI_CFG_LINK_CTRL_VAL_L1_EN) ==
				PCI_CFG_LINK_CTRL_VAL_L1_EN) {
		/* L1-ASPM enabled; disable(!) L0S */
		iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
		dev_printk(KERN_INFO, trans->dev,
			   "L1 Enabled; Disabling L0S\n");
	} else {
		/* L1-ASPM disabled; enable(!) L0S */
		iwl_clear_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
		dev_printk(KERN_INFO, trans->dev,
			   "L1 Disabled; Enabling L0S\n");
	}
712
	trans->pm_support = !(lctl & PCI_CFG_LINK_CTRL_VAL_L0S_EN);
E
Emmanuel Grumbach 已提交
713 714
}

715 716 717 718 719 720 721
/*
 * Start up NIC's basic functionality after it has been reset
 * (e.g. after platform boot, or shutdown via iwl_apm_stop())
 * NOTE:  This does not load uCode nor start the embedded processor
 */
static int iwl_apm_init(struct iwl_trans *trans)
{
D
Don Fry 已提交
722
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
723 724 725 726 727 728 729 730 731 732
	int ret = 0;
	IWL_DEBUG_INFO(trans, "Init card's basic functions\n");

	/*
	 * Use "set_bit" below rather than "write", to preserve any hardware
	 * bits already set by default after reset.
	 */

	/* Disable L0S exit timer (platform NMI Work/Around) */
	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
733
		    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
734 735 736 737 738 739

	/*
	 * Disable L0s without affecting L1;
	 *  don't wait for ICH L0s (ICH bug W/A)
	 */
	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
740
		    CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
741 742 743 744 745 746 747 748 749

	/* Set FH wait threshold to maximum (HW error during stress W/A) */
	iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);

	/*
	 * Enable HAP INTA (interrupt from management bus) to
	 * wake device's PCI Express link L1a -> L0s
	 */
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
750
		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
751

E
Emmanuel Grumbach 已提交
752
	iwl_apm_config(trans);
753 754

	/* Configure analog phase-lock-loop before activating to D0A */
755
	if (trans->cfg->base_params->pll_cfg_val)
756
		iwl_set_bit(trans, CSR_ANA_PLL_CFG,
757
			    trans->cfg->base_params->pll_cfg_val);
758 759 760 761 762 763 764 765 766 767 768 769 770

	/*
	 * Set "initialization complete" bit to move adapter from
	 * D0U* --> D0A* (powered-up active) state.
	 */
	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

	/*
	 * Wait for clock stabilization; once stabilized, access to
	 * device-internal resources is supported, e.g. iwl_write_prph()
	 * and accesses to uCode SRAM.
	 */
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
771 772
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000);
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
	if (ret < 0) {
		IWL_DEBUG_INFO(trans, "Failed to init the card\n");
		goto out;
	}

	/*
	 * Enable DMA clock and wait for it to stabilize.
	 *
	 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0" bits
	 * do not disable clocks.  This preserves any hardware bits already
	 * set by default in "CLK_CTRL_REG" after reset.
	 */
	iwl_write_prph(trans, APMG_CLK_EN_REG, APMG_CLK_VAL_DMA_CLK_RQT);
	udelay(20);

	/* Disable L1-Active */
	iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
			  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);

D
Don Fry 已提交
792
	set_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
793 794 795 796 797

out:
	return ret;
}

798 799 800 801 802 803 804 805
static int iwl_apm_stop_master(struct iwl_trans *trans)
{
	int ret = 0;

	/* stop device's busmaster DMA activity */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);

	ret = iwl_poll_bit(trans, CSR_RESET,
806 807
			   CSR_RESET_REG_FLAG_MASTER_DISABLED,
			   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
808 809 810 811 812 813 814 815 816 817
	if (ret)
		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");

	IWL_DEBUG_INFO(trans, "stop master\n");

	return ret;
}

static void iwl_apm_stop(struct iwl_trans *trans)
{
D
Don Fry 已提交
818
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
819 820
	IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");

D
Don Fry 已提交
821
	clear_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838

	/* Stop device's DMA activity */
	iwl_apm_stop_master(trans);

	/* Reset the entire device */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);

	udelay(10);

	/*
	 * Clear "initialization complete" bit to move adapter from
	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
	 */
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
}

839
static int iwl_nic_init(struct iwl_trans *trans)
840
{
J
Johannes Berg 已提交
841
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
842 843 844
	unsigned long flags;

	/* nic_init */
J
Johannes Berg 已提交
845
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
846
	iwl_apm_init(trans);
847 848

	/* Set interrupt coalescing calibration timer to default (512 usecs) */
849
	iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_CALIB_TIMEOUT_DEF);
850

J
Johannes Berg 已提交
851
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
852

853
	iwl_set_pwr_vmain(trans);
854

J
Johannes Berg 已提交
855
	iwl_op_mode_nic_config(trans->op_mode);
856

857
#ifndef CONFIG_IWLWIFI_IDI
858
	/* Allocate the RX queue, or reset if it is already allocated */
859
	iwl_rx_init(trans);
860
#endif
861 862

	/* Allocate or reset and init all Tx and Command queues */
863
	if (iwl_tx_init(trans))
864 865
		return -ENOMEM;

866
	if (trans->cfg->base_params->shadow_reg_enable) {
867
		/* enable shadow regs in HW */
868
		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
869
		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
870 871 872 873 874 875 876 877
	}

	return 0;
}

#define HW_READY_TIMEOUT (50)

/* Note: returns poll_bit return value, which is >= 0 if success */
878
static int iwl_set_hw_ready(struct iwl_trans *trans)
879 880 881
{
	int ret;

882
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
883
		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
884 885

	/* See if we got it */
886
	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
887 888 889
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   HW_READY_TIMEOUT);
890

891
	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
892 893 894 895
	return ret;
}

/* Note: returns standard 0/-ERROR code */
896
static int iwl_prepare_card_hw(struct iwl_trans *trans)
897 898 899
{
	int ret;

900
	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
901

902
	ret = iwl_set_hw_ready(trans);
903
	/* If the card is ready, exit 0 */
904 905 906 907
	if (ret >= 0)
		return 0;

	/* If HW is not ready, prepare the conditions to check again */
908
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
909
		    CSR_HW_IF_CONFIG_REG_PREPARE);
910

911
	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
912 913
			   ~CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE,
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE, 150000);
914 915 916 917 918

	if (ret < 0)
		return ret;

	/* HW should be ready by now, check again. */
919
	ret = iwl_set_hw_ready(trans);
920 921 922 923 924
	if (ret >= 0)
		return 0;
	return ret;
}

925 926 927
/*
 * ucode
 */
D
David Spinadel 已提交
928 929
static int iwl_load_section(struct iwl_trans *trans, u8 section_num,
			    const struct fw_desc *section)
930
{
931
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
D
David Spinadel 已提交
932 933 934
	dma_addr_t phy_addr = section->p_addr;
	u32 byte_cnt = section->len;
	u32 dst_addr = section->offset;
935 936
	int ret;

937
	trans_pcie->ucode_write_complete = false;
938 939

	iwl_write_direct32(trans,
940 941
			   FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
942 943

	iwl_write_direct32(trans,
944 945
			   FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
			   dst_addr);
946 947 948 949 950 951

	iwl_write_direct32(trans,
		FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
		phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);

	iwl_write_direct32(trans,
952 953 954
			   FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
			   (iwl_get_dma_hi_addr(phy_addr)
				<< FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
955 956

	iwl_write_direct32(trans,
957 958 959 960
			   FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
			   1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
			   1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
			   FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
961 962

	iwl_write_direct32(trans,
963 964 965 966
			   FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE	|
			   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE	|
			   FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
967

D
David Spinadel 已提交
968 969
	IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
		     section_num);
970 971
	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
				 trans_pcie->ucode_write_complete, 5 * HZ);
972
	if (!ret) {
D
David Spinadel 已提交
973 974
		IWL_ERR(trans, "Could not load the [%d] uCode section\n",
			section_num);
975 976 977 978 979 980
		return -ETIMEDOUT;
	}

	return 0;
}

981 982
static int iwl_load_given_ucode(struct iwl_trans *trans,
				const struct fw_img *image)
983 984
{
	int ret = 0;
D
David Spinadel 已提交
985
		int i;
986

D
David Spinadel 已提交
987 988 989
		for (i = 0; i < IWL_UCODE_SECTION_MAX; i++) {
			if (!image->sec[i].p_addr)
				break;
990

D
David Spinadel 已提交
991 992 993 994
			ret = iwl_load_section(trans, i, &image->sec[i]);
			if (ret)
				return ret;
		}
995 996 997 998 999 1000 1001

	/* Remove all resets to allow NIC to operate */
	iwl_write32(trans, CSR_RESET, 0);

	return 0;
}

1002 1003
static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
				   const struct fw_img *fw)
1004 1005
{
	int ret;
1006
	bool hw_rfkill;
1007

1008 1009
	/* This may fail if AMT took ownership of the device */
	if (iwl_prepare_card_hw(trans)) {
1010
		IWL_WARN(trans, "Exit HW not ready\n");
1011 1012 1013
		return -EIO;
	}

1014 1015
	iwl_enable_rfkill_int(trans);

1016
	/* If platform's RF_KILL switch is NOT set to KILL */
1017
	hw_rfkill = iwl_is_rfkill_set(trans);
1018
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1019
	if (hw_rfkill)
1020 1021
		return -ERFKILL;

1022
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1023

1024
	ret = iwl_nic_init(trans);
1025
	if (ret) {
1026
		IWL_ERR(trans, "Unable to init nic\n");
1027 1028 1029 1030
		return ret;
	}

	/* make sure rfkill handshake bits are cleared */
1031 1032
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1033 1034 1035
		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);

	/* clear (again), then enable host interrupts */
1036
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1037
	iwl_enable_interrupts(trans);
1038 1039

	/* really make sure rfkill handshake bits are cleared */
1040 1041
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1042

1043
	/* Load the given image to the HW */
1044
	return iwl_load_given_ucode(trans, fw);
1045 1046
}

1047 1048 1049
/*
 * Activate/Deactivate Tx DMA/FIFO channels according tx fifos mask
 */
1050
static void iwl_trans_txq_set_sched(struct iwl_trans *trans, u32 mask)
1051
{
J
Johannes Berg 已提交
1052 1053 1054
	struct iwl_trans_pcie __maybe_unused *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);

1055
	iwl_write_prph(trans, SCD_TXFACT, mask);
1056 1057
}

1058
static void iwl_tx_start(struct iwl_trans *trans)
1059
{
1060
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1061
	u32 a;
1062
	int chan;
1063 1064
	u32 reg_val;

1065 1066 1067 1068
	/* make sure all queue are not stopped/used */
	memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped));
	memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used));

1069
	trans_pcie->scd_base_addr =
1070
		iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
1071
	a = trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_LOWER_BOUND;
1072
	/* reset conext data memory */
1073
	for (; a < trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_UPPER_BOUND;
1074
		a += 4)
1075
		iwl_write_targ_mem(trans, a, 0);
1076
	/* reset tx status memory */
1077
	for (; a < trans_pcie->scd_base_addr + SCD_TX_STTS_MEM_UPPER_BOUND;
1078
		a += 4)
1079
		iwl_write_targ_mem(trans, a, 0);
1080
	for (; a < trans_pcie->scd_base_addr +
1081
	       SCD_TRANS_TBL_OFFSET_QUEUE(
1082
				trans->cfg->base_params->num_of_queues);
1083
	       a += 4)
1084
		iwl_write_targ_mem(trans, a, 0);
1085

1086
	iwl_write_prph(trans, SCD_DRAM_BASE_ADDR,
1087
		       trans_pcie->scd_bc_tbls.dma >> 10);
1088

1089 1090 1091 1092 1093
	/* The chain extension of the SCD doesn't work well. This feature is
	 * enabled by default by the HW, so we need to disable it manually.
	 */
	iwl_write_prph(trans, SCD_CHAINEXT_EN, 0);

1094 1095
	iwl_trans_ac_txq_enable(trans, trans_pcie->cmd_queue,
				trans_pcie->cmd_fifo);
1096

1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
	/* Activate all Tx DMA/FIFO channels */
	iwl_trans_txq_set_sched(trans, IWL_MASK(0, 7));

	/* Enable DMA channel */
	for (chan = 0; chan < FH_TCSR_CHNL_NUM ; chan++)
		iwl_write_direct32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(chan),
				   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
				   FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);

	/* Update FH chicken bits */
	reg_val = iwl_read_direct32(trans, FH_TX_CHICKEN_BITS_REG);
	iwl_write_direct32(trans, FH_TX_CHICKEN_BITS_REG,
			   reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);

1111
	/* Enable L1-Active */
1112
	iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
1113
			    APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1114 1115
}

1116 1117 1118 1119 1120 1121
static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans)
{
	iwl_reset_ict(trans);
	iwl_tx_start(trans);
}

1122 1123 1124
/**
 * iwlagn_txq_ctx_stop - Stop all Tx DMA channels
 */
1125
static int iwl_trans_tx_stop(struct iwl_trans *trans)
1126
{
1127
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1128
	int ch, txq_id, ret;
1129 1130 1131
	unsigned long flags;

	/* Turn off all Tx DMA fifos */
J
Johannes Berg 已提交
1132
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1133

1134
	iwl_trans_txq_set_sched(trans, 0);
1135 1136

	/* Stop each Tx DMA channel, and wait for it to be idle */
1137
	for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) {
1138
		iwl_write_direct32(trans,
1139
				   FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0);
1140
		ret = iwl_poll_direct_bit(trans, FH_TSSR_TX_STATUS_REG,
1141
			FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch), 1000);
1142
		if (ret < 0)
1143 1144 1145 1146 1147
			IWL_ERR(trans,
				"Failing on timeout while stopping DMA channel %d [0x%08x]",
				ch,
				iwl_read_direct32(trans,
						  FH_TSSR_TX_STATUS_REG));
1148
	}
J
Johannes Berg 已提交
1149
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1150

1151
	if (!trans_pcie->txq) {
1152
		IWL_WARN(trans, "Stopping tx queues that aren't allocated...");
1153 1154 1155 1156
		return 0;
	}

	/* Unmap DMA from host system and free skb's */
1157
	for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues;
1158
	     txq_id++)
1159
		iwl_tx_queue_unmap(trans, txq_id);
1160 1161 1162 1163

	return 0;
}

1164
static void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1165
{
1166
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1167
	unsigned long flags;
1168

1169
	/* tell the device to stop sending interrupts */
J
Johannes Berg 已提交
1170
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1171
	iwl_disable_interrupts(trans);
J
Johannes Berg 已提交
1172
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1173

1174
	/* device going down, Stop using ICT table */
1175
	iwl_disable_ict(trans);
1176 1177 1178 1179 1180 1181 1182 1183

	/*
	 * If a HW restart happens during firmware loading,
	 * then the firmware loading might call this function
	 * and later it might be called again due to the
	 * restart. So don't process again if the device is
	 * already dead.
	 */
D
Don Fry 已提交
1184
	if (test_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status)) {
1185
		iwl_trans_tx_stop(trans);
1186
#ifndef CONFIG_IWLWIFI_IDI
1187
		iwl_trans_rx_stop(trans);
1188
#endif
1189
		/* Power-down device's busmaster DMA clocks */
1190
		iwl_write_prph(trans, APMG_CLK_DIS_REG,
1191 1192 1193 1194 1195
			       APMG_CLK_VAL_DMA_CLK_RQT);
		udelay(5);
	}

	/* Make sure (redundant) we've released our request to stay awake */
1196
	iwl_clear_bit(trans, CSR_GP_CNTRL,
1197
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1198 1199

	/* Stop the device, and put it in low power state */
1200
	iwl_apm_stop(trans);
1201 1202 1203 1204

	/* Upon stop, the APM issues an interrupt if HW RF kill is set.
	 * Clean again the interrupt here
	 */
J
Johannes Berg 已提交
1205
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
1206
	iwl_disable_interrupts(trans);
J
Johannes Berg 已提交
1207
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1208

1209 1210
	iwl_enable_rfkill_int(trans);

1211
	/* wait to make sure we flush pending tasklet*/
J
Johannes Berg 已提交
1212
	synchronize_irq(trans_pcie->irq);
1213 1214
	tasklet_kill(&trans_pcie->irq_tasklet);

J
Johannes Berg 已提交
1215 1216
	cancel_work_sync(&trans_pcie->rx_replenish);

1217
	/* stop and reset the on-board processor */
1218
	iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_NEVO_RESET);
D
Don Fry 已提交
1219 1220 1221 1222 1223

	/* clear all status bits */
	clear_bit(STATUS_HCMD_ACTIVE, &trans_pcie->status);
	clear_bit(STATUS_INT_ENABLED, &trans_pcie->status);
	clear_bit(STATUS_DEVICE_ENABLED, &trans_pcie->status);
1224
	clear_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
1225 1226
}

1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
static void iwl_trans_pcie_wowlan_suspend(struct iwl_trans *trans)
{
	/* let the ucode operate on its own */
	iwl_write32(trans, CSR_UCODE_DRV_GP1_SET,
		    CSR_UCODE_DRV_GP1_BIT_D3_CFG_COMPLETE);

	iwl_disable_interrupts(trans);
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
}

1238
static int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
1239
			     struct iwl_device_cmd *dev_cmd, int txq_id)
1240
{
1241 1242
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
1243
	struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *) dev_cmd->payload;
1244
	struct iwl_cmd_meta *out_meta;
1245 1246
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
1247 1248 1249 1250 1251
	dma_addr_t phys_addr = 0;
	dma_addr_t txcmd_phys;
	dma_addr_t scratch_phys;
	u16 len, firstlen, secondlen;
	u8 wait_write_ptr = 0;
1252
	__le16 fc = hdr->frame_control;
1253
	u8 hdr_len = ieee80211_hdrlen(fc);
1254
	u16 __maybe_unused wifi_seq;
1255

1256
	txq = &trans_pcie->txq[txq_id];
1257 1258
	q = &txq->q;

1259 1260 1261 1262
	if (unlikely(!test_bit(txq_id, trans_pcie->queue_used))) {
		WARN_ON_ONCE(1);
		return -EINVAL;
	}
1263

1264
	spin_lock(&txq->lock);
1265

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	/* In AGG mode, the index in the ring must correspond to the WiFi
	 * sequence number. This is a HW requirements to help the SCD to parse
	 * the BA.
	 * Check here that the packets are in the right place on the ring.
	 */
#ifdef CONFIG_IWLWIFI_DEBUG
	wifi_seq = SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
	WARN_ONCE((iwl_read_prph(trans, SCD_AGGR_SEL) & BIT(txq_id)) &&
		  ((wifi_seq & 0xff) != q->write_ptr),
		  "Q: %d WiFi Seq %d tfdNum %d",
		  txq_id, wifi_seq, q->write_ptr);
#endif

1279
	/* Set up driver data for this TFD */
1280 1281
	txq->entries[q->write_ptr].skb = skb;
	txq->entries[q->write_ptr].cmd = dev_cmd;
1282 1283

	dev_cmd->hdr.cmd = REPLY_TX;
1284 1285 1286
	dev_cmd->hdr.sequence =
		cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
			    INDEX_TO_SEQ(q->write_ptr)));
1287 1288

	/* Set up first empty entry in queue's array of Tx/cmd buffers */
1289
	out_meta = &txq->entries[q->write_ptr].meta;
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309

	/*
	 * Use the first empty entry in this queue's command buffer array
	 * to contain the Tx command and MAC header concatenated together
	 * (payload data will be in another buffer).
	 * Size of this varies, due to varying MAC header length.
	 * If end is not dword aligned, we'll have 2 extra bytes at the end
	 * of the MAC header (device reads on dword boundaries).
	 * We'll tell device about this padding later.
	 */
	len = sizeof(struct iwl_tx_cmd) +
		sizeof(struct iwl_cmd_header) + hdr_len;
	firstlen = (len + 3) & ~3;

	/* Tell NIC about any 2-byte padding after MAC header */
	if (firstlen != len)
		tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK;

	/* Physical address of this Tx command's header (not MAC header!),
	 * within command buffer array. */
1310
	txcmd_phys = dma_map_single(trans->dev,
1311 1312
				    &dev_cmd->hdr, firstlen,
				    DMA_BIDIRECTIONAL);
1313
	if (unlikely(dma_mapping_error(trans->dev, txcmd_phys)))
1314
		goto out_err;
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
	dma_unmap_addr_set(out_meta, mapping, txcmd_phys);
	dma_unmap_len_set(out_meta, len, firstlen);

	if (!ieee80211_has_morefrags(fc)) {
		txq->need_update = 1;
	} else {
		wait_write_ptr = 1;
		txq->need_update = 0;
	}

	/* Set up TFD's 2nd entry to point directly to remainder of skb,
	 * if any (802.11 null frames have no payload). */
	secondlen = skb->len - hdr_len;
	if (secondlen > 0) {
1329
		phys_addr = dma_map_single(trans->dev, skb->data + hdr_len,
1330
					   secondlen, DMA_TO_DEVICE);
1331 1332
		if (unlikely(dma_mapping_error(trans->dev, phys_addr))) {
			dma_unmap_single(trans->dev,
1333 1334 1335
					 dma_unmap_addr(out_meta, mapping),
					 dma_unmap_len(out_meta, len),
					 DMA_BIDIRECTIONAL);
1336
			goto out_err;
1337 1338 1339 1340
		}
	}

	/* Attach buffers to TFD */
1341
	iwlagn_txq_attach_buf_to_tfd(trans, txq, txcmd_phys, firstlen, 1);
1342
	if (secondlen > 0)
1343
		iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr,
1344 1345 1346 1347 1348 1349
					     secondlen, 0);

	scratch_phys = txcmd_phys + sizeof(struct iwl_cmd_header) +
				offsetof(struct iwl_tx_cmd, scratch);

	/* take back ownership of DMA buffer to enable update */
1350
	dma_sync_single_for_cpu(trans->dev, txcmd_phys, firstlen,
1351
				DMA_BIDIRECTIONAL);
1352 1353 1354
	tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
	tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);

1355
	IWL_DEBUG_TX(trans, "sequence nr = 0X%x\n",
1356
		     le16_to_cpu(dev_cmd->hdr.sequence));
1357
	IWL_DEBUG_TX(trans, "tx_flags = 0X%x\n", le32_to_cpu(tx_cmd->tx_flags));
1358 1359

	/* Set up entry for this TFD in Tx byte-count array */
1360
	iwl_trans_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len));
1361

1362
	dma_sync_single_for_device(trans->dev, txcmd_phys, firstlen,
1363
				   DMA_BIDIRECTIONAL);
1364

1365
	trace_iwlwifi_dev_tx(trans->dev,
1366 1367 1368 1369 1370
			     &((struct iwl_tfd *)txq->tfds)[txq->q.write_ptr],
			     sizeof(struct iwl_tfd),
			     &dev_cmd->hdr, firstlen,
			     skb->data + hdr_len, secondlen);

1371
	/* start timer if queue currently empty */
1372 1373
	if (txq->need_update && q->read_ptr == q->write_ptr &&
	    trans_pcie->wd_timeout)
1374 1375
		mod_timer(&txq->stuck_timer, jiffies + trans_pcie->wd_timeout);

1376 1377
	/* Tell device the write index *just past* this latest filled TFD */
	q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd);
1378 1379
	iwl_txq_update_write_ptr(trans, txq);

1380 1381 1382 1383 1384 1385
	/*
	 * At this point the frame is "transmitted" successfully
	 * and we will get a TX status notification eventually,
	 * regardless of the value of ret. "ret" only indicates
	 * whether or not we should update the write pointer.
	 */
1386
	if (iwl_queue_space(q) < q->high_mark) {
1387 1388
		if (wait_write_ptr) {
			txq->need_update = 1;
1389
			iwl_txq_update_write_ptr(trans, txq);
1390
		} else {
1391
			iwl_stop_queue(trans, txq);
1392 1393
		}
	}
1394
	spin_unlock(&txq->lock);
1395
	return 0;
1396 1397 1398
 out_err:
	spin_unlock(&txq->lock);
	return -1;
1399 1400
}

1401
static int iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1402
{
1403
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1404
	int err;
1405
	bool hw_rfkill;
1406

1407 1408
	trans_pcie->inta_mask = CSR_INI_SET_MASK;

1409 1410 1411
	if (!trans_pcie->irq_requested) {
		tasklet_init(&trans_pcie->irq_tasklet, (void (*)(unsigned long))
			iwl_irq_tasklet, (unsigned long)trans);
1412

1413
		iwl_alloc_isr_ict(trans);
1414

J
Johannes Berg 已提交
1415
		err = request_irq(trans_pcie->irq, iwl_isr_ict, IRQF_SHARED,
1416
				  DRV_NAME, trans);
1417 1418
		if (err) {
			IWL_ERR(trans, "Error allocating IRQ %d\n",
J
Johannes Berg 已提交
1419
				trans_pcie->irq);
1420
			goto error;
1421 1422 1423 1424
		}

		INIT_WORK(&trans_pcie->rx_replenish, iwl_bg_rx_replenish);
		trans_pcie->irq_requested = true;
1425 1426
	}

1427 1428 1429
	err = iwl_prepare_card_hw(trans);
	if (err) {
		IWL_ERR(trans, "Error while preparing HW: %d", err);
1430
		goto err_free_irq;
1431
	}
1432 1433 1434

	iwl_apm_init(trans);

1435 1436 1437
	/* From now on, the op_mode will be kept updated about RF kill state */
	iwl_enable_rfkill_int(trans);

1438
	hw_rfkill = iwl_is_rfkill_set(trans);
1439
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1440

1441 1442
	return err;

1443
err_free_irq:
J
Johannes Berg 已提交
1444
	free_irq(trans_pcie->irq, trans);
1445 1446 1447 1448
error:
	iwl_free_isr_ict(trans);
	tasklet_kill(&trans_pcie->irq_tasklet);
	return err;
1449 1450
}

1451 1452
static void iwl_trans_pcie_stop_hw(struct iwl_trans *trans,
				   bool op_mode_leaving)
1453
{
1454
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1455
	bool hw_rfkill;
1456
	unsigned long flags;
1457

1458 1459
	iwl_apm_stop(trans);

1460 1461 1462
	spin_lock_irqsave(&trans_pcie->irq_lock, flags);
	iwl_disable_interrupts(trans);
	spin_unlock_irqrestore(&trans_pcie->irq_lock, flags);
1463

1464
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1465

1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
	if (!op_mode_leaving) {
		/*
		 * Even if we stop the HW, we still want the RF kill
		 * interrupt
		 */
		iwl_enable_rfkill_int(trans);

		/*
		 * Check again since the RF kill state may have changed while
		 * all the interrupts were disabled, in this case we couldn't
		 * receive the RF kill interrupt and update the state in the
		 * op_mode.
		 */
		hw_rfkill = iwl_is_rfkill_set(trans);
		iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
	}
1482 1483
}

1484 1485
static void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
				   struct sk_buff_head *skbs)
1486
{
1487 1488
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
1489 1490
	/* n_bd is usually 256 => n_bd - 1 = 0xff */
	int tfd_num = ssn & (txq->q.n_bd - 1);
1491
	int freed = 0;
1492

1493 1494
	spin_lock(&txq->lock);

1495
	if (txq->q.read_ptr != tfd_num) {
1496 1497
		IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n",
				   txq_id, txq->q.read_ptr, tfd_num, ssn);
1498
		freed = iwl_tx_queue_reclaim(trans, txq_id, tfd_num, skbs);
1499
		if (iwl_queue_space(&txq->q) > txq->q.low_mark)
1500
			iwl_wake_queue(trans, txq);
1501
	}
1502 1503

	spin_unlock(&txq->lock);
1504 1505
}

1506 1507
static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
{
1508
	writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1509 1510 1511 1512
}

static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
{
1513
	writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1514 1515 1516 1517
}

static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
{
1518
	return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1519 1520
}

1521
static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1522
				     const struct iwl_trans_config *trans_cfg)
1523 1524 1525 1526
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans_pcie->cmd_queue = trans_cfg->cmd_queue;
1527
	trans_pcie->cmd_fifo = trans_cfg->cmd_fifo;
1528 1529 1530 1531 1532 1533 1534
	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
		trans_pcie->n_no_reclaim_cmds = 0;
	else
		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
	if (trans_pcie->n_no_reclaim_cmds)
		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1535

1536 1537 1538 1539 1540
	trans_pcie->rx_buf_size_8k = trans_cfg->rx_buf_size_8k;
	if (trans_pcie->rx_buf_size_8k)
		trans_pcie->rx_page_order = get_order(8 * 1024);
	else
		trans_pcie->rx_page_order = get_order(4 * 1024);
1541 1542 1543

	trans_pcie->wd_timeout =
		msecs_to_jiffies(trans_cfg->queue_watchdog_timeout);
J
Johannes Berg 已提交
1544 1545

	trans_pcie->command_names = trans_cfg->command_names;
1546 1547
}

1548
void iwl_trans_pcie_free(struct iwl_trans *trans)
1549
{
1550
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1551

1552
	iwl_trans_pcie_tx_free(trans);
1553
#ifndef CONFIG_IWLWIFI_IDI
1554
	iwl_trans_pcie_rx_free(trans);
1555
#endif
1556
	if (trans_pcie->irq_requested == true) {
J
Johannes Berg 已提交
1557
		free_irq(trans_pcie->irq, trans);
1558 1559
		iwl_free_isr_ict(trans);
	}
1560 1561

	pci_disable_msi(trans_pcie->pci_dev);
1562
	iounmap(trans_pcie->hw_base);
1563 1564
	pci_release_regions(trans_pcie->pci_dev);
	pci_disable_device(trans_pcie->pci_dev);
1565
	kmem_cache_destroy(trans->dev_cmd_pool);
1566

1567
	kfree(trans);
1568 1569
}

D
Don Fry 已提交
1570 1571 1572 1573 1574
static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (state)
1575
		set_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
D
Don Fry 已提交
1576
	else
1577
		clear_bit(STATUS_TPOWER_PMI, &trans_pcie->status);
D
Don Fry 已提交
1578 1579
}

J
Johannes Berg 已提交
1580
#ifdef CONFIG_PM_SLEEP
1581 1582 1583 1584 1585 1586 1587
static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
{
	return 0;
}

static int iwl_trans_pcie_resume(struct iwl_trans *trans)
{
1588
	bool hw_rfkill;
1589

1590 1591
	iwl_enable_rfkill_int(trans);

1592
	hw_rfkill = iwl_is_rfkill_set(trans);
1593
	iwl_op_mode_hw_rf_kill(trans->op_mode, hw_rfkill);
1594

1595
	if (!hw_rfkill)
1596 1597
		iwl_enable_interrupts(trans);

1598 1599
	return 0;
}
J
Johannes Berg 已提交
1600
#endif /* CONFIG_PM_SLEEP */
1601

1602 1603 1604 1605
#define IWL_FLUSH_WAIT_MS	2000

static int iwl_trans_pcie_wait_tx_queue_empty(struct iwl_trans *trans)
{
1606
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1607 1608 1609 1610 1611 1612 1613
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	int cnt;
	unsigned long now = jiffies;
	int ret = 0;

	/* waiting for all the tx frames complete might take a while */
1614
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
W
Wey-Yi Guy 已提交
1615
		if (cnt == trans_pcie->cmd_queue)
1616
			continue;
1617
		txq = &trans_pcie->txq[cnt];
1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
		q = &txq->q;
		while (q->read_ptr != q->write_ptr && !time_after(jiffies,
		       now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS)))
			msleep(1);

		if (q->read_ptr != q->write_ptr) {
			IWL_ERR(trans, "fail to flush all tx fifo queues\n");
			ret = -ETIMEDOUT;
			break;
		}
	}
	return ret;
}

1632 1633
static const char *get_fh_string(int cmd)
{
J
Johannes Berg 已提交
1634
#define IWL_CMD(x) case x: return #x
1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647
	switch (cmd) {
	IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
	IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
	IWL_CMD(FH_RSCSR_CHNL0_WPTR);
	IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
	IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
	IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
	IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
	IWL_CMD(FH_TSSR_TX_STATUS_REG);
	IWL_CMD(FH_TSSR_TX_ERROR_REG);
	default:
		return "UNKNOWN";
	}
J
Johannes Berg 已提交
1648
#undef IWL_CMD
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
}

int iwl_dump_fh(struct iwl_trans *trans, char **buf, bool display)
{
	int i;
#ifdef CONFIG_IWLWIFI_DEBUG
	int pos = 0;
	size_t bufsz = 0;
#endif
	static const u32 fh_tbl[] = {
		FH_RSCSR_CHNL0_STTS_WPTR_REG,
		FH_RSCSR_CHNL0_RBDCB_BASE_REG,
		FH_RSCSR_CHNL0_WPTR,
		FH_MEM_RCSR_CHNL0_CONFIG_REG,
		FH_MEM_RSSR_SHARED_CTRL_REG,
		FH_MEM_RSSR_RX_STATUS_REG,
		FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
		FH_TSSR_TX_STATUS_REG,
		FH_TSSR_TX_ERROR_REG
	};
#ifdef CONFIG_IWLWIFI_DEBUG
	if (display) {
		bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
		*buf = kmalloc(bufsz, GFP_KERNEL);
		if (!*buf)
			return -ENOMEM;
		pos += scnprintf(*buf + pos, bufsz - pos,
				"FH register values:\n");
		for (i = 0; i < ARRAY_SIZE(fh_tbl); i++) {
			pos += scnprintf(*buf + pos, bufsz - pos,
				"  %34s: 0X%08x\n",
				get_fh_string(fh_tbl[i]),
1681
				iwl_read_direct32(trans, fh_tbl[i]));
1682 1683 1684 1685 1686 1687 1688 1689
		}
		return pos;
	}
#endif
	IWL_ERR(trans, "FH register values:\n");
	for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++) {
		IWL_ERR(trans, "  %34s: 0X%08x\n",
			get_fh_string(fh_tbl[i]),
1690
			iwl_read_direct32(trans, fh_tbl[i]));
1691 1692 1693 1694 1695 1696
	}
	return 0;
}

static const char *get_csr_string(int cmd)
{
J
Johannes Berg 已提交
1697
#define IWL_CMD(x) case x: return #x
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
	switch (cmd) {
	IWL_CMD(CSR_HW_IF_CONFIG_REG);
	IWL_CMD(CSR_INT_COALESCING);
	IWL_CMD(CSR_INT);
	IWL_CMD(CSR_INT_MASK);
	IWL_CMD(CSR_FH_INT_STATUS);
	IWL_CMD(CSR_GPIO_IN);
	IWL_CMD(CSR_RESET);
	IWL_CMD(CSR_GP_CNTRL);
	IWL_CMD(CSR_HW_REV);
	IWL_CMD(CSR_EEPROM_REG);
	IWL_CMD(CSR_EEPROM_GP);
	IWL_CMD(CSR_OTP_GP_REG);
	IWL_CMD(CSR_GIO_REG);
	IWL_CMD(CSR_GP_UCODE_REG);
	IWL_CMD(CSR_GP_DRIVER_REG);
	IWL_CMD(CSR_UCODE_DRV_GP1);
	IWL_CMD(CSR_UCODE_DRV_GP2);
	IWL_CMD(CSR_LED_REG);
	IWL_CMD(CSR_DRAM_INT_TBL_REG);
	IWL_CMD(CSR_GIO_CHICKEN_BITS);
	IWL_CMD(CSR_ANA_PLL_CFG);
	IWL_CMD(CSR_HW_REV_WA_REG);
	IWL_CMD(CSR_DBG_HPET_MEM_REG);
	default:
		return "UNKNOWN";
	}
J
Johannes Berg 已提交
1725
#undef IWL_CMD
1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
}

void iwl_dump_csr(struct iwl_trans *trans)
{
	int i;
	static const u32 csr_tbl[] = {
		CSR_HW_IF_CONFIG_REG,
		CSR_INT_COALESCING,
		CSR_INT,
		CSR_INT_MASK,
		CSR_FH_INT_STATUS,
		CSR_GPIO_IN,
		CSR_RESET,
		CSR_GP_CNTRL,
		CSR_HW_REV,
		CSR_EEPROM_REG,
		CSR_EEPROM_GP,
		CSR_OTP_GP_REG,
		CSR_GIO_REG,
		CSR_GP_UCODE_REG,
		CSR_GP_DRIVER_REG,
		CSR_UCODE_DRV_GP1,
		CSR_UCODE_DRV_GP2,
		CSR_LED_REG,
		CSR_DRAM_INT_TBL_REG,
		CSR_GIO_CHICKEN_BITS,
		CSR_ANA_PLL_CFG,
		CSR_HW_REV_WA_REG,
		CSR_DBG_HPET_MEM_REG
	};
	IWL_ERR(trans, "CSR values:\n");
	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
		"CSR_INT_PERIODIC_REG)\n");
	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
		IWL_ERR(trans, "  %25s: 0X%08x\n",
			get_csr_string(csr_tbl[i]),
1762
			iwl_read32(trans, csr_tbl[i]));
1763 1764 1765
	}
}

1766 1767 1768
#ifdef CONFIG_IWLWIFI_DEBUGFS
/* create and remove of files */
#define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
1769
	if (!debugfs_create_file(#name, mode, parent, trans,		\
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
				 &iwl_dbgfs_##name##_ops))		\
		return -ENOMEM;						\
} while (0)

/* file operation */
#define DEBUGFS_READ_FUNC(name)                                         \
static ssize_t iwl_dbgfs_##name##_read(struct file *file,               \
					char __user *user_buf,          \
					size_t count, loff_t *ppos);

#define DEBUGFS_WRITE_FUNC(name)                                        \
static ssize_t iwl_dbgfs_##name##_write(struct file *file,              \
					const char __user *user_buf,    \
					size_t count, loff_t *ppos);


#define DEBUGFS_READ_FILE_OPS(name)					\
	DEBUGFS_READ_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.read = iwl_dbgfs_##name##_read,				\
1790
	.open = simple_open,						\
1791 1792 1793
	.llseek = generic_file_llseek,					\
};

1794 1795 1796 1797
#define DEBUGFS_WRITE_FILE_OPS(name)                                    \
	DEBUGFS_WRITE_FUNC(name);                                       \
static const struct file_operations iwl_dbgfs_##name##_ops = {          \
	.write = iwl_dbgfs_##name##_write,                              \
1798
	.open = simple_open,						\
1799 1800 1801
	.llseek = generic_file_llseek,					\
};

1802 1803 1804 1805 1806 1807
#define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
	DEBUGFS_READ_FUNC(name);					\
	DEBUGFS_WRITE_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.write = iwl_dbgfs_##name##_write,				\
	.read = iwl_dbgfs_##name##_read,				\
1808
	.open = simple_open,						\
1809 1810 1811 1812
	.llseek = generic_file_llseek,					\
};

static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
1813 1814
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
1815
{
1816
	struct iwl_trans *trans = file->private_data;
1817
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1818 1819 1820 1821 1822 1823
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	char *buf;
	int pos = 0;
	int cnt;
	int ret;
1824 1825
	size_t bufsz;

1826
	bufsz = sizeof(char) * 64 * trans->cfg->base_params->num_of_queues;
1827

J
Johannes Berg 已提交
1828
	if (!trans_pcie->txq)
1829
		return -EAGAIN;
J
Johannes Berg 已提交
1830

1831 1832 1833 1834
	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

1835
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
1836
		txq = &trans_pcie->txq[cnt];
1837 1838
		q = &txq->q;
		pos += scnprintf(buf + pos, bufsz - pos,
1839
				"hwq %.2d: read=%u write=%u use=%d stop=%d\n",
1840
				cnt, q->read_ptr, q->write_ptr,
1841 1842
				!!test_bit(cnt, trans_pcie->queue_used),
				!!test_bit(cnt, trans_pcie->queue_stopped));
1843 1844 1845 1846 1847 1848 1849
	}
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
1850 1851 1852
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
{
1853
	struct iwl_trans *trans = file->private_data;
1854
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1855
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
	char buf[256];
	int pos = 0;
	const size_t bufsz = sizeof(buf);

	pos += scnprintf(buf + pos, bufsz - pos, "read: %u\n",
						rxq->read);
	pos += scnprintf(buf + pos, bufsz - pos, "write: %u\n",
						rxq->write);
	pos += scnprintf(buf + pos, bufsz - pos, "free_count: %u\n",
						rxq->free_count);
	if (rxq->rb_stts) {
		pos += scnprintf(buf + pos, bufsz - pos, "closed_rb_num: %u\n",
			 le16_to_cpu(rxq->rb_stts->closed_rb_num) &  0x0FFF);
	} else {
		pos += scnprintf(buf + pos, bufsz - pos,
					"closed_rb_num: Not Allocated\n");
	}
	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
}

1876 1877
static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
					char __user *user_buf,
1878 1879
					size_t count, loff_t *ppos)
{
1880
	struct iwl_trans *trans = file->private_data;
1881
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1882 1883 1884 1885 1886 1887 1888 1889
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	int pos = 0;
	char *buf;
	int bufsz = 24 * 64; /* 24 items * 64 char per item */
	ssize_t ret;

	buf = kzalloc(bufsz, GFP_KERNEL);
J
Johannes Berg 已提交
1890
	if (!buf)
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
		return -ENOMEM;

	pos += scnprintf(buf + pos, bufsz - pos,
			"Interrupt Statistics Report:\n");

	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
		isr_stats->hw);
	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
		isr_stats->sw);
	if (isr_stats->sw || isr_stats->hw) {
		pos += scnprintf(buf + pos, bufsz - pos,
			"\tLast Restarting Code:  0x%X\n",
			isr_stats->err_code);
	}
#ifdef CONFIG_IWLWIFI_DEBUG
	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
		isr_stats->sch);
	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
		isr_stats->alive);
#endif
	pos += scnprintf(buf + pos, bufsz - pos,
		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);

	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
		isr_stats->ctkill);

	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
		isr_stats->wakeup);

	pos += scnprintf(buf + pos, bufsz - pos,
		"Rx command responses:\t\t %u\n", isr_stats->rx);

	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
		isr_stats->tx);

	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
		isr_stats->unhandled);

	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
					 const char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
1939
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	char buf[8];
	int buf_size;
	u32 reset_flag;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%x", &reset_flag) != 1)
		return -EFAULT;
	if (reset_flag == 0)
		memset(isr_stats, 0, sizeof(*isr_stats));

	return count;
}

1958
static ssize_t iwl_dbgfs_csr_write(struct file *file,
1959 1960
				   const char __user *user_buf,
				   size_t count, loff_t *ppos)
1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979
{
	struct iwl_trans *trans = file->private_data;
	char buf[8];
	int buf_size;
	int csr;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%d", &csr) != 1)
		return -EFAULT;

	iwl_dump_csr(trans);

	return count;
}

static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
1980 1981
				     char __user *user_buf,
				     size_t count, loff_t *ppos)
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
{
	struct iwl_trans *trans = file->private_data;
	char *buf;
	int pos = 0;
	ssize_t ret = -EFAULT;

	ret = pos = iwl_dump_fh(trans, &buf, true);
	if (buf) {
		ret = simple_read_from_buffer(user_buf,
					      count, ppos, buf, pos);
		kfree(buf);
	}

	return ret;
}

1998 1999 2000 2001 2002 2003 2004 2005 2006
static ssize_t iwl_dbgfs_fw_restart_write(struct file *file,
					  const char __user *user_buf,
					  size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;

	if (!trans->op_mode)
		return -EAGAIN;

2007
	local_bh_disable();
2008
	iwl_op_mode_nic_error(trans->op_mode);
2009
	local_bh_enable();
2010 2011 2012 2013

	return count;
}

2014
DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2015
DEBUGFS_READ_FILE_OPS(fh_reg);
2016 2017
DEBUGFS_READ_FILE_OPS(rx_queue);
DEBUGFS_READ_FILE_OPS(tx_queue);
2018
DEBUGFS_WRITE_FILE_OPS(csr);
2019
DEBUGFS_WRITE_FILE_OPS(fw_restart);
2020 2021 2022 2023 2024 2025

/*
 * Create the debugfs files and directories
 *
 */
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
2026
					 struct dentry *dir)
2027 2028 2029
{
	DEBUGFS_ADD_FILE(rx_queue, dir, S_IRUSR);
	DEBUGFS_ADD_FILE(tx_queue, dir, S_IRUSR);
2030
	DEBUGFS_ADD_FILE(interrupt, dir, S_IWUSR | S_IRUSR);
2031 2032
	DEBUGFS_ADD_FILE(csr, dir, S_IWUSR);
	DEBUGFS_ADD_FILE(fh_reg, dir, S_IRUSR);
2033
	DEBUGFS_ADD_FILE(fw_restart, dir, S_IWUSR);
2034 2035 2036 2037
	return 0;
}
#else
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
2038 2039 2040 2041
					 struct dentry *dir)
{
	return 0;
}
2042 2043
#endif /*CONFIG_IWLWIFI_DEBUGFS */

2044
static const struct iwl_trans_ops trans_ops_pcie = {
2045
	.start_hw = iwl_trans_pcie_start_hw,
2046
	.stop_hw = iwl_trans_pcie_stop_hw,
2047
	.fw_alive = iwl_trans_pcie_fw_alive,
2048
	.start_fw = iwl_trans_pcie_start_fw,
2049
	.stop_device = iwl_trans_pcie_stop_device,
2050

2051 2052
	.wowlan_suspend = iwl_trans_pcie_wowlan_suspend,

2053
	.send_cmd = iwl_trans_pcie_send_cmd,
2054

2055
	.tx = iwl_trans_pcie_tx,
2056
	.reclaim = iwl_trans_pcie_reclaim,
2057

2058
	.txq_disable = iwl_trans_pcie_txq_disable,
2059
	.txq_enable = iwl_trans_pcie_txq_enable,
2060

2061
	.dbgfs_register = iwl_trans_pcie_dbgfs_register,
2062 2063 2064

	.wait_tx_queue_empty = iwl_trans_pcie_wait_tx_queue_empty,

J
Johannes Berg 已提交
2065
#ifdef CONFIG_PM_SLEEP
2066 2067
	.suspend = iwl_trans_pcie_suspend,
	.resume = iwl_trans_pcie_resume,
J
Johannes Berg 已提交
2068
#endif
2069 2070 2071
	.write8 = iwl_trans_pcie_write8,
	.write32 = iwl_trans_pcie_write32,
	.read32 = iwl_trans_pcie_read32,
2072
	.configure = iwl_trans_pcie_configure,
D
Don Fry 已提交
2073
	.set_pmi = iwl_trans_pcie_set_pmi,
2074
};
2075

2076
struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
2077 2078
				       const struct pci_device_id *ent,
				       const struct iwl_cfg *cfg)
2079 2080 2081
{
	struct iwl_trans_pcie *trans_pcie;
	struct iwl_trans *trans;
2082
	char cmd_pool_name[100];
2083 2084 2085 2086
	u16 pci_cmd;
	int err;

	trans = kzalloc(sizeof(struct iwl_trans) +
2087
			sizeof(struct iwl_trans_pcie), GFP_KERNEL);
2088 2089 2090 2091 2092 2093 2094

	if (WARN_ON(!trans))
		return NULL;

	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans->ops = &trans_ops_pcie;
2095
	trans->cfg = cfg;
2096
	trans_pcie->trans = trans;
J
Johannes Berg 已提交
2097
	spin_lock_init(&trans_pcie->irq_lock);
2098
	init_waitqueue_head(&trans_pcie->ucode_write_waitq);
2099 2100 2101 2102

	/* W/A - seems to solve weird behavior. We need to remove this if we
	 * don't want to stay in L1 all the time. This wastes a lot of power */
	pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 |
2103
			       PCIE_LINK_STATE_CLKPM);
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118

	if (pci_enable_device(pdev)) {
		err = -ENODEV;
		goto out_no_pci;
	}

	pci_set_master(pdev);

	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(36));
	if (!err)
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(36));
	if (err) {
		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (!err)
			err = pci_set_consistent_dma_mask(pdev,
2119
							  DMA_BIT_MASK(32));
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133
		/* both attempts failed: */
		if (err) {
			dev_printk(KERN_ERR, &pdev->dev,
				   "No suitable DMA available.\n");
			goto out_pci_disable_device;
		}
	}

	err = pci_request_regions(pdev, DRV_NAME);
	if (err) {
		dev_printk(KERN_ERR, &pdev->dev, "pci_request_regions failed");
		goto out_pci_disable_device;
	}

2134
	trans_pcie->hw_base = pci_ioremap_bar(pdev, 0);
2135
	if (!trans_pcie->hw_base) {
2136
		dev_printk(KERN_ERR, &pdev->dev, "pci_ioremap_bar failed");
2137 2138 2139 2140 2141
		err = -ENODEV;
		goto out_pci_release_regions;
	}

	dev_printk(KERN_INFO, &pdev->dev,
2142 2143
		   "pci_resource_len = 0x%08llx\n",
		   (unsigned long long) pci_resource_len(pdev, 0));
2144
	dev_printk(KERN_INFO, &pdev->dev,
2145
		   "pci_resource_base = %p\n", trans_pcie->hw_base);
2146 2147

	dev_printk(KERN_INFO, &pdev->dev,
2148
		   "HW Revision ID = 0x%X\n", pdev->revision);
2149 2150 2151 2152 2153 2154 2155 2156

	/* We disable the RETRY_TIMEOUT register (0x41) to keep
	 * PCI Tx retries from interfering with C3 CPU state */
	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);

	err = pci_enable_msi(pdev);
	if (err)
		dev_printk(KERN_ERR, &pdev->dev,
2157
			   "pci_enable_msi failed(0X%x)", err);
2158 2159

	trans->dev = &pdev->dev;
J
Johannes Berg 已提交
2160
	trans_pcie->irq = pdev->irq;
2161
	trans_pcie->pci_dev = pdev;
2162
	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
E
Emmanuel Grumbach 已提交
2163
	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
2164 2165
	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
2166 2167 2168 2169 2170 2171 2172 2173 2174

	/* TODO: Move this away, not needed if not MSI */
	/* enable rfkill interrupt: hw bug w/a */
	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
	if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
		pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
		pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
	}

2175 2176
	/* Initialize the wait queue for commands */
	init_waitqueue_head(&trans->wait_command_queue);
2177
	spin_lock_init(&trans->reg_lock);
2178

2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
	snprintf(cmd_pool_name, sizeof(cmd_pool_name), "iwl_cmd_pool:%s",
		 dev_name(trans->dev));

	trans->dev_cmd_headroom = 0;
	trans->dev_cmd_pool =
		kmem_cache_create(cmd_pool_name,
				  sizeof(struct iwl_device_cmd)
				  + trans->dev_cmd_headroom,
				  sizeof(void *),
				  SLAB_HWCACHE_ALIGN,
				  NULL);

	if (!trans->dev_cmd_pool)
		goto out_pci_disable_msi;

2194 2195
	return trans;

2196 2197
out_pci_disable_msi:
	pci_disable_msi(pdev);
2198 2199 2200 2201 2202 2203 2204 2205
out_pci_release_regions:
	pci_release_regions(pdev);
out_pci_disable_device:
	pci_disable_device(pdev);
out_no_pci:
	kfree(trans);
	return NULL;
}