sata_sil24.c 37.6 KB
Newer Older
T
Tejun Heo 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * sata_sil24.c - Driver for Silicon Image 3124/3132 SATA-2 controllers
 *
 * Copyright 2005  Tejun Heo
 *
 * Based on preview driver from Silicon Image.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2, or (at your option) any
 * later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
22
#include <linux/gfp.h>
T
Tejun Heo 已提交
23 24 25 26 27
#include <linux/pci.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
28
#include <linux/device.h>
T
Tejun Heo 已提交
29
#include <scsi/scsi_host.h>
30
#include <scsi/scsi_cmnd.h>
T
Tejun Heo 已提交
31 32 33
#include <linux/libata.h>

#define DRV_NAME	"sata_sil24"
T
Tejun Heo 已提交
34
#define DRV_VERSION	"1.1"
T
Tejun Heo 已提交
35 36 37 38 39

/*
 * Port request block (PRB) 32 bytes
 */
struct sil24_prb {
40 41 42
	__le16	ctrl;
	__le16	prot;
	__le32	rx_cnt;
T
Tejun Heo 已提交
43 44 45 46 47 48 49
	u8	fis[6 * 4];
};

/*
 * Scatter gather entry (SGE) 16 bytes
 */
struct sil24_sge {
50 51 52
	__le64	addr;
	__le32	cnt;
	__le32	flags;
T
Tejun Heo 已提交
53 54 55 56
};


enum {
T
Tejun Heo 已提交
57 58 59
	SIL24_HOST_BAR		= 0,
	SIL24_PORT_BAR		= 2,

T
Tejun Heo 已提交
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
	/* sil24 fetches in chunks of 64bytes.  The first block
	 * contains the PRB and two SGEs.  From the second block, it's
	 * consisted of four SGEs and called SGT.  Calculate the
	 * number of SGTs that fit into one page.
	 */
	SIL24_PRB_SZ		= sizeof(struct sil24_prb)
				  + 2 * sizeof(struct sil24_sge),
	SIL24_MAX_SGT		= (PAGE_SIZE - SIL24_PRB_SZ)
				  / (4 * sizeof(struct sil24_sge)),

	/* This will give us one unused SGEs for ATA.  This extra SGE
	 * will be used to store CDB for ATAPI devices.
	 */
	SIL24_MAX_SGE		= 4 * SIL24_MAX_SGT + 1,

T
Tejun Heo 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
	/*
	 * Global controller registers (128 bytes @ BAR0)
	 */
		/* 32 bit regs */
	HOST_SLOT_STAT		= 0x00, /* 32 bit slot stat * 4 */
	HOST_CTRL		= 0x40,
	HOST_IRQ_STAT		= 0x44,
	HOST_PHY_CFG		= 0x48,
	HOST_BIST_CTRL		= 0x50,
	HOST_BIST_PTRN		= 0x54,
	HOST_BIST_STAT		= 0x58,
	HOST_MEM_BIST_STAT	= 0x5c,
	HOST_FLASH_CMD		= 0x70,
		/* 8 bit regs */
	HOST_FLASH_DATA		= 0x74,
	HOST_TRANSITION_DETECT	= 0x75,
	HOST_GPIO_CTRL		= 0x76,
	HOST_I2C_ADDR		= 0x78, /* 32 bit */
	HOST_I2C_DATA		= 0x7c,
	HOST_I2C_XFER_CNT	= 0x7e,
	HOST_I2C_CTRL		= 0x7f,

	/* HOST_SLOT_STAT bits */
	HOST_SSTAT_ATTN		= (1 << 31),

100 101 102 103 104 105
	/* HOST_CTRL bits */
	HOST_CTRL_M66EN		= (1 << 16), /* M66EN PCI bus signal */
	HOST_CTRL_TRDY		= (1 << 17), /* latched PCI TRDY */
	HOST_CTRL_STOP		= (1 << 18), /* latched PCI STOP */
	HOST_CTRL_DEVSEL	= (1 << 19), /* latched PCI DEVSEL */
	HOST_CTRL_REQ64		= (1 << 20), /* latched PCI REQ64 */
106
	HOST_CTRL_GLOBAL_RST	= (1 << 31), /* global reset */
107

T
Tejun Heo 已提交
108 109 110 111 112
	/*
	 * Port registers
	 * (8192 bytes @ +0x0000, +0x2000, +0x4000 and +0x6000 @ BAR2)
	 */
	PORT_REGS_SIZE		= 0x2000,
113

114
	PORT_LRAM		= 0x0000, /* 31 LRAM slots and PMP regs */
115
	PORT_LRAM_SLOT_SZ	= 0x0080, /* 32 bytes PRB + 2 SGE, ACT... */
T
Tejun Heo 已提交
116

117
	PORT_PMP		= 0x0f80, /* 8 bytes PMP * 16 (128 bytes) */
118 119 120 121
	PORT_PMP_STATUS		= 0x0000, /* port device status offset */
	PORT_PMP_QACTIVE	= 0x0004, /* port device QActive offset */
	PORT_PMP_SIZE		= 0x0008, /* 8 bytes per PMP */

T
Tejun Heo 已提交
122
		/* 32 bit regs */
123 124 125 126 127
	PORT_CTRL_STAT		= 0x1000, /* write: ctrl-set, read: stat */
	PORT_CTRL_CLR		= 0x1004, /* write: ctrl-clear */
	PORT_IRQ_STAT		= 0x1008, /* high: status, low: interrupt */
	PORT_IRQ_ENABLE_SET	= 0x1010, /* write: enable-set */
	PORT_IRQ_ENABLE_CLR	= 0x1014, /* write: enable-clear */
T
Tejun Heo 已提交
128
	PORT_ACTIVATE_UPPER_ADDR= 0x101c,
129 130
	PORT_EXEC_FIFO		= 0x1020, /* command execution fifo */
	PORT_CMD_ERR		= 0x1024, /* command error number */
T
Tejun Heo 已提交
131 132 133 134 135 136 137 138 139 140 141 142 143
	PORT_FIS_CFG		= 0x1028,
	PORT_FIFO_THRES		= 0x102c,
		/* 16 bit regs */
	PORT_DECODE_ERR_CNT	= 0x1040,
	PORT_DECODE_ERR_THRESH	= 0x1042,
	PORT_CRC_ERR_CNT	= 0x1044,
	PORT_CRC_ERR_THRESH	= 0x1046,
	PORT_HSHK_ERR_CNT	= 0x1048,
	PORT_HSHK_ERR_THRESH	= 0x104a,
		/* 32 bit regs */
	PORT_PHY_CFG		= 0x1050,
	PORT_SLOT_STAT		= 0x1800,
	PORT_CMD_ACTIVATE	= 0x1c00, /* 64 bit cmd activate * 31 (248 bytes) */
144
	PORT_CONTEXT		= 0x1e04,
T
Tejun Heo 已提交
145 146 147 148 149 150 151 152 153 154 155 156
	PORT_EXEC_DIAG		= 0x1e00, /* 32bit exec diag * 16 (64 bytes, 0-10 used on 3124) */
	PORT_PSD_DIAG		= 0x1e40, /* 32bit psd diag * 16 (64 bytes, 0-8 used on 3124) */
	PORT_SCONTROL		= 0x1f00,
	PORT_SSTATUS		= 0x1f04,
	PORT_SERROR		= 0x1f08,
	PORT_SACTIVE		= 0x1f0c,

	/* PORT_CTRL_STAT bits */
	PORT_CS_PORT_RST	= (1 << 0), /* port reset */
	PORT_CS_DEV_RST		= (1 << 1), /* device reset */
	PORT_CS_INIT		= (1 << 2), /* port initialize */
	PORT_CS_IRQ_WOC		= (1 << 3), /* interrupt write one to clear */
T
Tejun Heo 已提交
157
	PORT_CS_CDB16		= (1 << 5), /* 0=12b cdb, 1=16b cdb */
158
	PORT_CS_PMP_RESUME	= (1 << 6), /* PMP resume */
159
	PORT_CS_32BIT_ACTV	= (1 << 10), /* 32-bit activation */
160
	PORT_CS_PMP_EN		= (1 << 13), /* port multiplier enable */
161
	PORT_CS_RDY		= (1 << 31), /* port ready to accept commands */
T
Tejun Heo 已提交
162 163 164 165 166 167 168 169 170

	/* PORT_IRQ_STAT/ENABLE_SET/CLR */
	/* bits[11:0] are masked */
	PORT_IRQ_COMPLETE	= (1 << 0), /* command(s) completed */
	PORT_IRQ_ERROR		= (1 << 1), /* command execution error */
	PORT_IRQ_PORTRDY_CHG	= (1 << 2), /* port ready change */
	PORT_IRQ_PWR_CHG	= (1 << 3), /* power management change */
	PORT_IRQ_PHYRDY_CHG	= (1 << 4), /* PHY ready change */
	PORT_IRQ_COMWAKE	= (1 << 5), /* COMWAKE received */
171 172 173 174 175
	PORT_IRQ_UNK_FIS	= (1 << 6), /* unknown FIS received */
	PORT_IRQ_DEV_XCHG	= (1 << 7), /* device exchanged */
	PORT_IRQ_8B10B		= (1 << 8), /* 8b/10b decode error threshold */
	PORT_IRQ_CRC		= (1 << 9), /* CRC error threshold */
	PORT_IRQ_HANDSHAKE	= (1 << 10), /* handshake error threshold */
176
	PORT_IRQ_SDB_NOTIFY	= (1 << 11), /* SDB notify received */
T
Tejun Heo 已提交
177

178
	DEF_PORT_IRQ		= PORT_IRQ_COMPLETE | PORT_IRQ_ERROR |
179
				  PORT_IRQ_PHYRDY_CHG | PORT_IRQ_DEV_XCHG |
T
Tejun Heo 已提交
180
				  PORT_IRQ_UNK_FIS | PORT_IRQ_SDB_NOTIFY,
181

T
Tejun Heo 已提交
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	/* bits[27:16] are unmasked (raw) */
	PORT_IRQ_RAW_SHIFT	= 16,
	PORT_IRQ_MASKED_MASK	= 0x7ff,
	PORT_IRQ_RAW_MASK	= (0x7ff << PORT_IRQ_RAW_SHIFT),

	/* ENABLE_SET/CLR specific, intr steering - 2 bit field */
	PORT_IRQ_STEER_SHIFT	= 30,
	PORT_IRQ_STEER_MASK	= (3 << PORT_IRQ_STEER_SHIFT),

	/* PORT_CMD_ERR constants */
	PORT_CERR_DEV		= 1, /* Error bit in D2H Register FIS */
	PORT_CERR_SDB		= 2, /* Error bit in SDB FIS */
	PORT_CERR_DATA		= 3, /* Error in data FIS not detected by dev */
	PORT_CERR_SEND		= 4, /* Initial cmd FIS transmission failure */
	PORT_CERR_INCONSISTENT	= 5, /* Protocol mismatch */
	PORT_CERR_DIRECTION	= 6, /* Data direction mismatch */
	PORT_CERR_UNDERRUN	= 7, /* Ran out of SGEs while writing */
	PORT_CERR_OVERRUN	= 8, /* Ran out of SGEs while reading */
	PORT_CERR_PKT_PROT	= 11, /* DIR invalid in 1st PIO setup of ATAPI */
	PORT_CERR_SGT_BOUNDARY	= 16, /* PLD ecode 00 - SGT not on qword boundary */
	PORT_CERR_SGT_TGTABRT	= 17, /* PLD ecode 01 - target abort */
	PORT_CERR_SGT_MSTABRT	= 18, /* PLD ecode 10 - master abort */
	PORT_CERR_SGT_PCIPERR	= 19, /* PLD ecode 11 - PCI parity err while fetching SGT */
	PORT_CERR_CMD_BOUNDARY	= 24, /* ctrl[15:13] 001 - PRB not on qword boundary */
	PORT_CERR_CMD_TGTABRT	= 25, /* ctrl[15:13] 010 - target abort */
	PORT_CERR_CMD_MSTABRT	= 26, /* ctrl[15:13] 100 - master abort */
	PORT_CERR_CMD_PCIPERR	= 27, /* ctrl[15:13] 110 - PCI parity err while fetching PRB */
	PORT_CERR_XFR_UNDEF	= 32, /* PSD ecode 00 - undefined */
	PORT_CERR_XFR_TGTABRT	= 33, /* PSD ecode 01 - target abort */
T
Tejun Heo 已提交
211
	PORT_CERR_XFR_MSTABRT	= 34, /* PSD ecode 10 - master abort */
T
Tejun Heo 已提交
212
	PORT_CERR_XFR_PCIPERR	= 35, /* PSD ecode 11 - PCI prity err during transfer */
213
	PORT_CERR_SENDSERVICE	= 36, /* FIS received while sending service */
T
Tejun Heo 已提交
214

T
Tejun Heo 已提交
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
	/* bits of PRB control field */
	PRB_CTRL_PROTOCOL	= (1 << 0), /* override def. ATA protocol */
	PRB_CTRL_PACKET_READ	= (1 << 4), /* PACKET cmd read */
	PRB_CTRL_PACKET_WRITE	= (1 << 5), /* PACKET cmd write */
	PRB_CTRL_NIEN		= (1 << 6), /* Mask completion irq */
	PRB_CTRL_SRST		= (1 << 7), /* Soft reset request (ign BSY?) */

	/* PRB protocol field */
	PRB_PROT_PACKET		= (1 << 0),
	PRB_PROT_TCQ		= (1 << 1),
	PRB_PROT_NCQ		= (1 << 2),
	PRB_PROT_READ		= (1 << 3),
	PRB_PROT_WRITE		= (1 << 4),
	PRB_PROT_TRANSPARENT	= (1 << 5),

T
Tejun Heo 已提交
230 231 232 233
	/*
	 * Other constants
	 */
	SGE_TRM			= (1 << 31), /* Last SGE in chain */
T
Tejun Heo 已提交
234 235 236 237
	SGE_LNK			= (1 << 30), /* linked list
						Points to SGT, not SGE */
	SGE_DRD			= (1 << 29), /* discard data read (/dev/null)
						data address ignored */
T
Tejun Heo 已提交
238

239 240
	SIL24_MAX_CMDS		= 31,

T
Tejun Heo 已提交
241 242 243
	/* board id */
	BID_SIL3124		= 0,
	BID_SIL3132		= 1,
244
	BID_SIL3131		= 2,
T
Tejun Heo 已提交
245

246
	/* host flags */
247 248 249
	SIL24_COMMON_FLAGS	= ATA_FLAG_SATA | ATA_FLAG_PIO_DMA |
				  ATA_FLAG_NCQ | ATA_FLAG_ACPI_SATA |
				  ATA_FLAG_AN | ATA_FLAG_PMP,
250
	SIL24_FLAG_PCIX_IRQ_WOC	= (1 << 24), /* IRQ loss errata on PCI-X */
251

T
Tejun Heo 已提交
252 253 254
	IRQ_STAT_4PORTS		= 0xf,
};

T
Tejun Heo 已提交
255
struct sil24_ata_block {
T
Tejun Heo 已提交
256
	struct sil24_prb prb;
T
Tejun Heo 已提交
257
	struct sil24_sge sge[SIL24_MAX_SGE];
T
Tejun Heo 已提交
258 259
};

T
Tejun Heo 已提交
260 261 262
struct sil24_atapi_block {
	struct sil24_prb prb;
	u8 cdb[16];
T
Tejun Heo 已提交
263
	struct sil24_sge sge[SIL24_MAX_SGE];
T
Tejun Heo 已提交
264 265 266 267 268 269 270
};

union sil24_cmd_block {
	struct sil24_ata_block ata;
	struct sil24_atapi_block atapi;
};

J
Joe Perches 已提交
271
static const struct sil24_cerr_info {
272 273 274
	unsigned int err_mask, action;
	const char *desc;
} sil24_cerr_db[] = {
275
	[0]			= { AC_ERR_DEV, 0,
276
				    "device error" },
277
	[PORT_CERR_DEV]		= { AC_ERR_DEV, 0,
278
				    "device error via D2H FIS" },
279
	[PORT_CERR_SDB]		= { AC_ERR_DEV, 0,
280
				    "device error via SDB FIS" },
T
Tejun Heo 已提交
281
	[PORT_CERR_DATA]	= { AC_ERR_ATA_BUS, ATA_EH_RESET,
282
				    "error in data FIS" },
T
Tejun Heo 已提交
283
	[PORT_CERR_SEND]	= { AC_ERR_ATA_BUS, ATA_EH_RESET,
284
				    "failed to transmit command FIS" },
T
Tejun Heo 已提交
285
	[PORT_CERR_INCONSISTENT] = { AC_ERR_HSM, ATA_EH_RESET,
286
				     "protocol mismatch" },
T
Tejun Heo 已提交
287
	[PORT_CERR_DIRECTION]	= { AC_ERR_HSM, ATA_EH_RESET,
288
				    "data directon mismatch" },
T
Tejun Heo 已提交
289
	[PORT_CERR_UNDERRUN]	= { AC_ERR_HSM, ATA_EH_RESET,
290
				    "ran out of SGEs while writing" },
T
Tejun Heo 已提交
291
	[PORT_CERR_OVERRUN]	= { AC_ERR_HSM, ATA_EH_RESET,
292
				    "ran out of SGEs while reading" },
T
Tejun Heo 已提交
293
	[PORT_CERR_PKT_PROT]	= { AC_ERR_HSM, ATA_EH_RESET,
294
				    "invalid data directon for ATAPI CDB" },
T
Tejun Heo 已提交
295
	[PORT_CERR_SGT_BOUNDARY] = { AC_ERR_SYSTEM, ATA_EH_RESET,
T
Tejun Heo 已提交
296
				     "SGT not on qword boundary" },
T
Tejun Heo 已提交
297
	[PORT_CERR_SGT_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
298
				    "PCI target abort while fetching SGT" },
T
Tejun Heo 已提交
299
	[PORT_CERR_SGT_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
300
				    "PCI master abort while fetching SGT" },
T
Tejun Heo 已提交
301
	[PORT_CERR_SGT_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
302
				    "PCI parity error while fetching SGT" },
T
Tejun Heo 已提交
303
	[PORT_CERR_CMD_BOUNDARY] = { AC_ERR_SYSTEM, ATA_EH_RESET,
304
				     "PRB not on qword boundary" },
T
Tejun Heo 已提交
305
	[PORT_CERR_CMD_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
306
				    "PCI target abort while fetching PRB" },
T
Tejun Heo 已提交
307
	[PORT_CERR_CMD_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
308
				    "PCI master abort while fetching PRB" },
T
Tejun Heo 已提交
309
	[PORT_CERR_CMD_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
310
				    "PCI parity error while fetching PRB" },
T
Tejun Heo 已提交
311
	[PORT_CERR_XFR_UNDEF]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
312
				    "undefined error while transferring data" },
T
Tejun Heo 已提交
313
	[PORT_CERR_XFR_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
314
				    "PCI target abort while transferring data" },
T
Tejun Heo 已提交
315
	[PORT_CERR_XFR_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
316
				    "PCI master abort while transferring data" },
T
Tejun Heo 已提交
317
	[PORT_CERR_XFR_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
318
				    "PCI parity error while transferring data" },
T
Tejun Heo 已提交
319
	[PORT_CERR_SENDSERVICE]	= { AC_ERR_HSM, ATA_EH_RESET,
320 321 322
				    "FIS received while sending service FIS" },
};

T
Tejun Heo 已提交
323 324 325 326 327 328 329
/*
 * ap->private_data
 *
 * The preview driver always returned 0 for status.  We emulate it
 * here from the previous interrupt.
 */
struct sil24_port_priv {
T
Tejun Heo 已提交
330
	union sil24_cmd_block *cmd_block;	/* 32 cmd blocks */
T
Tejun Heo 已提交
331
	dma_addr_t cmd_block_dma;		/* DMA base addr for them */
T
Tejun Heo 已提交
332
	int do_port_rst;
T
Tejun Heo 已提交
333 334
};

335
static void sil24_dev_config(struct ata_device *dev);
T
Tejun Heo 已提交
336 337
static int sil24_scr_read(struct ata_link *link, unsigned sc_reg, u32 *val);
static int sil24_scr_write(struct ata_link *link, unsigned sc_reg, u32 val);
T
Tejun Heo 已提交
338
static int sil24_qc_defer(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
339
static void sil24_qc_prep(struct ata_queued_cmd *qc);
340
static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc);
341
static bool sil24_qc_fill_rtf(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
342 343
static void sil24_pmp_attach(struct ata_port *ap);
static void sil24_pmp_detach(struct ata_port *ap);
344 345
static void sil24_freeze(struct ata_port *ap);
static void sil24_thaw(struct ata_port *ap);
346 347 348 349 350 351
static int sil24_softreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline);
static int sil24_hardreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline);
static int sil24_pmp_hardreset(struct ata_link *link, unsigned int *class,
			       unsigned long deadline);
352 353
static void sil24_error_handler(struct ata_port *ap);
static void sil24_post_internal_cmd(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
354 355
static int sil24_port_start(struct ata_port *ap);
static int sil24_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
356
#ifdef CONFIG_PM
357
static int sil24_pci_device_resume(struct pci_dev *pdev);
T
Tejun Heo 已提交
358
static int sil24_port_resume(struct ata_port *ap);
359
#endif
T
Tejun Heo 已提交
360

361
static const struct pci_device_id sil24_pci_tbl[] = {
362 363 364
	{ PCI_VDEVICE(CMD, 0x3124), BID_SIL3124 },
	{ PCI_VDEVICE(INTEL, 0x3124), BID_SIL3124 },
	{ PCI_VDEVICE(CMD, 0x3132), BID_SIL3132 },
365
	{ PCI_VDEVICE(CMD, 0x0242), BID_SIL3132 },
366
	{ PCI_VDEVICE(CMD, 0x0244), BID_SIL3132 },
367 368 369
	{ PCI_VDEVICE(CMD, 0x3131), BID_SIL3131 },
	{ PCI_VDEVICE(CMD, 0x3531), BID_SIL3131 },

T
Tejun Heo 已提交
370
	{ } /* terminate list */
T
Tejun Heo 已提交
371 372 373 374 375 376
};

static struct pci_driver sil24_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= sil24_pci_tbl,
	.probe			= sil24_init_one,
377
	.remove			= ata_pci_remove_one,
378
#ifdef CONFIG_PM
379 380
	.suspend		= ata_pci_device_suspend,
	.resume			= sil24_pci_device_resume,
381
#endif
T
Tejun Heo 已提交
382 383
};

384
static struct scsi_host_template sil24_sht = {
385
	ATA_NCQ_SHT(DRV_NAME),
386
	.can_queue		= SIL24_MAX_CMDS,
T
Tejun Heo 已提交
387
	.sg_tablesize		= SIL24_MAX_SGE,
T
Tejun Heo 已提交
388 389 390
	.dma_boundary		= ATA_DMA_BOUNDARY,
};

391 392
static struct ata_port_operations sil24_ops = {
	.inherits		= &sata_pmp_port_ops,
T
Tejun Heo 已提交
393

T
Tejun Heo 已提交
394
	.qc_defer		= sil24_qc_defer,
T
Tejun Heo 已提交
395 396
	.qc_prep		= sil24_qc_prep,
	.qc_issue		= sil24_qc_issue,
397
	.qc_fill_rtf		= sil24_qc_fill_rtf,
T
Tejun Heo 已提交
398

399 400
	.freeze			= sil24_freeze,
	.thaw			= sil24_thaw,
401 402
	.softreset		= sil24_softreset,
	.hardreset		= sil24_hardreset,
T
Tejun Heo 已提交
403
	.pmp_softreset		= sil24_softreset,
404
	.pmp_hardreset		= sil24_pmp_hardreset,
405 406 407
	.error_handler		= sil24_error_handler,
	.post_internal_cmd	= sil24_post_internal_cmd,
	.dev_config		= sil24_dev_config,
T
Tejun Heo 已提交
408 409 410

	.scr_read		= sil24_scr_read,
	.scr_write		= sil24_scr_write,
T
Tejun Heo 已提交
411 412 413
	.pmp_attach		= sil24_pmp_attach,
	.pmp_detach		= sil24_pmp_detach,

T
Tejun Heo 已提交
414
	.port_start		= sil24_port_start,
T
Tejun Heo 已提交
415 416 417
#ifdef CONFIG_PM
	.port_resume		= sil24_port_resume,
#endif
T
Tejun Heo 已提交
418 419
};

420
static bool sata_sil24_msi;    /* Disable MSI */
421 422 423
module_param_named(msi, sata_sil24_msi, bool, S_IRUGO);
MODULE_PARM_DESC(msi, "Enable MSI (Default: false)");

424
/*
J
Jeff Garzik 已提交
425
 * Use bits 30-31 of port_flags to encode available port numbers.
426 427 428 429 430
 * Current maxium is 4.
 */
#define SIL24_NPORTS2FLAG(nports)	((((unsigned)(nports) - 1) & 0x3) << 30)
#define SIL24_FLAG2NPORTS(flag)		((((flag) >> 30) & 0x3) + 1)

431
static const struct ata_port_info sil24_port_info[] = {
T
Tejun Heo 已提交
432 433
	/* sil_3124 */
	{
J
Jeff Garzik 已提交
434
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(4) |
435
				  SIL24_FLAG_PCIX_IRQ_WOC,
436 437 438
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
T
Tejun Heo 已提交
439 440
		.port_ops	= &sil24_ops,
	},
441
	/* sil_3132 */
T
Tejun Heo 已提交
442
	{
J
Jeff Garzik 已提交
443
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(2),
444 445 446
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
447 448 449 450
		.port_ops	= &sil24_ops,
	},
	/* sil_3131/sil_3531 */
	{
J
Jeff Garzik 已提交
451
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(1),
452 453 454
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
T
Tejun Heo 已提交
455 456 457 458
		.port_ops	= &sil24_ops,
	},
};

459 460 461 462 463 464 465
static int sil24_tag(int tag)
{
	if (unlikely(ata_tag_internal(tag)))
		return 0;
	return tag;
}

466 467 468 469 470 471 472 473 474 475
static unsigned long sil24_port_offset(struct ata_port *ap)
{
	return ap->port_no * PORT_REGS_SIZE;
}

static void __iomem *sil24_port_base(struct ata_port *ap)
{
	return ap->host->iomap[SIL24_PORT_BAR] + sil24_port_offset(ap);
}

476
static void sil24_dev_config(struct ata_device *dev)
T
Tejun Heo 已提交
477
{
478
	void __iomem *port = sil24_port_base(dev->link->ap);
T
Tejun Heo 已提交
479

480
	if (dev->cdb_len == 16)
T
Tejun Heo 已提交
481 482 483 484 485
		writel(PORT_CS_CDB16, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_CDB16, port + PORT_CTRL_CLR);
}

486
static void sil24_read_tf(struct ata_port *ap, int tag, struct ata_taskfile *tf)
487
{
488
	void __iomem *port = sil24_port_base(ap);
489
	struct sil24_prb __iomem *prb;
490
	u8 fis[6 * 4];
491

492 493 494
	prb = port + PORT_LRAM + sil24_tag(tag) * PORT_LRAM_SLOT_SZ;
	memcpy_fromio(fis, prb->fis, sizeof(fis));
	ata_tf_from_fis(fis, tf);
495 496
}

T
Tejun Heo 已提交
497 498 499 500 501 502 503
static int sil24_scr_map[] = {
	[SCR_CONTROL]	= 0,
	[SCR_STATUS]	= 1,
	[SCR_ERROR]	= 2,
	[SCR_ACTIVE]	= 3,
};

T
Tejun Heo 已提交
504
static int sil24_scr_read(struct ata_link *link, unsigned sc_reg, u32 *val)
T
Tejun Heo 已提交
505
{
T
Tejun Heo 已提交
506
	void __iomem *scr_addr = sil24_port_base(link->ap) + PORT_SCONTROL;
507

T
Tejun Heo 已提交
508
	if (sc_reg < ARRAY_SIZE(sil24_scr_map)) {
509 510
		*val = readl(scr_addr + sil24_scr_map[sc_reg] * 4);
		return 0;
T
Tejun Heo 已提交
511
	}
512
	return -EINVAL;
T
Tejun Heo 已提交
513 514
}

T
Tejun Heo 已提交
515
static int sil24_scr_write(struct ata_link *link, unsigned sc_reg, u32 val)
T
Tejun Heo 已提交
516
{
T
Tejun Heo 已提交
517
	void __iomem *scr_addr = sil24_port_base(link->ap) + PORT_SCONTROL;
518

T
Tejun Heo 已提交
519 520
	if (sc_reg < ARRAY_SIZE(sil24_scr_map)) {
		writel(val, scr_addr + sil24_scr_map[sc_reg] * 4);
521
		return 0;
T
Tejun Heo 已提交
522
	}
523
	return -EINVAL;
T
Tejun Heo 已提交
524 525
}

T
Tejun Heo 已提交
526 527
static void sil24_config_port(struct ata_port *ap)
{
528
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
529 530 531 532 533 534 535 536

	/* configure IRQ WoC */
	if (ap->flags & SIL24_FLAG_PCIX_IRQ_WOC)
		writel(PORT_CS_IRQ_WOC, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_IRQ_WOC, port + PORT_CTRL_CLR);

	/* zero error counters. */
537 538 539 540 541 542
	writew(0x8000, port + PORT_DECODE_ERR_THRESH);
	writew(0x8000, port + PORT_CRC_ERR_THRESH);
	writew(0x8000, port + PORT_HSHK_ERR_THRESH);
	writew(0x0000, port + PORT_DECODE_ERR_CNT);
	writew(0x0000, port + PORT_CRC_ERR_CNT);
	writew(0x0000, port + PORT_HSHK_ERR_CNT);
T
Tejun Heo 已提交
543 544 545 546 547 548 549 550

	/* always use 64bit activation */
	writel(PORT_CS_32BIT_ACTV, port + PORT_CTRL_CLR);

	/* clear port multiplier enable and resume bits */
	writel(PORT_CS_PMP_EN | PORT_CS_PMP_RESUME, port + PORT_CTRL_CLR);
}

T
Tejun Heo 已提交
551 552
static void sil24_config_pmp(struct ata_port *ap, int attached)
{
553
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
554 555 556 557 558 559 560 561 562

	if (attached)
		writel(PORT_CS_PMP_EN, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_PMP_EN, port + PORT_CTRL_CLR);
}

static void sil24_clear_pmp(struct ata_port *ap)
{
563
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
564 565 566 567 568 569 570 571 572 573 574 575
	int i;

	writel(PORT_CS_PMP_RESUME, port + PORT_CTRL_CLR);

	for (i = 0; i < SATA_PMP_MAX_PORTS; i++) {
		void __iomem *pmp_base = port + PORT_PMP + i * PORT_PMP_SIZE;

		writel(0, pmp_base + PORT_PMP_STATUS);
		writel(0, pmp_base + PORT_PMP_QACTIVE);
	}
}

576 577
static int sil24_init_port(struct ata_port *ap)
{
578
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
579
	struct sil24_port_priv *pp = ap->private_data;
580 581
	u32 tmp;

T
Tejun Heo 已提交
582
	/* clear PMP error status */
T
Tejun Heo 已提交
583
	if (sata_pmp_attached(ap))
T
Tejun Heo 已提交
584 585
		sil24_clear_pmp(ap);

586
	writel(PORT_CS_INIT, port + PORT_CTRL_STAT);
587
	ata_wait_register(ap, port + PORT_CTRL_STAT,
588
			  PORT_CS_INIT, PORT_CS_INIT, 10, 100);
589
	tmp = ata_wait_register(ap, port + PORT_CTRL_STAT,
590 591
				PORT_CS_RDY, 0, 10, 100);

T
Tejun Heo 已提交
592 593
	if ((tmp & (PORT_CS_INIT | PORT_CS_RDY)) != PORT_CS_RDY) {
		pp->do_port_rst = 1;
T
Tejun Heo 已提交
594
		ap->link.eh_context.i.action |= ATA_EH_RESET;
595
		return -EIO;
T
Tejun Heo 已提交
596 597
	}

598 599 600
	return 0;
}

601 602 603 604
static int sil24_exec_polled_cmd(struct ata_port *ap, int pmp,
				 const struct ata_taskfile *tf,
				 int is_cmd, u32 ctrl,
				 unsigned long timeout_msec)
T
Tejun Heo 已提交
605
{
606
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
607
	struct sil24_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
608
	struct sil24_prb *prb = &pp->cmd_block[0].ata.prb;
T
Tejun Heo 已提交
609
	dma_addr_t paddr = pp->cmd_block_dma;
610 611 612 613 614 615 616 617 618 619
	u32 irq_enabled, irq_mask, irq_stat;
	int rc;

	prb->ctrl = cpu_to_le16(ctrl);
	ata_tf_to_fis(tf, pmp, is_cmd, prb->fis);

	/* temporarily plug completion and error interrupts */
	irq_enabled = readl(port + PORT_IRQ_ENABLE_SET);
	writel(PORT_IRQ_COMPLETE | PORT_IRQ_ERROR, port + PORT_IRQ_ENABLE_CLR);

620 621 622 623 624
	/*
	 * The barrier is required to ensure that writes to cmd_block reach
	 * the memory before the write to PORT_CMD_ACTIVATE.
	 */
	wmb();
625 626 627 628
	writel((u32)paddr, port + PORT_CMD_ACTIVATE);
	writel((u64)paddr >> 32, port + PORT_CMD_ACTIVATE + 4);

	irq_mask = (PORT_IRQ_COMPLETE | PORT_IRQ_ERROR) << PORT_IRQ_RAW_SHIFT;
629
	irq_stat = ata_wait_register(ap, port + PORT_IRQ_STAT, irq_mask, 0x0,
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
				     10, timeout_msec);

	writel(irq_mask, port + PORT_IRQ_STAT); /* clear IRQs */
	irq_stat >>= PORT_IRQ_RAW_SHIFT;

	if (irq_stat & PORT_IRQ_COMPLETE)
		rc = 0;
	else {
		/* force port into known state */
		sil24_init_port(ap);

		if (irq_stat & PORT_IRQ_ERROR)
			rc = -EIO;
		else
			rc = -EBUSY;
	}

	/* restore IRQ enabled */
	writel(irq_enabled, port + PORT_IRQ_ENABLE_SET);

	return rc;
}

T
Tejun Heo 已提交
653 654
static int sil24_softreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline)
655
{
T
Tejun Heo 已提交
656
	struct ata_port *ap = link->ap;
T
Tejun Heo 已提交
657
	int pmp = sata_srst_pmp(link);
658
	unsigned long timeout_msec = 0;
659
	struct ata_taskfile tf;
660
	const char *reason;
661
	int rc;
T
Tejun Heo 已提交
662

663 664
	DPRINTK("ENTER\n");

665 666
	/* put the port into known state */
	if (sil24_init_port(ap)) {
667
		reason = "port not ready";
668 669 670
		goto err;
	}

671
	/* do SRST */
672 673
	if (time_after(deadline, jiffies))
		timeout_msec = jiffies_to_msecs(deadline - jiffies);
T
Tejun Heo 已提交
674

T
Tejun Heo 已提交
675
	ata_tf_init(link->device, &tf);	/* doesn't really matter */
676 677
	rc = sil24_exec_polled_cmd(ap, pmp, &tf, 0, PRB_CTRL_SRST,
				   timeout_msec);
678 679 680 681 682
	if (rc == -EBUSY) {
		reason = "timeout";
		goto err;
	} else if (rc) {
		reason = "SRST command error";
683
		goto err;
684
	}
685

686 687
	sil24_read_tf(ap, 0, &tf);
	*class = ata_dev_classify(&tf);
688

689
	DPRINTK("EXIT, class=%u\n", *class);
T
Tejun Heo 已提交
690
	return 0;
691 692

 err:
693
	ata_link_err(link, "softreset failed (%s)\n", reason);
694
	return -EIO;
T
Tejun Heo 已提交
695 696
}

T
Tejun Heo 已提交
697
static int sil24_hardreset(struct ata_link *link, unsigned int *class,
698
			   unsigned long deadline)
T
Tejun Heo 已提交
699
{
T
Tejun Heo 已提交
700
	struct ata_port *ap = link->ap;
701
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
702 703
	struct sil24_port_priv *pp = ap->private_data;
	int did_port_rst = 0;
704
	const char *reason;
705
	int tout_msec, rc;
706 707
	u32 tmp;

T
Tejun Heo 已提交
708 709 710 711 712
 retry:
	/* Sometimes, DEV_RST is not enough to recover the controller.
	 * This happens often after PM DMA CS errata.
	 */
	if (pp->do_port_rst) {
713 714
		ata_port_warn(ap,
			      "controller in dubious state, performing PORT_RST\n");
T
Tejun Heo 已提交
715 716

		writel(PORT_CS_PORT_RST, port + PORT_CTRL_STAT);
717
		ata_msleep(ap, 10);
T
Tejun Heo 已提交
718
		writel(PORT_CS_PORT_RST, port + PORT_CTRL_CLR);
719
		ata_wait_register(ap, port + PORT_CTRL_STAT, PORT_CS_RDY, 0,
T
Tejun Heo 已提交
720 721 722 723 724 725 726 727 728 729
				  10, 5000);

		/* restore port configuration */
		sil24_config_port(ap);
		sil24_config_pmp(ap, ap->nr_pmp_links);

		pp->do_port_rst = 0;
		did_port_rst = 1;
	}

730
	/* sil24 does the right thing(tm) without any protection */
T
Tejun Heo 已提交
731
	sata_set_spd(link);
732 733

	tout_msec = 100;
T
Tejun Heo 已提交
734
	if (ata_link_online(link))
735 736 737
		tout_msec = 5000;

	writel(PORT_CS_DEV_RST, port + PORT_CTRL_STAT);
738
	tmp = ata_wait_register(ap, port + PORT_CTRL_STAT,
739 740
				PORT_CS_DEV_RST, PORT_CS_DEV_RST, 10,
				tout_msec);
741

742 743
	/* SStatus oscillates between zero and valid status after
	 * DEV_RST, debounce it.
744
	 */
T
Tejun Heo 已提交
745
	rc = sata_link_debounce(link, sata_deb_timing_long, deadline);
746 747 748 749
	if (rc) {
		reason = "PHY debouncing failed";
		goto err;
	}
750 751

	if (tmp & PORT_CS_DEV_RST) {
T
Tejun Heo 已提交
752
		if (ata_link_offline(link))
753 754 755 756 757
			return 0;
		reason = "link not ready";
		goto err;
	}

758 759 760 761 762
	/* Sil24 doesn't store signature FIS after hardreset, so we
	 * can't wait for BSY to clear.  Some devices take a long time
	 * to get ready and those devices will choke if we don't wait
	 * for BSY clearance here.  Tell libata to perform follow-up
	 * softreset.
763
	 */
764
	return -EAGAIN;
765 766

 err:
T
Tejun Heo 已提交
767 768 769 770 771
	if (!did_port_rst) {
		pp->do_port_rst = 1;
		goto retry;
	}

772
	ata_link_err(link, "hardreset failed (%s)\n", reason);
773
	return -EIO;
T
Tejun Heo 已提交
774 775
}

T
Tejun Heo 已提交
776
static inline void sil24_fill_sg(struct ata_queued_cmd *qc,
T
Tejun Heo 已提交
777
				 struct sil24_sge *sge)
T
Tejun Heo 已提交
778
{
779
	struct scatterlist *sg;
J
Jeff Garzik 已提交
780
	struct sil24_sge *last_sge = NULL;
T
Tejun Heo 已提交
781
	unsigned int si;
T
Tejun Heo 已提交
782

T
Tejun Heo 已提交
783
	for_each_sg(qc->sg, sg, qc->n_elem, si) {
T
Tejun Heo 已提交
784 785
		sge->addr = cpu_to_le64(sg_dma_address(sg));
		sge->cnt = cpu_to_le32(sg_dma_len(sg));
J
Jeff Garzik 已提交
786 787 788
		sge->flags = 0;

		last_sge = sge;
789
		sge++;
T
Tejun Heo 已提交
790
	}
J
Jeff Garzik 已提交
791

T
Tejun Heo 已提交
792
	last_sge->flags = cpu_to_le32(SGE_TRM);
T
Tejun Heo 已提交
793 794
}

T
Tejun Heo 已提交
795 796 797 798 799
static int sil24_qc_defer(struct ata_queued_cmd *qc)
{
	struct ata_link *link = qc->dev->link;
	struct ata_port *ap = link->ap;
	u8 prot = qc->tf.protocol;
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819

	/*
	 * There is a bug in the chip:
	 * Port LRAM Causes the PRB/SGT Data to be Corrupted
	 * If the host issues a read request for LRAM and SActive registers
	 * while active commands are available in the port, PRB/SGT data in
	 * the LRAM can become corrupted. This issue applies only when
	 * reading from, but not writing to, the LRAM.
	 *
	 * Therefore, reading LRAM when there is no particular error [and
	 * other commands may be outstanding] is prohibited.
	 *
	 * To avoid this bug there are two situations where a command must run
	 * exclusive of any other commands on the port:
	 *
	 * - ATAPI commands which check the sense data
	 * - Passthrough ATA commands which always have ATA_QCFLAG_RESULT_TF
	 *   set.
	 *
 	 */
T
Tejun Heo 已提交
820
	int is_excl = (ata_is_atapi(prot) ||
821 822
		       (qc->flags & ATA_QCFLAG_RESULT_TF));

T
Tejun Heo 已提交
823 824 825 826 827 828 829
	if (unlikely(ap->excl_link)) {
		if (link == ap->excl_link) {
			if (ap->nr_active_links)
				return ATA_DEFER_PORT;
			qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
		} else
			return ATA_DEFER_PORT;
830
	} else if (unlikely(is_excl)) {
T
Tejun Heo 已提交
831 832 833 834 835 836 837 838 839
		ap->excl_link = link;
		if (ap->nr_active_links)
			return ATA_DEFER_PORT;
		qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
	}

	return ata_std_qc_defer(qc);
}

T
Tejun Heo 已提交
840 841 842 843
static void sil24_qc_prep(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;
	struct sil24_port_priv *pp = ap->private_data;
844
	union sil24_cmd_block *cb;
T
Tejun Heo 已提交
845 846
	struct sil24_prb *prb;
	struct sil24_sge *sge;
847
	u16 ctrl = 0;
T
Tejun Heo 已提交
848

849 850
	cb = &pp->cmd_block[sil24_tag(qc->tag)];

T
Tejun Heo 已提交
851
	if (!ata_is_atapi(qc->tf.protocol)) {
T
Tejun Heo 已提交
852 853
		prb = &cb->ata.prb;
		sge = cb->ata.sge;
854 855 856 857 858 859 860 861 862 863 864
		if (ata_is_data(qc->tf.protocol)) {
			u16 prot = 0;
			ctrl = PRB_CTRL_PROTOCOL;
			if (ata_is_ncq(qc->tf.protocol))
				prot |= PRB_PROT_NCQ;
			if (qc->tf.flags & ATA_TFLAG_WRITE)
				prot |= PRB_PROT_WRITE;
			else
				prot |= PRB_PROT_READ;
			prb->prot = cpu_to_le16(prot);
		}
T
Tejun Heo 已提交
865
	} else {
T
Tejun Heo 已提交
866 867
		prb = &cb->atapi.prb;
		sge = cb->atapi.sge;
D
Dan Carpenter 已提交
868
		memset(cb->atapi.cdb, 0, sizeof(cb->atapi.cdb));
869
		memcpy(cb->atapi.cdb, qc->cdb, qc->dev->cdb_len);
T
Tejun Heo 已提交
870

T
Tejun Heo 已提交
871
		if (ata_is_data(qc->tf.protocol)) {
T
Tejun Heo 已提交
872
			if (qc->tf.flags & ATA_TFLAG_WRITE)
873
				ctrl = PRB_CTRL_PACKET_WRITE;
T
Tejun Heo 已提交
874
			else
875 876
				ctrl = PRB_CTRL_PACKET_READ;
		}
T
Tejun Heo 已提交
877 878
	}

879
	prb->ctrl = cpu_to_le16(ctrl);
T
Tejun Heo 已提交
880
	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, prb->fis);
T
Tejun Heo 已提交
881 882

	if (qc->flags & ATA_QCFLAG_DMAMAP)
T
Tejun Heo 已提交
883
		sil24_fill_sg(qc, sge);
T
Tejun Heo 已提交
884 885
}

886
static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc)
T
Tejun Heo 已提交
887 888 889
{
	struct ata_port *ap = qc->ap;
	struct sil24_port_priv *pp = ap->private_data;
890
	void __iomem *port = sil24_port_base(ap);
891 892 893
	unsigned int tag = sil24_tag(qc->tag);
	dma_addr_t paddr;
	void __iomem *activate;
T
Tejun Heo 已提交
894

895 896 897
	paddr = pp->cmd_block_dma + tag * sizeof(*pp->cmd_block);
	activate = port + PORT_CMD_ACTIVATE + tag * 8;

898 899 900 901 902
	/*
	 * The barrier is required to ensure that writes to cmd_block reach
	 * the memory before the write to PORT_CMD_ACTIVATE.
	 */
	wmb();
903 904
	writel((u32)paddr, activate);
	writel((u64)paddr >> 32, activate + 4);
T
Tejun Heo 已提交
905

T
Tejun Heo 已提交
906 907 908
	return 0;
}

909 910 911 912 913 914
static bool sil24_qc_fill_rtf(struct ata_queued_cmd *qc)
{
	sil24_read_tf(qc->ap, qc->tag, &qc->result_tf);
	return true;
}

T
Tejun Heo 已提交
915 916
static void sil24_pmp_attach(struct ata_port *ap)
{
917 918
	u32 *gscr = ap->link.device->gscr;

T
Tejun Heo 已提交
919 920
	sil24_config_pmp(ap, 1);
	sil24_init_port(ap);
921 922 923

	if (sata_pmp_gscr_vendor(gscr) == 0x11ab &&
	    sata_pmp_gscr_devid(gscr) == 0x4140) {
924
		ata_port_info(ap,
925 926 927
			"disabling NCQ support due to sil24-mv4140 quirk\n");
		ap->flags &= ~ATA_FLAG_NCQ;
	}
T
Tejun Heo 已提交
928 929 930 931 932 933
}

static void sil24_pmp_detach(struct ata_port *ap)
{
	sil24_init_port(ap);
	sil24_config_pmp(ap, 0);
934 935

	ap->flags |= ATA_FLAG_NCQ;
T
Tejun Heo 已提交
936 937 938 939 940 941 942 943 944
}

static int sil24_pmp_hardreset(struct ata_link *link, unsigned int *class,
			       unsigned long deadline)
{
	int rc;

	rc = sil24_init_port(link->ap);
	if (rc) {
945
		ata_link_err(link, "hardreset failed (port not ready)\n");
T
Tejun Heo 已提交
946 947 948
		return rc;
	}

949
	return sata_std_hardreset(link, class, deadline);
T
Tejun Heo 已提交
950 951
}

952
static void sil24_freeze(struct ata_port *ap)
953
{
954
	void __iomem *port = sil24_port_base(ap);
955

956 957 958 959
	/* Port-wide IRQ mask in HOST_CTRL doesn't really work, clear
	 * PORT_IRQ_ENABLE instead.
	 */
	writel(0xffff, port + PORT_IRQ_ENABLE_CLR);
960 961
}

962
static void sil24_thaw(struct ata_port *ap)
T
Tejun Heo 已提交
963
{
964
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
965 966
	u32 tmp;

967 968 969
	/* clear IRQ */
	tmp = readl(port + PORT_IRQ_STAT);
	writel(tmp, port + PORT_IRQ_STAT);
T
Tejun Heo 已提交
970

971 972
	/* turn IRQ back on */
	writel(DEF_PORT_IRQ, port + PORT_IRQ_ENABLE_SET);
T
Tejun Heo 已提交
973 974
}

975
static void sil24_error_intr(struct ata_port *ap)
976
{
977
	void __iomem *port = sil24_port_base(ap);
978
	struct sil24_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
979 980 981 982
	struct ata_queued_cmd *qc = NULL;
	struct ata_link *link;
	struct ata_eh_info *ehi;
	int abort = 0, freeze = 0;
983
	u32 irq_stat;
984

985
	/* on error, we need to clear IRQ explicitly */
986
	irq_stat = readl(port + PORT_IRQ_STAT);
987
	writel(irq_stat, port + PORT_IRQ_STAT);
988

989
	/* first, analyze and record host port events */
T
Tejun Heo 已提交
990 991
	link = &ap->link;
	ehi = &link->eh_info;
992
	ata_ehi_clear_desc(ehi);
993

994
	ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat);
995

T
Tejun Heo 已提交
996 997
	if (irq_stat & PORT_IRQ_SDB_NOTIFY) {
		ata_ehi_push_desc(ehi, "SDB notify");
998
		sata_async_notification(ap);
T
Tejun Heo 已提交
999 1000
	}

1001 1002
	if (irq_stat & (PORT_IRQ_PHYRDY_CHG | PORT_IRQ_DEV_XCHG)) {
		ata_ehi_hotplugged(ehi);
T
Tejun Heo 已提交
1003 1004 1005
		ata_ehi_push_desc(ehi, "%s",
				  irq_stat & PORT_IRQ_PHYRDY_CHG ?
				  "PHY RDY changed" : "device exchanged");
1006
		freeze = 1;
1007 1008
	}

1009 1010
	if (irq_stat & PORT_IRQ_UNK_FIS) {
		ehi->err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1011
		ehi->action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1012
		ata_ehi_push_desc(ehi, "unknown FIS");
1013 1014 1015 1016 1017
		freeze = 1;
	}

	/* deal with command error */
	if (irq_stat & PORT_IRQ_ERROR) {
J
Joe Perches 已提交
1018
		const struct sil24_cerr_info *ci = NULL;
1019
		unsigned int err_mask = 0, action = 0;
T
Tejun Heo 已提交
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
		u32 context, cerr;
		int pmp;

		abort = 1;

		/* DMA Context Switch Failure in Port Multiplier Mode
		 * errata.  If we have active commands to 3 or more
		 * devices, any error condition on active devices can
		 * corrupt DMA context switching.
		 */
		if (ap->nr_active_links >= 3) {
			ehi->err_mask |= AC_ERR_OTHER;
T
Tejun Heo 已提交
1032
			ehi->action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1033
			ata_ehi_push_desc(ehi, "PMP DMA CS errata");
T
Tejun Heo 已提交
1034
			pp->do_port_rst = 1;
T
Tejun Heo 已提交
1035 1036 1037 1038
			freeze = 1;
		}

		/* find out the offending link and qc */
T
Tejun Heo 已提交
1039
		if (sata_pmp_attached(ap)) {
T
Tejun Heo 已提交
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
			context = readl(port + PORT_CONTEXT);
			pmp = (context >> 5) & 0xf;

			if (pmp < ap->nr_pmp_links) {
				link = &ap->pmp_link[pmp];
				ehi = &link->eh_info;
				qc = ata_qc_from_tag(ap, link->active_tag);

				ata_ehi_clear_desc(ehi);
				ata_ehi_push_desc(ehi, "irq_stat 0x%08x",
						  irq_stat);
			} else {
				err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1053
				action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1054 1055 1056 1057
				freeze = 1;
			}
		} else
			qc = ata_qc_from_tag(ap, link->active_tag);
1058 1059 1060 1061 1062 1063 1064 1065 1066

		/* analyze CMD_ERR */
		cerr = readl(port + PORT_CMD_ERR);
		if (cerr < ARRAY_SIZE(sil24_cerr_db))
			ci = &sil24_cerr_db[cerr];

		if (ci && ci->desc) {
			err_mask |= ci->err_mask;
			action |= ci->action;
T
Tejun Heo 已提交
1067
			if (action & ATA_EH_RESET)
1068
				freeze = 1;
T
Tejun Heo 已提交
1069
			ata_ehi_push_desc(ehi, "%s", ci->desc);
1070 1071
		} else {
			err_mask |= AC_ERR_OTHER;
T
Tejun Heo 已提交
1072
			action |= ATA_EH_RESET;
1073
			freeze = 1;
T
Tejun Heo 已提交
1074
			ata_ehi_push_desc(ehi, "unknown command error %d",
1075 1076 1077 1078
					  cerr);
		}

		/* record error info */
1079
		if (qc)
1080
			qc->err_mask |= err_mask;
1081
		else
1082 1083 1084
			ehi->err_mask |= err_mask;

		ehi->action |= action;
T
Tejun Heo 已提交
1085 1086

		/* if PMP, resume */
T
Tejun Heo 已提交
1087
		if (sata_pmp_attached(ap))
T
Tejun Heo 已提交
1088
			writel(PORT_CS_PMP_RESUME, port + PORT_CTRL_STAT);
1089
	}
1090 1091 1092 1093

	/* freeze or abort */
	if (freeze)
		ata_port_freeze(ap);
T
Tejun Heo 已提交
1094 1095 1096 1097 1098 1099
	else if (abort) {
		if (qc)
			ata_link_abort(qc->dev->link);
		else
			ata_port_abort(ap);
	}
1100 1101
}

T
Tejun Heo 已提交
1102 1103
static inline void sil24_host_intr(struct ata_port *ap)
{
1104
	void __iomem *port = sil24_port_base(ap);
1105 1106
	u32 slot_stat, qc_active;
	int rc;
T
Tejun Heo 已提交
1107

1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
	/* If PCIX_IRQ_WOC, there's an inherent race window between
	 * clearing IRQ pending status and reading PORT_SLOT_STAT
	 * which may cause spurious interrupts afterwards.  This is
	 * unavoidable and much better than losing interrupts which
	 * happens if IRQ pending is cleared after reading
	 * PORT_SLOT_STAT.
	 */
	if (ap->flags & SIL24_FLAG_PCIX_IRQ_WOC)
		writel(PORT_IRQ_COMPLETE, port + PORT_IRQ_STAT);

T
Tejun Heo 已提交
1118
	slot_stat = readl(port + PORT_SLOT_STAT);
1119

1120 1121 1122 1123 1124
	if (unlikely(slot_stat & HOST_SSTAT_ATTN)) {
		sil24_error_intr(ap);
		return;
	}

1125
	qc_active = slot_stat & ~HOST_SSTAT_ATTN;
1126
	rc = ata_qc_complete_multiple(ap, qc_active);
1127 1128 1129
	if (rc > 0)
		return;
	if (rc < 0) {
T
Tejun Heo 已提交
1130
		struct ata_eh_info *ehi = &ap->link.eh_info;
1131
		ehi->err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1132
		ehi->action |= ATA_EH_RESET;
1133
		ata_port_freeze(ap);
1134 1135 1136
		return;
	}

1137 1138
	/* spurious interrupts are expected if PCIX_IRQ_WOC */
	if (!(ap->flags & SIL24_FLAG_PCIX_IRQ_WOC) && ata_ratelimit())
1139 1140
		ata_port_info(ap,
			"spurious interrupt (slot_stat 0x%x active_tag %d sactive 0x%x)\n",
T
Tejun Heo 已提交
1141
			slot_stat, ap->link.active_tag, ap->link.sactive);
T
Tejun Heo 已提交
1142 1143
}

1144
static irqreturn_t sil24_interrupt(int irq, void *dev_instance)
T
Tejun Heo 已提交
1145
{
J
Jeff Garzik 已提交
1146
	struct ata_host *host = dev_instance;
T
Tejun Heo 已提交
1147
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
T
Tejun Heo 已提交
1148 1149 1150 1151
	unsigned handled = 0;
	u32 status;
	int i;

T
Tejun Heo 已提交
1152
	status = readl(host_base + HOST_IRQ_STAT);
T
Tejun Heo 已提交
1153

1154 1155 1156 1157 1158 1159
	if (status == 0xffffffff) {
		printk(KERN_ERR DRV_NAME ": IRQ status == 0xffffffff, "
		       "PCI fault or device removal?\n");
		goto out;
	}

T
Tejun Heo 已提交
1160 1161 1162
	if (!(status & IRQ_STAT_4PORTS))
		goto out;

J
Jeff Garzik 已提交
1163
	spin_lock(&host->lock);
T
Tejun Heo 已提交
1164

J
Jeff Garzik 已提交
1165
	for (i = 0; i < host->n_ports; i++)
T
Tejun Heo 已提交
1166
		if (status & (1 << i)) {
T
Tejun Heo 已提交
1167 1168
			sil24_host_intr(host->ports[i]);
			handled++;
T
Tejun Heo 已提交
1169 1170
		}

J
Jeff Garzik 已提交
1171
	spin_unlock(&host->lock);
T
Tejun Heo 已提交
1172 1173 1174 1175
 out:
	return IRQ_RETVAL(handled);
}

1176 1177
static void sil24_error_handler(struct ata_port *ap)
{
T
Tejun Heo 已提交
1178 1179
	struct sil24_port_priv *pp = ap->private_data;

T
Tejun Heo 已提交
1180
	if (sil24_init_port(ap))
1181 1182
		ata_eh_freeze_port(ap);

1183
	sata_pmp_error_handler(ap);
T
Tejun Heo 已提交
1184 1185

	pp->do_port_rst = 0;
1186 1187 1188 1189 1190 1191 1192
}

static void sil24_post_internal_cmd(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;

	/* make DMA engine forget about the failed command */
T
Tejun Heo 已提交
1193 1194
	if ((qc->flags & ATA_QCFLAG_FAILED) && sil24_init_port(ap))
		ata_eh_freeze_port(ap);
1195 1196
}

T
Tejun Heo 已提交
1197 1198
static int sil24_port_start(struct ata_port *ap)
{
J
Jeff Garzik 已提交
1199
	struct device *dev = ap->host->dev;
T
Tejun Heo 已提交
1200
	struct sil24_port_priv *pp;
T
Tejun Heo 已提交
1201
	union sil24_cmd_block *cb;
1202
	size_t cb_size = sizeof(*cb) * SIL24_MAX_CMDS;
T
Tejun Heo 已提交
1203 1204
	dma_addr_t cb_dma;

1205
	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
T
Tejun Heo 已提交
1206
	if (!pp)
1207
		return -ENOMEM;
T
Tejun Heo 已提交
1208

1209
	cb = dmam_alloc_coherent(dev, cb_size, &cb_dma, GFP_KERNEL);
1210
	if (!cb)
1211
		return -ENOMEM;
T
Tejun Heo 已提交
1212 1213 1214 1215 1216 1217 1218
	memset(cb, 0, cb_size);

	pp->cmd_block = cb;
	pp->cmd_block_dma = cb_dma;

	ap->private_data = pp;

1219 1220 1221
	ata_port_pbar_desc(ap, SIL24_HOST_BAR, -1, "host");
	ata_port_pbar_desc(ap, SIL24_PORT_BAR, sil24_port_offset(ap), "port");

T
Tejun Heo 已提交
1222 1223 1224
	return 0;
}

1225
static void sil24_init_controller(struct ata_host *host)
1226
{
1227
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
	u32 tmp;
	int i;

	/* GPIO off */
	writel(0, host_base + HOST_FLASH_CMD);

	/* clear global reset & mask interrupts during initialization */
	writel(0, host_base + HOST_CTRL);

	/* init ports */
1238
	for (i = 0; i < host->n_ports; i++) {
T
Tejun Heo 已提交
1239
		struct ata_port *ap = host->ports[i];
1240 1241
		void __iomem *port = sil24_port_base(ap);

1242 1243 1244 1245 1246 1247 1248 1249

		/* Initial PHY setting */
		writel(0x20c, port + PORT_PHY_CFG);

		/* Clear port RST */
		tmp = readl(port + PORT_CTRL_STAT);
		if (tmp & PORT_CS_PORT_RST) {
			writel(PORT_CS_PORT_RST, port + PORT_CTRL_CLR);
1250
			tmp = ata_wait_register(NULL, port + PORT_CTRL_STAT,
1251 1252 1253
						PORT_CS_PORT_RST,
						PORT_CS_PORT_RST, 10, 100);
			if (tmp & PORT_CS_PORT_RST)
1254 1255
				dev_err(host->dev,
					"failed to clear port RST\n");
1256 1257
		}

T
Tejun Heo 已提交
1258 1259
		/* configure port */
		sil24_config_port(ap);
1260 1261 1262 1263 1264 1265
	}

	/* Turn on interrupts */
	writel(IRQ_STAT_4PORTS, host_base + HOST_CTRL);
}

T
Tejun Heo 已提交
1266 1267
static int sil24_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
T
Tejun Heo 已提交
1268
	extern int __MARKER__sil24_cmd_block_is_sized_wrongly;
1269 1270 1271 1272
	struct ata_port_info pi = sil24_port_info[ent->driver_data];
	const struct ata_port_info *ppi[] = { &pi, NULL };
	void __iomem * const *iomap;
	struct ata_host *host;
1273
	int rc;
1274
	u32 tmp;
T
Tejun Heo 已提交
1275

T
Tejun Heo 已提交
1276 1277 1278 1279
	/* cause link error if sil24_cmd_block is sized wrongly */
	if (sizeof(union sil24_cmd_block) != PAGE_SIZE)
		__MARKER__sil24_cmd_block_is_sized_wrongly = 1;

1280
	ata_print_version_once(&pdev->dev, DRV_VERSION);
T
Tejun Heo 已提交
1281

1282
	/* acquire resources */
1283
	rc = pcim_enable_device(pdev);
T
Tejun Heo 已提交
1284 1285 1286
	if (rc)
		return rc;

T
Tejun Heo 已提交
1287 1288 1289
	rc = pcim_iomap_regions(pdev,
				(1 << SIL24_HOST_BAR) | (1 << SIL24_PORT_BAR),
				DRV_NAME);
T
Tejun Heo 已提交
1290
	if (rc)
1291
		return rc;
1292
	iomap = pcim_iomap_table(pdev);
T
Tejun Heo 已提交
1293

1294 1295 1296 1297
	/* apply workaround for completion IRQ loss on PCI-X errata */
	if (pi.flags & SIL24_FLAG_PCIX_IRQ_WOC) {
		tmp = readl(iomap[SIL24_HOST_BAR] + HOST_CTRL);
		if (tmp & (HOST_CTRL_TRDY | HOST_CTRL_STOP | HOST_CTRL_DEVSEL))
1298 1299
			dev_info(&pdev->dev,
				 "Applying completion IRQ loss on PCI-X errata fix\n");
1300 1301 1302
		else
			pi.flags &= ~SIL24_FLAG_PCIX_IRQ_WOC;
	}
T
Tejun Heo 已提交
1303

1304 1305 1306 1307 1308 1309
	/* allocate and fill host */
	host = ata_host_alloc_pinfo(&pdev->dev, ppi,
				    SIL24_FLAG2NPORTS(ppi[0]->flags));
	if (!host)
		return -ENOMEM;
	host->iomap = iomap;
T
Tejun Heo 已提交
1310

1311
	/* configure and activate the device */
1312 1313
	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
T
Tejun Heo 已提交
1314
		if (rc) {
1315
			rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1316
			if (rc) {
1317 1318
				dev_err(&pdev->dev,
					"64-bit DMA enable failed\n");
1319
				return rc;
T
Tejun Heo 已提交
1320 1321 1322
			}
		}
	} else {
1323
		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1324
		if (rc) {
1325
			dev_err(&pdev->dev, "32-bit DMA enable failed\n");
1326
			return rc;
T
Tejun Heo 已提交
1327
		}
1328
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1329
		if (rc) {
1330 1331
			dev_err(&pdev->dev,
				"32-bit consistent DMA enable failed\n");
1332
			return rc;
T
Tejun Heo 已提交
1333
		}
T
Tejun Heo 已提交
1334 1335
	}

1336 1337 1338 1339 1340
	/* Set max read request size to 4096.  This slightly increases
	 * write throughput for pci-e variants.
	 */
	pcie_set_readrq(pdev, 4096);

1341
	sil24_init_controller(host);
T
Tejun Heo 已提交
1342

1343
	if (sata_sil24_msi && !pci_enable_msi(pdev)) {
1344
		dev_info(&pdev->dev, "Using MSI\n");
1345 1346 1347
		pci_intx(pdev, 0);
	}

T
Tejun Heo 已提交
1348
	pci_set_master(pdev);
1349 1350
	return ata_host_activate(host, pdev->irq, sil24_interrupt, IRQF_SHARED,
				 &sil24_sht);
T
Tejun Heo 已提交
1351 1352
}

1353
#ifdef CONFIG_PM
1354 1355
static int sil24_pci_device_resume(struct pci_dev *pdev)
{
J
Jeff Garzik 已提交
1356
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
T
Tejun Heo 已提交
1357
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
1358
	int rc;
1359

1360 1361 1362
	rc = ata_pci_device_do_resume(pdev);
	if (rc)
		return rc;
1363 1364

	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND)
T
Tejun Heo 已提交
1365
		writel(HOST_CTRL_GLOBAL_RST, host_base + HOST_CTRL);
1366

1367
	sil24_init_controller(host);
1368

J
Jeff Garzik 已提交
1369
	ata_host_resume(host);
1370 1371 1372

	return 0;
}
T
Tejun Heo 已提交
1373 1374 1375 1376 1377 1378

static int sil24_port_resume(struct ata_port *ap)
{
	sil24_config_pmp(ap, ap->nr_pmp_links);
	return 0;
}
1379
#endif
1380

A
Axel Lin 已提交
1381
module_pci_driver(sil24_pci_driver);
T
Tejun Heo 已提交
1382 1383 1384 1385 1386

MODULE_AUTHOR("Tejun Heo");
MODULE_DESCRIPTION("Silicon Image 3124/3132 SATA low-level driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, sil24_pci_tbl);