sata_sil24.c 37.9 KB
Newer Older
T
Tejun Heo 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * sata_sil24.c - Driver for Silicon Image 3124/3132 SATA-2 controllers
 *
 * Copyright 2005  Tejun Heo
 *
 * Based on preview driver from Silicon Image.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2, or (at your option) any
 * later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
22
#include <linux/gfp.h>
T
Tejun Heo 已提交
23 24 25 26 27
#include <linux/pci.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
28
#include <linux/device.h>
T
Tejun Heo 已提交
29
#include <scsi/scsi_host.h>
30
#include <scsi/scsi_cmnd.h>
T
Tejun Heo 已提交
31 32 33
#include <linux/libata.h>

#define DRV_NAME	"sata_sil24"
T
Tejun Heo 已提交
34
#define DRV_VERSION	"1.1"
T
Tejun Heo 已提交
35 36 37 38 39

/*
 * Port request block (PRB) 32 bytes
 */
struct sil24_prb {
40 41 42
	__le16	ctrl;
	__le16	prot;
	__le32	rx_cnt;
T
Tejun Heo 已提交
43 44 45 46 47 48 49
	u8	fis[6 * 4];
};

/*
 * Scatter gather entry (SGE) 16 bytes
 */
struct sil24_sge {
50 51 52
	__le64	addr;
	__le32	cnt;
	__le32	flags;
T
Tejun Heo 已提交
53 54 55 56
};


enum {
T
Tejun Heo 已提交
57 58 59
	SIL24_HOST_BAR		= 0,
	SIL24_PORT_BAR		= 2,

T
Tejun Heo 已提交
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
	/* sil24 fetches in chunks of 64bytes.  The first block
	 * contains the PRB and two SGEs.  From the second block, it's
	 * consisted of four SGEs and called SGT.  Calculate the
	 * number of SGTs that fit into one page.
	 */
	SIL24_PRB_SZ		= sizeof(struct sil24_prb)
				  + 2 * sizeof(struct sil24_sge),
	SIL24_MAX_SGT		= (PAGE_SIZE - SIL24_PRB_SZ)
				  / (4 * sizeof(struct sil24_sge)),

	/* This will give us one unused SGEs for ATA.  This extra SGE
	 * will be used to store CDB for ATAPI devices.
	 */
	SIL24_MAX_SGE		= 4 * SIL24_MAX_SGT + 1,

T
Tejun Heo 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
	/*
	 * Global controller registers (128 bytes @ BAR0)
	 */
		/* 32 bit regs */
	HOST_SLOT_STAT		= 0x00, /* 32 bit slot stat * 4 */
	HOST_CTRL		= 0x40,
	HOST_IRQ_STAT		= 0x44,
	HOST_PHY_CFG		= 0x48,
	HOST_BIST_CTRL		= 0x50,
	HOST_BIST_PTRN		= 0x54,
	HOST_BIST_STAT		= 0x58,
	HOST_MEM_BIST_STAT	= 0x5c,
	HOST_FLASH_CMD		= 0x70,
		/* 8 bit regs */
	HOST_FLASH_DATA		= 0x74,
	HOST_TRANSITION_DETECT	= 0x75,
	HOST_GPIO_CTRL		= 0x76,
	HOST_I2C_ADDR		= 0x78, /* 32 bit */
	HOST_I2C_DATA		= 0x7c,
	HOST_I2C_XFER_CNT	= 0x7e,
	HOST_I2C_CTRL		= 0x7f,

	/* HOST_SLOT_STAT bits */
	HOST_SSTAT_ATTN		= (1 << 31),

100 101 102 103 104 105
	/* HOST_CTRL bits */
	HOST_CTRL_M66EN		= (1 << 16), /* M66EN PCI bus signal */
	HOST_CTRL_TRDY		= (1 << 17), /* latched PCI TRDY */
	HOST_CTRL_STOP		= (1 << 18), /* latched PCI STOP */
	HOST_CTRL_DEVSEL	= (1 << 19), /* latched PCI DEVSEL */
	HOST_CTRL_REQ64		= (1 << 20), /* latched PCI REQ64 */
106
	HOST_CTRL_GLOBAL_RST	= (1 << 31), /* global reset */
107

T
Tejun Heo 已提交
108 109 110 111 112
	/*
	 * Port registers
	 * (8192 bytes @ +0x0000, +0x2000, +0x4000 and +0x6000 @ BAR2)
	 */
	PORT_REGS_SIZE		= 0x2000,
113

114
	PORT_LRAM		= 0x0000, /* 31 LRAM slots and PMP regs */
115
	PORT_LRAM_SLOT_SZ	= 0x0080, /* 32 bytes PRB + 2 SGE, ACT... */
T
Tejun Heo 已提交
116

117
	PORT_PMP		= 0x0f80, /* 8 bytes PMP * 16 (128 bytes) */
118 119 120 121
	PORT_PMP_STATUS		= 0x0000, /* port device status offset */
	PORT_PMP_QACTIVE	= 0x0004, /* port device QActive offset */
	PORT_PMP_SIZE		= 0x0008, /* 8 bytes per PMP */

T
Tejun Heo 已提交
122
		/* 32 bit regs */
123 124 125 126 127
	PORT_CTRL_STAT		= 0x1000, /* write: ctrl-set, read: stat */
	PORT_CTRL_CLR		= 0x1004, /* write: ctrl-clear */
	PORT_IRQ_STAT		= 0x1008, /* high: status, low: interrupt */
	PORT_IRQ_ENABLE_SET	= 0x1010, /* write: enable-set */
	PORT_IRQ_ENABLE_CLR	= 0x1014, /* write: enable-clear */
T
Tejun Heo 已提交
128
	PORT_ACTIVATE_UPPER_ADDR= 0x101c,
129 130
	PORT_EXEC_FIFO		= 0x1020, /* command execution fifo */
	PORT_CMD_ERR		= 0x1024, /* command error number */
T
Tejun Heo 已提交
131 132 133 134 135 136 137 138 139 140 141 142 143
	PORT_FIS_CFG		= 0x1028,
	PORT_FIFO_THRES		= 0x102c,
		/* 16 bit regs */
	PORT_DECODE_ERR_CNT	= 0x1040,
	PORT_DECODE_ERR_THRESH	= 0x1042,
	PORT_CRC_ERR_CNT	= 0x1044,
	PORT_CRC_ERR_THRESH	= 0x1046,
	PORT_HSHK_ERR_CNT	= 0x1048,
	PORT_HSHK_ERR_THRESH	= 0x104a,
		/* 32 bit regs */
	PORT_PHY_CFG		= 0x1050,
	PORT_SLOT_STAT		= 0x1800,
	PORT_CMD_ACTIVATE	= 0x1c00, /* 64 bit cmd activate * 31 (248 bytes) */
144
	PORT_CONTEXT		= 0x1e04,
T
Tejun Heo 已提交
145 146 147 148 149 150 151 152 153 154 155 156
	PORT_EXEC_DIAG		= 0x1e00, /* 32bit exec diag * 16 (64 bytes, 0-10 used on 3124) */
	PORT_PSD_DIAG		= 0x1e40, /* 32bit psd diag * 16 (64 bytes, 0-8 used on 3124) */
	PORT_SCONTROL		= 0x1f00,
	PORT_SSTATUS		= 0x1f04,
	PORT_SERROR		= 0x1f08,
	PORT_SACTIVE		= 0x1f0c,

	/* PORT_CTRL_STAT bits */
	PORT_CS_PORT_RST	= (1 << 0), /* port reset */
	PORT_CS_DEV_RST		= (1 << 1), /* device reset */
	PORT_CS_INIT		= (1 << 2), /* port initialize */
	PORT_CS_IRQ_WOC		= (1 << 3), /* interrupt write one to clear */
T
Tejun Heo 已提交
157
	PORT_CS_CDB16		= (1 << 5), /* 0=12b cdb, 1=16b cdb */
158
	PORT_CS_PMP_RESUME	= (1 << 6), /* PMP resume */
159
	PORT_CS_32BIT_ACTV	= (1 << 10), /* 32-bit activation */
160
	PORT_CS_PMP_EN		= (1 << 13), /* port multiplier enable */
161
	PORT_CS_RDY		= (1 << 31), /* port ready to accept commands */
T
Tejun Heo 已提交
162 163 164 165 166 167 168 169 170

	/* PORT_IRQ_STAT/ENABLE_SET/CLR */
	/* bits[11:0] are masked */
	PORT_IRQ_COMPLETE	= (1 << 0), /* command(s) completed */
	PORT_IRQ_ERROR		= (1 << 1), /* command execution error */
	PORT_IRQ_PORTRDY_CHG	= (1 << 2), /* port ready change */
	PORT_IRQ_PWR_CHG	= (1 << 3), /* power management change */
	PORT_IRQ_PHYRDY_CHG	= (1 << 4), /* PHY ready change */
	PORT_IRQ_COMWAKE	= (1 << 5), /* COMWAKE received */
171 172 173 174 175
	PORT_IRQ_UNK_FIS	= (1 << 6), /* unknown FIS received */
	PORT_IRQ_DEV_XCHG	= (1 << 7), /* device exchanged */
	PORT_IRQ_8B10B		= (1 << 8), /* 8b/10b decode error threshold */
	PORT_IRQ_CRC		= (1 << 9), /* CRC error threshold */
	PORT_IRQ_HANDSHAKE	= (1 << 10), /* handshake error threshold */
176
	PORT_IRQ_SDB_NOTIFY	= (1 << 11), /* SDB notify received */
T
Tejun Heo 已提交
177

178
	DEF_PORT_IRQ		= PORT_IRQ_COMPLETE | PORT_IRQ_ERROR |
179
				  PORT_IRQ_PHYRDY_CHG | PORT_IRQ_DEV_XCHG |
T
Tejun Heo 已提交
180
				  PORT_IRQ_UNK_FIS | PORT_IRQ_SDB_NOTIFY,
181

T
Tejun Heo 已提交
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	/* bits[27:16] are unmasked (raw) */
	PORT_IRQ_RAW_SHIFT	= 16,
	PORT_IRQ_MASKED_MASK	= 0x7ff,
	PORT_IRQ_RAW_MASK	= (0x7ff << PORT_IRQ_RAW_SHIFT),

	/* ENABLE_SET/CLR specific, intr steering - 2 bit field */
	PORT_IRQ_STEER_SHIFT	= 30,
	PORT_IRQ_STEER_MASK	= (3 << PORT_IRQ_STEER_SHIFT),

	/* PORT_CMD_ERR constants */
	PORT_CERR_DEV		= 1, /* Error bit in D2H Register FIS */
	PORT_CERR_SDB		= 2, /* Error bit in SDB FIS */
	PORT_CERR_DATA		= 3, /* Error in data FIS not detected by dev */
	PORT_CERR_SEND		= 4, /* Initial cmd FIS transmission failure */
	PORT_CERR_INCONSISTENT	= 5, /* Protocol mismatch */
	PORT_CERR_DIRECTION	= 6, /* Data direction mismatch */
	PORT_CERR_UNDERRUN	= 7, /* Ran out of SGEs while writing */
	PORT_CERR_OVERRUN	= 8, /* Ran out of SGEs while reading */
	PORT_CERR_PKT_PROT	= 11, /* DIR invalid in 1st PIO setup of ATAPI */
	PORT_CERR_SGT_BOUNDARY	= 16, /* PLD ecode 00 - SGT not on qword boundary */
	PORT_CERR_SGT_TGTABRT	= 17, /* PLD ecode 01 - target abort */
	PORT_CERR_SGT_MSTABRT	= 18, /* PLD ecode 10 - master abort */
	PORT_CERR_SGT_PCIPERR	= 19, /* PLD ecode 11 - PCI parity err while fetching SGT */
	PORT_CERR_CMD_BOUNDARY	= 24, /* ctrl[15:13] 001 - PRB not on qword boundary */
	PORT_CERR_CMD_TGTABRT	= 25, /* ctrl[15:13] 010 - target abort */
	PORT_CERR_CMD_MSTABRT	= 26, /* ctrl[15:13] 100 - master abort */
	PORT_CERR_CMD_PCIPERR	= 27, /* ctrl[15:13] 110 - PCI parity err while fetching PRB */
	PORT_CERR_XFR_UNDEF	= 32, /* PSD ecode 00 - undefined */
	PORT_CERR_XFR_TGTABRT	= 33, /* PSD ecode 01 - target abort */
T
Tejun Heo 已提交
211
	PORT_CERR_XFR_MSTABRT	= 34, /* PSD ecode 10 - master abort */
T
Tejun Heo 已提交
212
	PORT_CERR_XFR_PCIPERR	= 35, /* PSD ecode 11 - PCI prity err during transfer */
213
	PORT_CERR_SENDSERVICE	= 36, /* FIS received while sending service */
T
Tejun Heo 已提交
214

T
Tejun Heo 已提交
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
	/* bits of PRB control field */
	PRB_CTRL_PROTOCOL	= (1 << 0), /* override def. ATA protocol */
	PRB_CTRL_PACKET_READ	= (1 << 4), /* PACKET cmd read */
	PRB_CTRL_PACKET_WRITE	= (1 << 5), /* PACKET cmd write */
	PRB_CTRL_NIEN		= (1 << 6), /* Mask completion irq */
	PRB_CTRL_SRST		= (1 << 7), /* Soft reset request (ign BSY?) */

	/* PRB protocol field */
	PRB_PROT_PACKET		= (1 << 0),
	PRB_PROT_TCQ		= (1 << 1),
	PRB_PROT_NCQ		= (1 << 2),
	PRB_PROT_READ		= (1 << 3),
	PRB_PROT_WRITE		= (1 << 4),
	PRB_PROT_TRANSPARENT	= (1 << 5),

T
Tejun Heo 已提交
230 231 232 233
	/*
	 * Other constants
	 */
	SGE_TRM			= (1 << 31), /* Last SGE in chain */
T
Tejun Heo 已提交
234 235 236 237
	SGE_LNK			= (1 << 30), /* linked list
						Points to SGT, not SGE */
	SGE_DRD			= (1 << 29), /* discard data read (/dev/null)
						data address ignored */
T
Tejun Heo 已提交
238

239 240
	SIL24_MAX_CMDS		= 31,

T
Tejun Heo 已提交
241 242 243
	/* board id */
	BID_SIL3124		= 0,
	BID_SIL3132		= 1,
244
	BID_SIL3131		= 2,
T
Tejun Heo 已提交
245

246
	/* host flags */
247 248 249
	SIL24_COMMON_FLAGS	= ATA_FLAG_SATA | ATA_FLAG_PIO_DMA |
				  ATA_FLAG_NCQ | ATA_FLAG_ACPI_SATA |
				  ATA_FLAG_AN | ATA_FLAG_PMP,
250
	SIL24_FLAG_PCIX_IRQ_WOC	= (1 << 24), /* IRQ loss errata on PCI-X */
251

T
Tejun Heo 已提交
252 253 254
	IRQ_STAT_4PORTS		= 0xf,
};

T
Tejun Heo 已提交
255
struct sil24_ata_block {
T
Tejun Heo 已提交
256
	struct sil24_prb prb;
T
Tejun Heo 已提交
257
	struct sil24_sge sge[SIL24_MAX_SGE];
T
Tejun Heo 已提交
258 259
};

T
Tejun Heo 已提交
260 261 262
struct sil24_atapi_block {
	struct sil24_prb prb;
	u8 cdb[16];
T
Tejun Heo 已提交
263
	struct sil24_sge sge[SIL24_MAX_SGE];
T
Tejun Heo 已提交
264 265 266 267 268 269 270
};

union sil24_cmd_block {
	struct sil24_ata_block ata;
	struct sil24_atapi_block atapi;
};

271 272 273 274
static struct sil24_cerr_info {
	unsigned int err_mask, action;
	const char *desc;
} sil24_cerr_db[] = {
275
	[0]			= { AC_ERR_DEV, 0,
276
				    "device error" },
277
	[PORT_CERR_DEV]		= { AC_ERR_DEV, 0,
278
				    "device error via D2H FIS" },
279
	[PORT_CERR_SDB]		= { AC_ERR_DEV, 0,
280
				    "device error via SDB FIS" },
T
Tejun Heo 已提交
281
	[PORT_CERR_DATA]	= { AC_ERR_ATA_BUS, ATA_EH_RESET,
282
				    "error in data FIS" },
T
Tejun Heo 已提交
283
	[PORT_CERR_SEND]	= { AC_ERR_ATA_BUS, ATA_EH_RESET,
284
				    "failed to transmit command FIS" },
T
Tejun Heo 已提交
285
	[PORT_CERR_INCONSISTENT] = { AC_ERR_HSM, ATA_EH_RESET,
286
				     "protocol mismatch" },
T
Tejun Heo 已提交
287
	[PORT_CERR_DIRECTION]	= { AC_ERR_HSM, ATA_EH_RESET,
288
				    "data directon mismatch" },
T
Tejun Heo 已提交
289
	[PORT_CERR_UNDERRUN]	= { AC_ERR_HSM, ATA_EH_RESET,
290
				    "ran out of SGEs while writing" },
T
Tejun Heo 已提交
291
	[PORT_CERR_OVERRUN]	= { AC_ERR_HSM, ATA_EH_RESET,
292
				    "ran out of SGEs while reading" },
T
Tejun Heo 已提交
293
	[PORT_CERR_PKT_PROT]	= { AC_ERR_HSM, ATA_EH_RESET,
294
				    "invalid data directon for ATAPI CDB" },
T
Tejun Heo 已提交
295
	[PORT_CERR_SGT_BOUNDARY] = { AC_ERR_SYSTEM, ATA_EH_RESET,
T
Tejun Heo 已提交
296
				     "SGT not on qword boundary" },
T
Tejun Heo 已提交
297
	[PORT_CERR_SGT_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
298
				    "PCI target abort while fetching SGT" },
T
Tejun Heo 已提交
299
	[PORT_CERR_SGT_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
300
				    "PCI master abort while fetching SGT" },
T
Tejun Heo 已提交
301
	[PORT_CERR_SGT_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
302
				    "PCI parity error while fetching SGT" },
T
Tejun Heo 已提交
303
	[PORT_CERR_CMD_BOUNDARY] = { AC_ERR_SYSTEM, ATA_EH_RESET,
304
				     "PRB not on qword boundary" },
T
Tejun Heo 已提交
305
	[PORT_CERR_CMD_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
306
				    "PCI target abort while fetching PRB" },
T
Tejun Heo 已提交
307
	[PORT_CERR_CMD_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
308
				    "PCI master abort while fetching PRB" },
T
Tejun Heo 已提交
309
	[PORT_CERR_CMD_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
310
				    "PCI parity error while fetching PRB" },
T
Tejun Heo 已提交
311
	[PORT_CERR_XFR_UNDEF]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
312
				    "undefined error while transferring data" },
T
Tejun Heo 已提交
313
	[PORT_CERR_XFR_TGTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
314
				    "PCI target abort while transferring data" },
T
Tejun Heo 已提交
315
	[PORT_CERR_XFR_MSTABRT]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
316
				    "PCI master abort while transferring data" },
T
Tejun Heo 已提交
317
	[PORT_CERR_XFR_PCIPERR]	= { AC_ERR_HOST_BUS, ATA_EH_RESET,
318
				    "PCI parity error while transferring data" },
T
Tejun Heo 已提交
319
	[PORT_CERR_SENDSERVICE]	= { AC_ERR_HSM, ATA_EH_RESET,
320 321 322
				    "FIS received while sending service FIS" },
};

T
Tejun Heo 已提交
323 324 325 326 327 328 329
/*
 * ap->private_data
 *
 * The preview driver always returned 0 for status.  We emulate it
 * here from the previous interrupt.
 */
struct sil24_port_priv {
T
Tejun Heo 已提交
330
	union sil24_cmd_block *cmd_block;	/* 32 cmd blocks */
T
Tejun Heo 已提交
331
	dma_addr_t cmd_block_dma;		/* DMA base addr for them */
T
Tejun Heo 已提交
332
	int do_port_rst;
T
Tejun Heo 已提交
333 334
};

335
static void sil24_dev_config(struct ata_device *dev);
T
Tejun Heo 已提交
336 337
static int sil24_scr_read(struct ata_link *link, unsigned sc_reg, u32 *val);
static int sil24_scr_write(struct ata_link *link, unsigned sc_reg, u32 val);
T
Tejun Heo 已提交
338
static int sil24_qc_defer(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
339
static void sil24_qc_prep(struct ata_queued_cmd *qc);
340
static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc);
341
static bool sil24_qc_fill_rtf(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
342 343
static void sil24_pmp_attach(struct ata_port *ap);
static void sil24_pmp_detach(struct ata_port *ap);
344 345
static void sil24_freeze(struct ata_port *ap);
static void sil24_thaw(struct ata_port *ap);
346 347 348 349 350 351
static int sil24_softreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline);
static int sil24_hardreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline);
static int sil24_pmp_hardreset(struct ata_link *link, unsigned int *class,
			       unsigned long deadline);
352 353
static void sil24_error_handler(struct ata_port *ap);
static void sil24_post_internal_cmd(struct ata_queued_cmd *qc);
T
Tejun Heo 已提交
354 355
static int sil24_port_start(struct ata_port *ap);
static int sil24_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
356
#ifdef CONFIG_PM
357
static int sil24_pci_device_resume(struct pci_dev *pdev);
T
Tejun Heo 已提交
358
static int sil24_port_resume(struct ata_port *ap);
359
#endif
T
Tejun Heo 已提交
360

361
static const struct pci_device_id sil24_pci_tbl[] = {
362 363 364
	{ PCI_VDEVICE(CMD, 0x3124), BID_SIL3124 },
	{ PCI_VDEVICE(INTEL, 0x3124), BID_SIL3124 },
	{ PCI_VDEVICE(CMD, 0x3132), BID_SIL3132 },
365
	{ PCI_VDEVICE(CMD, 0x0242), BID_SIL3132 },
366
	{ PCI_VDEVICE(CMD, 0x0244), BID_SIL3132 },
367 368 369
	{ PCI_VDEVICE(CMD, 0x3131), BID_SIL3131 },
	{ PCI_VDEVICE(CMD, 0x3531), BID_SIL3131 },

T
Tejun Heo 已提交
370
	{ } /* terminate list */
T
Tejun Heo 已提交
371 372 373 374 375 376
};

static struct pci_driver sil24_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= sil24_pci_tbl,
	.probe			= sil24_init_one,
377
	.remove			= ata_pci_remove_one,
378
#ifdef CONFIG_PM
379 380
	.suspend		= ata_pci_device_suspend,
	.resume			= sil24_pci_device_resume,
381
#endif
T
Tejun Heo 已提交
382 383
};

384
static struct scsi_host_template sil24_sht = {
385
	ATA_NCQ_SHT(DRV_NAME),
386
	.can_queue		= SIL24_MAX_CMDS,
T
Tejun Heo 已提交
387
	.sg_tablesize		= SIL24_MAX_SGE,
T
Tejun Heo 已提交
388 389 390
	.dma_boundary		= ATA_DMA_BOUNDARY,
};

391 392
static struct ata_port_operations sil24_ops = {
	.inherits		= &sata_pmp_port_ops,
T
Tejun Heo 已提交
393

T
Tejun Heo 已提交
394
	.qc_defer		= sil24_qc_defer,
T
Tejun Heo 已提交
395 396
	.qc_prep		= sil24_qc_prep,
	.qc_issue		= sil24_qc_issue,
397
	.qc_fill_rtf		= sil24_qc_fill_rtf,
T
Tejun Heo 已提交
398

399 400
	.freeze			= sil24_freeze,
	.thaw			= sil24_thaw,
401 402
	.softreset		= sil24_softreset,
	.hardreset		= sil24_hardreset,
T
Tejun Heo 已提交
403
	.pmp_softreset		= sil24_softreset,
404
	.pmp_hardreset		= sil24_pmp_hardreset,
405 406 407
	.error_handler		= sil24_error_handler,
	.post_internal_cmd	= sil24_post_internal_cmd,
	.dev_config		= sil24_dev_config,
T
Tejun Heo 已提交
408 409 410

	.scr_read		= sil24_scr_read,
	.scr_write		= sil24_scr_write,
T
Tejun Heo 已提交
411 412 413
	.pmp_attach		= sil24_pmp_attach,
	.pmp_detach		= sil24_pmp_detach,

T
Tejun Heo 已提交
414
	.port_start		= sil24_port_start,
T
Tejun Heo 已提交
415 416 417
#ifdef CONFIG_PM
	.port_resume		= sil24_port_resume,
#endif
T
Tejun Heo 已提交
418 419
};

420 421 422 423
static int sata_sil24_msi;    /* Disable MSI */
module_param_named(msi, sata_sil24_msi, bool, S_IRUGO);
MODULE_PARM_DESC(msi, "Enable MSI (Default: false)");

424
/*
J
Jeff Garzik 已提交
425
 * Use bits 30-31 of port_flags to encode available port numbers.
426 427 428 429 430
 * Current maxium is 4.
 */
#define SIL24_NPORTS2FLAG(nports)	((((unsigned)(nports) - 1) & 0x3) << 30)
#define SIL24_FLAG2NPORTS(flag)		((((flag) >> 30) & 0x3) + 1)

431
static const struct ata_port_info sil24_port_info[] = {
T
Tejun Heo 已提交
432 433
	/* sil_3124 */
	{
J
Jeff Garzik 已提交
434
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(4) |
435
				  SIL24_FLAG_PCIX_IRQ_WOC,
436 437 438
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
T
Tejun Heo 已提交
439 440
		.port_ops	= &sil24_ops,
	},
441
	/* sil_3132 */
T
Tejun Heo 已提交
442
	{
J
Jeff Garzik 已提交
443
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(2),
444 445 446
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
447 448 449 450
		.port_ops	= &sil24_ops,
	},
	/* sil_3131/sil_3531 */
	{
J
Jeff Garzik 已提交
451
		.flags		= SIL24_COMMON_FLAGS | SIL24_NPORTS2FLAG(1),
452 453 454
		.pio_mask	= ATA_PIO4,
		.mwdma_mask	= ATA_MWDMA2,
		.udma_mask	= ATA_UDMA5,
T
Tejun Heo 已提交
455 456 457 458
		.port_ops	= &sil24_ops,
	},
};

459 460 461 462 463 464 465
static int sil24_tag(int tag)
{
	if (unlikely(ata_tag_internal(tag)))
		return 0;
	return tag;
}

466 467 468 469 470 471 472 473 474 475
static unsigned long sil24_port_offset(struct ata_port *ap)
{
	return ap->port_no * PORT_REGS_SIZE;
}

static void __iomem *sil24_port_base(struct ata_port *ap)
{
	return ap->host->iomap[SIL24_PORT_BAR] + sil24_port_offset(ap);
}

476
static void sil24_dev_config(struct ata_device *dev)
T
Tejun Heo 已提交
477
{
478
	void __iomem *port = sil24_port_base(dev->link->ap);
T
Tejun Heo 已提交
479

480
	if (dev->cdb_len == 16)
T
Tejun Heo 已提交
481 482 483 484 485
		writel(PORT_CS_CDB16, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_CDB16, port + PORT_CTRL_CLR);
}

486
static void sil24_read_tf(struct ata_port *ap, int tag, struct ata_taskfile *tf)
487
{
488
	void __iomem *port = sil24_port_base(ap);
489
	struct sil24_prb __iomem *prb;
490
	u8 fis[6 * 4];
491

492 493 494
	prb = port + PORT_LRAM + sil24_tag(tag) * PORT_LRAM_SLOT_SZ;
	memcpy_fromio(fis, prb->fis, sizeof(fis));
	ata_tf_from_fis(fis, tf);
495 496
}

T
Tejun Heo 已提交
497 498 499 500 501 502 503
static int sil24_scr_map[] = {
	[SCR_CONTROL]	= 0,
	[SCR_STATUS]	= 1,
	[SCR_ERROR]	= 2,
	[SCR_ACTIVE]	= 3,
};

T
Tejun Heo 已提交
504
static int sil24_scr_read(struct ata_link *link, unsigned sc_reg, u32 *val)
T
Tejun Heo 已提交
505
{
T
Tejun Heo 已提交
506
	void __iomem *scr_addr = sil24_port_base(link->ap) + PORT_SCONTROL;
507

T
Tejun Heo 已提交
508
	if (sc_reg < ARRAY_SIZE(sil24_scr_map)) {
509
		void __iomem *addr;
T
Tejun Heo 已提交
510
		addr = scr_addr + sil24_scr_map[sc_reg] * 4;
511 512
		*val = readl(scr_addr + sil24_scr_map[sc_reg] * 4);
		return 0;
T
Tejun Heo 已提交
513
	}
514
	return -EINVAL;
T
Tejun Heo 已提交
515 516
}

T
Tejun Heo 已提交
517
static int sil24_scr_write(struct ata_link *link, unsigned sc_reg, u32 val)
T
Tejun Heo 已提交
518
{
T
Tejun Heo 已提交
519
	void __iomem *scr_addr = sil24_port_base(link->ap) + PORT_SCONTROL;
520

T
Tejun Heo 已提交
521
	if (sc_reg < ARRAY_SIZE(sil24_scr_map)) {
522
		void __iomem *addr;
T
Tejun Heo 已提交
523 524
		addr = scr_addr + sil24_scr_map[sc_reg] * 4;
		writel(val, scr_addr + sil24_scr_map[sc_reg] * 4);
525
		return 0;
T
Tejun Heo 已提交
526
	}
527
	return -EINVAL;
T
Tejun Heo 已提交
528 529
}

T
Tejun Heo 已提交
530 531
static void sil24_config_port(struct ata_port *ap)
{
532
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
533 534 535 536 537 538 539 540

	/* configure IRQ WoC */
	if (ap->flags & SIL24_FLAG_PCIX_IRQ_WOC)
		writel(PORT_CS_IRQ_WOC, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_IRQ_WOC, port + PORT_CTRL_CLR);

	/* zero error counters. */
541 542 543 544 545 546
	writew(0x8000, port + PORT_DECODE_ERR_THRESH);
	writew(0x8000, port + PORT_CRC_ERR_THRESH);
	writew(0x8000, port + PORT_HSHK_ERR_THRESH);
	writew(0x0000, port + PORT_DECODE_ERR_CNT);
	writew(0x0000, port + PORT_CRC_ERR_CNT);
	writew(0x0000, port + PORT_HSHK_ERR_CNT);
T
Tejun Heo 已提交
547 548 549 550 551 552 553 554

	/* always use 64bit activation */
	writel(PORT_CS_32BIT_ACTV, port + PORT_CTRL_CLR);

	/* clear port multiplier enable and resume bits */
	writel(PORT_CS_PMP_EN | PORT_CS_PMP_RESUME, port + PORT_CTRL_CLR);
}

T
Tejun Heo 已提交
555 556
static void sil24_config_pmp(struct ata_port *ap, int attached)
{
557
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
558 559 560 561 562 563 564 565 566

	if (attached)
		writel(PORT_CS_PMP_EN, port + PORT_CTRL_STAT);
	else
		writel(PORT_CS_PMP_EN, port + PORT_CTRL_CLR);
}

static void sil24_clear_pmp(struct ata_port *ap)
{
567
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
568 569 570 571 572 573 574 575 576 577 578 579
	int i;

	writel(PORT_CS_PMP_RESUME, port + PORT_CTRL_CLR);

	for (i = 0; i < SATA_PMP_MAX_PORTS; i++) {
		void __iomem *pmp_base = port + PORT_PMP + i * PORT_PMP_SIZE;

		writel(0, pmp_base + PORT_PMP_STATUS);
		writel(0, pmp_base + PORT_PMP_QACTIVE);
	}
}

580 581
static int sil24_init_port(struct ata_port *ap)
{
582
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
583
	struct sil24_port_priv *pp = ap->private_data;
584 585
	u32 tmp;

T
Tejun Heo 已提交
586
	/* clear PMP error status */
T
Tejun Heo 已提交
587
	if (sata_pmp_attached(ap))
T
Tejun Heo 已提交
588 589
		sil24_clear_pmp(ap);

590
	writel(PORT_CS_INIT, port + PORT_CTRL_STAT);
591
	ata_wait_register(ap, port + PORT_CTRL_STAT,
592
			  PORT_CS_INIT, PORT_CS_INIT, 10, 100);
593
	tmp = ata_wait_register(ap, port + PORT_CTRL_STAT,
594 595
				PORT_CS_RDY, 0, 10, 100);

T
Tejun Heo 已提交
596 597
	if ((tmp & (PORT_CS_INIT | PORT_CS_RDY)) != PORT_CS_RDY) {
		pp->do_port_rst = 1;
T
Tejun Heo 已提交
598
		ap->link.eh_context.i.action |= ATA_EH_RESET;
599
		return -EIO;
T
Tejun Heo 已提交
600 601
	}

602 603 604
	return 0;
}

605 606 607 608
static int sil24_exec_polled_cmd(struct ata_port *ap, int pmp,
				 const struct ata_taskfile *tf,
				 int is_cmd, u32 ctrl,
				 unsigned long timeout_msec)
T
Tejun Heo 已提交
609
{
610
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
611
	struct sil24_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
612
	struct sil24_prb *prb = &pp->cmd_block[0].ata.prb;
T
Tejun Heo 已提交
613
	dma_addr_t paddr = pp->cmd_block_dma;
614 615 616 617 618 619 620 621 622 623
	u32 irq_enabled, irq_mask, irq_stat;
	int rc;

	prb->ctrl = cpu_to_le16(ctrl);
	ata_tf_to_fis(tf, pmp, is_cmd, prb->fis);

	/* temporarily plug completion and error interrupts */
	irq_enabled = readl(port + PORT_IRQ_ENABLE_SET);
	writel(PORT_IRQ_COMPLETE | PORT_IRQ_ERROR, port + PORT_IRQ_ENABLE_CLR);

624 625 626 627 628
	/*
	 * The barrier is required to ensure that writes to cmd_block reach
	 * the memory before the write to PORT_CMD_ACTIVATE.
	 */
	wmb();
629 630 631 632
	writel((u32)paddr, port + PORT_CMD_ACTIVATE);
	writel((u64)paddr >> 32, port + PORT_CMD_ACTIVATE + 4);

	irq_mask = (PORT_IRQ_COMPLETE | PORT_IRQ_ERROR) << PORT_IRQ_RAW_SHIFT;
633
	irq_stat = ata_wait_register(ap, port + PORT_IRQ_STAT, irq_mask, 0x0,
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
				     10, timeout_msec);

	writel(irq_mask, port + PORT_IRQ_STAT); /* clear IRQs */
	irq_stat >>= PORT_IRQ_RAW_SHIFT;

	if (irq_stat & PORT_IRQ_COMPLETE)
		rc = 0;
	else {
		/* force port into known state */
		sil24_init_port(ap);

		if (irq_stat & PORT_IRQ_ERROR)
			rc = -EIO;
		else
			rc = -EBUSY;
	}

	/* restore IRQ enabled */
	writel(irq_enabled, port + PORT_IRQ_ENABLE_SET);

	return rc;
}

T
Tejun Heo 已提交
657 658
static int sil24_softreset(struct ata_link *link, unsigned int *class,
			   unsigned long deadline)
659
{
T
Tejun Heo 已提交
660
	struct ata_port *ap = link->ap;
T
Tejun Heo 已提交
661
	int pmp = sata_srst_pmp(link);
662
	unsigned long timeout_msec = 0;
663
	struct ata_taskfile tf;
664
	const char *reason;
665
	int rc;
T
Tejun Heo 已提交
666

667 668
	DPRINTK("ENTER\n");

669 670
	/* put the port into known state */
	if (sil24_init_port(ap)) {
671
		reason = "port not ready";
672 673 674
		goto err;
	}

675
	/* do SRST */
676 677
	if (time_after(deadline, jiffies))
		timeout_msec = jiffies_to_msecs(deadline - jiffies);
T
Tejun Heo 已提交
678

T
Tejun Heo 已提交
679
	ata_tf_init(link->device, &tf);	/* doesn't really matter */
680 681
	rc = sil24_exec_polled_cmd(ap, pmp, &tf, 0, PRB_CTRL_SRST,
				   timeout_msec);
682 683 684 685 686
	if (rc == -EBUSY) {
		reason = "timeout";
		goto err;
	} else if (rc) {
		reason = "SRST command error";
687
		goto err;
688
	}
689

690 691
	sil24_read_tf(ap, 0, &tf);
	*class = ata_dev_classify(&tf);
692

693
	DPRINTK("EXIT, class=%u\n", *class);
T
Tejun Heo 已提交
694
	return 0;
695 696

 err:
697
	ata_link_err(link, "softreset failed (%s)\n", reason);
698
	return -EIO;
T
Tejun Heo 已提交
699 700
}

T
Tejun Heo 已提交
701
static int sil24_hardreset(struct ata_link *link, unsigned int *class,
702
			   unsigned long deadline)
T
Tejun Heo 已提交
703
{
T
Tejun Heo 已提交
704
	struct ata_port *ap = link->ap;
705
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
706 707
	struct sil24_port_priv *pp = ap->private_data;
	int did_port_rst = 0;
708
	const char *reason;
709
	int tout_msec, rc;
710 711
	u32 tmp;

T
Tejun Heo 已提交
712 713 714 715 716
 retry:
	/* Sometimes, DEV_RST is not enough to recover the controller.
	 * This happens often after PM DMA CS errata.
	 */
	if (pp->do_port_rst) {
717 718
		ata_port_warn(ap,
			      "controller in dubious state, performing PORT_RST\n");
T
Tejun Heo 已提交
719 720

		writel(PORT_CS_PORT_RST, port + PORT_CTRL_STAT);
721
		ata_msleep(ap, 10);
T
Tejun Heo 已提交
722
		writel(PORT_CS_PORT_RST, port + PORT_CTRL_CLR);
723
		ata_wait_register(ap, port + PORT_CTRL_STAT, PORT_CS_RDY, 0,
T
Tejun Heo 已提交
724 725 726 727 728 729 730 731 732 733
				  10, 5000);

		/* restore port configuration */
		sil24_config_port(ap);
		sil24_config_pmp(ap, ap->nr_pmp_links);

		pp->do_port_rst = 0;
		did_port_rst = 1;
	}

734
	/* sil24 does the right thing(tm) without any protection */
T
Tejun Heo 已提交
735
	sata_set_spd(link);
736 737

	tout_msec = 100;
T
Tejun Heo 已提交
738
	if (ata_link_online(link))
739 740 741
		tout_msec = 5000;

	writel(PORT_CS_DEV_RST, port + PORT_CTRL_STAT);
742
	tmp = ata_wait_register(ap, port + PORT_CTRL_STAT,
743 744
				PORT_CS_DEV_RST, PORT_CS_DEV_RST, 10,
				tout_msec);
745

746 747
	/* SStatus oscillates between zero and valid status after
	 * DEV_RST, debounce it.
748
	 */
T
Tejun Heo 已提交
749
	rc = sata_link_debounce(link, sata_deb_timing_long, deadline);
750 751 752 753
	if (rc) {
		reason = "PHY debouncing failed";
		goto err;
	}
754 755

	if (tmp & PORT_CS_DEV_RST) {
T
Tejun Heo 已提交
756
		if (ata_link_offline(link))
757 758 759 760 761
			return 0;
		reason = "link not ready";
		goto err;
	}

762 763 764 765 766
	/* Sil24 doesn't store signature FIS after hardreset, so we
	 * can't wait for BSY to clear.  Some devices take a long time
	 * to get ready and those devices will choke if we don't wait
	 * for BSY clearance here.  Tell libata to perform follow-up
	 * softreset.
767
	 */
768
	return -EAGAIN;
769 770

 err:
T
Tejun Heo 已提交
771 772 773 774 775
	if (!did_port_rst) {
		pp->do_port_rst = 1;
		goto retry;
	}

776
	ata_link_err(link, "hardreset failed (%s)\n", reason);
777
	return -EIO;
T
Tejun Heo 已提交
778 779
}

T
Tejun Heo 已提交
780
static inline void sil24_fill_sg(struct ata_queued_cmd *qc,
T
Tejun Heo 已提交
781
				 struct sil24_sge *sge)
T
Tejun Heo 已提交
782
{
783
	struct scatterlist *sg;
J
Jeff Garzik 已提交
784
	struct sil24_sge *last_sge = NULL;
T
Tejun Heo 已提交
785
	unsigned int si;
T
Tejun Heo 已提交
786

T
Tejun Heo 已提交
787
	for_each_sg(qc->sg, sg, qc->n_elem, si) {
T
Tejun Heo 已提交
788 789
		sge->addr = cpu_to_le64(sg_dma_address(sg));
		sge->cnt = cpu_to_le32(sg_dma_len(sg));
J
Jeff Garzik 已提交
790 791 792
		sge->flags = 0;

		last_sge = sge;
793
		sge++;
T
Tejun Heo 已提交
794
	}
J
Jeff Garzik 已提交
795

T
Tejun Heo 已提交
796
	last_sge->flags = cpu_to_le32(SGE_TRM);
T
Tejun Heo 已提交
797 798
}

T
Tejun Heo 已提交
799 800 801 802 803
static int sil24_qc_defer(struct ata_queued_cmd *qc)
{
	struct ata_link *link = qc->dev->link;
	struct ata_port *ap = link->ap;
	u8 prot = qc->tf.protocol;
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823

	/*
	 * There is a bug in the chip:
	 * Port LRAM Causes the PRB/SGT Data to be Corrupted
	 * If the host issues a read request for LRAM and SActive registers
	 * while active commands are available in the port, PRB/SGT data in
	 * the LRAM can become corrupted. This issue applies only when
	 * reading from, but not writing to, the LRAM.
	 *
	 * Therefore, reading LRAM when there is no particular error [and
	 * other commands may be outstanding] is prohibited.
	 *
	 * To avoid this bug there are two situations where a command must run
	 * exclusive of any other commands on the port:
	 *
	 * - ATAPI commands which check the sense data
	 * - Passthrough ATA commands which always have ATA_QCFLAG_RESULT_TF
	 *   set.
	 *
 	 */
T
Tejun Heo 已提交
824
	int is_excl = (ata_is_atapi(prot) ||
825 826
		       (qc->flags & ATA_QCFLAG_RESULT_TF));

T
Tejun Heo 已提交
827 828 829 830 831 832 833
	if (unlikely(ap->excl_link)) {
		if (link == ap->excl_link) {
			if (ap->nr_active_links)
				return ATA_DEFER_PORT;
			qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
		} else
			return ATA_DEFER_PORT;
834
	} else if (unlikely(is_excl)) {
T
Tejun Heo 已提交
835 836 837 838 839 840 841 842 843
		ap->excl_link = link;
		if (ap->nr_active_links)
			return ATA_DEFER_PORT;
		qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
	}

	return ata_std_qc_defer(qc);
}

T
Tejun Heo 已提交
844 845 846 847
static void sil24_qc_prep(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;
	struct sil24_port_priv *pp = ap->private_data;
848
	union sil24_cmd_block *cb;
T
Tejun Heo 已提交
849 850
	struct sil24_prb *prb;
	struct sil24_sge *sge;
851
	u16 ctrl = 0;
T
Tejun Heo 已提交
852

853 854
	cb = &pp->cmd_block[sil24_tag(qc->tag)];

T
Tejun Heo 已提交
855
	if (!ata_is_atapi(qc->tf.protocol)) {
T
Tejun Heo 已提交
856 857
		prb = &cb->ata.prb;
		sge = cb->ata.sge;
858 859 860 861 862 863 864 865 866 867 868
		if (ata_is_data(qc->tf.protocol)) {
			u16 prot = 0;
			ctrl = PRB_CTRL_PROTOCOL;
			if (ata_is_ncq(qc->tf.protocol))
				prot |= PRB_PROT_NCQ;
			if (qc->tf.flags & ATA_TFLAG_WRITE)
				prot |= PRB_PROT_WRITE;
			else
				prot |= PRB_PROT_READ;
			prb->prot = cpu_to_le16(prot);
		}
T
Tejun Heo 已提交
869
	} else {
T
Tejun Heo 已提交
870 871
		prb = &cb->atapi.prb;
		sge = cb->atapi.sge;
D
Dan Carpenter 已提交
872
		memset(cb->atapi.cdb, 0, sizeof(cb->atapi.cdb));
873
		memcpy(cb->atapi.cdb, qc->cdb, qc->dev->cdb_len);
T
Tejun Heo 已提交
874

T
Tejun Heo 已提交
875
		if (ata_is_data(qc->tf.protocol)) {
T
Tejun Heo 已提交
876
			if (qc->tf.flags & ATA_TFLAG_WRITE)
877
				ctrl = PRB_CTRL_PACKET_WRITE;
T
Tejun Heo 已提交
878
			else
879 880
				ctrl = PRB_CTRL_PACKET_READ;
		}
T
Tejun Heo 已提交
881 882
	}

883
	prb->ctrl = cpu_to_le16(ctrl);
T
Tejun Heo 已提交
884
	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, prb->fis);
T
Tejun Heo 已提交
885 886

	if (qc->flags & ATA_QCFLAG_DMAMAP)
T
Tejun Heo 已提交
887
		sil24_fill_sg(qc, sge);
T
Tejun Heo 已提交
888 889
}

890
static unsigned int sil24_qc_issue(struct ata_queued_cmd *qc)
T
Tejun Heo 已提交
891 892 893
{
	struct ata_port *ap = qc->ap;
	struct sil24_port_priv *pp = ap->private_data;
894
	void __iomem *port = sil24_port_base(ap);
895 896 897
	unsigned int tag = sil24_tag(qc->tag);
	dma_addr_t paddr;
	void __iomem *activate;
T
Tejun Heo 已提交
898

899 900 901
	paddr = pp->cmd_block_dma + tag * sizeof(*pp->cmd_block);
	activate = port + PORT_CMD_ACTIVATE + tag * 8;

902 903 904 905 906
	/*
	 * The barrier is required to ensure that writes to cmd_block reach
	 * the memory before the write to PORT_CMD_ACTIVATE.
	 */
	wmb();
907 908
	writel((u32)paddr, activate);
	writel((u64)paddr >> 32, activate + 4);
T
Tejun Heo 已提交
909

T
Tejun Heo 已提交
910 911 912
	return 0;
}

913 914 915 916 917 918
static bool sil24_qc_fill_rtf(struct ata_queued_cmd *qc)
{
	sil24_read_tf(qc->ap, qc->tag, &qc->result_tf);
	return true;
}

T
Tejun Heo 已提交
919 920
static void sil24_pmp_attach(struct ata_port *ap)
{
921 922
	u32 *gscr = ap->link.device->gscr;

T
Tejun Heo 已提交
923 924
	sil24_config_pmp(ap, 1);
	sil24_init_port(ap);
925 926 927

	if (sata_pmp_gscr_vendor(gscr) == 0x11ab &&
	    sata_pmp_gscr_devid(gscr) == 0x4140) {
928
		ata_port_info(ap,
929 930 931
			"disabling NCQ support due to sil24-mv4140 quirk\n");
		ap->flags &= ~ATA_FLAG_NCQ;
	}
T
Tejun Heo 已提交
932 933 934 935 936 937
}

static void sil24_pmp_detach(struct ata_port *ap)
{
	sil24_init_port(ap);
	sil24_config_pmp(ap, 0);
938 939

	ap->flags |= ATA_FLAG_NCQ;
T
Tejun Heo 已提交
940 941 942 943 944 945 946 947 948
}

static int sil24_pmp_hardreset(struct ata_link *link, unsigned int *class,
			       unsigned long deadline)
{
	int rc;

	rc = sil24_init_port(link->ap);
	if (rc) {
949
		ata_link_err(link, "hardreset failed (port not ready)\n");
T
Tejun Heo 已提交
950 951 952
		return rc;
	}

953
	return sata_std_hardreset(link, class, deadline);
T
Tejun Heo 已提交
954 955
}

956
static void sil24_freeze(struct ata_port *ap)
957
{
958
	void __iomem *port = sil24_port_base(ap);
959

960 961 962 963
	/* Port-wide IRQ mask in HOST_CTRL doesn't really work, clear
	 * PORT_IRQ_ENABLE instead.
	 */
	writel(0xffff, port + PORT_IRQ_ENABLE_CLR);
964 965
}

966
static void sil24_thaw(struct ata_port *ap)
T
Tejun Heo 已提交
967
{
968
	void __iomem *port = sil24_port_base(ap);
T
Tejun Heo 已提交
969 970
	u32 tmp;

971 972 973
	/* clear IRQ */
	tmp = readl(port + PORT_IRQ_STAT);
	writel(tmp, port + PORT_IRQ_STAT);
T
Tejun Heo 已提交
974

975 976
	/* turn IRQ back on */
	writel(DEF_PORT_IRQ, port + PORT_IRQ_ENABLE_SET);
T
Tejun Heo 已提交
977 978
}

979
static void sil24_error_intr(struct ata_port *ap)
980
{
981
	void __iomem *port = sil24_port_base(ap);
982
	struct sil24_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
983 984 985 986
	struct ata_queued_cmd *qc = NULL;
	struct ata_link *link;
	struct ata_eh_info *ehi;
	int abort = 0, freeze = 0;
987
	u32 irq_stat;
988

989
	/* on error, we need to clear IRQ explicitly */
990
	irq_stat = readl(port + PORT_IRQ_STAT);
991
	writel(irq_stat, port + PORT_IRQ_STAT);
992

993
	/* first, analyze and record host port events */
T
Tejun Heo 已提交
994 995
	link = &ap->link;
	ehi = &link->eh_info;
996
	ata_ehi_clear_desc(ehi);
997

998
	ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat);
999

T
Tejun Heo 已提交
1000 1001
	if (irq_stat & PORT_IRQ_SDB_NOTIFY) {
		ata_ehi_push_desc(ehi, "SDB notify");
1002
		sata_async_notification(ap);
T
Tejun Heo 已提交
1003 1004
	}

1005 1006
	if (irq_stat & (PORT_IRQ_PHYRDY_CHG | PORT_IRQ_DEV_XCHG)) {
		ata_ehi_hotplugged(ehi);
T
Tejun Heo 已提交
1007 1008 1009
		ata_ehi_push_desc(ehi, "%s",
				  irq_stat & PORT_IRQ_PHYRDY_CHG ?
				  "PHY RDY changed" : "device exchanged");
1010
		freeze = 1;
1011 1012
	}

1013 1014
	if (irq_stat & PORT_IRQ_UNK_FIS) {
		ehi->err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1015
		ehi->action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1016
		ata_ehi_push_desc(ehi, "unknown FIS");
1017 1018 1019 1020 1021 1022 1023
		freeze = 1;
	}

	/* deal with command error */
	if (irq_stat & PORT_IRQ_ERROR) {
		struct sil24_cerr_info *ci = NULL;
		unsigned int err_mask = 0, action = 0;
T
Tejun Heo 已提交
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
		u32 context, cerr;
		int pmp;

		abort = 1;

		/* DMA Context Switch Failure in Port Multiplier Mode
		 * errata.  If we have active commands to 3 or more
		 * devices, any error condition on active devices can
		 * corrupt DMA context switching.
		 */
		if (ap->nr_active_links >= 3) {
			ehi->err_mask |= AC_ERR_OTHER;
T
Tejun Heo 已提交
1036
			ehi->action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1037
			ata_ehi_push_desc(ehi, "PMP DMA CS errata");
T
Tejun Heo 已提交
1038
			pp->do_port_rst = 1;
T
Tejun Heo 已提交
1039 1040 1041 1042
			freeze = 1;
		}

		/* find out the offending link and qc */
T
Tejun Heo 已提交
1043
		if (sata_pmp_attached(ap)) {
T
Tejun Heo 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
			context = readl(port + PORT_CONTEXT);
			pmp = (context >> 5) & 0xf;

			if (pmp < ap->nr_pmp_links) {
				link = &ap->pmp_link[pmp];
				ehi = &link->eh_info;
				qc = ata_qc_from_tag(ap, link->active_tag);

				ata_ehi_clear_desc(ehi);
				ata_ehi_push_desc(ehi, "irq_stat 0x%08x",
						  irq_stat);
			} else {
				err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1057
				action |= ATA_EH_RESET;
T
Tejun Heo 已提交
1058 1059 1060 1061
				freeze = 1;
			}
		} else
			qc = ata_qc_from_tag(ap, link->active_tag);
1062 1063 1064 1065 1066 1067 1068 1069 1070

		/* analyze CMD_ERR */
		cerr = readl(port + PORT_CMD_ERR);
		if (cerr < ARRAY_SIZE(sil24_cerr_db))
			ci = &sil24_cerr_db[cerr];

		if (ci && ci->desc) {
			err_mask |= ci->err_mask;
			action |= ci->action;
T
Tejun Heo 已提交
1071
			if (action & ATA_EH_RESET)
1072
				freeze = 1;
T
Tejun Heo 已提交
1073
			ata_ehi_push_desc(ehi, "%s", ci->desc);
1074 1075
		} else {
			err_mask |= AC_ERR_OTHER;
T
Tejun Heo 已提交
1076
			action |= ATA_EH_RESET;
1077
			freeze = 1;
T
Tejun Heo 已提交
1078
			ata_ehi_push_desc(ehi, "unknown command error %d",
1079 1080 1081 1082
					  cerr);
		}

		/* record error info */
1083
		if (qc)
1084
			qc->err_mask |= err_mask;
1085
		else
1086 1087 1088
			ehi->err_mask |= err_mask;

		ehi->action |= action;
T
Tejun Heo 已提交
1089 1090

		/* if PMP, resume */
T
Tejun Heo 已提交
1091
		if (sata_pmp_attached(ap))
T
Tejun Heo 已提交
1092
			writel(PORT_CS_PMP_RESUME, port + PORT_CTRL_STAT);
1093
	}
1094 1095 1096 1097

	/* freeze or abort */
	if (freeze)
		ata_port_freeze(ap);
T
Tejun Heo 已提交
1098 1099 1100 1101 1102 1103
	else if (abort) {
		if (qc)
			ata_link_abort(qc->dev->link);
		else
			ata_port_abort(ap);
	}
1104 1105
}

T
Tejun Heo 已提交
1106 1107
static inline void sil24_host_intr(struct ata_port *ap)
{
1108
	void __iomem *port = sil24_port_base(ap);
1109 1110
	u32 slot_stat, qc_active;
	int rc;
T
Tejun Heo 已提交
1111

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	/* If PCIX_IRQ_WOC, there's an inherent race window between
	 * clearing IRQ pending status and reading PORT_SLOT_STAT
	 * which may cause spurious interrupts afterwards.  This is
	 * unavoidable and much better than losing interrupts which
	 * happens if IRQ pending is cleared after reading
	 * PORT_SLOT_STAT.
	 */
	if (ap->flags & SIL24_FLAG_PCIX_IRQ_WOC)
		writel(PORT_IRQ_COMPLETE, port + PORT_IRQ_STAT);

T
Tejun Heo 已提交
1122
	slot_stat = readl(port + PORT_SLOT_STAT);
1123

1124 1125 1126 1127 1128
	if (unlikely(slot_stat & HOST_SSTAT_ATTN)) {
		sil24_error_intr(ap);
		return;
	}

1129
	qc_active = slot_stat & ~HOST_SSTAT_ATTN;
1130
	rc = ata_qc_complete_multiple(ap, qc_active);
1131 1132 1133
	if (rc > 0)
		return;
	if (rc < 0) {
T
Tejun Heo 已提交
1134
		struct ata_eh_info *ehi = &ap->link.eh_info;
1135
		ehi->err_mask |= AC_ERR_HSM;
T
Tejun Heo 已提交
1136
		ehi->action |= ATA_EH_RESET;
1137
		ata_port_freeze(ap);
1138 1139 1140
		return;
	}

1141 1142
	/* spurious interrupts are expected if PCIX_IRQ_WOC */
	if (!(ap->flags & SIL24_FLAG_PCIX_IRQ_WOC) && ata_ratelimit())
1143 1144
		ata_port_info(ap,
			"spurious interrupt (slot_stat 0x%x active_tag %d sactive 0x%x)\n",
T
Tejun Heo 已提交
1145
			slot_stat, ap->link.active_tag, ap->link.sactive);
T
Tejun Heo 已提交
1146 1147
}

1148
static irqreturn_t sil24_interrupt(int irq, void *dev_instance)
T
Tejun Heo 已提交
1149
{
J
Jeff Garzik 已提交
1150
	struct ata_host *host = dev_instance;
T
Tejun Heo 已提交
1151
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
T
Tejun Heo 已提交
1152 1153 1154 1155
	unsigned handled = 0;
	u32 status;
	int i;

T
Tejun Heo 已提交
1156
	status = readl(host_base + HOST_IRQ_STAT);
T
Tejun Heo 已提交
1157

1158 1159 1160 1161 1162 1163
	if (status == 0xffffffff) {
		printk(KERN_ERR DRV_NAME ": IRQ status == 0xffffffff, "
		       "PCI fault or device removal?\n");
		goto out;
	}

T
Tejun Heo 已提交
1164 1165 1166
	if (!(status & IRQ_STAT_4PORTS))
		goto out;

J
Jeff Garzik 已提交
1167
	spin_lock(&host->lock);
T
Tejun Heo 已提交
1168

J
Jeff Garzik 已提交
1169
	for (i = 0; i < host->n_ports; i++)
T
Tejun Heo 已提交
1170
		if (status & (1 << i)) {
T
Tejun Heo 已提交
1171 1172
			sil24_host_intr(host->ports[i]);
			handled++;
T
Tejun Heo 已提交
1173 1174
		}

J
Jeff Garzik 已提交
1175
	spin_unlock(&host->lock);
T
Tejun Heo 已提交
1176 1177 1178 1179
 out:
	return IRQ_RETVAL(handled);
}

1180 1181
static void sil24_error_handler(struct ata_port *ap)
{
T
Tejun Heo 已提交
1182 1183
	struct sil24_port_priv *pp = ap->private_data;

T
Tejun Heo 已提交
1184
	if (sil24_init_port(ap))
1185 1186
		ata_eh_freeze_port(ap);

1187
	sata_pmp_error_handler(ap);
T
Tejun Heo 已提交
1188 1189

	pp->do_port_rst = 0;
1190 1191 1192 1193 1194 1195 1196
}

static void sil24_post_internal_cmd(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;

	/* make DMA engine forget about the failed command */
T
Tejun Heo 已提交
1197 1198
	if ((qc->flags & ATA_QCFLAG_FAILED) && sil24_init_port(ap))
		ata_eh_freeze_port(ap);
1199 1200
}

T
Tejun Heo 已提交
1201 1202
static int sil24_port_start(struct ata_port *ap)
{
J
Jeff Garzik 已提交
1203
	struct device *dev = ap->host->dev;
T
Tejun Heo 已提交
1204
	struct sil24_port_priv *pp;
T
Tejun Heo 已提交
1205
	union sil24_cmd_block *cb;
1206
	size_t cb_size = sizeof(*cb) * SIL24_MAX_CMDS;
T
Tejun Heo 已提交
1207 1208
	dma_addr_t cb_dma;

1209
	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
T
Tejun Heo 已提交
1210
	if (!pp)
1211
		return -ENOMEM;
T
Tejun Heo 已提交
1212

1213
	cb = dmam_alloc_coherent(dev, cb_size, &cb_dma, GFP_KERNEL);
1214
	if (!cb)
1215
		return -ENOMEM;
T
Tejun Heo 已提交
1216 1217 1218 1219 1220 1221 1222
	memset(cb, 0, cb_size);

	pp->cmd_block = cb;
	pp->cmd_block_dma = cb_dma;

	ap->private_data = pp;

1223 1224 1225
	ata_port_pbar_desc(ap, SIL24_HOST_BAR, -1, "host");
	ata_port_pbar_desc(ap, SIL24_PORT_BAR, sil24_port_offset(ap), "port");

T
Tejun Heo 已提交
1226 1227 1228
	return 0;
}

1229
static void sil24_init_controller(struct ata_host *host)
1230
{
1231
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	u32 tmp;
	int i;

	/* GPIO off */
	writel(0, host_base + HOST_FLASH_CMD);

	/* clear global reset & mask interrupts during initialization */
	writel(0, host_base + HOST_CTRL);

	/* init ports */
1242
	for (i = 0; i < host->n_ports; i++) {
T
Tejun Heo 已提交
1243
		struct ata_port *ap = host->ports[i];
1244 1245
		void __iomem *port = sil24_port_base(ap);

1246 1247 1248 1249 1250 1251 1252 1253

		/* Initial PHY setting */
		writel(0x20c, port + PORT_PHY_CFG);

		/* Clear port RST */
		tmp = readl(port + PORT_CTRL_STAT);
		if (tmp & PORT_CS_PORT_RST) {
			writel(PORT_CS_PORT_RST, port + PORT_CTRL_CLR);
1254
			tmp = ata_wait_register(NULL, port + PORT_CTRL_STAT,
1255 1256 1257
						PORT_CS_PORT_RST,
						PORT_CS_PORT_RST, 10, 100);
			if (tmp & PORT_CS_PORT_RST)
1258 1259
				dev_err(host->dev,
					"failed to clear port RST\n");
1260 1261
		}

T
Tejun Heo 已提交
1262 1263
		/* configure port */
		sil24_config_port(ap);
1264 1265 1266 1267 1268 1269
	}

	/* Turn on interrupts */
	writel(IRQ_STAT_4PORTS, host_base + HOST_CTRL);
}

T
Tejun Heo 已提交
1270 1271
static int sil24_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
T
Tejun Heo 已提交
1272
	extern int __MARKER__sil24_cmd_block_is_sized_wrongly;
1273
	static int printed_version;
1274 1275 1276 1277
	struct ata_port_info pi = sil24_port_info[ent->driver_data];
	const struct ata_port_info *ppi[] = { &pi, NULL };
	void __iomem * const *iomap;
	struct ata_host *host;
1278
	int rc;
1279
	u32 tmp;
T
Tejun Heo 已提交
1280

T
Tejun Heo 已提交
1281 1282 1283 1284
	/* cause link error if sil24_cmd_block is sized wrongly */
	if (sizeof(union sil24_cmd_block) != PAGE_SIZE)
		__MARKER__sil24_cmd_block_is_sized_wrongly = 1;

T
Tejun Heo 已提交
1285
	if (!printed_version++)
1286
		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
T
Tejun Heo 已提交
1287

1288
	/* acquire resources */
1289
	rc = pcim_enable_device(pdev);
T
Tejun Heo 已提交
1290 1291 1292
	if (rc)
		return rc;

T
Tejun Heo 已提交
1293 1294 1295
	rc = pcim_iomap_regions(pdev,
				(1 << SIL24_HOST_BAR) | (1 << SIL24_PORT_BAR),
				DRV_NAME);
T
Tejun Heo 已提交
1296
	if (rc)
1297
		return rc;
1298
	iomap = pcim_iomap_table(pdev);
T
Tejun Heo 已提交
1299

1300 1301 1302 1303
	/* apply workaround for completion IRQ loss on PCI-X errata */
	if (pi.flags & SIL24_FLAG_PCIX_IRQ_WOC) {
		tmp = readl(iomap[SIL24_HOST_BAR] + HOST_CTRL);
		if (tmp & (HOST_CTRL_TRDY | HOST_CTRL_STOP | HOST_CTRL_DEVSEL))
1304 1305
			dev_info(&pdev->dev,
				 "Applying completion IRQ loss on PCI-X errata fix\n");
1306 1307 1308
		else
			pi.flags &= ~SIL24_FLAG_PCIX_IRQ_WOC;
	}
T
Tejun Heo 已提交
1309

1310 1311 1312 1313 1314 1315
	/* allocate and fill host */
	host = ata_host_alloc_pinfo(&pdev->dev, ppi,
				    SIL24_FLAG2NPORTS(ppi[0]->flags));
	if (!host)
		return -ENOMEM;
	host->iomap = iomap;
T
Tejun Heo 已提交
1316

1317
	/* configure and activate the device */
1318 1319
	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
T
Tejun Heo 已提交
1320
		if (rc) {
1321
			rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1322
			if (rc) {
1323 1324
				dev_err(&pdev->dev,
					"64-bit DMA enable failed\n");
1325
				return rc;
T
Tejun Heo 已提交
1326 1327 1328
			}
		}
	} else {
1329
		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1330
		if (rc) {
1331
			dev_err(&pdev->dev, "32-bit DMA enable failed\n");
1332
			return rc;
T
Tejun Heo 已提交
1333
		}
1334
		rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
T
Tejun Heo 已提交
1335
		if (rc) {
1336 1337
			dev_err(&pdev->dev,
				"32-bit consistent DMA enable failed\n");
1338
			return rc;
T
Tejun Heo 已提交
1339
		}
T
Tejun Heo 已提交
1340 1341
	}

1342 1343 1344 1345 1346
	/* Set max read request size to 4096.  This slightly increases
	 * write throughput for pci-e variants.
	 */
	pcie_set_readrq(pdev, 4096);

1347
	sil24_init_controller(host);
T
Tejun Heo 已提交
1348

1349
	if (sata_sil24_msi && !pci_enable_msi(pdev)) {
1350
		dev_info(&pdev->dev, "Using MSI\n");
1351 1352 1353
		pci_intx(pdev, 0);
	}

T
Tejun Heo 已提交
1354
	pci_set_master(pdev);
1355 1356
	return ata_host_activate(host, pdev->irq, sil24_interrupt, IRQF_SHARED,
				 &sil24_sht);
T
Tejun Heo 已提交
1357 1358
}

1359
#ifdef CONFIG_PM
1360 1361
static int sil24_pci_device_resume(struct pci_dev *pdev)
{
J
Jeff Garzik 已提交
1362
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
T
Tejun Heo 已提交
1363
	void __iomem *host_base = host->iomap[SIL24_HOST_BAR];
1364
	int rc;
1365

1366 1367 1368
	rc = ata_pci_device_do_resume(pdev);
	if (rc)
		return rc;
1369 1370

	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND)
T
Tejun Heo 已提交
1371
		writel(HOST_CTRL_GLOBAL_RST, host_base + HOST_CTRL);
1372

1373
	sil24_init_controller(host);
1374

J
Jeff Garzik 已提交
1375
	ata_host_resume(host);
1376 1377 1378

	return 0;
}
T
Tejun Heo 已提交
1379 1380 1381 1382 1383 1384

static int sil24_port_resume(struct ata_port *ap)
{
	sil24_config_pmp(ap, ap->nr_pmp_links);
	return 0;
}
1385
#endif
1386

T
Tejun Heo 已提交
1387 1388
static int __init sil24_init(void)
{
1389
	return pci_register_driver(&sil24_pci_driver);
T
Tejun Heo 已提交
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
}

static void __exit sil24_exit(void)
{
	pci_unregister_driver(&sil24_pci_driver);
}

MODULE_AUTHOR("Tejun Heo");
MODULE_DESCRIPTION("Silicon Image 3124/3132 SATA low-level driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, sil24_pci_tbl);

module_init(sil24_init);
module_exit(sil24_exit);