ahci.c 64.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *  ahci.c - AHCI SATA support
 *
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
L
Linus Torvalds 已提交
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39 40 41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
44
#include <linux/dmi.h>
L
Linus Torvalds 已提交
45
#include <scsi/scsi_host.h>
46
#include <scsi/scsi_cmnd.h>
L
Linus Torvalds 已提交
47 48 49
#include <linux/libata.h>

#define DRV_NAME	"ahci"
T
Tejun Heo 已提交
50
#define DRV_VERSION	"3.0"
L
Linus Torvalds 已提交
51

52 53 54 55
static int ahci_skip_host_reset;
module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");

56 57 58
static int ahci_enable_alpm(struct ata_port *ap,
		enum link_pm policy);
static void ahci_disable_alpm(struct ata_port *ap);
L
Linus Torvalds 已提交
59 60 61

enum {
	AHCI_PCI_BAR		= 5,
62
	AHCI_MAX_PORTS		= 32,
L
Linus Torvalds 已提交
63 64
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
65
	AHCI_USE_CLUSTERING	= 1,
T
Tejun Heo 已提交
66
	AHCI_MAX_CMDS		= 32,
67
	AHCI_CMD_SZ		= 32,
T
Tejun Heo 已提交
68
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
L
Linus Torvalds 已提交
69
	AHCI_RX_FIS_SZ		= 256,
70
	AHCI_CMD_TBL_CDB	= 0x40,
71 72 73 74
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
L
Linus Torvalds 已提交
75 76 77 78
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
79
	AHCI_CMD_PREFETCH	= (1 << 7),
T
Tejun Heo 已提交
80 81
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
L
Linus Torvalds 已提交
82 83

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
84
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
T
Tejun Heo 已提交
85
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
L
Linus Torvalds 已提交
86 87

	board_ahci		= 0,
T
Tejun Heo 已提交
88 89 90 91
	board_ahci_vt8251	= 1,
	board_ahci_ign_iferr	= 2,
	board_ahci_sb600	= 3,
	board_ahci_mv		= 4,
92
	board_ahci_sb700	= 5,
L
Linus Torvalds 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
107
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
T
Tejun Heo 已提交
108
	HOST_CAP_PMP		= (1 << 17), /* Port Multiplier support */
T
Tejun Heo 已提交
109
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
110
	HOST_CAP_ALPM		= (1 << 26), /* Aggressive Link PM support */
111
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
T
Tejun Heo 已提交
112
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
113
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
114
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
L
Linus Torvalds 已提交
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
T
Tejun Heo 已提交
131
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
L
Linus Torvalds 已提交
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

T
Tejun Heo 已提交
153 154 155
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
156
				  PORT_IRQ_PHYRDY |
T
Tejun Heo 已提交
157 158
				  PORT_IRQ_UNK_FIS |
				  PORT_IRQ_BAD_PMP,
T
Tejun Heo 已提交
159 160 161 162 163 164
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
L
Linus Torvalds 已提交
165 166

	/* PORT_CMD bits */
167 168
	PORT_CMD_ASP		= (1 << 27), /* Aggressive Slumber/Partial */
	PORT_CMD_ALPE		= (1 << 26), /* Aggressive Link PM enable */
169
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
T
Tejun Heo 已提交
170
	PORT_CMD_PMP		= (1 << 17), /* PMP attached */
L
Linus Torvalds 已提交
171 172 173
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
T
Tejun Heo 已提交
174
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
L
Linus Torvalds 已提交
175 176 177 178
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

179
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
L
Linus Torvalds 已提交
180 181 182
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
183

184 185 186 187 188 189 190
	/* hpriv->flags bits */
	AHCI_HFLAG_NO_NCQ		= (1 << 0),
	AHCI_HFLAG_IGN_IRQ_IF_ERR	= (1 << 1), /* ignore IRQ_IF_ERR */
	AHCI_HFLAG_IGN_SERR_INTERNAL	= (1 << 2), /* ignore SERR_INTERNAL */
	AHCI_HFLAG_32BIT_ONLY		= (1 << 3), /* force 32bit */
	AHCI_HFLAG_MV_PATA		= (1 << 4), /* PATA port */
	AHCI_HFLAG_NO_MSI		= (1 << 5), /* no PCI MSI */
T
Tejun Heo 已提交
191
	AHCI_HFLAG_NO_PMP		= (1 << 6), /* no PMP */
192
	AHCI_HFLAG_NO_HOTPLUG		= (1 << 7), /* ignore PxSERR.DIAG.N */
193
	AHCI_HFLAG_SECT255		= (1 << 8), /* max 255 sectors */
194

195
	/* ap->flags bits */
T
Tejun Heo 已提交
196 197 198

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
199 200
					  ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
					  ATA_FLAG_IPM,
201
	AHCI_LFLAG_COMMON		= ATA_LFLAG_SKIP_D2H_BSY,
202 203

	ICH_MAP				= 0x90, /* ICH MAP register */
L
Linus Torvalds 已提交
204 205 206
};

struct ahci_cmd_hdr {
A
Al Viro 已提交
207 208 209 210 211
	__le32			opts;
	__le32			status;
	__le32			tbl_addr;
	__le32			tbl_addr_hi;
	__le32			reserved[4];
L
Linus Torvalds 已提交
212 213 214
};

struct ahci_sg {
A
Al Viro 已提交
215 216 217 218
	__le32			addr;
	__le32			addr_hi;
	__le32			reserved;
	__le32			flags_size;
L
Linus Torvalds 已提交
219 220 221
};

struct ahci_host_priv {
222
	unsigned int		flags;		/* AHCI_HFLAG_* */
223 224 225 226
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
L
Linus Torvalds 已提交
227 228 229
};

struct ahci_port_priv {
T
Tejun Heo 已提交
230
	struct ata_link		*active_link;
L
Linus Torvalds 已提交
231 232 233 234 235 236
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
237 238 239
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
240
	unsigned int		ncq_saw_sdb:1;
241
	u32 			intr_mask;	/* interrupts to enable */
L
Linus Torvalds 已提交
242 243
};

244 245
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
246
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
247
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
L
Linus Torvalds 已提交
248 249 250 251 252 253
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
T
Tejun Heo 已提交
254 255
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
T
Tejun Heo 已提交
256 257
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
T
Tejun Heo 已提交
258
static void ahci_error_handler(struct ata_port *ap);
259
static void ahci_vt8251_error_handler(struct ata_port *ap);
260
static void ahci_p5wdh_error_handler(struct ata_port *ap);
T
Tejun Heo 已提交
261
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
262
static int ahci_port_resume(struct ata_port *ap);
263
static void ahci_dev_config(struct ata_device *dev);
264 265 266
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
267
#ifdef CONFIG_PM
268 269 270
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
271
#endif
L
Linus Torvalds 已提交
272

273 274 275 276 277
static struct class_device_attribute *ahci_shost_attrs[] = {
	&class_device_attr_link_power_management_policy,
	NULL
};

278
static struct scsi_host_template ahci_sht = {
L
Linus Torvalds 已提交
279 280 281 282
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
T
Tejun Heo 已提交
283 284
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
L
Linus Torvalds 已提交
285 286 287 288 289 290 291 292
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
T
Tejun Heo 已提交
293
	.slave_destroy		= ata_scsi_slave_destroy,
L
Linus Torvalds 已提交
294
	.bios_param		= ata_std_bios_param,
295
	.shost_attrs		= ahci_shost_attrs,
L
Linus Torvalds 已提交
296 297
};

J
Jeff Garzik 已提交
298
static const struct ata_port_operations ahci_ops = {
L
Linus Torvalds 已提交
299 300 301 302
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

303 304
	.dev_config		= ahci_dev_config,

L
Linus Torvalds 已提交
305 306
	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
307
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
L
Linus Torvalds 已提交
308 309 310 311 312 313 314 315
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

T
Tejun Heo 已提交
316 317 318 319 320 321
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
322 323 324
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

325
#ifdef CONFIG_PM
326 327
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
328
#endif
329 330
	.enable_pm		= ahci_enable_alpm,
	.disable_pm		= ahci_disable_alpm,
331

L
Linus Torvalds 已提交
332 333 334 335
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

336 337 338 339 340 341 342
static const struct ata_port_operations ahci_vt8251_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
343
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
344 345 346 347 348 349 350 351 352 353 354 355 356 357
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
358 359 360
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

361
#ifdef CONFIG_PM
362 363
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
364
#endif
365 366 367 368 369

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
static const struct ata_port_operations ahci_p5wdh_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_p5wdh_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

#ifdef CONFIG_PM
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
#endif

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

404 405
#define AHCI_HFLAGS(flags)	.private_data	= (void *)(flags)

406
static const struct ata_port_info ahci_port_info[] = {
L
Linus Torvalds 已提交
407 408
	/* board_ahci */
	{
T
Tejun Heo 已提交
409
		.flags		= AHCI_FLAG_COMMON,
410
		.link_flags	= AHCI_LFLAG_COMMON,
411
		.pio_mask	= 0x1f, /* pio0-4 */
412
		.udma_mask	= ATA_UDMA6,
L
Linus Torvalds 已提交
413 414
		.port_ops	= &ahci_ops,
	},
415 416
	/* board_ahci_vt8251 */
	{
T
Tejun Heo 已提交
417
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
418
		.flags		= AHCI_FLAG_COMMON,
419
		.link_flags	= AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
420
		.pio_mask	= 0x1f, /* pio0-4 */
421
		.udma_mask	= ATA_UDMA6,
422
		.port_ops	= &ahci_vt8251_ops,
423
	},
424 425
	/* board_ahci_ign_iferr */
	{
426 427
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
		.flags		= AHCI_FLAG_COMMON,
428
		.link_flags	= AHCI_LFLAG_COMMON,
429
		.pio_mask	= 0x1f, /* pio0-4 */
430
		.udma_mask	= ATA_UDMA6,
431 432
		.port_ops	= &ahci_ops,
	},
433 434
	/* board_ahci_sb600 */
	{
435
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
436
				 AHCI_HFLAG_SECT255 | AHCI_HFLAG_NO_PMP),
437
		.flags		= AHCI_FLAG_COMMON,
438
		.link_flags	= AHCI_LFLAG_COMMON,
439
		.pio_mask	= 0x1f, /* pio0-4 */
440
		.udma_mask	= ATA_UDMA6,
441 442
		.port_ops	= &ahci_ops,
	},
443 444
	/* board_ahci_mv */
	{
445 446
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
				 AHCI_HFLAG_MV_PATA),
447
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
448
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
449
		.link_flags	= AHCI_LFLAG_COMMON,
450 451 452 453
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
454 455 456 457 458 459 460 461 462 463
	/* board_ahci_sb700 */
	{
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
				 AHCI_HFLAG_NO_PMP),
		.flags		= AHCI_FLAG_COMMON,
		.link_flags	= AHCI_LFLAG_COMMON,
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
L
Linus Torvalds 已提交
464 465
};

466
static const struct pci_device_id ahci_pci_tbl[] = {
J
Jeff Garzik 已提交
467
	/* Intel */
468 469 470 471 472
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
473
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
474 475 476 477
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
T
Tejun Heo 已提交
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
495 496
	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
497 498
	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
	{ PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
J
Jeff Garzik 已提交
499

500 501 502
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
J
Jeff Garzik 已提交
503 504

	/* ATI */
505
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
506 507 508 509 510 511
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb700 }, /* ATI SB700/800 */
J
Jeff Garzik 已提交
512 513

	/* VIA */
514
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
T
Tejun Heo 已提交
515
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
J
Jeff Garzik 已提交
516 517

	/* NVIDIA */
518 519 520 521
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
522 523 524 525 526 527 528 529
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
530 531 532 533 534 535 536 537
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
562 563 564 565
	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci },		/* MCP79 */
P
Peer Chen 已提交
566 567 568 569 570 571 572 573
	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },		/* MCP79 */
574 575 576 577 578 579 580 581 582 583 584 585
	{ PCI_VDEVICE(NVIDIA, 0x0bc8), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bc9), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bca), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcb), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcc), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcd), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bce), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcf), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd0), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd1), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd2), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd3), board_ahci },		/* MCP7B */
J
Jeff Garzik 已提交
586

J
Jeff Garzik 已提交
587
	/* SiS */
588 589 590
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
J
Jeff Garzik 已提交
591

592 593 594
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */

595 596
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
597
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
598

L
Linus Torvalds 已提交
599 600 601 602 603 604 605 606
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
607
	.remove			= ata_pci_remove_one,
608
#ifdef CONFIG_PM
609 610
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
611
#endif
L
Linus Torvalds 已提交
612 613 614
};


615 616 617 618 619
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

620 621
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
L
Linus Torvalds 已提交
622
{
623
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
624

625 626 627 628 629 630
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
L
Linus Torvalds 已提交
631 632
}

633 634 635 636 637 638 639 640 641 642 643 644 645 646
static void ahci_enable_ahci(void __iomem *mmio)
{
	u32 tmp;

	/* turn on AHCI_EN */
	tmp = readl(mmio + HOST_CTL);
	if (!(tmp & HOST_AHCI_EN)) {
		tmp |= HOST_AHCI_EN;
		writel(tmp, mmio + HOST_CTL);
		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
		WARN_ON(!(tmp & HOST_AHCI_EN));
	}
}

647 648
/**
 *	ahci_save_initial_config - Save and fixup initial config values
649 650
 *	@pdev: target PCI device
 *	@hpriv: host private area to store config values
651 652 653 654 655 656 657 658 659 660 661
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
662 663
static void ahci_save_initial_config(struct pci_dev *pdev,
				     struct ahci_host_priv *hpriv)
664
{
665
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
666
	u32 cap, port_map;
667
	int i;
668

669 670 671
	/* make sure AHCI mode is enabled before accessing CAP */
	ahci_enable_ahci(mmio);

672 673 674 675 676 677
	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

678
	/* some chips have errata preventing 64bit use */
679
	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
T
Tejun Heo 已提交
680 681 682 683 684
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

685
	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
686 687 688 689 690
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

691
	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
T
Tejun Heo 已提交
692 693 694 695 696
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do PMP, turning off CAP_PMP\n");
		cap &= ~HOST_CAP_PMP;
	}

697 698 699 700 701
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
702
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
703 704 705 706 707 708 709 710
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
			   hpriv->port_map,
			   hpriv->port_map & 0xf);

		port_map &= 0xf;
	}

711
	/* cross check port_map and cap.n_ports */
T
Tejun Heo 已提交
712
	if (port_map) {
T
Tejun Heo 已提交
713
		int map_ports = 0;
714

T
Tejun Heo 已提交
715 716 717
		for (i = 0; i < AHCI_MAX_PORTS; i++)
			if (port_map & (1 << i))
				map_ports++;
718

T
Tejun Heo 已提交
719 720
		/* If PI has more ports than n_ports, whine, clear
		 * port_map and let it be generated from n_ports.
721
		 */
T
Tejun Heo 已提交
722
		if (map_ports > ahci_nr_ports(cap)) {
723
			dev_printk(KERN_WARNING, &pdev->dev,
T
Tejun Heo 已提交
724 725 726
				   "implemented port map (0x%x) contains more "
				   "ports than nr_ports (%u), using nr_ports\n",
				   port_map, ahci_nr_ports(cap));
T
Tejun Heo 已提交
727 728 729 730 731 732
			port_map = 0;
		}
	}

	/* fabricate port_map from cap.nr_ports */
	if (!port_map) {
733
		port_map = (1 << ahci_nr_ports(cap)) - 1;
T
Tejun Heo 已提交
734 735 736 737 738
		dev_printk(KERN_WARNING, &pdev->dev,
			   "forcing PORTS_IMPL to 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
739 740
	}

741 742 743 744 745 746 747
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
748
 *	@host: target ATA host
749 750 751 752 753 754
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
755
static void ahci_restore_initial_config(struct ata_host *host)
756
{
757 758 759
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

760 761 762 763 764
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

T
Tejun Heo 已提交
765
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
L
Linus Torvalds 已提交
766
{
T
Tejun Heo 已提交
767 768 769 770 771 772 773 774
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
L
Linus Torvalds 已提交
775

T
Tejun Heo 已提交
776 777 778
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
779
	return 0;
L
Linus Torvalds 已提交
780 781
}

T
Tejun Heo 已提交
782
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
L
Linus Torvalds 已提交
783
{
T
Tejun Heo 已提交
784 785 786 787 788 789
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
L
Linus Torvalds 已提交
790
	}
T
Tejun Heo 已提交
791 792
	return -EINVAL;
}
L
Linus Torvalds 已提交
793

T
Tejun Heo 已提交
794 795 796 797 798 799 800 801 802 803
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
L
Linus Torvalds 已提交
804 805
}

806
static void ahci_start_engine(struct ata_port *ap)
807
{
808
	void __iomem *port_mmio = ahci_port_base(ap);
809 810
	u32 tmp;

811
	/* start DMA */
812
	tmp = readl(port_mmio + PORT_CMD);
813 814 815 816 817
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

818
static int ahci_stop_engine(struct ata_port *ap)
819
{
820
	void __iomem *port_mmio = ahci_port_base(ap);
821 822 823 824
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

825
	/* check if the HBA is idle */
826 827 828
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

829
	/* setting HBA to idle */
830 831 832
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

833
	/* wait for engine to stop. This could be as long as 500 msec */
834
	tmp = ata_wait_register(port_mmio + PORT_CMD,
835
				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
836
	if (tmp & PORT_CMD_LIST_ON)
837 838 839 840 841
		return -EIO;

	return 0;
}

842
static void ahci_start_fis_rx(struct ata_port *ap)
843
{
844 845 846
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
847 848 849
	u32 tmp;

	/* set FIS registers */
850 851 852 853
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
854

855 856 857 858
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
859 860 861 862 863 864 865 866 867 868

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

869
static int ahci_stop_fis_rx(struct ata_port *ap)
870
{
871
	void __iomem *port_mmio = ahci_port_base(ap);
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

888
static void ahci_power_up(struct ata_port *ap)
889
{
890 891
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
892 893 894 895 896
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
897
	if (hpriv->cap & HOST_CAP_SSS) {
898 899 900 901 902 903 904 905
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
static void ahci_disable_alpm(struct ata_port *ap)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;

	/* IPM bits should be disabled by libata-core */
	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/* disable ALPM and ASP */
	cmd &= ~PORT_CMD_ASP;
	cmd &= ~PORT_CMD_ALPE;

	/* force the interface back to active */
	cmd |= PORT_CMD_ICC_ACTIVE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* wait 10ms to be sure we've come out of any low power state */
	msleep(10);

	/* clear out any PhyRdy stuff from interrupt status */
	writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);

	/* go ahead and clean out PhyRdy Change from Serror too */
	ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));

	/*
 	 * Clear flag to indicate that we should ignore all PhyRdy
 	 * state changes
 	 */
	hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;

	/*
 	 * Enable interrupts on Phy Ready.
 	 */
	pp->intr_mask |= PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * don't change the link pm policy - we can be called
 	 * just to turn of link pm temporarily
 	 */
}

static int ahci_enable_alpm(struct ata_port *ap,
	enum link_pm policy)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;
	u32 asp;

	/* Make sure the host is capable of link power management */
	if (!(hpriv->cap & HOST_CAP_ALPM))
		return -EINVAL;

	switch (policy) {
	case MAX_PERFORMANCE:
	case NOT_AVAILABLE:
		/*
 		 * if we came here with NOT_AVAILABLE,
 		 * it just means this is the first time we
 		 * have tried to enable - default to max performance,
 		 * and let the user go to lower power modes on request.
 		 */
		ahci_disable_alpm(ap);
		return 0;
	case MIN_POWER:
		/* configure HBA to enter SLUMBER */
		asp = PORT_CMD_ASP;
		break;
	case MEDIUM_POWER:
		/* configure HBA to enter PARTIAL */
		asp = 0;
		break;
	default:
		return -EINVAL;
	}

	/*
 	 * Disable interrupts on Phy Ready. This keeps us from
 	 * getting woken up due to spurious phy ready interrupts
	 * TBD - Hot plug should be done via polling now, is
	 * that even supported?
 	 */
	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * Set a flag to indicate that we should ignore all PhyRdy
 	 * state changes since these can happen now whenever we
 	 * change link state
 	 */
	hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;

	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/*
 	 * Set ASP based on Policy
 	 */
	cmd |= asp;

	/*
 	 * Setting this bit will instruct the HBA to aggressively
 	 * enter a lower power link state when it's appropriate and
 	 * based on the value set above for ASP
 	 */
	cmd |= PORT_CMD_ALPE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* IPM bits should be set by libata-core */
	return 0;
}

1030
#ifdef CONFIG_PM
1031
static void ahci_power_down(struct ata_port *ap)
1032
{
1033 1034
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
1035 1036
	u32 cmd, scontrol;

1037
	if (!(hpriv->cap & HOST_CAP_SSS))
1038
		return;
1039

1040 1041 1042 1043
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
1044

1045 1046 1047 1048
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
1049
}
1050
#endif
1051

1052
static void ahci_start_port(struct ata_port *ap)
1053 1054
{
	/* enable FIS reception */
1055
	ahci_start_fis_rx(ap);
1056 1057

	/* enable DMA */
1058
	ahci_start_engine(ap);
1059 1060
}

1061
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
1062 1063 1064 1065
{
	int rc;

	/* disable DMA */
1066
	rc = ahci_stop_engine(ap);
1067 1068 1069 1070 1071 1072
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
1073
	rc = ahci_stop_fis_rx(ap);
1074 1075 1076 1077 1078 1079 1080 1081
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

1082
static int ahci_reset_controller(struct ata_host *host)
1083
{
1084
	struct pci_dev *pdev = to_pci_dev(host->dev);
T
Tejun Heo 已提交
1085
	struct ahci_host_priv *hpriv = host->private_data;
1086
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1087
	u32 tmp;
1088

1089 1090 1091
	/* we must be in AHCI mode, before using anything
	 * AHCI-specific, such as HOST_RESET.
	 */
1092
	ahci_enable_ahci(mmio);
1093 1094

	/* global controller reset */
1095 1096 1097 1098 1099 1100
	if (!ahci_skip_host_reset) {
		tmp = readl(mmio + HOST_CTL);
		if ((tmp & HOST_RESET) == 0) {
			writel(tmp | HOST_RESET, mmio + HOST_CTL);
			readl(mmio + HOST_CTL); /* flush */
		}
1101

1102 1103 1104 1105
		/* reset must complete within 1 second, or
		 * the hardware should be considered fried.
		 */
		ssleep(1);
1106

1107 1108 1109 1110 1111 1112
		tmp = readl(mmio + HOST_CTL);
		if (tmp & HOST_RESET) {
			dev_printk(KERN_ERR, host->dev,
				   "controller reset failed (0x%x)\n", tmp);
			return -EIO;
		}
1113

1114 1115
		/* turn on AHCI mode */
		ahci_enable_ahci(mmio);
1116

1117 1118 1119 1120 1121 1122 1123
		/* Some registers might be cleared on reset.  Restore
		 * initial values.
		 */
		ahci_restore_initial_config(host);
	} else
		dev_printk(KERN_INFO, host->dev,
			   "skipping global host reset\n");
1124 1125 1126 1127 1128 1129

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
T
Tejun Heo 已提交
1130 1131 1132 1133
		if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
			tmp16 |= hpriv->port_map;
			pci_write_config_word(pdev, 0x92, tmp16);
		}
1134 1135 1136 1137 1138
	}

	return 0;
}

1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

1167
static void ahci_init_controller(struct ata_host *host)
1168
{
1169
	struct ahci_host_priv *hpriv = host->private_data;
1170 1171
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1172
	int i;
1173
	void __iomem *port_mmio;
1174 1175
	u32 tmp;

1176
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
		port_mmio = __ahci_port_base(host, 4);

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

1188 1189
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
1190

1191
		port_mmio = ahci_port_base(ap);
1192
		if (ata_port_is_dummy(ap))
1193 1194
			continue;

1195
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
1196 1197 1198 1199 1200 1201 1202 1203 1204
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

1205 1206 1207 1208 1209 1210 1211 1212
static void ahci_dev_config(struct ata_device *dev)
{
	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;

	if (hpriv->flags & AHCI_HFLAG_SECT255)
		dev->max_sectors = 255;
}

1213
static unsigned int ahci_dev_classify(struct ata_port *ap)
L
Linus Torvalds 已提交
1214
{
1215
	void __iomem *port_mmio = ahci_port_base(ap);
L
Linus Torvalds 已提交
1216
	struct ata_taskfile tf;
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	u32 tmp;

	tmp = readl(port_mmio + PORT_SIG);
	tf.lbah		= (tmp >> 24)	& 0xff;
	tf.lbam		= (tmp >> 16)	& 0xff;
	tf.lbal		= (tmp >> 8)	& 0xff;
	tf.nsect	= (tmp)		& 0xff;

	return ata_dev_classify(&tf);
}

T
Tejun Heo 已提交
1228 1229
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts)
1230
{
T
Tejun Heo 已提交
1231 1232 1233 1234 1235 1236 1237 1238
	dma_addr_t cmd_tbl_dma;

	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;

	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
	pp->cmd_slot[tag].status = 0;
	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1239 1240
}

1241
static int ahci_kick_engine(struct ata_port *ap, int force_restart)
T
Tejun Heo 已提交
1242
{
T
Tejun Heo 已提交
1243
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
J
Jeff Garzik 已提交
1244
	struct ahci_host_priv *hpriv = ap->host->private_data;
1245
	u32 tmp;
1246
	int busy, rc;
1247

1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	/* do we need to kick the port? */
	busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
	if (!busy && !force_restart)
		return 0;

	/* stop engine */
	rc = ahci_stop_engine(ap);
	if (rc)
		goto out_restart;

	/* need to do CLO? */
	if (!busy) {
		rc = 0;
		goto out_restart;
	}

	if (!(hpriv->cap & HOST_CAP_CLO)) {
		rc = -EOPNOTSUPP;
		goto out_restart;
	}
1268

1269
	/* perform CLO */
1270 1271 1272 1273
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_CLO;
	writel(tmp, port_mmio + PORT_CMD);

1274
	rc = 0;
1275 1276 1277
	tmp = ata_wait_register(port_mmio + PORT_CMD,
				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
	if (tmp & PORT_CMD_CLO)
1278
		rc = -EIO;
1279

1280 1281 1282 1283
	/* restart engine */
 out_restart:
	ahci_start_engine(ap);
	return rc;
1284 1285
}

1286 1287 1288
static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
				struct ata_taskfile *tf, int is_cmd, u16 flags,
				unsigned long timeout_msec)
1289
{
1290
	const u32 cmd_fis_len = 5; /* five dwords */
T
Tejun Heo 已提交
1291
	struct ahci_port_priv *pp = ap->private_data;
1292
	void __iomem *port_mmio = ahci_port_base(ap);
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315
	u8 *fis = pp->cmd_tbl;
	u32 tmp;

	/* prep the command */
	ata_tf_to_fis(tf, pmp, is_cmd, fis);
	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));

	/* issue & wait */
	writel(1, port_mmio + PORT_CMD_ISSUE);

	if (timeout_msec) {
		tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
					1, timeout_msec);
		if (tmp & 0x1) {
			ahci_kick_engine(ap, 1);
			return -EBUSY;
		}
	} else
		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1316
static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1317
			     int pmp, unsigned long deadline)
1318
{
T
Tejun Heo 已提交
1319
	struct ata_port *ap = link->ap;
T
Tejun Heo 已提交
1320
	const char *reason = NULL;
1321
	unsigned long now, msecs;
T
Tejun Heo 已提交
1322 1323 1324 1325 1326
	struct ata_taskfile tf;
	int rc;

	DPRINTK("ENTER\n");

T
Tejun Heo 已提交
1327
	if (ata_link_offline(link)) {
1328 1329 1330 1331 1332
		DPRINTK("PHY reports no device\n");
		*class = ATA_DEV_NONE;
		return 0;
	}

T
Tejun Heo 已提交
1333
	/* prepare for SRST (AHCI-1.1 10.4.1) */
1334
	rc = ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1335
	if (rc && rc != -EOPNOTSUPP)
T
Tejun Heo 已提交
1336
		ata_link_printk(link, KERN_WARNING,
T
Tejun Heo 已提交
1337
				"failed to reset engine (errno=%d)\n", rc);
T
Tejun Heo 已提交
1338

T
Tejun Heo 已提交
1339
	ata_tf_init(link->device, &tf);
T
Tejun Heo 已提交
1340 1341

	/* issue the first D2H Register FIS */
1342 1343 1344 1345 1346
	msecs = 0;
	now = jiffies;
	if (time_after(now, deadline))
		msecs = jiffies_to_msecs(deadline - now);

T
Tejun Heo 已提交
1347
	tf.ctl |= ATA_SRST;
1348
	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1349
				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
T
Tejun Heo 已提交
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
		rc = -EIO;
		reason = "1st FIS failed";
		goto fail;
	}

	/* spec says at least 5us, but be generous and sleep for 1ms */
	msleep(1);

	/* issue the second D2H Register FIS */
	tf.ctl &= ~ATA_SRST;
1360
	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
T
Tejun Heo 已提交
1361

1362 1363
	/* wait a while before checking status */
	ata_wait_after_reset(ap, deadline);
T
Tejun Heo 已提交
1364

T
Tejun Heo 已提交
1365 1366 1367 1368 1369
	rc = ata_wait_ready(ap, deadline);
	/* link occupied, -ENODEV too is an error */
	if (rc) {
		reason = "device not ready";
		goto fail;
T
Tejun Heo 已提交
1370
	}
T
Tejun Heo 已提交
1371
	*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1372 1373 1374 1375 1376

	DPRINTK("EXIT, class=%u\n", *class);
	return 0;

 fail:
T
Tejun Heo 已提交
1377
	ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
T
Tejun Heo 已提交
1378 1379 1380
	return rc;
}

T
Tejun Heo 已提交
1381
static int ahci_softreset(struct ata_link *link, unsigned int *class,
1382 1383
			  unsigned long deadline)
{
T
Tejun Heo 已提交
1384 1385 1386 1387 1388 1389
	int pmp = 0;

	if (link->ap->flags & ATA_FLAG_PMP)
		pmp = SATA_PMP_CTRL_PORT;

	return ahci_do_softreset(link, class, pmp, deadline);
1390 1391
}

T
Tejun Heo 已提交
1392
static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1393
			  unsigned long deadline)
1394
{
T
Tejun Heo 已提交
1395
	struct ata_port *ap = link->ap;
1396 1397 1398
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
1399 1400 1401
	int rc;

	DPRINTK("ENTER\n");
L
Linus Torvalds 已提交
1402

1403
	ahci_stop_engine(ap);
1404 1405

	/* clear D2H reception area to properly wait for D2H FIS */
T
Tejun Heo 已提交
1406
	ata_tf_init(link->device, &tf);
1407
	tf.command = 0x80;
1408
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1409

T
Tejun Heo 已提交
1410
	rc = sata_std_hardreset(link, class, deadline);
1411

1412
	ahci_start_engine(ap);
L
Linus Torvalds 已提交
1413

T
Tejun Heo 已提交
1414
	if (rc == 0 && ata_link_online(link))
1415
		*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1416
	if (rc != -EAGAIN && *class == ATA_DEV_UNKNOWN)
1417
		*class = ATA_DEV_NONE;
L
Linus Torvalds 已提交
1418

1419 1420 1421 1422
	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
	return rc;
}

T
Tejun Heo 已提交
1423
static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
1424
				 unsigned long deadline)
1425
{
T
Tejun Heo 已提交
1426
	struct ata_port *ap = link->ap;
1427
	u32 serror;
1428 1429 1430 1431
	int rc;

	DPRINTK("ENTER\n");

1432
	ahci_stop_engine(ap);
1433

T
Tejun Heo 已提交
1434
	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
1435
				 deadline);
1436 1437

	/* vt8251 needs SError cleared for the port to operate */
1438 1439
	ahci_scr_read(ap, SCR_ERROR, &serror);
	ahci_scr_write(ap, SCR_ERROR, serror);
1440

1441
	ahci_start_engine(ap);
1442 1443 1444 1445 1446 1447 1448 1449 1450

	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);

	/* vt8251 doesn't clear BSY on signature FIS reception,
	 * request follow-up softreset.
	 */
	return rc ?: -EAGAIN;
}

1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
				unsigned long deadline)
{
	struct ata_port *ap = link->ap;
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
	int rc;

	ahci_stop_engine(ap);

	/* clear D2H reception area to properly wait for D2H FIS */
	ata_tf_init(link->device, &tf);
	tf.command = 0x80;
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);

	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
				 deadline);

	ahci_start_engine(ap);

	if (rc || ata_link_offline(link))
		return rc;

	/* spec mandates ">= 2ms" before checking status */
	msleep(150);

	/* The pseudo configuration device on SIMG4726 attached to
	 * ASUS P5W-DH Deluxe doesn't send signature FIS after
	 * hardreset if no device is attached to the first downstream
	 * port && the pseudo device locks up on SRST w/ PMP==0.  To
	 * work around this, wait for !BSY only briefly.  If BSY isn't
	 * cleared, perform CLO and proceed to IDENTIFY (achieved by
	 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
	 *
	 * Wait for two seconds.  Devices attached to downstream port
	 * which can't process the following IDENTIFY after this will
	 * have to be reset again.  For most cases, this should
	 * suffice while making probing snappish enough.
	 */
	rc = ata_wait_ready(ap, jiffies + 2 * HZ);
	if (rc)
		ahci_kick_engine(ap, 0);

	return 0;
}

T
Tejun Heo 已提交
1498
static void ahci_postreset(struct ata_link *link, unsigned int *class)
1499
{
T
Tejun Heo 已提交
1500
	struct ata_port *ap = link->ap;
1501
	void __iomem *port_mmio = ahci_port_base(ap);
1502 1503
	u32 new_tmp, tmp;

T
Tejun Heo 已提交
1504
	ata_std_postreset(link, class);
1505 1506 1507

	/* Make sure port's ATAPI bit is set appropriately */
	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1508
	if (*class == ATA_DEV_ATAPI)
1509 1510 1511 1512 1513 1514 1515
		new_tmp |= PORT_CMD_ATAPI;
	else
		new_tmp &= ~PORT_CMD_ATAPI;
	if (new_tmp != tmp) {
		writel(new_tmp, port_mmio + PORT_CMD);
		readl(port_mmio + PORT_CMD); /* flush */
	}
L
Linus Torvalds 已提交
1516 1517
}

T
Tejun Heo 已提交
1518 1519 1520 1521 1522 1523
static int ahci_pmp_softreset(struct ata_link *link, unsigned int *class,
			      unsigned long deadline)
{
	return ahci_do_softreset(link, class, link->pmp, deadline);
}

L
Linus Torvalds 已提交
1524 1525
static u8 ahci_check_status(struct ata_port *ap)
{
T
Tejun Heo 已提交
1526
	void __iomem *mmio = ap->ioaddr.cmd_addr;
L
Linus Torvalds 已提交
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538

	return readl(mmio + PORT_TFDATA) & 0xFF;
}

static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
{
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;

	ata_tf_from_fis(d2h_fis, tf);
}

T
Tejun Heo 已提交
1539
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
L
Linus Torvalds 已提交
1540
{
1541
	struct scatterlist *sg;
T
Tejun Heo 已提交
1542 1543
	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
	unsigned int si;
L
Linus Torvalds 已提交
1544 1545 1546 1547 1548 1549

	VPRINTK("ENTER\n");

	/*
	 * Next, the S/G list.
	 */
T
Tejun Heo 已提交
1550
	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1551 1552 1553
		dma_addr_t addr = sg_dma_address(sg);
		u32 sg_len = sg_dma_len(sg);

T
Tejun Heo 已提交
1554 1555 1556
		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
L
Linus Torvalds 已提交
1557
	}
1558

T
Tejun Heo 已提交
1559
	return si;
L
Linus Torvalds 已提交
1560 1561 1562 1563
}

static void ahci_qc_prep(struct ata_queued_cmd *qc)
{
1564 1565
	struct ata_port *ap = qc->ap;
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1566
	int is_atapi = ata_is_atapi(qc->tf.protocol);
T
Tejun Heo 已提交
1567
	void *cmd_tbl;
L
Linus Torvalds 已提交
1568 1569
	u32 opts;
	const u32 cmd_fis_len = 5; /* five dwords */
1570
	unsigned int n_elem;
L
Linus Torvalds 已提交
1571 1572 1573 1574 1575

	/*
	 * Fill in command table information.  First, the header,
	 * a SATA Register - Host to Device command FIS.
	 */
T
Tejun Heo 已提交
1576 1577
	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;

T
Tejun Heo 已提交
1578
	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1579
	if (is_atapi) {
T
Tejun Heo 已提交
1580 1581
		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1582
	}
L
Linus Torvalds 已提交
1583

1584 1585
	n_elem = 0;
	if (qc->flags & ATA_QCFLAG_DMAMAP)
T
Tejun Heo 已提交
1586
		n_elem = ahci_fill_sg(qc, cmd_tbl);
L
Linus Torvalds 已提交
1587

1588 1589 1590
	/*
	 * Fill in command slot information.
	 */
T
Tejun Heo 已提交
1591
	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1592 1593 1594
	if (qc->tf.flags & ATA_TFLAG_WRITE)
		opts |= AHCI_CMD_WRITE;
	if (is_atapi)
1595
		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1596

T
Tejun Heo 已提交
1597
	ahci_fill_cmd_slot(pp, qc->tag, opts);
L
Linus Torvalds 已提交
1598 1599
}

T
Tejun Heo 已提交
1600
static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
L
Linus Torvalds 已提交
1601
{
1602
	struct ahci_host_priv *hpriv = ap->host->private_data;
T
Tejun Heo 已提交
1603
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1604 1605 1606 1607
	struct ata_eh_info *host_ehi = &ap->link.eh_info;
	struct ata_link *link = NULL;
	struct ata_queued_cmd *active_qc;
	struct ata_eh_info *active_ehi;
T
Tejun Heo 已提交
1608
	u32 serror;
L
Linus Torvalds 已提交
1609

T
Tejun Heo 已提交
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
	/* determine active link */
	ata_port_for_each_link(link, ap)
		if (ata_link_active(link))
			break;
	if (!link)
		link = &ap->link;

	active_qc = ata_qc_from_tag(ap, link->active_tag);
	active_ehi = &link->eh_info;

	/* record irq stat */
	ata_ehi_clear_desc(host_ehi);
	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
L
Linus Torvalds 已提交
1623

T
Tejun Heo 已提交
1624
	/* AHCI needs SError cleared; otherwise, it might lock up */
1625
	ahci_scr_read(ap, SCR_ERROR, &serror);
T
Tejun Heo 已提交
1626
	ahci_scr_write(ap, SCR_ERROR, serror);
T
Tejun Heo 已提交
1627
	host_ehi->serror |= serror;
T
Tejun Heo 已提交
1628

1629
	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1630
	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1631 1632
		irq_stat &= ~PORT_IRQ_IF_ERR;

1633
	if (irq_stat & PORT_IRQ_TF_ERR) {
T
Tejun Heo 已提交
1634 1635 1636 1637 1638 1639 1640 1641 1642
		/* If qc is active, charge it; otherwise, the active
		 * link.  There's no active qc on NCQ errors.  It will
		 * be determined by EH by reading log page 10h.
		 */
		if (active_qc)
			active_qc->err_mask |= AC_ERR_DEV;
		else
			active_ehi->err_mask |= AC_ERR_DEV;

1643
		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
T
Tejun Heo 已提交
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
			host_ehi->serror &= ~SERR_INTERNAL;
	}

	if (irq_stat & PORT_IRQ_UNK_FIS) {
		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);

		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi,
				  "unknown FIS %08x %08x %08x %08x" ,
				  unk[0], unk[1], unk[2], unk[3]);
	}

	if (ap->nr_pmp_links && (irq_stat & PORT_IRQ_BAD_PMP)) {
		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1661
	}
T
Tejun Heo 已提交
1662 1663

	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
T
Tejun Heo 已提交
1664 1665 1666
		host_ehi->err_mask |= AC_ERR_HOST_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "host bus error");
L
Linus Torvalds 已提交
1667 1668
	}

T
Tejun Heo 已提交
1669
	if (irq_stat & PORT_IRQ_IF_ERR) {
T
Tejun Heo 已提交
1670 1671 1672
		host_ehi->err_mask |= AC_ERR_ATA_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "interface fatal error");
T
Tejun Heo 已提交
1673
	}
L
Linus Torvalds 已提交
1674

T
Tejun Heo 已提交
1675
	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
T
Tejun Heo 已提交
1676 1677 1678
		ata_ehi_hotplugged(host_ehi);
		ata_ehi_push_desc(host_ehi, "%s",
			irq_stat & PORT_IRQ_CONNECT ?
T
Tejun Heo 已提交
1679 1680 1681 1682
			"connection status changed" : "PHY RDY changed");
	}

	/* okay, let's hand over to EH */
1683

T
Tejun Heo 已提交
1684 1685 1686 1687
	if (irq_stat & PORT_IRQ_FREEZE)
		ata_port_freeze(ap);
	else
		ata_port_abort(ap);
L
Linus Torvalds 已提交
1688 1689
}

1690
static void ahci_port_intr(struct ata_port *ap)
L
Linus Torvalds 已提交
1691
{
1692
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
T
Tejun Heo 已提交
1693
	struct ata_eh_info *ehi = &ap->link.eh_info;
1694
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1695
	struct ahci_host_priv *hpriv = ap->host->private_data;
1696
	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
T
Tejun Heo 已提交
1697
	u32 status, qc_active;
1698
	int rc;
L
Linus Torvalds 已提交
1699 1700 1701 1702

	status = readl(port_mmio + PORT_IRQ_STAT);
	writel(status, port_mmio + PORT_IRQ_STAT);

1703 1704 1705 1706
	/* ignore BAD_PMP while resetting */
	if (unlikely(resetting))
		status &= ~PORT_IRQ_BAD_PMP;

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	/* If we are getting PhyRdy, this is
 	 * just a power state change, we should
 	 * clear out this, plus the PhyRdy/Comm
 	 * Wake bits from Serror
 	 */
	if ((hpriv->flags & AHCI_HFLAG_NO_HOTPLUG) &&
		(status & PORT_IRQ_PHYRDY)) {
		status &= ~PORT_IRQ_PHYRDY;
		ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));
	}

T
Tejun Heo 已提交
1718 1719 1720
	if (unlikely(status & PORT_IRQ_ERROR)) {
		ahci_error_intr(ap, status);
		return;
L
Linus Torvalds 已提交
1721 1722
	}

1723
	if (status & PORT_IRQ_SDB_FIS) {
T
Tejun Heo 已提交
1724 1725 1726 1727 1728 1729 1730 1731
		/* If SNotification is available, leave notification
		 * handling to sata_async_notification().  If not,
		 * emulate it by snooping SDB FIS RX area.
		 *
		 * Snooping FIS RX area is probably cheaper than
		 * poking SNotification but some constrollers which
		 * implement SNotification, ICH9 for example, don't
		 * store AN SDB FIS into receive area.
1732
		 */
T
Tejun Heo 已提交
1733
		if (hpriv->cap & HOST_CAP_SNTF)
1734
			sata_async_notification(ap);
T
Tejun Heo 已提交
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
		else {
			/* If the 'N' bit in word 0 of the FIS is set,
			 * we just received asynchronous notification.
			 * Tell libata about it.
			 */
			const __le32 *f = pp->rx_fis + RX_FIS_SDB;
			u32 f0 = le32_to_cpu(f[0]);

			if (f0 & (1 << 15))
				sata_async_notification(ap);
		}
1746 1747
	}

T
Tejun Heo 已提交
1748 1749
	/* pp->active_link is valid iff any command is in flight */
	if (ap->qc_active && pp->active_link->sactive)
T
Tejun Heo 已提交
1750 1751 1752 1753 1754
		qc_active = readl(port_mmio + PORT_SCR_ACT);
	else
		qc_active = readl(port_mmio + PORT_CMD_ISSUE);

	rc = ata_qc_complete_multiple(ap, qc_active, NULL);
1755

1756 1757
	/* while resetting, invalid completions are expected */
	if (unlikely(rc < 0 && !resetting)) {
T
Tejun Heo 已提交
1758 1759 1760
		ehi->err_mask |= AC_ERR_HSM;
		ehi->action |= ATA_EH_SOFTRESET;
		ata_port_freeze(ap);
L
Linus Torvalds 已提交
1761 1762 1763 1764 1765 1766 1767 1768
	}
}

static void ahci_irq_clear(struct ata_port *ap)
{
	/* TODO */
}

1769
static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
L
Linus Torvalds 已提交
1770
{
J
Jeff Garzik 已提交
1771
	struct ata_host *host = dev_instance;
L
Linus Torvalds 已提交
1772 1773
	struct ahci_host_priv *hpriv;
	unsigned int i, handled = 0;
1774
	void __iomem *mmio;
L
Linus Torvalds 已提交
1775 1776 1777 1778
	u32 irq_stat, irq_ack = 0;

	VPRINTK("ENTER\n");

J
Jeff Garzik 已提交
1779
	hpriv = host->private_data;
T
Tejun Heo 已提交
1780
	mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
1781 1782 1783 1784 1785 1786 1787

	/* sigh.  0xffffffff is a valid return from h/w */
	irq_stat = readl(mmio + HOST_IRQ_STAT);
	irq_stat &= hpriv->port_map;
	if (!irq_stat)
		return IRQ_NONE;

1788
	spin_lock(&host->lock);
L
Linus Torvalds 已提交
1789

1790
	for (i = 0; i < host->n_ports; i++) {
L
Linus Torvalds 已提交
1791 1792
		struct ata_port *ap;

1793 1794 1795
		if (!(irq_stat & (1 << i)))
			continue;

J
Jeff Garzik 已提交
1796
		ap = host->ports[i];
1797
		if (ap) {
1798
			ahci_port_intr(ap);
1799 1800 1801
			VPRINTK("port %u\n", i);
		} else {
			VPRINTK("port %u (no irq)\n", i);
1802
			if (ata_ratelimit())
J
Jeff Garzik 已提交
1803
				dev_printk(KERN_WARNING, host->dev,
1804
					"interrupt on disabled port %u\n", i);
L
Linus Torvalds 已提交
1805
		}
1806 1807

		irq_ack |= (1 << i);
L
Linus Torvalds 已提交
1808 1809 1810 1811 1812 1813 1814
	}

	if (irq_ack) {
		writel(irq_ack, mmio + HOST_IRQ_STAT);
		handled = 1;
	}

J
Jeff Garzik 已提交
1815
	spin_unlock(&host->lock);
L
Linus Torvalds 已提交
1816 1817 1818 1819 1820 1821

	VPRINTK("EXIT\n");

	return IRQ_RETVAL(handled);
}

1822
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
L
Linus Torvalds 已提交
1823 1824
{
	struct ata_port *ap = qc->ap;
1825
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1826 1827 1828 1829 1830 1831 1832
	struct ahci_port_priv *pp = ap->private_data;

	/* Keep track of the currently active link.  It will be used
	 * in completion path to determine whether NCQ phase is in
	 * progress.
	 */
	pp->active_link = qc->dev->link;
L
Linus Torvalds 已提交
1833

T
Tejun Heo 已提交
1834 1835 1836
	if (qc->tf.protocol == ATA_PROT_NCQ)
		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
L
Linus Torvalds 已提交
1837 1838 1839 1840 1841
	readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1842 1843
static void ahci_freeze(struct ata_port *ap)
{
1844
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1845 1846 1847 1848 1849 1850 1851

	/* turn IRQ off */
	writel(0, port_mmio + PORT_IRQ_MASK);
}

static void ahci_thaw(struct ata_port *ap)
{
T
Tejun Heo 已提交
1852
	void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
1853
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1854
	u32 tmp;
1855
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1856 1857 1858 1859

	/* clear IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	writel(tmp, port_mmio + PORT_IRQ_STAT);
1860
	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
T
Tejun Heo 已提交
1861

1862 1863
	/* turn IRQ back on */
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1864 1865 1866 1867
}

static void ahci_error_handler(struct ata_port *ap)
{
1868
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
T
Tejun Heo 已提交
1869
		/* restart engine */
1870 1871
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
T
Tejun Heo 已提交
1872 1873 1874
	}

	/* perform recovery */
T
Tejun Heo 已提交
1875 1876 1877 1878
	sata_pmp_do_eh(ap, ata_std_prereset, ahci_softreset,
		       ahci_hardreset, ahci_postreset,
		       sata_pmp_std_prereset, ahci_pmp_softreset,
		       sata_pmp_std_hardreset, sata_pmp_std_postreset);
T
Tejun Heo 已提交
1879 1880
}

1881 1882 1883 1884
static void ahci_vt8251_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
1885 1886
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
1887 1888 1889 1890 1891 1892 1893
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
		  ahci_postreset);
}

1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
static void ahci_p5wdh_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_p5wdh_hardreset,
		  ahci_postreset);
}

T
Tejun Heo 已提交
1907 1908 1909 1910
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;

1911 1912 1913
	/* make DMA engine forget about the failed command */
	if (qc->flags & ATA_QCFLAG_FAILED)
		ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1914 1915
}

T
Tejun Heo 已提交
1916 1917 1918
static void ahci_pmp_attach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1919
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1920 1921 1922 1923 1924
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd |= PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1925 1926 1927

	pp->intr_mask |= PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1928 1929 1930 1931 1932
}

static void ahci_pmp_detach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1933
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1934 1935 1936 1937 1938
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd &= ~PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1939 1940 1941

	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1942 1943
}

1944 1945 1946 1947 1948
static int ahci_port_resume(struct ata_port *ap)
{
	ahci_power_up(ap);
	ahci_start_port(ap);

T
Tejun Heo 已提交
1949 1950 1951 1952 1953
	if (ap->nr_pmp_links)
		ahci_pmp_attach(ap);
	else
		ahci_pmp_detach(ap);

1954 1955 1956
	return 0;
}

1957
#ifdef CONFIG_PM
1958 1959 1960 1961 1962
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
{
	const char *emsg = NULL;
	int rc;

1963
	rc = ahci_deinit_port(ap, &emsg);
1964
	if (rc == 0)
1965
		ahci_power_down(ap);
1966
	else {
1967
		ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
1968
		ahci_start_port(ap);
1969 1970 1971 1972 1973 1974 1975
	}

	return rc;
}

static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
{
J
Jeff Garzik 已提交
1976
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
T
Tejun Heo 已提交
1977
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1978 1979
	u32 ctl;

1980
	if (mesg.event & PM_EVENT_SLEEP) {
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995
		/* AHCI spec rev1.1 section 8.3.3:
		 * Software must disable interrupts prior to requesting a
		 * transition of the HBA to D3 state.
		 */
		ctl = readl(mmio + HOST_CTL);
		ctl &= ~HOST_IRQ_EN;
		writel(ctl, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	return ata_pci_device_suspend(pdev, mesg);
}

static int ahci_pci_device_resume(struct pci_dev *pdev)
{
J
Jeff Garzik 已提交
1996
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
1997 1998
	int rc;

1999 2000 2001
	rc = ata_pci_device_do_resume(pdev);
	if (rc)
		return rc;
2002 2003

	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2004
		rc = ahci_reset_controller(host);
2005 2006 2007
		if (rc)
			return rc;

2008
		ahci_init_controller(host);
2009 2010
	}

J
Jeff Garzik 已提交
2011
	ata_host_resume(host);
2012 2013 2014

	return 0;
}
2015
#endif
2016

2017 2018
static int ahci_port_start(struct ata_port *ap)
{
J
Jeff Garzik 已提交
2019
	struct device *dev = ap->host->dev;
2020 2021 2022 2023
	struct ahci_port_priv *pp;
	void *mem;
	dma_addr_t mem_dma;

2024
	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2025 2026 2027
	if (!pp)
		return -ENOMEM;

2028 2029 2030
	mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
				  GFP_KERNEL);
	if (!mem)
2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
		return -ENOMEM;
	memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);

	/*
	 * First item in chunk of DMA memory: 32-slot command table,
	 * 32 bytes each in size
	 */
	pp->cmd_slot = mem;
	pp->cmd_slot_dma = mem_dma;

	mem += AHCI_CMD_SLOT_SZ;
	mem_dma += AHCI_CMD_SLOT_SZ;

	/*
	 * Second item: Received-FIS area
	 */
	pp->rx_fis = mem;
	pp->rx_fis_dma = mem_dma;

	mem += AHCI_RX_FIS_SZ;
	mem_dma += AHCI_RX_FIS_SZ;

	/*
	 * Third item: data area for storing a single command
	 * and its scatter-gather table
	 */
	pp->cmd_tbl = mem;
	pp->cmd_tbl_dma = mem_dma;

2060
	/*
2061 2062 2063
	 * Save off initial list of interrupts to be enabled.
	 * This could be changed later
	 */
2064 2065
	pp->intr_mask = DEF_PORT_IRQ;

2066 2067
	ap->private_data = pp;

2068 2069
	/* engage engines, captain */
	return ahci_port_resume(ap);
2070 2071 2072 2073
}

static void ahci_port_stop(struct ata_port *ap)
{
2074 2075
	const char *emsg = NULL;
	int rc;
2076

2077
	/* de-initialize port */
2078
	rc = ahci_deinit_port(ap, &emsg);
2079 2080
	if (rc)
		ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
2081 2082
}

2083
static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
L
Linus Torvalds 已提交
2084 2085 2086 2087 2088 2089 2090 2091 2092
{
	int rc;

	if (using_dac &&
	    !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
		if (rc) {
			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
			if (rc) {
2093 2094
				dev_printk(KERN_ERR, &pdev->dev,
					   "64-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2095 2096 2097 2098 2099 2100
				return rc;
			}
		}
	} else {
		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2101 2102
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2103 2104 2105 2106
			return rc;
		}
		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2107 2108
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit consistent DMA enable failed\n");
L
Linus Torvalds 已提交
2109 2110 2111 2112 2113 2114
			return rc;
		}
	}
	return 0;
}

2115
static void ahci_print_info(struct ata_host *host)
L
Linus Torvalds 已提交
2116
{
2117 2118 2119
	struct ahci_host_priv *hpriv = host->private_data;
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137
	u32 vers, cap, impl, speed;
	const char *speed_s;
	u16 cc;
	const char *scc_s;

	vers = readl(mmio + HOST_VERSION);
	cap = hpriv->cap;
	impl = hpriv->port_map;

	speed = (cap >> 20) & 0xf;
	if (speed == 1)
		speed_s = "1.5";
	else if (speed == 2)
		speed_s = "3";
	else
		speed_s = "?";

	pci_read_config_word(pdev, 0x0a, &cc);
2138
	if (cc == PCI_CLASS_STORAGE_IDE)
L
Linus Torvalds 已提交
2139
		scc_s = "IDE";
2140
	else if (cc == PCI_CLASS_STORAGE_SATA)
L
Linus Torvalds 已提交
2141
		scc_s = "SATA";
2142
	else if (cc == PCI_CLASS_STORAGE_RAID)
L
Linus Torvalds 已提交
2143 2144 2145 2146
		scc_s = "RAID";
	else
		scc_s = "unknown";

2147 2148
	dev_printk(KERN_INFO, &pdev->dev,
		"AHCI %02x%02x.%02x%02x "
L
Linus Torvalds 已提交
2149
		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2150
		,
L
Linus Torvalds 已提交
2151

2152 2153 2154 2155
		(vers >> 24) & 0xff,
		(vers >> 16) & 0xff,
		(vers >> 8) & 0xff,
		vers & 0xff,
L
Linus Torvalds 已提交
2156 2157 2158 2159 2160 2161 2162

		((cap >> 8) & 0x1f) + 1,
		(cap & 0x1f) + 1,
		speed_s,
		impl,
		scc_s);

2163 2164
	dev_printk(KERN_INFO, &pdev->dev,
		"flags: "
T
Tejun Heo 已提交
2165 2166
		"%s%s%s%s%s%s%s"
		"%s%s%s%s%s%s%s\n"
2167
		,
L
Linus Torvalds 已提交
2168 2169 2170

		cap & (1 << 31) ? "64bit " : "",
		cap & (1 << 30) ? "ncq " : "",
T
Tejun Heo 已提交
2171
		cap & (1 << 29) ? "sntf " : "",
L
Linus Torvalds 已提交
2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
		cap & (1 << 28) ? "ilck " : "",
		cap & (1 << 27) ? "stag " : "",
		cap & (1 << 26) ? "pm " : "",
		cap & (1 << 25) ? "led " : "",

		cap & (1 << 24) ? "clo " : "",
		cap & (1 << 19) ? "nz " : "",
		cap & (1 << 18) ? "only " : "",
		cap & (1 << 17) ? "pmp " : "",
		cap & (1 << 15) ? "pio " : "",
		cap & (1 << 14) ? "slum " : "",
		cap & (1 << 13) ? "part " : ""
		);
}

2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231
/* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
 * hardwired to on-board SIMG 4726.  The chipset is ICH8 and doesn't
 * support PMP and the 4726 either directly exports the device
 * attached to the first downstream port or acts as a hardware storage
 * controller and emulate a single ATA device (can be RAID 0/1 or some
 * other configuration).
 *
 * When there's no device attached to the first downstream port of the
 * 4726, "Config Disk" appears, which is a pseudo ATA device to
 * configure the 4726.  However, ATA emulation of the device is very
 * lame.  It doesn't send signature D2H Reg FIS after the initial
 * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
 *
 * The following function works around the problem by always using
 * hardreset on the port and not depending on receiving signature FIS
 * afterward.  If signature FIS isn't received soon, ATA class is
 * assumed without follow-up softreset.
 */
static void ahci_p5wdh_workaround(struct ata_host *host)
{
	static struct dmi_system_id sysids[] = {
		{
			.ident = "P5W DH Deluxe",
			.matches = {
				DMI_MATCH(DMI_SYS_VENDOR,
					  "ASUSTEK COMPUTER INC"),
				DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
			},
		},
		{ }
	};
	struct pci_dev *pdev = to_pci_dev(host->dev);

	if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
	    dmi_check_system(sysids)) {
		struct ata_port *ap = host->ports[1];

		dev_printk(KERN_INFO, &pdev->dev, "enabling ASUS P5W DH "
			   "Deluxe on-board SIMG4726 workaround\n");

		ap->ops = &ahci_p5wdh_ops;
		ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
	}
}

2232
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
L
Linus Torvalds 已提交
2233 2234
{
	static int printed_version;
2235 2236
	struct ata_port_info pi = ahci_port_info[ent->driver_data];
	const struct ata_port_info *ppi[] = { &pi, NULL };
2237
	struct device *dev = &pdev->dev;
L
Linus Torvalds 已提交
2238
	struct ahci_host_priv *hpriv;
2239
	struct ata_host *host;
T
Tejun Heo 已提交
2240
	int n_ports, i, rc;
L
Linus Torvalds 已提交
2241 2242 2243

	VPRINTK("ENTER\n");

T
Tejun Heo 已提交
2244 2245
	WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);

L
Linus Torvalds 已提交
2246
	if (!printed_version++)
2247
		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
L
Linus Torvalds 已提交
2248

2249
	/* acquire resources */
2250
	rc = pcim_enable_device(pdev);
L
Linus Torvalds 已提交
2251 2252 2253
	if (rc)
		return rc;

T
Tejun Heo 已提交
2254 2255 2256 2257
	/* AHCI controllers often implement SFF compatible interface.
	 * Grab all PCI BARs just in case.
	 */
	rc = pcim_iomap_regions_request_all(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
T
Tejun Heo 已提交
2258
	if (rc == -EBUSY)
2259
		pcim_pin_device(pdev);
T
Tejun Heo 已提交
2260
	if (rc)
2261
		return rc;
L
Linus Torvalds 已提交
2262

2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
	if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
	    (pdev->device == 0x2652 || pdev->device == 0x2653)) {
		u8 map;

		/* ICH6s share the same PCI ID for both piix and ahci
		 * modes.  Enabling ahci mode while MAP indicates
		 * combined mode is a bad idea.  Yield to ata_piix.
		 */
		pci_read_config_byte(pdev, ICH_MAP, &map);
		if (map & 0x3) {
			dev_printk(KERN_INFO, &pdev->dev, "controller is in "
				   "combined mode, can't enable AHCI mode\n");
			return -ENODEV;
		}
	}

2279 2280 2281
	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
	if (!hpriv)
		return -ENOMEM;
2282 2283 2284 2285
	hpriv->flags |= (unsigned long)pi.private_data;

	if ((hpriv->flags & AHCI_HFLAG_NO_MSI) || pci_enable_msi(pdev))
		pci_intx(pdev, 1);
L
Linus Torvalds 已提交
2286

2287
	/* save initial config */
2288
	ahci_save_initial_config(pdev, hpriv);
L
Linus Torvalds 已提交
2289

2290
	/* prepare host */
2291
	if (hpriv->cap & HOST_CAP_NCQ)
2292
		pi.flags |= ATA_FLAG_NCQ;
L
Linus Torvalds 已提交
2293

T
Tejun Heo 已提交
2294 2295 2296
	if (hpriv->cap & HOST_CAP_PMP)
		pi.flags |= ATA_FLAG_PMP;

T
Tejun Heo 已提交
2297 2298 2299 2300 2301 2302 2303 2304
	/* CAP.NP sometimes indicate the index of the last enabled
	 * port, at other times, that of the last possible port, so
	 * determining the maximum port number requires looking at
	 * both CAP.NP and port_map.
	 */
	n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));

	host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
2305 2306 2307 2308 2309 2310
	if (!host)
		return -ENOMEM;
	host->iomap = pcim_iomap_table(pdev);
	host->private_data = hpriv;

	for (i = 0; i < host->n_ports; i++) {
2311 2312
		struct ata_port *ap = host->ports[i];
		void __iomem *port_mmio = ahci_port_base(ap);
2313

2314 2315 2316 2317
		ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
		ata_port_pbar_desc(ap, AHCI_PCI_BAR,
				   0x100 + ap->port_no * 0x80, "port");

2318 2319 2320
		/* set initial link pm policy */
		ap->pm_policy = NOT_AVAILABLE;

2321
		/* standard SATA port setup */
T
Tejun Heo 已提交
2322
		if (hpriv->port_map & (1 << i))
2323
			ap->ioaddr.cmd_addr = port_mmio;
2324 2325 2326 2327

		/* disabled/not-implemented port */
		else
			ap->ops = &ata_dummy_port_ops;
2328
	}
2329

2330 2331 2332
	/* apply workaround for ASUS P5W DH Deluxe mainboard */
	ahci_p5wdh_workaround(host);

2333 2334
	/* initialize adapter */
	rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
L
Linus Torvalds 已提交
2335
	if (rc)
2336
		return rc;
L
Linus Torvalds 已提交
2337

2338 2339 2340
	rc = ahci_reset_controller(host);
	if (rc)
		return rc;
L
Linus Torvalds 已提交
2341

2342 2343
	ahci_init_controller(host);
	ahci_print_info(host);
L
Linus Torvalds 已提交
2344

2345 2346 2347
	pci_set_master(pdev);
	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
				 &ahci_sht);
2348
}
L
Linus Torvalds 已提交
2349 2350 2351

static int __init ahci_init(void)
{
2352
	return pci_register_driver(&ahci_pci_driver);
L
Linus Torvalds 已提交
2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364
}

static void __exit ahci_exit(void)
{
	pci_unregister_driver(&ahci_pci_driver);
}


MODULE_AUTHOR("Jeff Garzik");
MODULE_DESCRIPTION("AHCI SATA low-level driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
2365
MODULE_VERSION(DRV_VERSION);
L
Linus Torvalds 已提交
2366 2367 2368

module_init(ahci_init);
module_exit(ahci_exit);