ahci.c 62.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *  ahci.c - AHCI SATA support
 *
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
L
Linus Torvalds 已提交
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39 40 41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
44
#include <linux/dmi.h>
L
Linus Torvalds 已提交
45
#include <scsi/scsi_host.h>
46
#include <scsi/scsi_cmnd.h>
L
Linus Torvalds 已提交
47 48 49
#include <linux/libata.h>

#define DRV_NAME	"ahci"
T
Tejun Heo 已提交
50
#define DRV_VERSION	"3.0"
L
Linus Torvalds 已提交
51

52 53 54
static int ahci_enable_alpm(struct ata_port *ap,
		enum link_pm policy);
static void ahci_disable_alpm(struct ata_port *ap);
L
Linus Torvalds 已提交
55 56 57

enum {
	AHCI_PCI_BAR		= 5,
58
	AHCI_MAX_PORTS		= 32,
L
Linus Torvalds 已提交
59 60
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
61
	AHCI_USE_CLUSTERING	= 1,
T
Tejun Heo 已提交
62
	AHCI_MAX_CMDS		= 32,
63
	AHCI_CMD_SZ		= 32,
T
Tejun Heo 已提交
64
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
L
Linus Torvalds 已提交
65
	AHCI_RX_FIS_SZ		= 256,
66
	AHCI_CMD_TBL_CDB	= 0x40,
67 68 69 70
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
L
Linus Torvalds 已提交
71 72 73 74
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
75
	AHCI_CMD_PREFETCH	= (1 << 7),
T
Tejun Heo 已提交
76 77
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
L
Linus Torvalds 已提交
78 79

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
80
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
T
Tejun Heo 已提交
81
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
L
Linus Torvalds 已提交
82 83

	board_ahci		= 0,
T
Tejun Heo 已提交
84 85 86 87
	board_ahci_vt8251	= 1,
	board_ahci_ign_iferr	= 2,
	board_ahci_sb600	= 3,
	board_ahci_mv		= 4,
L
Linus Torvalds 已提交
88 89 90 91 92 93 94 95 96 97 98 99 100 101

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
102
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
T
Tejun Heo 已提交
103
	HOST_CAP_PMP		= (1 << 17), /* Port Multiplier support */
T
Tejun Heo 已提交
104
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
105
	HOST_CAP_ALPM		= (1 << 26), /* Aggressive Link PM support */
106
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
T
Tejun Heo 已提交
107
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
108
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
109
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
L
Linus Torvalds 已提交
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
T
Tejun Heo 已提交
126
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
L
Linus Torvalds 已提交
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

T
Tejun Heo 已提交
148 149 150
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
151
				  PORT_IRQ_PHYRDY |
T
Tejun Heo 已提交
152 153
				  PORT_IRQ_UNK_FIS |
				  PORT_IRQ_BAD_PMP,
T
Tejun Heo 已提交
154 155 156 157 158 159
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
L
Linus Torvalds 已提交
160 161

	/* PORT_CMD bits */
162 163
	PORT_CMD_ASP		= (1 << 27), /* Aggressive Slumber/Partial */
	PORT_CMD_ALPE		= (1 << 26), /* Aggressive Link PM enable */
164
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
T
Tejun Heo 已提交
165
	PORT_CMD_PMP		= (1 << 17), /* PMP attached */
L
Linus Torvalds 已提交
166 167 168
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
T
Tejun Heo 已提交
169
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
L
Linus Torvalds 已提交
170 171 172 173
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

174
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
L
Linus Torvalds 已提交
175 176 177
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
178

179 180 181 182 183 184 185
	/* hpriv->flags bits */
	AHCI_HFLAG_NO_NCQ		= (1 << 0),
	AHCI_HFLAG_IGN_IRQ_IF_ERR	= (1 << 1), /* ignore IRQ_IF_ERR */
	AHCI_HFLAG_IGN_SERR_INTERNAL	= (1 << 2), /* ignore SERR_INTERNAL */
	AHCI_HFLAG_32BIT_ONLY		= (1 << 3), /* force 32bit */
	AHCI_HFLAG_MV_PATA		= (1 << 4), /* PATA port */
	AHCI_HFLAG_NO_MSI		= (1 << 5), /* no PCI MSI */
T
Tejun Heo 已提交
186
	AHCI_HFLAG_NO_PMP		= (1 << 6), /* no PMP */
187
	AHCI_HFLAG_NO_HOTPLUG		= (1 << 7), /* ignore PxSERR.DIAG.N */
188

189
	/* ap->flags bits */
T
Tejun Heo 已提交
190 191 192

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
193 194
					  ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
					  ATA_FLAG_IPM,
195
	AHCI_LFLAG_COMMON		= ATA_LFLAG_SKIP_D2H_BSY,
196 197

	ICH_MAP				= 0x90, /* ICH MAP register */
L
Linus Torvalds 已提交
198 199 200
};

struct ahci_cmd_hdr {
A
Al Viro 已提交
201 202 203 204 205
	__le32			opts;
	__le32			status;
	__le32			tbl_addr;
	__le32			tbl_addr_hi;
	__le32			reserved[4];
L
Linus Torvalds 已提交
206 207 208
};

struct ahci_sg {
A
Al Viro 已提交
209 210 211 212
	__le32			addr;
	__le32			addr_hi;
	__le32			reserved;
	__le32			flags_size;
L
Linus Torvalds 已提交
213 214 215
};

struct ahci_host_priv {
216
	unsigned int		flags;		/* AHCI_HFLAG_* */
217 218 219 220
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
L
Linus Torvalds 已提交
221 222 223
};

struct ahci_port_priv {
T
Tejun Heo 已提交
224
	struct ata_link		*active_link;
L
Linus Torvalds 已提交
225 226 227 228 229 230
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
231 232 233
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
234
	unsigned int		ncq_saw_sdb:1;
235
	u32 			intr_mask;	/* interrupts to enable */
L
Linus Torvalds 已提交
236 237
};

238 239
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
240
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
241
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
L
Linus Torvalds 已提交
242 243 244 245 246 247
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
T
Tejun Heo 已提交
248 249
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
T
Tejun Heo 已提交
250 251
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
T
Tejun Heo 已提交
252
static void ahci_error_handler(struct ata_port *ap);
253
static void ahci_vt8251_error_handler(struct ata_port *ap);
254
static void ahci_p5wdh_error_handler(struct ata_port *ap);
T
Tejun Heo 已提交
255
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
256
static int ahci_port_resume(struct ata_port *ap);
257 258 259
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
260
#ifdef CONFIG_PM
261 262 263
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
264
#endif
L
Linus Torvalds 已提交
265

266 267 268 269 270
static struct class_device_attribute *ahci_shost_attrs[] = {
	&class_device_attr_link_power_management_policy,
	NULL
};

271
static struct scsi_host_template ahci_sht = {
L
Linus Torvalds 已提交
272 273 274 275
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
T
Tejun Heo 已提交
276 277
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
L
Linus Torvalds 已提交
278 279 280 281 282 283 284 285
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
T
Tejun Heo 已提交
286
	.slave_destroy		= ata_scsi_slave_destroy,
L
Linus Torvalds 已提交
287
	.bios_param		= ata_std_bios_param,
288
	.shost_attrs		= ahci_shost_attrs,
L
Linus Torvalds 已提交
289 290
};

J
Jeff Garzik 已提交
291
static const struct ata_port_operations ahci_ops = {
L
Linus Torvalds 已提交
292 293 294 295 296 297
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
298
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
L
Linus Torvalds 已提交
299 300 301 302 303 304 305 306
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

T
Tejun Heo 已提交
307 308 309 310 311 312
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
313 314 315
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

316
#ifdef CONFIG_PM
317 318
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
319
#endif
320 321
	.enable_pm		= ahci_enable_alpm,
	.disable_pm		= ahci_disable_alpm,
322

L
Linus Torvalds 已提交
323 324 325 326
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

327 328 329 330 331 332 333
static const struct ata_port_operations ahci_vt8251_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
334
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
335 336 337 338 339 340 341 342 343 344 345 346 347 348
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
349 350 351
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

352
#ifdef CONFIG_PM
353 354
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
355
#endif
356 357 358 359 360

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394
static const struct ata_port_operations ahci_p5wdh_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_p5wdh_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

#ifdef CONFIG_PM
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
#endif

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

395 396
#define AHCI_HFLAGS(flags)	.private_data	= (void *)(flags)

397
static const struct ata_port_info ahci_port_info[] = {
L
Linus Torvalds 已提交
398 399
	/* board_ahci */
	{
T
Tejun Heo 已提交
400
		.flags		= AHCI_FLAG_COMMON,
401
		.link_flags	= AHCI_LFLAG_COMMON,
402
		.pio_mask	= 0x1f, /* pio0-4 */
403
		.udma_mask	= ATA_UDMA6,
L
Linus Torvalds 已提交
404 405
		.port_ops	= &ahci_ops,
	},
406 407
	/* board_ahci_vt8251 */
	{
T
Tejun Heo 已提交
408
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
409
		.flags		= AHCI_FLAG_COMMON,
410
		.link_flags	= AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
411
		.pio_mask	= 0x1f, /* pio0-4 */
412
		.udma_mask	= ATA_UDMA6,
413
		.port_ops	= &ahci_vt8251_ops,
414
	},
415 416
	/* board_ahci_ign_iferr */
	{
417 418
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
		.flags		= AHCI_FLAG_COMMON,
419
		.link_flags	= AHCI_LFLAG_COMMON,
420
		.pio_mask	= 0x1f, /* pio0-4 */
421
		.udma_mask	= ATA_UDMA6,
422 423
		.port_ops	= &ahci_ops,
	},
424 425
	/* board_ahci_sb600 */
	{
426
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
T
Tejun Heo 已提交
427
				 AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_PMP),
428
		.flags		= AHCI_FLAG_COMMON,
429
		.link_flags	= AHCI_LFLAG_COMMON,
430
		.pio_mask	= 0x1f, /* pio0-4 */
431
		.udma_mask	= ATA_UDMA6,
432 433
		.port_ops	= &ahci_ops,
	},
434 435
	/* board_ahci_mv */
	{
436 437
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
				 AHCI_HFLAG_MV_PATA),
438
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
439
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
440
		.link_flags	= AHCI_LFLAG_COMMON,
441 442 443 444
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
L
Linus Torvalds 已提交
445 446
};

447
static const struct pci_device_id ahci_pci_tbl[] = {
J
Jeff Garzik 已提交
448
	/* Intel */
449 450 451 452 453
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
454
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
455 456 457 458
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
T
Tejun Heo 已提交
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
476 477
	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
J
Jeff Garzik 已提交
478

479 480 481
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
J
Jeff Garzik 已提交
482 483

	/* ATI */
484
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
H
henry su 已提交
485 486 487 488 489 490
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
J
Jeff Garzik 已提交
491 492

	/* VIA */
493
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
T
Tejun Heo 已提交
494
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
J
Jeff Garzik 已提交
495 496

	/* NVIDIA */
497 498 499 500
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
501 502 503 504 505 506 507 508
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
509 510 511 512 513 514 515 516
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
541 542 543 544
	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci },		/* MCP79 */
P
Peer Chen 已提交
545 546 547 548 549 550 551 552
	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },		/* MCP79 */
J
Jeff Garzik 已提交
553

J
Jeff Garzik 已提交
554
	/* SiS */
555 556 557
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
J
Jeff Garzik 已提交
558

559 560 561
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */

562 563
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
564
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
565

L
Linus Torvalds 已提交
566 567 568 569 570 571 572 573
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
574
	.remove			= ata_pci_remove_one,
575
#ifdef CONFIG_PM
576 577
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
578
#endif
L
Linus Torvalds 已提交
579 580 581
};


582 583 584 585 586
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

587 588
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
L
Linus Torvalds 已提交
589
{
590
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
591

592 593 594 595 596 597
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
L
Linus Torvalds 已提交
598 599
}

600 601 602 603 604 605 606 607 608 609 610 611 612 613
static void ahci_enable_ahci(void __iomem *mmio)
{
	u32 tmp;

	/* turn on AHCI_EN */
	tmp = readl(mmio + HOST_CTL);
	if (!(tmp & HOST_AHCI_EN)) {
		tmp |= HOST_AHCI_EN;
		writel(tmp, mmio + HOST_CTL);
		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
		WARN_ON(!(tmp & HOST_AHCI_EN));
	}
}

614 615
/**
 *	ahci_save_initial_config - Save and fixup initial config values
616 617
 *	@pdev: target PCI device
 *	@hpriv: host private area to store config values
618 619 620 621 622 623 624 625 626 627 628
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
629 630
static void ahci_save_initial_config(struct pci_dev *pdev,
				     struct ahci_host_priv *hpriv)
631
{
632
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
633
	u32 cap, port_map;
634
	int i;
635

636 637 638
	/* make sure AHCI mode is enabled before accessing CAP */
	ahci_enable_ahci(mmio);

639 640 641 642 643 644
	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

645
	/* some chips have errata preventing 64bit use */
646
	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
T
Tejun Heo 已提交
647 648 649 650 651
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

652
	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
653 654 655 656 657
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

T
Tejun Heo 已提交
658 659 660 661 662 663
	if ((cap && HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do PMP, turning off CAP_PMP\n");
		cap &= ~HOST_CAP_PMP;
	}

664 665 666 667 668
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
669
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
670 671 672 673 674 675 676 677
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
			   hpriv->port_map,
			   hpriv->port_map & 0xf);

		port_map &= 0xf;
	}

678
	/* cross check port_map and cap.n_ports */
T
Tejun Heo 已提交
679
	if (port_map) {
680 681 682 683 684 685 686 687 688 689
		u32 tmp_port_map = port_map;
		int n_ports = ahci_nr_ports(cap);

		for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
			if (tmp_port_map & (1 << i)) {
				n_ports--;
				tmp_port_map &= ~(1 << i);
			}
		}

T
Tejun Heo 已提交
690 691
		/* If n_ports and port_map are inconsistent, whine and
		 * clear port_map and let it be generated from n_ports.
692
		 */
T
Tejun Heo 已提交
693
		if (n_ports || tmp_port_map) {
694
			dev_printk(KERN_WARNING, &pdev->dev,
695
				   "nr_ports (%u) and implemented port map "
T
Tejun Heo 已提交
696
				   "(0x%x) don't match, using nr_ports\n",
697
				   ahci_nr_ports(cap), port_map);
T
Tejun Heo 已提交
698 699 700 701 702 703
			port_map = 0;
		}
	}

	/* fabricate port_map from cap.nr_ports */
	if (!port_map) {
704
		port_map = (1 << ahci_nr_ports(cap)) - 1;
T
Tejun Heo 已提交
705 706 707 708 709
		dev_printk(KERN_WARNING, &pdev->dev,
			   "forcing PORTS_IMPL to 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
710 711
	}

712 713 714 715 716 717 718
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
719
 *	@host: target ATA host
720 721 722 723 724 725
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
726
static void ahci_restore_initial_config(struct ata_host *host)
727
{
728 729 730
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

731 732 733 734 735
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

T
Tejun Heo 已提交
736
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
L
Linus Torvalds 已提交
737
{
T
Tejun Heo 已提交
738 739 740 741 742 743 744 745
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
L
Linus Torvalds 已提交
746

T
Tejun Heo 已提交
747 748 749
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
750
	return 0;
L
Linus Torvalds 已提交
751 752
}

T
Tejun Heo 已提交
753
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
L
Linus Torvalds 已提交
754
{
T
Tejun Heo 已提交
755 756 757 758 759 760
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
L
Linus Torvalds 已提交
761
	}
T
Tejun Heo 已提交
762 763
	return -EINVAL;
}
L
Linus Torvalds 已提交
764

T
Tejun Heo 已提交
765 766 767 768 769 770 771 772 773 774
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
L
Linus Torvalds 已提交
775 776
}

777
static void ahci_start_engine(struct ata_port *ap)
778
{
779
	void __iomem *port_mmio = ahci_port_base(ap);
780 781
	u32 tmp;

782
	/* start DMA */
783
	tmp = readl(port_mmio + PORT_CMD);
784 785 786 787 788
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

789
static int ahci_stop_engine(struct ata_port *ap)
790
{
791
	void __iomem *port_mmio = ahci_port_base(ap);
792 793 794 795
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

796
	/* check if the HBA is idle */
797 798 799
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

800
	/* setting HBA to idle */
801 802 803
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

804
	/* wait for engine to stop. This could be as long as 500 msec */
805
	tmp = ata_wait_register(port_mmio + PORT_CMD,
806
				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
807
	if (tmp & PORT_CMD_LIST_ON)
808 809 810 811 812
		return -EIO;

	return 0;
}

813
static void ahci_start_fis_rx(struct ata_port *ap)
814
{
815 816 817
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
818 819 820
	u32 tmp;

	/* set FIS registers */
821 822 823 824
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
825

826 827 828 829
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
830 831 832 833 834 835 836 837 838 839

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

840
static int ahci_stop_fis_rx(struct ata_port *ap)
841
{
842
	void __iomem *port_mmio = ahci_port_base(ap);
843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

859
static void ahci_power_up(struct ata_port *ap)
860
{
861 862
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
863 864 865 866 867
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
868
	if (hpriv->cap & HOST_CAP_SSS) {
869 870 871 872 873 874 875 876
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
static void ahci_disable_alpm(struct ata_port *ap)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;

	/* IPM bits should be disabled by libata-core */
	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/* disable ALPM and ASP */
	cmd &= ~PORT_CMD_ASP;
	cmd &= ~PORT_CMD_ALPE;

	/* force the interface back to active */
	cmd |= PORT_CMD_ICC_ACTIVE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* wait 10ms to be sure we've come out of any low power state */
	msleep(10);

	/* clear out any PhyRdy stuff from interrupt status */
	writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);

	/* go ahead and clean out PhyRdy Change from Serror too */
	ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));

	/*
 	 * Clear flag to indicate that we should ignore all PhyRdy
 	 * state changes
 	 */
	hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;

	/*
 	 * Enable interrupts on Phy Ready.
 	 */
	pp->intr_mask |= PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * don't change the link pm policy - we can be called
 	 * just to turn of link pm temporarily
 	 */
}

static int ahci_enable_alpm(struct ata_port *ap,
	enum link_pm policy)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;
	u32 asp;

	/* Make sure the host is capable of link power management */
	if (!(hpriv->cap & HOST_CAP_ALPM))
		return -EINVAL;

	switch (policy) {
	case MAX_PERFORMANCE:
	case NOT_AVAILABLE:
		/*
 		 * if we came here with NOT_AVAILABLE,
 		 * it just means this is the first time we
 		 * have tried to enable - default to max performance,
 		 * and let the user go to lower power modes on request.
 		 */
		ahci_disable_alpm(ap);
		return 0;
	case MIN_POWER:
		/* configure HBA to enter SLUMBER */
		asp = PORT_CMD_ASP;
		break;
	case MEDIUM_POWER:
		/* configure HBA to enter PARTIAL */
		asp = 0;
		break;
	default:
		return -EINVAL;
	}

	/*
 	 * Disable interrupts on Phy Ready. This keeps us from
 	 * getting woken up due to spurious phy ready interrupts
	 * TBD - Hot plug should be done via polling now, is
	 * that even supported?
 	 */
	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * Set a flag to indicate that we should ignore all PhyRdy
 	 * state changes since these can happen now whenever we
 	 * change link state
 	 */
	hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;

	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/*
 	 * Set ASP based on Policy
 	 */
	cmd |= asp;

	/*
 	 * Setting this bit will instruct the HBA to aggressively
 	 * enter a lower power link state when it's appropriate and
 	 * based on the value set above for ASP
 	 */
	cmd |= PORT_CMD_ALPE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* IPM bits should be set by libata-core */
	return 0;
}

1001
#ifdef CONFIG_PM
1002
static void ahci_power_down(struct ata_port *ap)
1003
{
1004 1005
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
1006 1007
	u32 cmd, scontrol;

1008
	if (!(hpriv->cap & HOST_CAP_SSS))
1009
		return;
1010

1011 1012 1013 1014
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
1015

1016 1017 1018 1019
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
1020
}
1021
#endif
1022

1023
static void ahci_start_port(struct ata_port *ap)
1024 1025
{
	/* enable FIS reception */
1026
	ahci_start_fis_rx(ap);
1027 1028

	/* enable DMA */
1029
	ahci_start_engine(ap);
1030 1031
}

1032
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
1033 1034 1035 1036
{
	int rc;

	/* disable DMA */
1037
	rc = ahci_stop_engine(ap);
1038 1039 1040 1041 1042 1043
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
1044
	rc = ahci_stop_fis_rx(ap);
1045 1046 1047 1048 1049 1050 1051 1052
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

1053
static int ahci_reset_controller(struct ata_host *host)
1054
{
1055
	struct pci_dev *pdev = to_pci_dev(host->dev);
T
Tejun Heo 已提交
1056
	struct ahci_host_priv *hpriv = host->private_data;
1057
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1058
	u32 tmp;
1059

1060 1061 1062
	/* we must be in AHCI mode, before using anything
	 * AHCI-specific, such as HOST_RESET.
	 */
1063
	ahci_enable_ahci(mmio);
1064 1065

	/* global controller reset */
1066
	tmp = readl(mmio + HOST_CTL);
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	if ((tmp & HOST_RESET) == 0) {
		writel(tmp | HOST_RESET, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	/* reset must complete within 1 second, or
	 * the hardware should be considered fried.
	 */
	ssleep(1);

	tmp = readl(mmio + HOST_CTL);
	if (tmp & HOST_RESET) {
1079
		dev_printk(KERN_ERR, host->dev,
1080 1081 1082 1083
			   "controller reset failed (0x%x)\n", tmp);
		return -EIO;
	}

1084
	/* turn on AHCI mode */
1085
	ahci_enable_ahci(mmio);
1086

1087
	/* some registers might be cleared on reset.  restore initial values */
1088
	ahci_restore_initial_config(host);
1089 1090 1091 1092 1093 1094

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
T
Tejun Heo 已提交
1095 1096 1097 1098
		if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
			tmp16 |= hpriv->port_map;
			pci_write_config_word(pdev, 0x92, tmp16);
		}
1099 1100 1101 1102 1103
	}

	return 0;
}

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

1132
static void ahci_init_controller(struct ata_host *host)
1133
{
1134
	struct ahci_host_priv *hpriv = host->private_data;
1135 1136
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1137
	int i;
1138
	void __iomem *port_mmio;
1139 1140
	u32 tmp;

1141
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
		port_mmio = __ahci_port_base(host, 4);

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

1153 1154
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
1155

1156
		port_mmio = ahci_port_base(ap);
1157
		if (ata_port_is_dummy(ap))
1158 1159
			continue;

1160
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
1161 1162 1163 1164 1165 1166 1167 1168 1169
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

1170
static unsigned int ahci_dev_classify(struct ata_port *ap)
L
Linus Torvalds 已提交
1171
{
1172
	void __iomem *port_mmio = ahci_port_base(ap);
L
Linus Torvalds 已提交
1173
	struct ata_taskfile tf;
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
	u32 tmp;

	tmp = readl(port_mmio + PORT_SIG);
	tf.lbah		= (tmp >> 24)	& 0xff;
	tf.lbam		= (tmp >> 16)	& 0xff;
	tf.lbal		= (tmp >> 8)	& 0xff;
	tf.nsect	= (tmp)		& 0xff;

	return ata_dev_classify(&tf);
}

T
Tejun Heo 已提交
1185 1186
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts)
1187
{
T
Tejun Heo 已提交
1188 1189 1190 1191 1192 1193 1194 1195
	dma_addr_t cmd_tbl_dma;

	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;

	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
	pp->cmd_slot[tag].status = 0;
	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1196 1197
}

1198
static int ahci_kick_engine(struct ata_port *ap, int force_restart)
T
Tejun Heo 已提交
1199
{
T
Tejun Heo 已提交
1200
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
J
Jeff Garzik 已提交
1201
	struct ahci_host_priv *hpriv = ap->host->private_data;
1202
	u32 tmp;
1203
	int busy, rc;
1204

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
	/* do we need to kick the port? */
	busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
	if (!busy && !force_restart)
		return 0;

	/* stop engine */
	rc = ahci_stop_engine(ap);
	if (rc)
		goto out_restart;

	/* need to do CLO? */
	if (!busy) {
		rc = 0;
		goto out_restart;
	}

	if (!(hpriv->cap & HOST_CAP_CLO)) {
		rc = -EOPNOTSUPP;
		goto out_restart;
	}
1225

1226
	/* perform CLO */
1227 1228 1229 1230
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_CLO;
	writel(tmp, port_mmio + PORT_CMD);

1231
	rc = 0;
1232 1233 1234
	tmp = ata_wait_register(port_mmio + PORT_CMD,
				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
	if (tmp & PORT_CMD_CLO)
1235
		rc = -EIO;
1236

1237 1238 1239 1240
	/* restart engine */
 out_restart:
	ahci_start_engine(ap);
	return rc;
1241 1242
}

1243 1244 1245
static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
				struct ata_taskfile *tf, int is_cmd, u16 flags,
				unsigned long timeout_msec)
1246
{
1247
	const u32 cmd_fis_len = 5; /* five dwords */
T
Tejun Heo 已提交
1248
	struct ahci_port_priv *pp = ap->private_data;
1249
	void __iomem *port_mmio = ahci_port_base(ap);
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
	u8 *fis = pp->cmd_tbl;
	u32 tmp;

	/* prep the command */
	ata_tf_to_fis(tf, pmp, is_cmd, fis);
	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));

	/* issue & wait */
	writel(1, port_mmio + PORT_CMD_ISSUE);

	if (timeout_msec) {
		tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
					1, timeout_msec);
		if (tmp & 0x1) {
			ahci_kick_engine(ap, 1);
			return -EBUSY;
		}
	} else
		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1273
static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1274
			     int pmp, unsigned long deadline)
1275
{
T
Tejun Heo 已提交
1276
	struct ata_port *ap = link->ap;
T
Tejun Heo 已提交
1277
	const char *reason = NULL;
1278
	unsigned long now, msecs;
T
Tejun Heo 已提交
1279 1280 1281 1282 1283
	struct ata_taskfile tf;
	int rc;

	DPRINTK("ENTER\n");

T
Tejun Heo 已提交
1284
	if (ata_link_offline(link)) {
1285 1286 1287 1288 1289
		DPRINTK("PHY reports no device\n");
		*class = ATA_DEV_NONE;
		return 0;
	}

T
Tejun Heo 已提交
1290
	/* prepare for SRST (AHCI-1.1 10.4.1) */
1291
	rc = ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1292
	if (rc && rc != -EOPNOTSUPP)
T
Tejun Heo 已提交
1293
		ata_link_printk(link, KERN_WARNING,
T
Tejun Heo 已提交
1294
				"failed to reset engine (errno=%d)\n", rc);
T
Tejun Heo 已提交
1295

T
Tejun Heo 已提交
1296
	ata_tf_init(link->device, &tf);
T
Tejun Heo 已提交
1297 1298

	/* issue the first D2H Register FIS */
1299 1300 1301 1302 1303
	msecs = 0;
	now = jiffies;
	if (time_after(now, deadline))
		msecs = jiffies_to_msecs(deadline - now);

T
Tejun Heo 已提交
1304
	tf.ctl |= ATA_SRST;
1305
	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1306
				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
T
Tejun Heo 已提交
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
		rc = -EIO;
		reason = "1st FIS failed";
		goto fail;
	}

	/* spec says at least 5us, but be generous and sleep for 1ms */
	msleep(1);

	/* issue the second D2H Register FIS */
	tf.ctl &= ~ATA_SRST;
1317
	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
T
Tejun Heo 已提交
1318

1319 1320
	/* wait a while before checking status */
	ata_wait_after_reset(ap, deadline);
T
Tejun Heo 已提交
1321

T
Tejun Heo 已提交
1322 1323 1324 1325 1326
	rc = ata_wait_ready(ap, deadline);
	/* link occupied, -ENODEV too is an error */
	if (rc) {
		reason = "device not ready";
		goto fail;
T
Tejun Heo 已提交
1327
	}
T
Tejun Heo 已提交
1328
	*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1329 1330 1331 1332 1333

	DPRINTK("EXIT, class=%u\n", *class);
	return 0;

 fail:
T
Tejun Heo 已提交
1334
	ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
T
Tejun Heo 已提交
1335 1336 1337
	return rc;
}

T
Tejun Heo 已提交
1338
static int ahci_softreset(struct ata_link *link, unsigned int *class,
1339 1340
			  unsigned long deadline)
{
T
Tejun Heo 已提交
1341 1342 1343 1344 1345 1346
	int pmp = 0;

	if (link->ap->flags & ATA_FLAG_PMP)
		pmp = SATA_PMP_CTRL_PORT;

	return ahci_do_softreset(link, class, pmp, deadline);
1347 1348
}

T
Tejun Heo 已提交
1349
static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1350
			  unsigned long deadline)
1351
{
T
Tejun Heo 已提交
1352
	struct ata_port *ap = link->ap;
1353 1354 1355
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
1356 1357 1358
	int rc;

	DPRINTK("ENTER\n");
L
Linus Torvalds 已提交
1359

1360
	ahci_stop_engine(ap);
1361 1362

	/* clear D2H reception area to properly wait for D2H FIS */
T
Tejun Heo 已提交
1363
	ata_tf_init(link->device, &tf);
1364
	tf.command = 0x80;
1365
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1366

T
Tejun Heo 已提交
1367
	rc = sata_std_hardreset(link, class, deadline);
1368

1369
	ahci_start_engine(ap);
L
Linus Torvalds 已提交
1370

T
Tejun Heo 已提交
1371
	if (rc == 0 && ata_link_online(link))
1372
		*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1373
	if (rc != -EAGAIN && *class == ATA_DEV_UNKNOWN)
1374
		*class = ATA_DEV_NONE;
L
Linus Torvalds 已提交
1375

1376 1377 1378 1379
	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
	return rc;
}

T
Tejun Heo 已提交
1380
static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
1381
				 unsigned long deadline)
1382
{
T
Tejun Heo 已提交
1383
	struct ata_port *ap = link->ap;
1384
	u32 serror;
1385 1386 1387 1388
	int rc;

	DPRINTK("ENTER\n");

1389
	ahci_stop_engine(ap);
1390

T
Tejun Heo 已提交
1391
	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
1392
				 deadline);
1393 1394

	/* vt8251 needs SError cleared for the port to operate */
1395 1396
	ahci_scr_read(ap, SCR_ERROR, &serror);
	ahci_scr_write(ap, SCR_ERROR, serror);
1397

1398
	ahci_start_engine(ap);
1399 1400 1401 1402 1403 1404 1405 1406 1407

	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);

	/* vt8251 doesn't clear BSY on signature FIS reception,
	 * request follow-up softreset.
	 */
	return rc ?: -EAGAIN;
}

1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
				unsigned long deadline)
{
	struct ata_port *ap = link->ap;
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
	int rc;

	ahci_stop_engine(ap);

	/* clear D2H reception area to properly wait for D2H FIS */
	ata_tf_init(link->device, &tf);
	tf.command = 0x80;
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);

	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
				 deadline);

	ahci_start_engine(ap);

	if (rc || ata_link_offline(link))
		return rc;

	/* spec mandates ">= 2ms" before checking status */
	msleep(150);

	/* The pseudo configuration device on SIMG4726 attached to
	 * ASUS P5W-DH Deluxe doesn't send signature FIS after
	 * hardreset if no device is attached to the first downstream
	 * port && the pseudo device locks up on SRST w/ PMP==0.  To
	 * work around this, wait for !BSY only briefly.  If BSY isn't
	 * cleared, perform CLO and proceed to IDENTIFY (achieved by
	 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
	 *
	 * Wait for two seconds.  Devices attached to downstream port
	 * which can't process the following IDENTIFY after this will
	 * have to be reset again.  For most cases, this should
	 * suffice while making probing snappish enough.
	 */
	rc = ata_wait_ready(ap, jiffies + 2 * HZ);
	if (rc)
		ahci_kick_engine(ap, 0);

	return 0;
}

T
Tejun Heo 已提交
1455
static void ahci_postreset(struct ata_link *link, unsigned int *class)
1456
{
T
Tejun Heo 已提交
1457
	struct ata_port *ap = link->ap;
1458
	void __iomem *port_mmio = ahci_port_base(ap);
1459 1460
	u32 new_tmp, tmp;

T
Tejun Heo 已提交
1461
	ata_std_postreset(link, class);
1462 1463 1464

	/* Make sure port's ATAPI bit is set appropriately */
	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1465
	if (*class == ATA_DEV_ATAPI)
1466 1467 1468 1469 1470 1471 1472
		new_tmp |= PORT_CMD_ATAPI;
	else
		new_tmp &= ~PORT_CMD_ATAPI;
	if (new_tmp != tmp) {
		writel(new_tmp, port_mmio + PORT_CMD);
		readl(port_mmio + PORT_CMD); /* flush */
	}
L
Linus Torvalds 已提交
1473 1474
}

T
Tejun Heo 已提交
1475 1476 1477 1478 1479 1480
static int ahci_pmp_softreset(struct ata_link *link, unsigned int *class,
			      unsigned long deadline)
{
	return ahci_do_softreset(link, class, link->pmp, deadline);
}

L
Linus Torvalds 已提交
1481 1482
static u8 ahci_check_status(struct ata_port *ap)
{
T
Tejun Heo 已提交
1483
	void __iomem *mmio = ap->ioaddr.cmd_addr;
L
Linus Torvalds 已提交
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495

	return readl(mmio + PORT_TFDATA) & 0xFF;
}

static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
{
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;

	ata_tf_from_fis(d2h_fis, tf);
}

T
Tejun Heo 已提交
1496
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
L
Linus Torvalds 已提交
1497
{
1498
	struct scatterlist *sg;
T
Tejun Heo 已提交
1499 1500
	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
	unsigned int si;
L
Linus Torvalds 已提交
1501 1502 1503 1504 1505 1506

	VPRINTK("ENTER\n");

	/*
	 * Next, the S/G list.
	 */
T
Tejun Heo 已提交
1507
	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1508 1509 1510
		dma_addr_t addr = sg_dma_address(sg);
		u32 sg_len = sg_dma_len(sg);

T
Tejun Heo 已提交
1511 1512 1513
		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
L
Linus Torvalds 已提交
1514
	}
1515

T
Tejun Heo 已提交
1516
	return si;
L
Linus Torvalds 已提交
1517 1518 1519 1520
}

static void ahci_qc_prep(struct ata_queued_cmd *qc)
{
1521 1522
	struct ata_port *ap = qc->ap;
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1523
	int is_atapi = ata_is_atapi(qc->tf.protocol);
T
Tejun Heo 已提交
1524
	void *cmd_tbl;
L
Linus Torvalds 已提交
1525 1526
	u32 opts;
	const u32 cmd_fis_len = 5; /* five dwords */
1527
	unsigned int n_elem;
L
Linus Torvalds 已提交
1528 1529 1530 1531 1532

	/*
	 * Fill in command table information.  First, the header,
	 * a SATA Register - Host to Device command FIS.
	 */
T
Tejun Heo 已提交
1533 1534
	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;

T
Tejun Heo 已提交
1535
	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1536
	if (is_atapi) {
T
Tejun Heo 已提交
1537 1538
		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1539
	}
L
Linus Torvalds 已提交
1540

1541 1542
	n_elem = 0;
	if (qc->flags & ATA_QCFLAG_DMAMAP)
T
Tejun Heo 已提交
1543
		n_elem = ahci_fill_sg(qc, cmd_tbl);
L
Linus Torvalds 已提交
1544

1545 1546 1547
	/*
	 * Fill in command slot information.
	 */
T
Tejun Heo 已提交
1548
	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1549 1550 1551
	if (qc->tf.flags & ATA_TFLAG_WRITE)
		opts |= AHCI_CMD_WRITE;
	if (is_atapi)
1552
		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1553

T
Tejun Heo 已提交
1554
	ahci_fill_cmd_slot(pp, qc->tag, opts);
L
Linus Torvalds 已提交
1555 1556
}

T
Tejun Heo 已提交
1557
static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
L
Linus Torvalds 已提交
1558
{
1559
	struct ahci_host_priv *hpriv = ap->host->private_data;
T
Tejun Heo 已提交
1560
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1561 1562 1563 1564
	struct ata_eh_info *host_ehi = &ap->link.eh_info;
	struct ata_link *link = NULL;
	struct ata_queued_cmd *active_qc;
	struct ata_eh_info *active_ehi;
T
Tejun Heo 已提交
1565
	u32 serror;
L
Linus Torvalds 已提交
1566

T
Tejun Heo 已提交
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
	/* determine active link */
	ata_port_for_each_link(link, ap)
		if (ata_link_active(link))
			break;
	if (!link)
		link = &ap->link;

	active_qc = ata_qc_from_tag(ap, link->active_tag);
	active_ehi = &link->eh_info;

	/* record irq stat */
	ata_ehi_clear_desc(host_ehi);
	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
L
Linus Torvalds 已提交
1580

T
Tejun Heo 已提交
1581
	/* AHCI needs SError cleared; otherwise, it might lock up */
1582
	ahci_scr_read(ap, SCR_ERROR, &serror);
T
Tejun Heo 已提交
1583
	ahci_scr_write(ap, SCR_ERROR, serror);
T
Tejun Heo 已提交
1584
	host_ehi->serror |= serror;
T
Tejun Heo 已提交
1585

1586
	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1587
	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1588 1589
		irq_stat &= ~PORT_IRQ_IF_ERR;

1590
	if (irq_stat & PORT_IRQ_TF_ERR) {
T
Tejun Heo 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599
		/* If qc is active, charge it; otherwise, the active
		 * link.  There's no active qc on NCQ errors.  It will
		 * be determined by EH by reading log page 10h.
		 */
		if (active_qc)
			active_qc->err_mask |= AC_ERR_DEV;
		else
			active_ehi->err_mask |= AC_ERR_DEV;

1600
		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
T
Tejun Heo 已提交
1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617
			host_ehi->serror &= ~SERR_INTERNAL;
	}

	if (irq_stat & PORT_IRQ_UNK_FIS) {
		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);

		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi,
				  "unknown FIS %08x %08x %08x %08x" ,
				  unk[0], unk[1], unk[2], unk[3]);
	}

	if (ap->nr_pmp_links && (irq_stat & PORT_IRQ_BAD_PMP)) {
		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1618
	}
T
Tejun Heo 已提交
1619 1620

	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
T
Tejun Heo 已提交
1621 1622 1623
		host_ehi->err_mask |= AC_ERR_HOST_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "host bus error");
L
Linus Torvalds 已提交
1624 1625
	}

T
Tejun Heo 已提交
1626
	if (irq_stat & PORT_IRQ_IF_ERR) {
T
Tejun Heo 已提交
1627 1628 1629
		host_ehi->err_mask |= AC_ERR_ATA_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "interface fatal error");
T
Tejun Heo 已提交
1630
	}
L
Linus Torvalds 已提交
1631

T
Tejun Heo 已提交
1632
	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
T
Tejun Heo 已提交
1633 1634 1635
		ata_ehi_hotplugged(host_ehi);
		ata_ehi_push_desc(host_ehi, "%s",
			irq_stat & PORT_IRQ_CONNECT ?
T
Tejun Heo 已提交
1636 1637 1638 1639
			"connection status changed" : "PHY RDY changed");
	}

	/* okay, let's hand over to EH */
1640

T
Tejun Heo 已提交
1641 1642 1643 1644
	if (irq_stat & PORT_IRQ_FREEZE)
		ata_port_freeze(ap);
	else
		ata_port_abort(ap);
L
Linus Torvalds 已提交
1645 1646
}

1647
static void ahci_port_intr(struct ata_port *ap)
L
Linus Torvalds 已提交
1648
{
1649
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
T
Tejun Heo 已提交
1650
	struct ata_eh_info *ehi = &ap->link.eh_info;
1651
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1652
	struct ahci_host_priv *hpriv = ap->host->private_data;
1653
	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
T
Tejun Heo 已提交
1654
	u32 status, qc_active;
1655
	int rc;
L
Linus Torvalds 已提交
1656 1657 1658 1659

	status = readl(port_mmio + PORT_IRQ_STAT);
	writel(status, port_mmio + PORT_IRQ_STAT);

1660 1661 1662 1663
	/* ignore BAD_PMP while resetting */
	if (unlikely(resetting))
		status &= ~PORT_IRQ_BAD_PMP;

1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
	/* If we are getting PhyRdy, this is
 	 * just a power state change, we should
 	 * clear out this, plus the PhyRdy/Comm
 	 * Wake bits from Serror
 	 */
	if ((hpriv->flags & AHCI_HFLAG_NO_HOTPLUG) &&
		(status & PORT_IRQ_PHYRDY)) {
		status &= ~PORT_IRQ_PHYRDY;
		ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));
	}

T
Tejun Heo 已提交
1675 1676 1677
	if (unlikely(status & PORT_IRQ_ERROR)) {
		ahci_error_intr(ap, status);
		return;
L
Linus Torvalds 已提交
1678 1679
	}

1680
	if (status & PORT_IRQ_SDB_FIS) {
T
Tejun Heo 已提交
1681 1682 1683 1684 1685 1686 1687 1688
		/* If SNotification is available, leave notification
		 * handling to sata_async_notification().  If not,
		 * emulate it by snooping SDB FIS RX area.
		 *
		 * Snooping FIS RX area is probably cheaper than
		 * poking SNotification but some constrollers which
		 * implement SNotification, ICH9 for example, don't
		 * store AN SDB FIS into receive area.
1689
		 */
T
Tejun Heo 已提交
1690
		if (hpriv->cap & HOST_CAP_SNTF)
1691
			sata_async_notification(ap);
T
Tejun Heo 已提交
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
		else {
			/* If the 'N' bit in word 0 of the FIS is set,
			 * we just received asynchronous notification.
			 * Tell libata about it.
			 */
			const __le32 *f = pp->rx_fis + RX_FIS_SDB;
			u32 f0 = le32_to_cpu(f[0]);

			if (f0 & (1 << 15))
				sata_async_notification(ap);
		}
1703 1704
	}

T
Tejun Heo 已提交
1705 1706
	/* pp->active_link is valid iff any command is in flight */
	if (ap->qc_active && pp->active_link->sactive)
T
Tejun Heo 已提交
1707 1708 1709 1710 1711
		qc_active = readl(port_mmio + PORT_SCR_ACT);
	else
		qc_active = readl(port_mmio + PORT_CMD_ISSUE);

	rc = ata_qc_complete_multiple(ap, qc_active, NULL);
1712

1713 1714
	/* while resetting, invalid completions are expected */
	if (unlikely(rc < 0 && !resetting)) {
T
Tejun Heo 已提交
1715 1716 1717
		ehi->err_mask |= AC_ERR_HSM;
		ehi->action |= ATA_EH_SOFTRESET;
		ata_port_freeze(ap);
L
Linus Torvalds 已提交
1718 1719 1720 1721 1722 1723 1724 1725
	}
}

static void ahci_irq_clear(struct ata_port *ap)
{
	/* TODO */
}

1726
static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
L
Linus Torvalds 已提交
1727
{
J
Jeff Garzik 已提交
1728
	struct ata_host *host = dev_instance;
L
Linus Torvalds 已提交
1729 1730
	struct ahci_host_priv *hpriv;
	unsigned int i, handled = 0;
1731
	void __iomem *mmio;
L
Linus Torvalds 已提交
1732 1733 1734 1735
	u32 irq_stat, irq_ack = 0;

	VPRINTK("ENTER\n");

J
Jeff Garzik 已提交
1736
	hpriv = host->private_data;
T
Tejun Heo 已提交
1737
	mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
1738 1739 1740 1741 1742 1743 1744

	/* sigh.  0xffffffff is a valid return from h/w */
	irq_stat = readl(mmio + HOST_IRQ_STAT);
	irq_stat &= hpriv->port_map;
	if (!irq_stat)
		return IRQ_NONE;

1745
	spin_lock(&host->lock);
L
Linus Torvalds 已提交
1746

1747
	for (i = 0; i < host->n_ports; i++) {
L
Linus Torvalds 已提交
1748 1749
		struct ata_port *ap;

1750 1751 1752
		if (!(irq_stat & (1 << i)))
			continue;

J
Jeff Garzik 已提交
1753
		ap = host->ports[i];
1754
		if (ap) {
1755
			ahci_port_intr(ap);
1756 1757 1758
			VPRINTK("port %u\n", i);
		} else {
			VPRINTK("port %u (no irq)\n", i);
1759
			if (ata_ratelimit())
J
Jeff Garzik 已提交
1760
				dev_printk(KERN_WARNING, host->dev,
1761
					"interrupt on disabled port %u\n", i);
L
Linus Torvalds 已提交
1762
		}
1763 1764

		irq_ack |= (1 << i);
L
Linus Torvalds 已提交
1765 1766 1767 1768 1769 1770 1771
	}

	if (irq_ack) {
		writel(irq_ack, mmio + HOST_IRQ_STAT);
		handled = 1;
	}

J
Jeff Garzik 已提交
1772
	spin_unlock(&host->lock);
L
Linus Torvalds 已提交
1773 1774 1775 1776 1777 1778

	VPRINTK("EXIT\n");

	return IRQ_RETVAL(handled);
}

1779
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
L
Linus Torvalds 已提交
1780 1781
{
	struct ata_port *ap = qc->ap;
1782
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1783 1784 1785 1786 1787 1788 1789
	struct ahci_port_priv *pp = ap->private_data;

	/* Keep track of the currently active link.  It will be used
	 * in completion path to determine whether NCQ phase is in
	 * progress.
	 */
	pp->active_link = qc->dev->link;
L
Linus Torvalds 已提交
1790

T
Tejun Heo 已提交
1791 1792 1793
	if (qc->tf.protocol == ATA_PROT_NCQ)
		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798
	readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1799 1800
static void ahci_freeze(struct ata_port *ap)
{
1801
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1802 1803 1804 1805 1806 1807 1808

	/* turn IRQ off */
	writel(0, port_mmio + PORT_IRQ_MASK);
}

static void ahci_thaw(struct ata_port *ap)
{
T
Tejun Heo 已提交
1809
	void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
1810
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1811
	u32 tmp;
1812
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1813 1814 1815 1816

	/* clear IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	writel(tmp, port_mmio + PORT_IRQ_STAT);
1817
	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
T
Tejun Heo 已提交
1818

1819 1820
	/* turn IRQ back on */
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1821 1822 1823 1824
}

static void ahci_error_handler(struct ata_port *ap)
{
1825
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
T
Tejun Heo 已提交
1826
		/* restart engine */
1827 1828
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
T
Tejun Heo 已提交
1829 1830 1831
	}

	/* perform recovery */
T
Tejun Heo 已提交
1832 1833 1834 1835
	sata_pmp_do_eh(ap, ata_std_prereset, ahci_softreset,
		       ahci_hardreset, ahci_postreset,
		       sata_pmp_std_prereset, ahci_pmp_softreset,
		       sata_pmp_std_hardreset, sata_pmp_std_postreset);
T
Tejun Heo 已提交
1836 1837
}

1838 1839 1840 1841
static void ahci_vt8251_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
1842 1843
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
1844 1845 1846 1847 1848 1849 1850
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
		  ahci_postreset);
}

1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
static void ahci_p5wdh_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_p5wdh_hardreset,
		  ahci_postreset);
}

T
Tejun Heo 已提交
1864 1865 1866 1867
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;

1868 1869 1870
	/* make DMA engine forget about the failed command */
	if (qc->flags & ATA_QCFLAG_FAILED)
		ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1871 1872
}

T
Tejun Heo 已提交
1873 1874 1875
static void ahci_pmp_attach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1876
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1877 1878 1879 1880 1881
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd |= PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1882 1883 1884

	pp->intr_mask |= PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1885 1886 1887 1888 1889
}

static void ahci_pmp_detach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1890
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1891 1892 1893 1894 1895
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd &= ~PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1896 1897 1898

	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1899 1900
}

1901 1902 1903 1904 1905
static int ahci_port_resume(struct ata_port *ap)
{
	ahci_power_up(ap);
	ahci_start_port(ap);

T
Tejun Heo 已提交
1906 1907 1908 1909 1910
	if (ap->nr_pmp_links)
		ahci_pmp_attach(ap);
	else
		ahci_pmp_detach(ap);

1911 1912 1913
	return 0;
}

1914
#ifdef CONFIG_PM
1915 1916 1917 1918 1919
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
{
	const char *emsg = NULL;
	int rc;

1920
	rc = ahci_deinit_port(ap, &emsg);
1921
	if (rc == 0)
1922
		ahci_power_down(ap);
1923
	else {
1924
		ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
1925
		ahci_start_port(ap);
1926 1927 1928 1929 1930 1931 1932
	}

	return rc;
}

static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
{
J
Jeff Garzik 已提交
1933
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
T
Tejun Heo 已提交
1934
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
	u32 ctl;

	if (mesg.event == PM_EVENT_SUSPEND) {
		/* AHCI spec rev1.1 section 8.3.3:
		 * Software must disable interrupts prior to requesting a
		 * transition of the HBA to D3 state.
		 */
		ctl = readl(mmio + HOST_CTL);
		ctl &= ~HOST_IRQ_EN;
		writel(ctl, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	return ata_pci_device_suspend(pdev, mesg);
}

static int ahci_pci_device_resume(struct pci_dev *pdev)
{
J
Jeff Garzik 已提交
1953
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
1954 1955
	int rc;

1956 1957 1958
	rc = ata_pci_device_do_resume(pdev);
	if (rc)
		return rc;
1959 1960

	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1961
		rc = ahci_reset_controller(host);
1962 1963 1964
		if (rc)
			return rc;

1965
		ahci_init_controller(host);
1966 1967
	}

J
Jeff Garzik 已提交
1968
	ata_host_resume(host);
1969 1970 1971

	return 0;
}
1972
#endif
1973

1974 1975
static int ahci_port_start(struct ata_port *ap)
{
J
Jeff Garzik 已提交
1976
	struct device *dev = ap->host->dev;
1977 1978 1979 1980 1981
	struct ahci_port_priv *pp;
	void *mem;
	dma_addr_t mem_dma;
	int rc;

1982
	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1983 1984 1985 1986
	if (!pp)
		return -ENOMEM;

	rc = ata_pad_alloc(ap, dev);
1987
	if (rc)
1988 1989
		return rc;

1990 1991 1992
	mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
				  GFP_KERNEL);
	if (!mem)
1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
		return -ENOMEM;
	memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);

	/*
	 * First item in chunk of DMA memory: 32-slot command table,
	 * 32 bytes each in size
	 */
	pp->cmd_slot = mem;
	pp->cmd_slot_dma = mem_dma;

	mem += AHCI_CMD_SLOT_SZ;
	mem_dma += AHCI_CMD_SLOT_SZ;

	/*
	 * Second item: Received-FIS area
	 */
	pp->rx_fis = mem;
	pp->rx_fis_dma = mem_dma;

	mem += AHCI_RX_FIS_SZ;
	mem_dma += AHCI_RX_FIS_SZ;

	/*
	 * Third item: data area for storing a single command
	 * and its scatter-gather table
	 */
	pp->cmd_tbl = mem;
	pp->cmd_tbl_dma = mem_dma;

2022
	/*
2023 2024 2025
	 * Save off initial list of interrupts to be enabled.
	 * This could be changed later
	 */
2026 2027
	pp->intr_mask = DEF_PORT_IRQ;

2028 2029
	ap->private_data = pp;

2030 2031
	/* engage engines, captain */
	return ahci_port_resume(ap);
2032 2033 2034 2035
}

static void ahci_port_stop(struct ata_port *ap)
{
2036 2037
	const char *emsg = NULL;
	int rc;
2038

2039
	/* de-initialize port */
2040
	rc = ahci_deinit_port(ap, &emsg);
2041 2042
	if (rc)
		ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
2043 2044
}

2045
static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
L
Linus Torvalds 已提交
2046 2047 2048 2049 2050 2051 2052 2053 2054
{
	int rc;

	if (using_dac &&
	    !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
		if (rc) {
			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
			if (rc) {
2055 2056
				dev_printk(KERN_ERR, &pdev->dev,
					   "64-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2057 2058 2059 2060 2061 2062
				return rc;
			}
		}
	} else {
		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2063 2064
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2065 2066 2067 2068
			return rc;
		}
		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2069 2070
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit consistent DMA enable failed\n");
L
Linus Torvalds 已提交
2071 2072 2073 2074 2075 2076
			return rc;
		}
	}
	return 0;
}

2077
static void ahci_print_info(struct ata_host *host)
L
Linus Torvalds 已提交
2078
{
2079 2080 2081
	struct ahci_host_priv *hpriv = host->private_data;
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
	u32 vers, cap, impl, speed;
	const char *speed_s;
	u16 cc;
	const char *scc_s;

	vers = readl(mmio + HOST_VERSION);
	cap = hpriv->cap;
	impl = hpriv->port_map;

	speed = (cap >> 20) & 0xf;
	if (speed == 1)
		speed_s = "1.5";
	else if (speed == 2)
		speed_s = "3";
	else
		speed_s = "?";

	pci_read_config_word(pdev, 0x0a, &cc);
2100
	if (cc == PCI_CLASS_STORAGE_IDE)
L
Linus Torvalds 已提交
2101
		scc_s = "IDE";
2102
	else if (cc == PCI_CLASS_STORAGE_SATA)
L
Linus Torvalds 已提交
2103
		scc_s = "SATA";
2104
	else if (cc == PCI_CLASS_STORAGE_RAID)
L
Linus Torvalds 已提交
2105 2106 2107 2108
		scc_s = "RAID";
	else
		scc_s = "unknown";

2109 2110
	dev_printk(KERN_INFO, &pdev->dev,
		"AHCI %02x%02x.%02x%02x "
L
Linus Torvalds 已提交
2111
		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2112
		,
L
Linus Torvalds 已提交
2113

2114 2115 2116 2117
		(vers >> 24) & 0xff,
		(vers >> 16) & 0xff,
		(vers >> 8) & 0xff,
		vers & 0xff,
L
Linus Torvalds 已提交
2118 2119 2120 2121 2122 2123 2124

		((cap >> 8) & 0x1f) + 1,
		(cap & 0x1f) + 1,
		speed_s,
		impl,
		scc_s);

2125 2126
	dev_printk(KERN_INFO, &pdev->dev,
		"flags: "
T
Tejun Heo 已提交
2127 2128
		"%s%s%s%s%s%s%s"
		"%s%s%s%s%s%s%s\n"
2129
		,
L
Linus Torvalds 已提交
2130 2131 2132

		cap & (1 << 31) ? "64bit " : "",
		cap & (1 << 30) ? "ncq " : "",
T
Tejun Heo 已提交
2133
		cap & (1 << 29) ? "sntf " : "",
L
Linus Torvalds 已提交
2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
		cap & (1 << 28) ? "ilck " : "",
		cap & (1 << 27) ? "stag " : "",
		cap & (1 << 26) ? "pm " : "",
		cap & (1 << 25) ? "led " : "",

		cap & (1 << 24) ? "clo " : "",
		cap & (1 << 19) ? "nz " : "",
		cap & (1 << 18) ? "only " : "",
		cap & (1 << 17) ? "pmp " : "",
		cap & (1 << 15) ? "pio " : "",
		cap & (1 << 14) ? "slum " : "",
		cap & (1 << 13) ? "part " : ""
		);
}

2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
/* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
 * hardwired to on-board SIMG 4726.  The chipset is ICH8 and doesn't
 * support PMP and the 4726 either directly exports the device
 * attached to the first downstream port or acts as a hardware storage
 * controller and emulate a single ATA device (can be RAID 0/1 or some
 * other configuration).
 *
 * When there's no device attached to the first downstream port of the
 * 4726, "Config Disk" appears, which is a pseudo ATA device to
 * configure the 4726.  However, ATA emulation of the device is very
 * lame.  It doesn't send signature D2H Reg FIS after the initial
 * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
 *
 * The following function works around the problem by always using
 * hardreset on the port and not depending on receiving signature FIS
 * afterward.  If signature FIS isn't received soon, ATA class is
 * assumed without follow-up softreset.
 */
static void ahci_p5wdh_workaround(struct ata_host *host)
{
	static struct dmi_system_id sysids[] = {
		{
			.ident = "P5W DH Deluxe",
			.matches = {
				DMI_MATCH(DMI_SYS_VENDOR,
					  "ASUSTEK COMPUTER INC"),
				DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
			},
		},
		{ }
	};
	struct pci_dev *pdev = to_pci_dev(host->dev);

	if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
	    dmi_check_system(sysids)) {
		struct ata_port *ap = host->ports[1];

		dev_printk(KERN_INFO, &pdev->dev, "enabling ASUS P5W DH "
			   "Deluxe on-board SIMG4726 workaround\n");

		ap->ops = &ahci_p5wdh_ops;
		ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
	}
}

2194
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
L
Linus Torvalds 已提交
2195 2196
{
	static int printed_version;
2197 2198
	struct ata_port_info pi = ahci_port_info[ent->driver_data];
	const struct ata_port_info *ppi[] = { &pi, NULL };
2199
	struct device *dev = &pdev->dev;
L
Linus Torvalds 已提交
2200
	struct ahci_host_priv *hpriv;
2201 2202
	struct ata_host *host;
	int i, rc;
L
Linus Torvalds 已提交
2203 2204 2205

	VPRINTK("ENTER\n");

T
Tejun Heo 已提交
2206 2207
	WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);

L
Linus Torvalds 已提交
2208
	if (!printed_version++)
2209
		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
L
Linus Torvalds 已提交
2210

2211
	/* acquire resources */
2212
	rc = pcim_enable_device(pdev);
L
Linus Torvalds 已提交
2213 2214 2215
	if (rc)
		return rc;

T
Tejun Heo 已提交
2216 2217
	rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
	if (rc == -EBUSY)
2218
		pcim_pin_device(pdev);
T
Tejun Heo 已提交
2219
	if (rc)
2220
		return rc;
L
Linus Torvalds 已提交
2221

2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
	if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
	    (pdev->device == 0x2652 || pdev->device == 0x2653)) {
		u8 map;

		/* ICH6s share the same PCI ID for both piix and ahci
		 * modes.  Enabling ahci mode while MAP indicates
		 * combined mode is a bad idea.  Yield to ata_piix.
		 */
		pci_read_config_byte(pdev, ICH_MAP, &map);
		if (map & 0x3) {
			dev_printk(KERN_INFO, &pdev->dev, "controller is in "
				   "combined mode, can't enable AHCI mode\n");
			return -ENODEV;
		}
	}

2238 2239 2240
	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
	if (!hpriv)
		return -ENOMEM;
2241 2242 2243 2244
	hpriv->flags |= (unsigned long)pi.private_data;

	if ((hpriv->flags & AHCI_HFLAG_NO_MSI) || pci_enable_msi(pdev))
		pci_intx(pdev, 1);
L
Linus Torvalds 已提交
2245

2246
	/* save initial config */
2247
	ahci_save_initial_config(pdev, hpriv);
L
Linus Torvalds 已提交
2248

2249
	/* prepare host */
2250
	if (hpriv->cap & HOST_CAP_NCQ)
2251
		pi.flags |= ATA_FLAG_NCQ;
L
Linus Torvalds 已提交
2252

T
Tejun Heo 已提交
2253 2254 2255
	if (hpriv->cap & HOST_CAP_PMP)
		pi.flags |= ATA_FLAG_PMP;

2256 2257 2258 2259 2260 2261 2262
	host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map));
	if (!host)
		return -ENOMEM;
	host->iomap = pcim_iomap_table(pdev);
	host->private_data = hpriv;

	for (i = 0; i < host->n_ports; i++) {
2263 2264
		struct ata_port *ap = host->ports[i];
		void __iomem *port_mmio = ahci_port_base(ap);
2265

2266 2267 2268 2269
		ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
		ata_port_pbar_desc(ap, AHCI_PCI_BAR,
				   0x100 + ap->port_no * 0x80, "port");

2270 2271 2272
		/* set initial link pm policy */
		ap->pm_policy = NOT_AVAILABLE;

2273
		/* standard SATA port setup */
T
Tejun Heo 已提交
2274
		if (hpriv->port_map & (1 << i))
2275
			ap->ioaddr.cmd_addr = port_mmio;
2276 2277 2278 2279

		/* disabled/not-implemented port */
		else
			ap->ops = &ata_dummy_port_ops;
2280
	}
2281

2282 2283 2284
	/* apply workaround for ASUS P5W DH Deluxe mainboard */
	ahci_p5wdh_workaround(host);

2285 2286
	/* initialize adapter */
	rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
L
Linus Torvalds 已提交
2287
	if (rc)
2288
		return rc;
L
Linus Torvalds 已提交
2289

2290 2291 2292
	rc = ahci_reset_controller(host);
	if (rc)
		return rc;
L
Linus Torvalds 已提交
2293

2294 2295
	ahci_init_controller(host);
	ahci_print_info(host);
L
Linus Torvalds 已提交
2296

2297 2298 2299
	pci_set_master(pdev);
	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
				 &ahci_sht);
2300
}
L
Linus Torvalds 已提交
2301 2302 2303

static int __init ahci_init(void)
{
2304
	return pci_register_driver(&ahci_pci_driver);
L
Linus Torvalds 已提交
2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316
}

static void __exit ahci_exit(void)
{
	pci_unregister_driver(&ahci_pci_driver);
}


MODULE_AUTHOR("Jeff Garzik");
MODULE_DESCRIPTION("AHCI SATA low-level driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
2317
MODULE_VERSION(DRV_VERSION);
L
Linus Torvalds 已提交
2318 2319 2320

module_init(ahci_init);
module_exit(ahci_exit);