ahci.c 64.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *  ahci.c - AHCI SATA support
 *
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
L
Linus Torvalds 已提交
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39 40 41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
44
#include <linux/dmi.h>
L
Linus Torvalds 已提交
45
#include <scsi/scsi_host.h>
46
#include <scsi/scsi_cmnd.h>
L
Linus Torvalds 已提交
47 48 49
#include <linux/libata.h>

#define DRV_NAME	"ahci"
T
Tejun Heo 已提交
50
#define DRV_VERSION	"3.0"
L
Linus Torvalds 已提交
51

52 53 54 55
static int ahci_skip_host_reset;
module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");

56 57 58
static int ahci_enable_alpm(struct ata_port *ap,
		enum link_pm policy);
static void ahci_disable_alpm(struct ata_port *ap);
L
Linus Torvalds 已提交
59 60 61

enum {
	AHCI_PCI_BAR		= 5,
62
	AHCI_MAX_PORTS		= 32,
L
Linus Torvalds 已提交
63 64
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
65
	AHCI_USE_CLUSTERING	= 1,
T
Tejun Heo 已提交
66
	AHCI_MAX_CMDS		= 32,
67
	AHCI_CMD_SZ		= 32,
T
Tejun Heo 已提交
68
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
L
Linus Torvalds 已提交
69
	AHCI_RX_FIS_SZ		= 256,
70
	AHCI_CMD_TBL_CDB	= 0x40,
71 72 73 74
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
L
Linus Torvalds 已提交
75 76 77 78
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
79
	AHCI_CMD_PREFETCH	= (1 << 7),
T
Tejun Heo 已提交
80 81
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
L
Linus Torvalds 已提交
82 83

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
84
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
T
Tejun Heo 已提交
85
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
L
Linus Torvalds 已提交
86 87

	board_ahci		= 0,
T
Tejun Heo 已提交
88 89 90 91
	board_ahci_vt8251	= 1,
	board_ahci_ign_iferr	= 2,
	board_ahci_sb600	= 3,
	board_ahci_mv		= 4,
92
	board_ahci_sb700	= 5,
L
Linus Torvalds 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
107
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
T
Tejun Heo 已提交
108
	HOST_CAP_PMP		= (1 << 17), /* Port Multiplier support */
T
Tejun Heo 已提交
109
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
110
	HOST_CAP_ALPM		= (1 << 26), /* Aggressive Link PM support */
111
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
T
Tejun Heo 已提交
112
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
113
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
114
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
L
Linus Torvalds 已提交
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
T
Tejun Heo 已提交
131
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
L
Linus Torvalds 已提交
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

T
Tejun Heo 已提交
153 154 155
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
156
				  PORT_IRQ_PHYRDY |
T
Tejun Heo 已提交
157 158
				  PORT_IRQ_UNK_FIS |
				  PORT_IRQ_BAD_PMP,
T
Tejun Heo 已提交
159 160 161 162 163 164
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
L
Linus Torvalds 已提交
165 166

	/* PORT_CMD bits */
167 168
	PORT_CMD_ASP		= (1 << 27), /* Aggressive Slumber/Partial */
	PORT_CMD_ALPE		= (1 << 26), /* Aggressive Link PM enable */
169
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
T
Tejun Heo 已提交
170
	PORT_CMD_PMP		= (1 << 17), /* PMP attached */
L
Linus Torvalds 已提交
171 172 173
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
T
Tejun Heo 已提交
174
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
L
Linus Torvalds 已提交
175 176 177 178
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

179
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
L
Linus Torvalds 已提交
180 181 182
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
183

184 185 186 187 188 189 190
	/* hpriv->flags bits */
	AHCI_HFLAG_NO_NCQ		= (1 << 0),
	AHCI_HFLAG_IGN_IRQ_IF_ERR	= (1 << 1), /* ignore IRQ_IF_ERR */
	AHCI_HFLAG_IGN_SERR_INTERNAL	= (1 << 2), /* ignore SERR_INTERNAL */
	AHCI_HFLAG_32BIT_ONLY		= (1 << 3), /* force 32bit */
	AHCI_HFLAG_MV_PATA		= (1 << 4), /* PATA port */
	AHCI_HFLAG_NO_MSI		= (1 << 5), /* no PCI MSI */
T
Tejun Heo 已提交
191
	AHCI_HFLAG_NO_PMP		= (1 << 6), /* no PMP */
192
	AHCI_HFLAG_NO_HOTPLUG		= (1 << 7), /* ignore PxSERR.DIAG.N */
193
	AHCI_HFLAG_SECT255		= (1 << 8), /* max 255 sectors */
194

195
	/* ap->flags bits */
T
Tejun Heo 已提交
196 197 198

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
199 200
					  ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
					  ATA_FLAG_IPM,
201
	AHCI_LFLAG_COMMON		= ATA_LFLAG_SKIP_D2H_BSY,
202 203

	ICH_MAP				= 0x90, /* ICH MAP register */
L
Linus Torvalds 已提交
204 205 206
};

struct ahci_cmd_hdr {
A
Al Viro 已提交
207 208 209 210 211
	__le32			opts;
	__le32			status;
	__le32			tbl_addr;
	__le32			tbl_addr_hi;
	__le32			reserved[4];
L
Linus Torvalds 已提交
212 213 214
};

struct ahci_sg {
A
Al Viro 已提交
215 216 217 218
	__le32			addr;
	__le32			addr_hi;
	__le32			reserved;
	__le32			flags_size;
L
Linus Torvalds 已提交
219 220 221
};

struct ahci_host_priv {
222
	unsigned int		flags;		/* AHCI_HFLAG_* */
223 224 225 226
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
L
Linus Torvalds 已提交
227 228 229
};

struct ahci_port_priv {
T
Tejun Heo 已提交
230
	struct ata_link		*active_link;
L
Linus Torvalds 已提交
231 232 233 234 235 236
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
237 238 239
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
240
	unsigned int		ncq_saw_sdb:1;
241
	u32 			intr_mask;	/* interrupts to enable */
L
Linus Torvalds 已提交
242 243
};

244 245
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
246
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
247
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
L
Linus Torvalds 已提交
248 249 250 251 252 253
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
T
Tejun Heo 已提交
254 255
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
T
Tejun Heo 已提交
256 257
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
T
Tejun Heo 已提交
258
static void ahci_error_handler(struct ata_port *ap);
259
static void ahci_vt8251_error_handler(struct ata_port *ap);
260
static void ahci_p5wdh_error_handler(struct ata_port *ap);
T
Tejun Heo 已提交
261
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
262
static int ahci_port_resume(struct ata_port *ap);
263
static void ahci_dev_config(struct ata_device *dev);
264 265 266
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
267
#ifdef CONFIG_PM
268 269 270
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
271
#endif
L
Linus Torvalds 已提交
272

273 274 275 276 277
static struct class_device_attribute *ahci_shost_attrs[] = {
	&class_device_attr_link_power_management_policy,
	NULL
};

278
static struct scsi_host_template ahci_sht = {
L
Linus Torvalds 已提交
279 280 281 282
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
T
Tejun Heo 已提交
283 284
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
L
Linus Torvalds 已提交
285 286 287 288 289 290 291 292
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
T
Tejun Heo 已提交
293
	.slave_destroy		= ata_scsi_slave_destroy,
L
Linus Torvalds 已提交
294
	.bios_param		= ata_std_bios_param,
295
	.shost_attrs		= ahci_shost_attrs,
L
Linus Torvalds 已提交
296 297
};

J
Jeff Garzik 已提交
298
static const struct ata_port_operations ahci_ops = {
L
Linus Torvalds 已提交
299 300 301 302
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

303 304
	.dev_config		= ahci_dev_config,

L
Linus Torvalds 已提交
305 306
	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
307
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
L
Linus Torvalds 已提交
308 309 310 311 312 313 314 315
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

T
Tejun Heo 已提交
316 317 318 319 320 321
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
322 323 324
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

325
#ifdef CONFIG_PM
326 327
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
328
#endif
329 330
	.enable_pm		= ahci_enable_alpm,
	.disable_pm		= ahci_disable_alpm,
331

L
Linus Torvalds 已提交
332 333 334 335
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

336 337 338 339 340 341 342
static const struct ata_port_operations ahci_vt8251_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

T
Tejun Heo 已提交
343
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
344 345 346 347 348 349 350 351 352 353 354 355 356 357
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

T
Tejun Heo 已提交
358 359 360
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

361
#ifdef CONFIG_PM
362 363
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
364
#endif
365 366 367 368 369

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
static const struct ata_port_operations ahci_p5wdh_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_p5wdh_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

#ifdef CONFIG_PM
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
#endif

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

404 405
#define AHCI_HFLAGS(flags)	.private_data	= (void *)(flags)

406
static const struct ata_port_info ahci_port_info[] = {
L
Linus Torvalds 已提交
407 408
	/* board_ahci */
	{
T
Tejun Heo 已提交
409
		.flags		= AHCI_FLAG_COMMON,
410
		.link_flags	= AHCI_LFLAG_COMMON,
411
		.pio_mask	= 0x1f, /* pio0-4 */
412
		.udma_mask	= ATA_UDMA6,
L
Linus Torvalds 已提交
413 414
		.port_ops	= &ahci_ops,
	},
415 416
	/* board_ahci_vt8251 */
	{
T
Tejun Heo 已提交
417
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
418
		.flags		= AHCI_FLAG_COMMON,
419
		.link_flags	= AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
420
		.pio_mask	= 0x1f, /* pio0-4 */
421
		.udma_mask	= ATA_UDMA6,
422
		.port_ops	= &ahci_vt8251_ops,
423
	},
424 425
	/* board_ahci_ign_iferr */
	{
426 427
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
		.flags		= AHCI_FLAG_COMMON,
428
		.link_flags	= AHCI_LFLAG_COMMON,
429
		.pio_mask	= 0x1f, /* pio0-4 */
430
		.udma_mask	= ATA_UDMA6,
431 432
		.port_ops	= &ahci_ops,
	},
433 434
	/* board_ahci_sb600 */
	{
435
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
436
				 AHCI_HFLAG_32BIT_ONLY |
437
				 AHCI_HFLAG_SECT255 | AHCI_HFLAG_NO_PMP),
438
		.flags		= AHCI_FLAG_COMMON,
439
		.link_flags	= AHCI_LFLAG_COMMON,
440
		.pio_mask	= 0x1f, /* pio0-4 */
441
		.udma_mask	= ATA_UDMA6,
442 443
		.port_ops	= &ahci_ops,
	},
444 445
	/* board_ahci_mv */
	{
446 447
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
				 AHCI_HFLAG_MV_PATA),
448
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
449
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
450
		.link_flags	= AHCI_LFLAG_COMMON,
451 452 453 454
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
455 456 457 458 459 460 461 462 463 464
	/* board_ahci_sb700 */
	{
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
				 AHCI_HFLAG_NO_PMP),
		.flags		= AHCI_FLAG_COMMON,
		.link_flags	= AHCI_LFLAG_COMMON,
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
L
Linus Torvalds 已提交
465 466
};

467
static const struct pci_device_id ahci_pci_tbl[] = {
J
Jeff Garzik 已提交
468
	/* Intel */
469 470 471 472 473
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
474
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
475 476 477 478
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
T
Tejun Heo 已提交
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
496 497
	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
498 499
	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
	{ PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
J
Jeff Garzik 已提交
500

501 502 503
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
J
Jeff Garzik 已提交
504 505

	/* ATI */
506
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
507 508 509 510 511 512
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb700 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb700 }, /* ATI SB700/800 */
J
Jeff Garzik 已提交
513 514

	/* VIA */
515
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
T
Tejun Heo 已提交
516
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
J
Jeff Garzik 已提交
517 518

	/* NVIDIA */
519 520 521 522
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
523 524 525 526 527 528 529 530
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
531 532 533 534 535 536 537 538
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
563 564 565 566
	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci },		/* MCP79 */
P
Peer Chen 已提交
567 568 569 570 571 572 573 574
	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },		/* MCP79 */
575 576 577 578 579 580 581 582 583 584 585 586
	{ PCI_VDEVICE(NVIDIA, 0x0bc8), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bc9), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bca), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcb), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcc), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcd), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bce), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bcf), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd0), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd1), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd2), board_ahci },		/* MCP7B */
	{ PCI_VDEVICE(NVIDIA, 0x0bd3), board_ahci },		/* MCP7B */
J
Jeff Garzik 已提交
587

J
Jeff Garzik 已提交
588
	/* SiS */
589 590 591
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
J
Jeff Garzik 已提交
592

593 594
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */
595
	{ PCI_VDEVICE(MARVELL, 0x6121), board_ahci_mv },	/* 6121 */
596

597 598
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
599
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
600

L
Linus Torvalds 已提交
601 602 603 604 605 606 607 608
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
609
	.remove			= ata_pci_remove_one,
610
#ifdef CONFIG_PM
611 612
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
613
#endif
L
Linus Torvalds 已提交
614 615 616
};


617 618 619 620 621
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

622 623
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
L
Linus Torvalds 已提交
624
{
625
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
626

627 628 629 630 631 632
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
L
Linus Torvalds 已提交
633 634
}

635 636 637 638 639 640 641 642 643 644 645 646 647 648
static void ahci_enable_ahci(void __iomem *mmio)
{
	u32 tmp;

	/* turn on AHCI_EN */
	tmp = readl(mmio + HOST_CTL);
	if (!(tmp & HOST_AHCI_EN)) {
		tmp |= HOST_AHCI_EN;
		writel(tmp, mmio + HOST_CTL);
		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
		WARN_ON(!(tmp & HOST_AHCI_EN));
	}
}

649 650
/**
 *	ahci_save_initial_config - Save and fixup initial config values
651 652
 *	@pdev: target PCI device
 *	@hpriv: host private area to store config values
653 654 655 656 657 658 659 660 661 662 663
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
664 665
static void ahci_save_initial_config(struct pci_dev *pdev,
				     struct ahci_host_priv *hpriv)
666
{
667
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
668
	u32 cap, port_map;
669
	int i;
670
	int mv;
671

672 673 674
	/* make sure AHCI mode is enabled before accessing CAP */
	ahci_enable_ahci(mmio);

675 676 677 678 679 680
	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

681
	/* some chips have errata preventing 64bit use */
682
	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
T
Tejun Heo 已提交
683 684 685 686 687
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

688
	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
689 690 691 692 693
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

694
	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
T
Tejun Heo 已提交
695 696 697 698 699
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do PMP, turning off CAP_PMP\n");
		cap &= ~HOST_CAP_PMP;
	}

700 701 702 703 704
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
705
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
706 707 708 709
		if (pdev->device == 0x6121)
			mv = 0x3;
		else
			mv = 0xf;
710 711
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
712 713
			   port_map,
			   port_map & mv);
714

715
		port_map &= mv;
716 717
	}

718
	/* cross check port_map and cap.n_ports */
T
Tejun Heo 已提交
719
	if (port_map) {
T
Tejun Heo 已提交
720
		int map_ports = 0;
721

T
Tejun Heo 已提交
722 723 724
		for (i = 0; i < AHCI_MAX_PORTS; i++)
			if (port_map & (1 << i))
				map_ports++;
725

T
Tejun Heo 已提交
726 727
		/* If PI has more ports than n_ports, whine, clear
		 * port_map and let it be generated from n_ports.
728
		 */
T
Tejun Heo 已提交
729
		if (map_ports > ahci_nr_ports(cap)) {
730
			dev_printk(KERN_WARNING, &pdev->dev,
T
Tejun Heo 已提交
731 732 733
				   "implemented port map (0x%x) contains more "
				   "ports than nr_ports (%u), using nr_ports\n",
				   port_map, ahci_nr_ports(cap));
T
Tejun Heo 已提交
734 735 736 737 738 739
			port_map = 0;
		}
	}

	/* fabricate port_map from cap.nr_ports */
	if (!port_map) {
740
		port_map = (1 << ahci_nr_ports(cap)) - 1;
T
Tejun Heo 已提交
741 742 743 744 745
		dev_printk(KERN_WARNING, &pdev->dev,
			   "forcing PORTS_IMPL to 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
746 747
	}

748 749 750 751 752 753 754
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
755
 *	@host: target ATA host
756 757 758 759 760 761
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
762
static void ahci_restore_initial_config(struct ata_host *host)
763
{
764 765 766
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

767 768 769 770 771
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

T
Tejun Heo 已提交
772
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
L
Linus Torvalds 已提交
773
{
T
Tejun Heo 已提交
774 775 776 777 778 779 780 781
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
L
Linus Torvalds 已提交
782

T
Tejun Heo 已提交
783 784 785
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
786
	return 0;
L
Linus Torvalds 已提交
787 788
}

T
Tejun Heo 已提交
789
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
L
Linus Torvalds 已提交
790
{
T
Tejun Heo 已提交
791 792 793 794 795 796
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
L
Linus Torvalds 已提交
797
	}
T
Tejun Heo 已提交
798 799
	return -EINVAL;
}
L
Linus Torvalds 已提交
800

T
Tejun Heo 已提交
801 802 803 804 805 806 807 808 809 810
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
L
Linus Torvalds 已提交
811 812
}

813
static void ahci_start_engine(struct ata_port *ap)
814
{
815
	void __iomem *port_mmio = ahci_port_base(ap);
816 817
	u32 tmp;

818
	/* start DMA */
819
	tmp = readl(port_mmio + PORT_CMD);
820 821 822 823 824
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

825
static int ahci_stop_engine(struct ata_port *ap)
826
{
827
	void __iomem *port_mmio = ahci_port_base(ap);
828 829 830 831
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

832
	/* check if the HBA is idle */
833 834 835
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

836
	/* setting HBA to idle */
837 838 839
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

840
	/* wait for engine to stop. This could be as long as 500 msec */
841
	tmp = ata_wait_register(port_mmio + PORT_CMD,
842
				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
843
	if (tmp & PORT_CMD_LIST_ON)
844 845 846 847 848
		return -EIO;

	return 0;
}

849
static void ahci_start_fis_rx(struct ata_port *ap)
850
{
851 852 853
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
854 855 856
	u32 tmp;

	/* set FIS registers */
857 858 859 860
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
861

862 863 864 865
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
866 867 868 869 870 871 872 873 874 875

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

876
static int ahci_stop_fis_rx(struct ata_port *ap)
877
{
878
	void __iomem *port_mmio = ahci_port_base(ap);
879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

895
static void ahci_power_up(struct ata_port *ap)
896
{
897 898
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
899 900 901 902 903
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
904
	if (hpriv->cap & HOST_CAP_SSS) {
905 906 907 908 909 910 911 912
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
static void ahci_disable_alpm(struct ata_port *ap)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;

	/* IPM bits should be disabled by libata-core */
	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/* disable ALPM and ASP */
	cmd &= ~PORT_CMD_ASP;
	cmd &= ~PORT_CMD_ALPE;

	/* force the interface back to active */
	cmd |= PORT_CMD_ICC_ACTIVE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* wait 10ms to be sure we've come out of any low power state */
	msleep(10);

	/* clear out any PhyRdy stuff from interrupt status */
	writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);

	/* go ahead and clean out PhyRdy Change from Serror too */
	ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));

	/*
 	 * Clear flag to indicate that we should ignore all PhyRdy
 	 * state changes
 	 */
	hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;

	/*
 	 * Enable interrupts on Phy Ready.
 	 */
	pp->intr_mask |= PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * don't change the link pm policy - we can be called
 	 * just to turn of link pm temporarily
 	 */
}

static int ahci_enable_alpm(struct ata_port *ap,
	enum link_pm policy)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;
	u32 asp;

	/* Make sure the host is capable of link power management */
	if (!(hpriv->cap & HOST_CAP_ALPM))
		return -EINVAL;

	switch (policy) {
	case MAX_PERFORMANCE:
	case NOT_AVAILABLE:
		/*
 		 * if we came here with NOT_AVAILABLE,
 		 * it just means this is the first time we
 		 * have tried to enable - default to max performance,
 		 * and let the user go to lower power modes on request.
 		 */
		ahci_disable_alpm(ap);
		return 0;
	case MIN_POWER:
		/* configure HBA to enter SLUMBER */
		asp = PORT_CMD_ASP;
		break;
	case MEDIUM_POWER:
		/* configure HBA to enter PARTIAL */
		asp = 0;
		break;
	default:
		return -EINVAL;
	}

	/*
 	 * Disable interrupts on Phy Ready. This keeps us from
 	 * getting woken up due to spurious phy ready interrupts
	 * TBD - Hot plug should be done via polling now, is
	 * that even supported?
 	 */
	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * Set a flag to indicate that we should ignore all PhyRdy
 	 * state changes since these can happen now whenever we
 	 * change link state
 	 */
	hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;

	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/*
 	 * Set ASP based on Policy
 	 */
	cmd |= asp;

	/*
 	 * Setting this bit will instruct the HBA to aggressively
 	 * enter a lower power link state when it's appropriate and
 	 * based on the value set above for ASP
 	 */
	cmd |= PORT_CMD_ALPE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* IPM bits should be set by libata-core */
	return 0;
}

1037
#ifdef CONFIG_PM
1038
static void ahci_power_down(struct ata_port *ap)
1039
{
1040 1041
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
1042 1043
	u32 cmd, scontrol;

1044
	if (!(hpriv->cap & HOST_CAP_SSS))
1045
		return;
1046

1047 1048 1049 1050
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
1051

1052 1053 1054 1055
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
1056
}
1057
#endif
1058

1059
static void ahci_start_port(struct ata_port *ap)
1060 1061
{
	/* enable FIS reception */
1062
	ahci_start_fis_rx(ap);
1063 1064

	/* enable DMA */
1065
	ahci_start_engine(ap);
1066 1067
}

1068
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
1069 1070 1071 1072
{
	int rc;

	/* disable DMA */
1073
	rc = ahci_stop_engine(ap);
1074 1075 1076 1077 1078 1079
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
1080
	rc = ahci_stop_fis_rx(ap);
1081 1082 1083 1084 1085 1086 1087 1088
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

1089
static int ahci_reset_controller(struct ata_host *host)
1090
{
1091
	struct pci_dev *pdev = to_pci_dev(host->dev);
T
Tejun Heo 已提交
1092
	struct ahci_host_priv *hpriv = host->private_data;
1093
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1094
	u32 tmp;
1095

1096 1097 1098
	/* we must be in AHCI mode, before using anything
	 * AHCI-specific, such as HOST_RESET.
	 */
1099
	ahci_enable_ahci(mmio);
1100 1101

	/* global controller reset */
1102 1103 1104 1105 1106 1107
	if (!ahci_skip_host_reset) {
		tmp = readl(mmio + HOST_CTL);
		if ((tmp & HOST_RESET) == 0) {
			writel(tmp | HOST_RESET, mmio + HOST_CTL);
			readl(mmio + HOST_CTL); /* flush */
		}
1108

1109 1110 1111 1112
		/* reset must complete within 1 second, or
		 * the hardware should be considered fried.
		 */
		ssleep(1);
1113

1114 1115 1116 1117 1118 1119
		tmp = readl(mmio + HOST_CTL);
		if (tmp & HOST_RESET) {
			dev_printk(KERN_ERR, host->dev,
				   "controller reset failed (0x%x)\n", tmp);
			return -EIO;
		}
1120

1121 1122
		/* turn on AHCI mode */
		ahci_enable_ahci(mmio);
1123

1124 1125 1126 1127 1128 1129 1130
		/* Some registers might be cleared on reset.  Restore
		 * initial values.
		 */
		ahci_restore_initial_config(host);
	} else
		dev_printk(KERN_INFO, host->dev,
			   "skipping global host reset\n");
1131 1132 1133 1134 1135 1136

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
T
Tejun Heo 已提交
1137 1138 1139 1140
		if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
			tmp16 |= hpriv->port_map;
			pci_write_config_word(pdev, 0x92, tmp16);
		}
1141 1142 1143 1144 1145
	}

	return 0;
}

1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

1174
static void ahci_init_controller(struct ata_host *host)
1175
{
1176
	struct ahci_host_priv *hpriv = host->private_data;
1177 1178
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1179
	int i;
1180
	void __iomem *port_mmio;
1181
	u32 tmp;
1182
	int mv;
1183

1184
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
1185 1186 1187 1188 1189
		if (pdev->device == 0x6121)
			mv = 2;
		else
			mv = 4;
		port_mmio = __ahci_port_base(host, mv);
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

1200 1201
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
1202

1203
		port_mmio = ahci_port_base(ap);
1204
		if (ata_port_is_dummy(ap))
1205 1206
			continue;

1207
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
1208 1209 1210 1211 1212 1213 1214 1215 1216
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

1217 1218 1219 1220
static void ahci_dev_config(struct ata_device *dev)
{
	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;

1221
	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1222
		dev->max_sectors = 255;
1223 1224 1225
		ata_dev_printk(dev, KERN_INFO,
			       "SB600 AHCI: limiting to 255 sectors per cmd\n");
	}
1226 1227
}

1228
static unsigned int ahci_dev_classify(struct ata_port *ap)
L
Linus Torvalds 已提交
1229
{
1230
	void __iomem *port_mmio = ahci_port_base(ap);
L
Linus Torvalds 已提交
1231
	struct ata_taskfile tf;
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
	u32 tmp;

	tmp = readl(port_mmio + PORT_SIG);
	tf.lbah		= (tmp >> 24)	& 0xff;
	tf.lbam		= (tmp >> 16)	& 0xff;
	tf.lbal		= (tmp >> 8)	& 0xff;
	tf.nsect	= (tmp)		& 0xff;

	return ata_dev_classify(&tf);
}

T
Tejun Heo 已提交
1243 1244
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts)
1245
{
T
Tejun Heo 已提交
1246 1247 1248 1249 1250 1251 1252 1253
	dma_addr_t cmd_tbl_dma;

	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;

	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
	pp->cmd_slot[tag].status = 0;
	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1254 1255
}

1256
static int ahci_kick_engine(struct ata_port *ap, int force_restart)
T
Tejun Heo 已提交
1257
{
T
Tejun Heo 已提交
1258
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
J
Jeff Garzik 已提交
1259
	struct ahci_host_priv *hpriv = ap->host->private_data;
1260
	u32 tmp;
1261
	int busy, rc;
1262

1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
	/* do we need to kick the port? */
	busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
	if (!busy && !force_restart)
		return 0;

	/* stop engine */
	rc = ahci_stop_engine(ap);
	if (rc)
		goto out_restart;

	/* need to do CLO? */
	if (!busy) {
		rc = 0;
		goto out_restart;
	}

	if (!(hpriv->cap & HOST_CAP_CLO)) {
		rc = -EOPNOTSUPP;
		goto out_restart;
	}
1283

1284
	/* perform CLO */
1285 1286 1287 1288
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_CLO;
	writel(tmp, port_mmio + PORT_CMD);

1289
	rc = 0;
1290 1291 1292
	tmp = ata_wait_register(port_mmio + PORT_CMD,
				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
	if (tmp & PORT_CMD_CLO)
1293
		rc = -EIO;
1294

1295 1296 1297 1298
	/* restart engine */
 out_restart:
	ahci_start_engine(ap);
	return rc;
1299 1300
}

1301 1302 1303
static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
				struct ata_taskfile *tf, int is_cmd, u16 flags,
				unsigned long timeout_msec)
1304
{
1305
	const u32 cmd_fis_len = 5; /* five dwords */
T
Tejun Heo 已提交
1306
	struct ahci_port_priv *pp = ap->private_data;
1307
	void __iomem *port_mmio = ahci_port_base(ap);
1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
	u8 *fis = pp->cmd_tbl;
	u32 tmp;

	/* prep the command */
	ata_tf_to_fis(tf, pmp, is_cmd, fis);
	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));

	/* issue & wait */
	writel(1, port_mmio + PORT_CMD_ISSUE);

	if (timeout_msec) {
		tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
					1, timeout_msec);
		if (tmp & 0x1) {
			ahci_kick_engine(ap, 1);
			return -EBUSY;
		}
	} else
		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1331
static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
1332
			     int pmp, unsigned long deadline)
1333
{
T
Tejun Heo 已提交
1334
	struct ata_port *ap = link->ap;
T
Tejun Heo 已提交
1335
	const char *reason = NULL;
1336
	unsigned long now, msecs;
T
Tejun Heo 已提交
1337 1338 1339 1340 1341
	struct ata_taskfile tf;
	int rc;

	DPRINTK("ENTER\n");

T
Tejun Heo 已提交
1342
	if (ata_link_offline(link)) {
1343 1344 1345 1346 1347
		DPRINTK("PHY reports no device\n");
		*class = ATA_DEV_NONE;
		return 0;
	}

T
Tejun Heo 已提交
1348
	/* prepare for SRST (AHCI-1.1 10.4.1) */
1349
	rc = ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1350
	if (rc && rc != -EOPNOTSUPP)
T
Tejun Heo 已提交
1351
		ata_link_printk(link, KERN_WARNING,
T
Tejun Heo 已提交
1352
				"failed to reset engine (errno=%d)\n", rc);
T
Tejun Heo 已提交
1353

T
Tejun Heo 已提交
1354
	ata_tf_init(link->device, &tf);
T
Tejun Heo 已提交
1355 1356

	/* issue the first D2H Register FIS */
1357 1358 1359 1360 1361
	msecs = 0;
	now = jiffies;
	if (time_after(now, deadline))
		msecs = jiffies_to_msecs(deadline - now);

T
Tejun Heo 已提交
1362
	tf.ctl |= ATA_SRST;
1363
	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1364
				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
T
Tejun Heo 已提交
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
		rc = -EIO;
		reason = "1st FIS failed";
		goto fail;
	}

	/* spec says at least 5us, but be generous and sleep for 1ms */
	msleep(1);

	/* issue the second D2H Register FIS */
	tf.ctl &= ~ATA_SRST;
1375
	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
T
Tejun Heo 已提交
1376

1377 1378
	/* wait a while before checking status */
	ata_wait_after_reset(ap, deadline);
T
Tejun Heo 已提交
1379

T
Tejun Heo 已提交
1380 1381 1382 1383 1384
	rc = ata_wait_ready(ap, deadline);
	/* link occupied, -ENODEV too is an error */
	if (rc) {
		reason = "device not ready";
		goto fail;
T
Tejun Heo 已提交
1385
	}
T
Tejun Heo 已提交
1386
	*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1387 1388 1389 1390 1391

	DPRINTK("EXIT, class=%u\n", *class);
	return 0;

 fail:
T
Tejun Heo 已提交
1392
	ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
T
Tejun Heo 已提交
1393 1394 1395
	return rc;
}

T
Tejun Heo 已提交
1396
static int ahci_softreset(struct ata_link *link, unsigned int *class,
1397 1398
			  unsigned long deadline)
{
T
Tejun Heo 已提交
1399 1400 1401 1402 1403 1404
	int pmp = 0;

	if (link->ap->flags & ATA_FLAG_PMP)
		pmp = SATA_PMP_CTRL_PORT;

	return ahci_do_softreset(link, class, pmp, deadline);
1405 1406
}

T
Tejun Heo 已提交
1407
static int ahci_hardreset(struct ata_link *link, unsigned int *class,
1408
			  unsigned long deadline)
1409
{
T
Tejun Heo 已提交
1410
	struct ata_port *ap = link->ap;
1411 1412 1413
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
1414 1415 1416
	int rc;

	DPRINTK("ENTER\n");
L
Linus Torvalds 已提交
1417

1418
	ahci_stop_engine(ap);
1419 1420

	/* clear D2H reception area to properly wait for D2H FIS */
T
Tejun Heo 已提交
1421
	ata_tf_init(link->device, &tf);
1422
	tf.command = 0x80;
1423
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1424

T
Tejun Heo 已提交
1425
	rc = sata_std_hardreset(link, class, deadline);
1426

1427
	ahci_start_engine(ap);
L
Linus Torvalds 已提交
1428

T
Tejun Heo 已提交
1429
	if (rc == 0 && ata_link_online(link))
1430
		*class = ahci_dev_classify(ap);
T
Tejun Heo 已提交
1431
	if (rc != -EAGAIN && *class == ATA_DEV_UNKNOWN)
1432
		*class = ATA_DEV_NONE;
L
Linus Torvalds 已提交
1433

1434 1435 1436 1437
	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
	return rc;
}

T
Tejun Heo 已提交
1438
static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
1439
				 unsigned long deadline)
1440
{
T
Tejun Heo 已提交
1441
	struct ata_port *ap = link->ap;
1442
	u32 serror;
1443 1444 1445 1446
	int rc;

	DPRINTK("ENTER\n");

1447
	ahci_stop_engine(ap);
1448

T
Tejun Heo 已提交
1449
	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
1450
				 deadline);
1451 1452

	/* vt8251 needs SError cleared for the port to operate */
1453 1454
	ahci_scr_read(ap, SCR_ERROR, &serror);
	ahci_scr_write(ap, SCR_ERROR, serror);
1455

1456
	ahci_start_engine(ap);
1457 1458 1459 1460 1461 1462 1463 1464 1465

	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);

	/* vt8251 doesn't clear BSY on signature FIS reception,
	 * request follow-up softreset.
	 */
	return rc ?: -EAGAIN;
}

1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
				unsigned long deadline)
{
	struct ata_port *ap = link->ap;
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
	int rc;

	ahci_stop_engine(ap);

	/* clear D2H reception area to properly wait for D2H FIS */
	ata_tf_init(link->device, &tf);
	tf.command = 0x80;
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);

	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
				 deadline);

	ahci_start_engine(ap);

	if (rc || ata_link_offline(link))
		return rc;

	/* spec mandates ">= 2ms" before checking status */
	msleep(150);

	/* The pseudo configuration device on SIMG4726 attached to
	 * ASUS P5W-DH Deluxe doesn't send signature FIS after
	 * hardreset if no device is attached to the first downstream
	 * port && the pseudo device locks up on SRST w/ PMP==0.  To
	 * work around this, wait for !BSY only briefly.  If BSY isn't
	 * cleared, perform CLO and proceed to IDENTIFY (achieved by
	 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
	 *
	 * Wait for two seconds.  Devices attached to downstream port
	 * which can't process the following IDENTIFY after this will
	 * have to be reset again.  For most cases, this should
	 * suffice while making probing snappish enough.
	 */
	rc = ata_wait_ready(ap, jiffies + 2 * HZ);
	if (rc)
		ahci_kick_engine(ap, 0);

	return 0;
}

T
Tejun Heo 已提交
1513
static void ahci_postreset(struct ata_link *link, unsigned int *class)
1514
{
T
Tejun Heo 已提交
1515
	struct ata_port *ap = link->ap;
1516
	void __iomem *port_mmio = ahci_port_base(ap);
1517 1518
	u32 new_tmp, tmp;

T
Tejun Heo 已提交
1519
	ata_std_postreset(link, class);
1520 1521 1522

	/* Make sure port's ATAPI bit is set appropriately */
	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1523
	if (*class == ATA_DEV_ATAPI)
1524 1525 1526 1527 1528 1529 1530
		new_tmp |= PORT_CMD_ATAPI;
	else
		new_tmp &= ~PORT_CMD_ATAPI;
	if (new_tmp != tmp) {
		writel(new_tmp, port_mmio + PORT_CMD);
		readl(port_mmio + PORT_CMD); /* flush */
	}
L
Linus Torvalds 已提交
1531 1532
}

T
Tejun Heo 已提交
1533 1534 1535 1536 1537 1538
static int ahci_pmp_softreset(struct ata_link *link, unsigned int *class,
			      unsigned long deadline)
{
	return ahci_do_softreset(link, class, link->pmp, deadline);
}

L
Linus Torvalds 已提交
1539 1540
static u8 ahci_check_status(struct ata_port *ap)
{
T
Tejun Heo 已提交
1541
	void __iomem *mmio = ap->ioaddr.cmd_addr;
L
Linus Torvalds 已提交
1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553

	return readl(mmio + PORT_TFDATA) & 0xFF;
}

static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
{
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;

	ata_tf_from_fis(d2h_fis, tf);
}

T
Tejun Heo 已提交
1554
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
L
Linus Torvalds 已提交
1555
{
1556
	struct scatterlist *sg;
T
Tejun Heo 已提交
1557 1558
	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
	unsigned int si;
L
Linus Torvalds 已提交
1559 1560 1561 1562 1563 1564

	VPRINTK("ENTER\n");

	/*
	 * Next, the S/G list.
	 */
T
Tejun Heo 已提交
1565
	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1566 1567 1568
		dma_addr_t addr = sg_dma_address(sg);
		u32 sg_len = sg_dma_len(sg);

T
Tejun Heo 已提交
1569 1570 1571
		ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
		ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
L
Linus Torvalds 已提交
1572
	}
1573

T
Tejun Heo 已提交
1574
	return si;
L
Linus Torvalds 已提交
1575 1576 1577 1578
}

static void ahci_qc_prep(struct ata_queued_cmd *qc)
{
1579 1580
	struct ata_port *ap = qc->ap;
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1581
	int is_atapi = ata_is_atapi(qc->tf.protocol);
T
Tejun Heo 已提交
1582
	void *cmd_tbl;
L
Linus Torvalds 已提交
1583 1584
	u32 opts;
	const u32 cmd_fis_len = 5; /* five dwords */
1585
	unsigned int n_elem;
L
Linus Torvalds 已提交
1586 1587 1588 1589 1590

	/*
	 * Fill in command table information.  First, the header,
	 * a SATA Register - Host to Device command FIS.
	 */
T
Tejun Heo 已提交
1591 1592
	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;

T
Tejun Heo 已提交
1593
	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1594
	if (is_atapi) {
T
Tejun Heo 已提交
1595 1596
		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1597
	}
L
Linus Torvalds 已提交
1598

1599 1600
	n_elem = 0;
	if (qc->flags & ATA_QCFLAG_DMAMAP)
T
Tejun Heo 已提交
1601
		n_elem = ahci_fill_sg(qc, cmd_tbl);
L
Linus Torvalds 已提交
1602

1603 1604 1605
	/*
	 * Fill in command slot information.
	 */
T
Tejun Heo 已提交
1606
	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
1607 1608 1609
	if (qc->tf.flags & ATA_TFLAG_WRITE)
		opts |= AHCI_CMD_WRITE;
	if (is_atapi)
1610
		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1611

T
Tejun Heo 已提交
1612
	ahci_fill_cmd_slot(pp, qc->tag, opts);
L
Linus Torvalds 已提交
1613 1614
}

T
Tejun Heo 已提交
1615
static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
L
Linus Torvalds 已提交
1616
{
1617
	struct ahci_host_priv *hpriv = ap->host->private_data;
T
Tejun Heo 已提交
1618
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1619 1620 1621 1622
	struct ata_eh_info *host_ehi = &ap->link.eh_info;
	struct ata_link *link = NULL;
	struct ata_queued_cmd *active_qc;
	struct ata_eh_info *active_ehi;
T
Tejun Heo 已提交
1623
	u32 serror;
L
Linus Torvalds 已提交
1624

T
Tejun Heo 已提交
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	/* determine active link */
	ata_port_for_each_link(link, ap)
		if (ata_link_active(link))
			break;
	if (!link)
		link = &ap->link;

	active_qc = ata_qc_from_tag(ap, link->active_tag);
	active_ehi = &link->eh_info;

	/* record irq stat */
	ata_ehi_clear_desc(host_ehi);
	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
L
Linus Torvalds 已提交
1638

T
Tejun Heo 已提交
1639
	/* AHCI needs SError cleared; otherwise, it might lock up */
1640
	ahci_scr_read(ap, SCR_ERROR, &serror);
T
Tejun Heo 已提交
1641
	ahci_scr_write(ap, SCR_ERROR, serror);
T
Tejun Heo 已提交
1642
	host_ehi->serror |= serror;
T
Tejun Heo 已提交
1643

1644
	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1645
	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1646 1647
		irq_stat &= ~PORT_IRQ_IF_ERR;

1648
	if (irq_stat & PORT_IRQ_TF_ERR) {
T
Tejun Heo 已提交
1649 1650 1651 1652 1653 1654 1655 1656 1657
		/* If qc is active, charge it; otherwise, the active
		 * link.  There's no active qc on NCQ errors.  It will
		 * be determined by EH by reading log page 10h.
		 */
		if (active_qc)
			active_qc->err_mask |= AC_ERR_DEV;
		else
			active_ehi->err_mask |= AC_ERR_DEV;

1658
		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
T
Tejun Heo 已提交
1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
			host_ehi->serror &= ~SERR_INTERNAL;
	}

	if (irq_stat & PORT_IRQ_UNK_FIS) {
		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);

		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi,
				  "unknown FIS %08x %08x %08x %08x" ,
				  unk[0], unk[1], unk[2], unk[3]);
	}

	if (ap->nr_pmp_links && (irq_stat & PORT_IRQ_BAD_PMP)) {
		active_ehi->err_mask |= AC_ERR_HSM;
		active_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(active_ehi, "incorrect PMP");
1676
	}
T
Tejun Heo 已提交
1677 1678

	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
T
Tejun Heo 已提交
1679 1680 1681
		host_ehi->err_mask |= AC_ERR_HOST_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "host bus error");
L
Linus Torvalds 已提交
1682 1683
	}

T
Tejun Heo 已提交
1684
	if (irq_stat & PORT_IRQ_IF_ERR) {
T
Tejun Heo 已提交
1685 1686 1687
		host_ehi->err_mask |= AC_ERR_ATA_BUS;
		host_ehi->action |= ATA_EH_SOFTRESET;
		ata_ehi_push_desc(host_ehi, "interface fatal error");
T
Tejun Heo 已提交
1688
	}
L
Linus Torvalds 已提交
1689

T
Tejun Heo 已提交
1690
	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
T
Tejun Heo 已提交
1691 1692 1693
		ata_ehi_hotplugged(host_ehi);
		ata_ehi_push_desc(host_ehi, "%s",
			irq_stat & PORT_IRQ_CONNECT ?
T
Tejun Heo 已提交
1694 1695 1696 1697
			"connection status changed" : "PHY RDY changed");
	}

	/* okay, let's hand over to EH */
1698

T
Tejun Heo 已提交
1699 1700 1701 1702
	if (irq_stat & PORT_IRQ_FREEZE)
		ata_port_freeze(ap);
	else
		ata_port_abort(ap);
L
Linus Torvalds 已提交
1703 1704
}

1705
static void ahci_port_intr(struct ata_port *ap)
L
Linus Torvalds 已提交
1706
{
1707
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
T
Tejun Heo 已提交
1708
	struct ata_eh_info *ehi = &ap->link.eh_info;
1709
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1710
	struct ahci_host_priv *hpriv = ap->host->private_data;
1711
	int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
T
Tejun Heo 已提交
1712
	u32 status, qc_active;
1713
	int rc;
L
Linus Torvalds 已提交
1714 1715 1716 1717

	status = readl(port_mmio + PORT_IRQ_STAT);
	writel(status, port_mmio + PORT_IRQ_STAT);

1718 1719 1720 1721
	/* ignore BAD_PMP while resetting */
	if (unlikely(resetting))
		status &= ~PORT_IRQ_BAD_PMP;

1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	/* If we are getting PhyRdy, this is
 	 * just a power state change, we should
 	 * clear out this, plus the PhyRdy/Comm
 	 * Wake bits from Serror
 	 */
	if ((hpriv->flags & AHCI_HFLAG_NO_HOTPLUG) &&
		(status & PORT_IRQ_PHYRDY)) {
		status &= ~PORT_IRQ_PHYRDY;
		ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));
	}

T
Tejun Heo 已提交
1733 1734 1735
	if (unlikely(status & PORT_IRQ_ERROR)) {
		ahci_error_intr(ap, status);
		return;
L
Linus Torvalds 已提交
1736 1737
	}

1738
	if (status & PORT_IRQ_SDB_FIS) {
T
Tejun Heo 已提交
1739 1740 1741 1742 1743 1744 1745 1746
		/* If SNotification is available, leave notification
		 * handling to sata_async_notification().  If not,
		 * emulate it by snooping SDB FIS RX area.
		 *
		 * Snooping FIS RX area is probably cheaper than
		 * poking SNotification but some constrollers which
		 * implement SNotification, ICH9 for example, don't
		 * store AN SDB FIS into receive area.
1747
		 */
T
Tejun Heo 已提交
1748
		if (hpriv->cap & HOST_CAP_SNTF)
1749
			sata_async_notification(ap);
T
Tejun Heo 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
		else {
			/* If the 'N' bit in word 0 of the FIS is set,
			 * we just received asynchronous notification.
			 * Tell libata about it.
			 */
			const __le32 *f = pp->rx_fis + RX_FIS_SDB;
			u32 f0 = le32_to_cpu(f[0]);

			if (f0 & (1 << 15))
				sata_async_notification(ap);
		}
1761 1762
	}

T
Tejun Heo 已提交
1763 1764
	/* pp->active_link is valid iff any command is in flight */
	if (ap->qc_active && pp->active_link->sactive)
T
Tejun Heo 已提交
1765 1766 1767 1768 1769
		qc_active = readl(port_mmio + PORT_SCR_ACT);
	else
		qc_active = readl(port_mmio + PORT_CMD_ISSUE);

	rc = ata_qc_complete_multiple(ap, qc_active, NULL);
1770

1771 1772
	/* while resetting, invalid completions are expected */
	if (unlikely(rc < 0 && !resetting)) {
T
Tejun Heo 已提交
1773 1774 1775
		ehi->err_mask |= AC_ERR_HSM;
		ehi->action |= ATA_EH_SOFTRESET;
		ata_port_freeze(ap);
L
Linus Torvalds 已提交
1776 1777 1778 1779 1780 1781 1782 1783
	}
}

static void ahci_irq_clear(struct ata_port *ap)
{
	/* TODO */
}

1784
static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
L
Linus Torvalds 已提交
1785
{
J
Jeff Garzik 已提交
1786
	struct ata_host *host = dev_instance;
L
Linus Torvalds 已提交
1787 1788
	struct ahci_host_priv *hpriv;
	unsigned int i, handled = 0;
1789
	void __iomem *mmio;
L
Linus Torvalds 已提交
1790 1791 1792 1793
	u32 irq_stat, irq_ack = 0;

	VPRINTK("ENTER\n");

J
Jeff Garzik 已提交
1794
	hpriv = host->private_data;
T
Tejun Heo 已提交
1795
	mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
1796 1797 1798 1799 1800 1801 1802

	/* sigh.  0xffffffff is a valid return from h/w */
	irq_stat = readl(mmio + HOST_IRQ_STAT);
	irq_stat &= hpriv->port_map;
	if (!irq_stat)
		return IRQ_NONE;

1803
	spin_lock(&host->lock);
L
Linus Torvalds 已提交
1804

1805
	for (i = 0; i < host->n_ports; i++) {
L
Linus Torvalds 已提交
1806 1807
		struct ata_port *ap;

1808 1809 1810
		if (!(irq_stat & (1 << i)))
			continue;

J
Jeff Garzik 已提交
1811
		ap = host->ports[i];
1812
		if (ap) {
1813
			ahci_port_intr(ap);
1814 1815 1816
			VPRINTK("port %u\n", i);
		} else {
			VPRINTK("port %u (no irq)\n", i);
1817
			if (ata_ratelimit())
J
Jeff Garzik 已提交
1818
				dev_printk(KERN_WARNING, host->dev,
1819
					"interrupt on disabled port %u\n", i);
L
Linus Torvalds 已提交
1820
		}
1821 1822

		irq_ack |= (1 << i);
L
Linus Torvalds 已提交
1823 1824 1825 1826 1827 1828 1829
	}

	if (irq_ack) {
		writel(irq_ack, mmio + HOST_IRQ_STAT);
		handled = 1;
	}

J
Jeff Garzik 已提交
1830
	spin_unlock(&host->lock);
L
Linus Torvalds 已提交
1831 1832 1833 1834 1835 1836

	VPRINTK("EXIT\n");

	return IRQ_RETVAL(handled);
}

1837
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
L
Linus Torvalds 已提交
1838 1839
{
	struct ata_port *ap = qc->ap;
1840
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1841 1842 1843 1844 1845 1846 1847
	struct ahci_port_priv *pp = ap->private_data;

	/* Keep track of the currently active link.  It will be used
	 * in completion path to determine whether NCQ phase is in
	 * progress.
	 */
	pp->active_link = qc->dev->link;
L
Linus Torvalds 已提交
1848

T
Tejun Heo 已提交
1849 1850 1851
	if (qc->tf.protocol == ATA_PROT_NCQ)
		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
L
Linus Torvalds 已提交
1852 1853 1854 1855 1856
	readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

T
Tejun Heo 已提交
1857 1858
static void ahci_freeze(struct ata_port *ap)
{
1859
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1860 1861 1862 1863 1864 1865 1866

	/* turn IRQ off */
	writel(0, port_mmio + PORT_IRQ_MASK);
}

static void ahci_thaw(struct ata_port *ap)
{
T
Tejun Heo 已提交
1867
	void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
1868
	void __iomem *port_mmio = ahci_port_base(ap);
T
Tejun Heo 已提交
1869
	u32 tmp;
1870
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1871 1872 1873 1874

	/* clear IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	writel(tmp, port_mmio + PORT_IRQ_STAT);
1875
	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
T
Tejun Heo 已提交
1876

1877 1878
	/* turn IRQ back on */
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1879 1880 1881 1882
}

static void ahci_error_handler(struct ata_port *ap)
{
1883
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
T
Tejun Heo 已提交
1884
		/* restart engine */
1885 1886
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
T
Tejun Heo 已提交
1887 1888 1889
	}

	/* perform recovery */
T
Tejun Heo 已提交
1890 1891 1892 1893
	sata_pmp_do_eh(ap, ata_std_prereset, ahci_softreset,
		       ahci_hardreset, ahci_postreset,
		       sata_pmp_std_prereset, ahci_pmp_softreset,
		       sata_pmp_std_hardreset, sata_pmp_std_postreset);
T
Tejun Heo 已提交
1894 1895
}

1896 1897 1898 1899
static void ahci_vt8251_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
1900 1901
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
1902 1903 1904 1905 1906 1907 1908
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
		  ahci_postreset);
}

1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921
static void ahci_p5wdh_error_handler(struct ata_port *ap)
{
	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
		/* restart engine */
		ahci_stop_engine(ap);
		ahci_start_engine(ap);
	}

	/* perform recovery */
	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_p5wdh_hardreset,
		  ahci_postreset);
}

T
Tejun Heo 已提交
1922 1923 1924 1925
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
{
	struct ata_port *ap = qc->ap;

1926 1927 1928
	/* make DMA engine forget about the failed command */
	if (qc->flags & ATA_QCFLAG_FAILED)
		ahci_kick_engine(ap, 1);
T
Tejun Heo 已提交
1929 1930
}

T
Tejun Heo 已提交
1931 1932 1933
static void ahci_pmp_attach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1934
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1935 1936 1937 1938 1939
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd |= PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1940 1941 1942

	pp->intr_mask |= PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1943 1944 1945 1946 1947
}

static void ahci_pmp_detach(struct ata_port *ap)
{
	void __iomem *port_mmio = ahci_port_base(ap);
1948
	struct ahci_port_priv *pp = ap->private_data;
T
Tejun Heo 已提交
1949 1950 1951 1952 1953
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD);
	cmd &= ~PORT_CMD_PMP;
	writel(cmd, port_mmio + PORT_CMD);
1954 1955 1956

	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
T
Tejun Heo 已提交
1957 1958
}

1959 1960 1961 1962 1963
static int ahci_port_resume(struct ata_port *ap)
{
	ahci_power_up(ap);
	ahci_start_port(ap);

T
Tejun Heo 已提交
1964 1965 1966 1967 1968
	if (ap->nr_pmp_links)
		ahci_pmp_attach(ap);
	else
		ahci_pmp_detach(ap);

1969 1970 1971
	return 0;
}

1972
#ifdef CONFIG_PM
1973 1974 1975 1976 1977
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
{
	const char *emsg = NULL;
	int rc;

1978
	rc = ahci_deinit_port(ap, &emsg);
1979
	if (rc == 0)
1980
		ahci_power_down(ap);
1981
	else {
1982
		ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
1983
		ahci_start_port(ap);
1984 1985 1986 1987 1988 1989 1990
	}

	return rc;
}

static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
{
J
Jeff Garzik 已提交
1991
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
T
Tejun Heo 已提交
1992
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1993 1994
	u32 ctl;

1995
	if (mesg.event & PM_EVENT_SLEEP) {
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
		/* AHCI spec rev1.1 section 8.3.3:
		 * Software must disable interrupts prior to requesting a
		 * transition of the HBA to D3 state.
		 */
		ctl = readl(mmio + HOST_CTL);
		ctl &= ~HOST_IRQ_EN;
		writel(ctl, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	return ata_pci_device_suspend(pdev, mesg);
}

static int ahci_pci_device_resume(struct pci_dev *pdev)
{
J
Jeff Garzik 已提交
2011
	struct ata_host *host = dev_get_drvdata(&pdev->dev);
2012 2013
	int rc;

2014 2015 2016
	rc = ata_pci_device_do_resume(pdev);
	if (rc)
		return rc;
2017 2018

	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2019
		rc = ahci_reset_controller(host);
2020 2021 2022
		if (rc)
			return rc;

2023
		ahci_init_controller(host);
2024 2025
	}

J
Jeff Garzik 已提交
2026
	ata_host_resume(host);
2027 2028 2029

	return 0;
}
2030
#endif
2031

2032 2033
static int ahci_port_start(struct ata_port *ap)
{
J
Jeff Garzik 已提交
2034
	struct device *dev = ap->host->dev;
2035 2036 2037 2038
	struct ahci_port_priv *pp;
	void *mem;
	dma_addr_t mem_dma;

2039
	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2040 2041 2042
	if (!pp)
		return -ENOMEM;

2043 2044 2045
	mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
				  GFP_KERNEL);
	if (!mem)
2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074
		return -ENOMEM;
	memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);

	/*
	 * First item in chunk of DMA memory: 32-slot command table,
	 * 32 bytes each in size
	 */
	pp->cmd_slot = mem;
	pp->cmd_slot_dma = mem_dma;

	mem += AHCI_CMD_SLOT_SZ;
	mem_dma += AHCI_CMD_SLOT_SZ;

	/*
	 * Second item: Received-FIS area
	 */
	pp->rx_fis = mem;
	pp->rx_fis_dma = mem_dma;

	mem += AHCI_RX_FIS_SZ;
	mem_dma += AHCI_RX_FIS_SZ;

	/*
	 * Third item: data area for storing a single command
	 * and its scatter-gather table
	 */
	pp->cmd_tbl = mem;
	pp->cmd_tbl_dma = mem_dma;

2075
	/*
2076 2077 2078
	 * Save off initial list of interrupts to be enabled.
	 * This could be changed later
	 */
2079 2080
	pp->intr_mask = DEF_PORT_IRQ;

2081 2082
	ap->private_data = pp;

2083 2084
	/* engage engines, captain */
	return ahci_port_resume(ap);
2085 2086 2087 2088
}

static void ahci_port_stop(struct ata_port *ap)
{
2089 2090
	const char *emsg = NULL;
	int rc;
2091

2092
	/* de-initialize port */
2093
	rc = ahci_deinit_port(ap, &emsg);
2094 2095
	if (rc)
		ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
2096 2097
}

2098
static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
L
Linus Torvalds 已提交
2099 2100 2101 2102 2103 2104 2105 2106 2107
{
	int rc;

	if (using_dac &&
	    !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
		if (rc) {
			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
			if (rc) {
2108 2109
				dev_printk(KERN_ERR, &pdev->dev,
					   "64-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2110 2111 2112 2113 2114 2115
				return rc;
			}
		}
	} else {
		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2116 2117
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit DMA enable failed\n");
L
Linus Torvalds 已提交
2118 2119 2120 2121
			return rc;
		}
		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
		if (rc) {
2122 2123
			dev_printk(KERN_ERR, &pdev->dev,
				   "32-bit consistent DMA enable failed\n");
L
Linus Torvalds 已提交
2124 2125 2126 2127 2128 2129
			return rc;
		}
	}
	return 0;
}

2130
static void ahci_print_info(struct ata_host *host)
L
Linus Torvalds 已提交
2131
{
2132 2133 2134
	struct ahci_host_priv *hpriv = host->private_data;
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
L
Linus Torvalds 已提交
2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
	u32 vers, cap, impl, speed;
	const char *speed_s;
	u16 cc;
	const char *scc_s;

	vers = readl(mmio + HOST_VERSION);
	cap = hpriv->cap;
	impl = hpriv->port_map;

	speed = (cap >> 20) & 0xf;
	if (speed == 1)
		speed_s = "1.5";
	else if (speed == 2)
		speed_s = "3";
	else
		speed_s = "?";

	pci_read_config_word(pdev, 0x0a, &cc);
2153
	if (cc == PCI_CLASS_STORAGE_IDE)
L
Linus Torvalds 已提交
2154
		scc_s = "IDE";
2155
	else if (cc == PCI_CLASS_STORAGE_SATA)
L
Linus Torvalds 已提交
2156
		scc_s = "SATA";
2157
	else if (cc == PCI_CLASS_STORAGE_RAID)
L
Linus Torvalds 已提交
2158 2159 2160 2161
		scc_s = "RAID";
	else
		scc_s = "unknown";

2162 2163
	dev_printk(KERN_INFO, &pdev->dev,
		"AHCI %02x%02x.%02x%02x "
L
Linus Torvalds 已提交
2164
		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
2165
		,
L
Linus Torvalds 已提交
2166

2167 2168 2169 2170
		(vers >> 24) & 0xff,
		(vers >> 16) & 0xff,
		(vers >> 8) & 0xff,
		vers & 0xff,
L
Linus Torvalds 已提交
2171 2172 2173 2174 2175 2176 2177

		((cap >> 8) & 0x1f) + 1,
		(cap & 0x1f) + 1,
		speed_s,
		impl,
		scc_s);

2178 2179
	dev_printk(KERN_INFO, &pdev->dev,
		"flags: "
T
Tejun Heo 已提交
2180 2181
		"%s%s%s%s%s%s%s"
		"%s%s%s%s%s%s%s\n"
2182
		,
L
Linus Torvalds 已提交
2183 2184 2185

		cap & (1 << 31) ? "64bit " : "",
		cap & (1 << 30) ? "ncq " : "",
T
Tejun Heo 已提交
2186
		cap & (1 << 29) ? "sntf " : "",
L
Linus Torvalds 已提交
2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201
		cap & (1 << 28) ? "ilck " : "",
		cap & (1 << 27) ? "stag " : "",
		cap & (1 << 26) ? "pm " : "",
		cap & (1 << 25) ? "led " : "",

		cap & (1 << 24) ? "clo " : "",
		cap & (1 << 19) ? "nz " : "",
		cap & (1 << 18) ? "only " : "",
		cap & (1 << 17) ? "pmp " : "",
		cap & (1 << 15) ? "pio " : "",
		cap & (1 << 14) ? "slum " : "",
		cap & (1 << 13) ? "part " : ""
		);
}

2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246
/* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
 * hardwired to on-board SIMG 4726.  The chipset is ICH8 and doesn't
 * support PMP and the 4726 either directly exports the device
 * attached to the first downstream port or acts as a hardware storage
 * controller and emulate a single ATA device (can be RAID 0/1 or some
 * other configuration).
 *
 * When there's no device attached to the first downstream port of the
 * 4726, "Config Disk" appears, which is a pseudo ATA device to
 * configure the 4726.  However, ATA emulation of the device is very
 * lame.  It doesn't send signature D2H Reg FIS after the initial
 * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
 *
 * The following function works around the problem by always using
 * hardreset on the port and not depending on receiving signature FIS
 * afterward.  If signature FIS isn't received soon, ATA class is
 * assumed without follow-up softreset.
 */
static void ahci_p5wdh_workaround(struct ata_host *host)
{
	static struct dmi_system_id sysids[] = {
		{
			.ident = "P5W DH Deluxe",
			.matches = {
				DMI_MATCH(DMI_SYS_VENDOR,
					  "ASUSTEK COMPUTER INC"),
				DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
			},
		},
		{ }
	};
	struct pci_dev *pdev = to_pci_dev(host->dev);

	if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
	    dmi_check_system(sysids)) {
		struct ata_port *ap = host->ports[1];

		dev_printk(KERN_INFO, &pdev->dev, "enabling ASUS P5W DH "
			   "Deluxe on-board SIMG4726 workaround\n");

		ap->ops = &ahci_p5wdh_ops;
		ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
	}
}

2247
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
L
Linus Torvalds 已提交
2248 2249
{
	static int printed_version;
2250 2251
	struct ata_port_info pi = ahci_port_info[ent->driver_data];
	const struct ata_port_info *ppi[] = { &pi, NULL };
2252
	struct device *dev = &pdev->dev;
L
Linus Torvalds 已提交
2253
	struct ahci_host_priv *hpriv;
2254
	struct ata_host *host;
T
Tejun Heo 已提交
2255
	int n_ports, i, rc;
L
Linus Torvalds 已提交
2256 2257 2258

	VPRINTK("ENTER\n");

T
Tejun Heo 已提交
2259 2260
	WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);

L
Linus Torvalds 已提交
2261
	if (!printed_version++)
2262
		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
L
Linus Torvalds 已提交
2263

2264
	/* acquire resources */
2265
	rc = pcim_enable_device(pdev);
L
Linus Torvalds 已提交
2266 2267 2268
	if (rc)
		return rc;

T
Tejun Heo 已提交
2269 2270 2271 2272
	/* AHCI controllers often implement SFF compatible interface.
	 * Grab all PCI BARs just in case.
	 */
	rc = pcim_iomap_regions_request_all(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
T
Tejun Heo 已提交
2273
	if (rc == -EBUSY)
2274
		pcim_pin_device(pdev);
T
Tejun Heo 已提交
2275
	if (rc)
2276
		return rc;
L
Linus Torvalds 已提交
2277

2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293
	if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
	    (pdev->device == 0x2652 || pdev->device == 0x2653)) {
		u8 map;

		/* ICH6s share the same PCI ID for both piix and ahci
		 * modes.  Enabling ahci mode while MAP indicates
		 * combined mode is a bad idea.  Yield to ata_piix.
		 */
		pci_read_config_byte(pdev, ICH_MAP, &map);
		if (map & 0x3) {
			dev_printk(KERN_INFO, &pdev->dev, "controller is in "
				   "combined mode, can't enable AHCI mode\n");
			return -ENODEV;
		}
	}

2294 2295 2296
	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
	if (!hpriv)
		return -ENOMEM;
2297 2298 2299 2300
	hpriv->flags |= (unsigned long)pi.private_data;

	if ((hpriv->flags & AHCI_HFLAG_NO_MSI) || pci_enable_msi(pdev))
		pci_intx(pdev, 1);
L
Linus Torvalds 已提交
2301

2302
	/* save initial config */
2303
	ahci_save_initial_config(pdev, hpriv);
L
Linus Torvalds 已提交
2304

2305
	/* prepare host */
2306
	if (hpriv->cap & HOST_CAP_NCQ)
2307
		pi.flags |= ATA_FLAG_NCQ;
L
Linus Torvalds 已提交
2308

T
Tejun Heo 已提交
2309 2310 2311
	if (hpriv->cap & HOST_CAP_PMP)
		pi.flags |= ATA_FLAG_PMP;

T
Tejun Heo 已提交
2312 2313 2314 2315 2316 2317 2318 2319
	/* CAP.NP sometimes indicate the index of the last enabled
	 * port, at other times, that of the last possible port, so
	 * determining the maximum port number requires looking at
	 * both CAP.NP and port_map.
	 */
	n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));

	host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
2320 2321 2322 2323 2324 2325
	if (!host)
		return -ENOMEM;
	host->iomap = pcim_iomap_table(pdev);
	host->private_data = hpriv;

	for (i = 0; i < host->n_ports; i++) {
2326 2327
		struct ata_port *ap = host->ports[i];
		void __iomem *port_mmio = ahci_port_base(ap);
2328

2329 2330 2331 2332
		ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
		ata_port_pbar_desc(ap, AHCI_PCI_BAR,
				   0x100 + ap->port_no * 0x80, "port");

2333 2334 2335
		/* set initial link pm policy */
		ap->pm_policy = NOT_AVAILABLE;

2336
		/* standard SATA port setup */
T
Tejun Heo 已提交
2337
		if (hpriv->port_map & (1 << i))
2338
			ap->ioaddr.cmd_addr = port_mmio;
2339 2340 2341 2342

		/* disabled/not-implemented port */
		else
			ap->ops = &ata_dummy_port_ops;
2343
	}
2344

2345 2346 2347
	/* apply workaround for ASUS P5W DH Deluxe mainboard */
	ahci_p5wdh_workaround(host);

2348 2349
	/* initialize adapter */
	rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
L
Linus Torvalds 已提交
2350
	if (rc)
2351
		return rc;
L
Linus Torvalds 已提交
2352

2353 2354 2355
	rc = ahci_reset_controller(host);
	if (rc)
		return rc;
L
Linus Torvalds 已提交
2356

2357 2358
	ahci_init_controller(host);
	ahci_print_info(host);
L
Linus Torvalds 已提交
2359

2360 2361 2362
	pci_set_master(pdev);
	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
				 &ahci_sht);
2363
}
L
Linus Torvalds 已提交
2364 2365 2366

static int __init ahci_init(void)
{
2367
	return pci_register_driver(&ahci_pci_driver);
L
Linus Torvalds 已提交
2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
}

static void __exit ahci_exit(void)
{
	pci_unregister_driver(&ahci_pci_driver);
}


MODULE_AUTHOR("Jeff Garzik");
MODULE_DESCRIPTION("AHCI SATA low-level driver");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
2380
MODULE_VERSION(DRV_VERSION);
L
Linus Torvalds 已提交
2381 2382 2383

module_init(ahci_init);
module_exit(ahci_exit);