block.c 72.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * Block driver for media (i.e., flash cards)
 *
 * Copyright 2002 Hewlett-Packard Company
5
 * Copyright 2005-2008 Pierre Ossman
L
Linus Torvalds 已提交
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
 *
 * Use consistent with the GNU GPL is permitted,
 * provided that this copyright notice is
 * preserved in its entirety in all copies and derived works.
 *
 * HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED,
 * AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS
 * FITNESS FOR ANY PARTICULAR PURPOSE.
 *
 * Many thanks to Alessandro Rubini and Jonathan Corbet!
 *
 * Author:  Andrew Christian
 *          28 May 2002
 */
#include <linux/moduleparam.h>
#include <linux/module.h>
#include <linux/init.h>

#include <linux/kernel.h>
#include <linux/fs.h>
26
#include <linux/slab.h>
L
Linus Torvalds 已提交
27 28 29 30
#include <linux/errno.h>
#include <linux/hdreg.h>
#include <linux/kdev_t.h>
#include <linux/blkdev.h>
31
#include <linux/cdev.h>
32
#include <linux/mutex.h>
33
#include <linux/scatterlist.h>
34
#include <linux/string_helpers.h>
35 36 37
#include <linux/delay.h>
#include <linux/capability.h>
#include <linux/compat.h>
38
#include <linux/pm_runtime.h>
39
#include <linux/idr.h>
40
#include <linux/debugfs.h>
L
Linus Torvalds 已提交
41

42
#include <linux/mmc/ioctl.h>
L
Linus Torvalds 已提交
43
#include <linux/mmc/card.h>
P
Pierre Ossman 已提交
44
#include <linux/mmc/host.h>
P
Pierre Ossman 已提交
45 46
#include <linux/mmc/mmc.h>
#include <linux/mmc/sd.h>
L
Linus Torvalds 已提交
47

48
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
49

50
#include "queue.h"
51
#include "block.h"
52
#include "core.h"
53
#include "card.h"
54
#include "host.h"
55
#include "bus.h"
56
#include "mmc_ops.h"
57
#include "quirks.h"
58
#include "sd_ops.h"
L
Linus Torvalds 已提交
59

60
MODULE_ALIAS("mmc:block");
61 62 63 64 65
#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
#define MODULE_PARAM_PREFIX "mmcblk."

66
#define MMC_BLK_TIMEOUT_MS  (10 * 60 * 1000)        /* 10 minute timeout */
67 68
#define MMC_SANITIZE_REQ_TIMEOUT 240000
#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
69

70
#define mmc_req_rel_wr(req)	((req->cmd_flags & REQ_FUA) && \
71
				  (rq_data_dir(req) == WRITE))
72
static DEFINE_MUTEX(block_mutex);
73

L
Linus Torvalds 已提交
74
/*
75 76
 * The defaults come from config options but can be overriden by module
 * or bootarg options.
L
Linus Torvalds 已提交
77
 */
78
static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
79

80 81
/*
 * We've only got one major, so number of mmcblk devices is
B
Ben Hutchings 已提交
82
 * limited to (1 << 20) / number of minors per device.  It is also
83
 * limited by the MAX_DEVICES below.
84 85 86
 */
static int max_devices;

B
Ben Hutchings 已提交
87 88
#define MAX_DEVICES 256

89
static DEFINE_IDA(mmc_blk_ida);
90
static DEFINE_IDA(mmc_rpmb_ida);
L
Linus Torvalds 已提交
91 92 93 94 95 96

/*
 * There is one mmc_blk_data per slot.
 */
struct mmc_blk_data {
	spinlock_t	lock;
97
	struct device	*parent;
L
Linus Torvalds 已提交
98 99
	struct gendisk	*disk;
	struct mmc_queue queue;
100
	struct list_head part;
101
	struct list_head rpmbs;
L
Linus Torvalds 已提交
102

103 104 105 106
	unsigned int	flags;
#define MMC_BLK_CMD23	(1 << 0)	/* Can do SET_BLOCK_COUNT for multiblock */
#define MMC_BLK_REL_WR	(1 << 1)	/* MMC Reliable write support */

L
Linus Torvalds 已提交
107
	unsigned int	usage;
108
	unsigned int	read_only;
109
	unsigned int	part_type;
110 111 112 113 114
	unsigned int	reset_done;
#define MMC_BLK_READ		BIT(0)
#define MMC_BLK_WRITE		BIT(1)
#define MMC_BLK_DISCARD		BIT(2)
#define MMC_BLK_SECDISCARD	BIT(3)
115 116 117

	/*
	 * Only set in main mmc_blk_data associated
118
	 * with mmc_card with dev_set_drvdata, and keeps
119 120 121 122
	 * track of the current selected device partition.
	 */
	unsigned int	part_curr;
	struct device_attribute force_ro;
123 124
	struct device_attribute power_ro_lock;
	int	area_type;
125 126 127 128

	/* debugfs files (only in main mmc_blk_data) */
	struct dentry *status_dentry;
	struct dentry *ext_csd_dentry;
L
Linus Torvalds 已提交
129 130
};

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
/* Device type for RPMB character devices */
static dev_t mmc_rpmb_devt;

/* Bus type for RPMB character devices */
static struct bus_type mmc_rpmb_bus_type = {
	.name = "mmc_rpmb",
};

/**
 * struct mmc_rpmb_data - special RPMB device type for these areas
 * @dev: the device for the RPMB area
 * @chrdev: character device for the RPMB area
 * @id: unique device ID number
 * @part_index: partition index (0 on first)
 * @md: parent MMC block device
 * @node: list item, so we can put this device on a list
 */
struct mmc_rpmb_data {
	struct device dev;
	struct cdev chrdev;
	int id;
	unsigned int part_index;
	struct mmc_blk_data *md;
	struct list_head node;
};

157
static DEFINE_MUTEX(open_lock);
L
Linus Torvalds 已提交
158

159 160 161
module_param(perdev_minors, int, 0444);
MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");

162
static inline int mmc_blk_part_switch(struct mmc_card *card,
163
				      unsigned int part_type);
164

L
Linus Torvalds 已提交
165 166 167 168
static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
{
	struct mmc_blk_data *md;

169
	mutex_lock(&open_lock);
L
Linus Torvalds 已提交
170 171 172 173 174
	md = disk->private_data;
	if (md && md->usage == 0)
		md = NULL;
	if (md)
		md->usage++;
175
	mutex_unlock(&open_lock);
L
Linus Torvalds 已提交
176 177 178 179

	return md;
}

180 181
static inline int mmc_get_devidx(struct gendisk *disk)
{
182
	int devidx = disk->first_minor / perdev_minors;
183 184 185
	return devidx;
}

L
Linus Torvalds 已提交
186 187
static void mmc_blk_put(struct mmc_blk_data *md)
{
188
	mutex_lock(&open_lock);
L
Linus Torvalds 已提交
189 190
	md->usage--;
	if (md->usage == 0) {
191
		int devidx = mmc_get_devidx(md->disk);
192
		blk_put_queue(md->queue.queue);
193
		ida_simple_remove(&mmc_blk_ida, devidx);
L
Linus Torvalds 已提交
194 195 196
		put_disk(md->disk);
		kfree(md);
	}
197
	mutex_unlock(&open_lock);
L
Linus Torvalds 已提交
198 199
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
static ssize_t power_ro_lock_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	int ret;
	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
	struct mmc_card *card = md->queue.card;
	int locked = 0;

	if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN)
		locked = 2;
	else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN)
		locked = 1;

	ret = snprintf(buf, PAGE_SIZE, "%d\n", locked);

215 216
	mmc_blk_put(md);

217 218 219 220 221 222 223 224
	return ret;
}

static ssize_t power_ro_lock_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
{
	int ret;
	struct mmc_blk_data *md, *part_md;
225 226
	struct mmc_queue *mq;
	struct request *req;
227 228 229 230 231 232 233 234 235
	unsigned long set;

	if (kstrtoul(buf, 0, &set))
		return -EINVAL;

	if (set != 1)
		return count;

	md = mmc_blk_get(dev_to_disk(dev));
236
	mq = &md->queue;
237

238 239
	/* Dispatch locking to the block layer */
	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, __GFP_RECLAIM);
240 241 242 243
	if (IS_ERR(req)) {
		count = PTR_ERR(req);
		goto out_put;
	}
244 245 246
	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
	blk_execute_rq(mq->queue, NULL, req, 0);
	ret = req_to_mmc_queue_req(req)->drv_op_result;
247
	blk_put_request(req);
248 249 250 251 252 253 254 255 256 257 258 259

	if (!ret) {
		pr_info("%s: Locking boot partition ro until next power on\n",
			md->disk->disk_name);
		set_disk_ro(md->disk, 1);

		list_for_each_entry(part_md, &md->part, part)
			if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) {
				pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name);
				set_disk_ro(part_md->disk, 1);
			}
	}
260
out_put:
261 262 263 264
	mmc_blk_put(md);
	return count;
}

265 266 267 268 269 270
static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	int ret;
	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));

271
	ret = snprintf(buf, PAGE_SIZE, "%d\n",
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
		       get_disk_ro(dev_to_disk(dev)) ^
		       md->read_only);
	mmc_blk_put(md);
	return ret;
}

static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr,
			      const char *buf, size_t count)
{
	int ret;
	char *end;
	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
	unsigned long set = simple_strtoul(buf, &end, 0);
	if (end == buf) {
		ret = -EINVAL;
		goto out;
	}

	set_disk_ro(dev_to_disk(dev), set || md->read_only);
	ret = count;
out:
	mmc_blk_put(md);
	return ret;
}

A
Al Viro 已提交
297
static int mmc_blk_open(struct block_device *bdev, fmode_t mode)
L
Linus Torvalds 已提交
298
{
A
Al Viro 已提交
299
	struct mmc_blk_data *md = mmc_blk_get(bdev->bd_disk);
L
Linus Torvalds 已提交
300 301
	int ret = -ENXIO;

302
	mutex_lock(&block_mutex);
L
Linus Torvalds 已提交
303 304
	if (md) {
		if (md->usage == 2)
A
Al Viro 已提交
305
			check_disk_change(bdev);
L
Linus Torvalds 已提交
306
		ret = 0;
P
Pierre Ossman 已提交
307

A
Al Viro 已提交
308
		if ((mode & FMODE_WRITE) && md->read_only) {
309
			mmc_blk_put(md);
P
Pierre Ossman 已提交
310
			ret = -EROFS;
311
		}
L
Linus Torvalds 已提交
312
	}
313
	mutex_unlock(&block_mutex);
L
Linus Torvalds 已提交
314 315 316 317

	return ret;
}

318
static void mmc_blk_release(struct gendisk *disk, fmode_t mode)
L
Linus Torvalds 已提交
319
{
A
Al Viro 已提交
320
	struct mmc_blk_data *md = disk->private_data;
L
Linus Torvalds 已提交
321

322
	mutex_lock(&block_mutex);
L
Linus Torvalds 已提交
323
	mmc_blk_put(md);
324
	mutex_unlock(&block_mutex);
L
Linus Torvalds 已提交
325 326 327
}

static int
328
mmc_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
L
Linus Torvalds 已提交
329
{
330 331 332 333
	geo->cylinders = get_capacity(bdev->bd_disk) / (4 * 16);
	geo->heads = 4;
	geo->sectors = 16;
	return 0;
L
Linus Torvalds 已提交
334 335
}

336 337 338 339
struct mmc_blk_ioc_data {
	struct mmc_ioc_cmd ic;
	unsigned char *buf;
	u64 buf_bytes;
340
	struct mmc_rpmb_data *rpmb;
341 342 343 344 345 346 347 348
};

static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user(
	struct mmc_ioc_cmd __user *user)
{
	struct mmc_blk_ioc_data *idata;
	int err;

349
	idata = kmalloc(sizeof(*idata), GFP_KERNEL);
350 351
	if (!idata) {
		err = -ENOMEM;
352
		goto out;
353 354 355 356
	}

	if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) {
		err = -EFAULT;
357
		goto idata_err;
358 359 360 361 362
	}

	idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks;
	if (idata->buf_bytes > MMC_IOC_MAX_BYTES) {
		err = -EOVERFLOW;
363
		goto idata_err;
364 365
	}

366 367
	if (!idata->buf_bytes) {
		idata->buf = NULL;
368
		return idata;
369
	}
370

371
	idata->buf = kmalloc(idata->buf_bytes, GFP_KERNEL);
372 373
	if (!idata->buf) {
		err = -ENOMEM;
374
		goto idata_err;
375 376 377 378 379 380 381 382 383 384 385 386
	}

	if (copy_from_user(idata->buf, (void __user *)(unsigned long)
					idata->ic.data_ptr, idata->buf_bytes)) {
		err = -EFAULT;
		goto copy_err;
	}

	return idata;

copy_err:
	kfree(idata->buf);
387
idata_err:
388
	kfree(idata);
389
out:
390 391 392
	return ERR_PTR(err);
}

393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr,
				      struct mmc_blk_ioc_data *idata)
{
	struct mmc_ioc_cmd *ic = &idata->ic;

	if (copy_to_user(&(ic_ptr->response), ic->response,
			 sizeof(ic->response)))
		return -EFAULT;

	if (!idata->ic.write_flag) {
		if (copy_to_user((void __user *)(unsigned long)ic->data_ptr,
				 idata->buf, idata->buf_bytes))
			return -EFAULT;
	}

	return 0;
}

411 412 413 414 415 416 417 418 419 420
static int ioctl_rpmb_card_status_poll(struct mmc_card *card, u32 *status,
				       u32 retries_max)
{
	int err;
	u32 retry_count = 0;

	if (!status || !retries_max)
		return -EINVAL;

	do {
421
		err = __mmc_send_status(card, status, 5);
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
		if (err)
			break;

		if (!R1_STATUS(*status) &&
				(R1_CURRENT_STATE(*status) != R1_STATE_PRG))
			break; /* RPMB programming operation complete */

		/*
		 * Rechedule to give the MMC device a chance to continue
		 * processing the previous command without being polled too
		 * frequently.
		 */
		usleep_range(1000, 5000);
	} while (++retry_count < retries_max);

	if (retry_count == retries_max)
		err = -EPERM;

	return err;
}

443 444 445 446
static int ioctl_do_sanitize(struct mmc_card *card)
{
	int err;

447
	if (!mmc_can_sanitize(card)) {
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
			pr_warn("%s: %s - SANITIZE is not supported\n",
				mmc_hostname(card->host), __func__);
			err = -EOPNOTSUPP;
			goto out;
	}

	pr_debug("%s: %s - SANITIZE IN PROGRESS...\n",
		mmc_hostname(card->host), __func__);

	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
					EXT_CSD_SANITIZE_START, 1,
					MMC_SANITIZE_REQ_TIMEOUT);

	if (err)
		pr_err("%s: %s - EXT_CSD_SANITIZE_START failed. err=%d\n",
		       mmc_hostname(card->host), __func__, err);

	pr_debug("%s: %s - SANITIZE COMPLETED\n", mmc_hostname(card->host),
					     __func__);
out:
	return err;
}

471 472
static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
			       struct mmc_blk_ioc_data *idata)
473
{
474 475 476
	struct mmc_command cmd = {};
	struct mmc_data data = {};
	struct mmc_request mrq = {};
477 478
	struct scatterlist sg;
	int err;
479
	unsigned int target_part;
480
	u32 status = 0;
481

482 483
	if (!card || !md || !idata)
		return -EINVAL;
484

485 486 487 488 489 490 491 492 493 494 495 496 497
	/*
	 * The RPMB accesses comes in from the character device, so we
	 * need to target these explicitly. Else we just target the
	 * partition type for the block device the ioctl() was issued
	 * on.
	 */
	if (idata->rpmb) {
		/* Support multiple RPMB partitions */
		target_part = idata->rpmb->part_index;
		target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB;
	} else {
		target_part = md->part_type;
	}
498

499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
	cmd.opcode = idata->ic.opcode;
	cmd.arg = idata->ic.arg;
	cmd.flags = idata->ic.flags;

	if (idata->buf_bytes) {
		data.sg = &sg;
		data.sg_len = 1;
		data.blksz = idata->ic.blksz;
		data.blocks = idata->ic.blocks;

		sg_init_one(data.sg, idata->buf, idata->buf_bytes);

		if (idata->ic.write_flag)
			data.flags = MMC_DATA_WRITE;
		else
			data.flags = MMC_DATA_READ;

		/* data.flags must already be set before doing this. */
		mmc_set_data_timeout(&data, card);

		/* Allow overriding the timeout_ns for empirical tuning. */
		if (idata->ic.data_timeout_ns)
			data.timeout_ns = idata->ic.data_timeout_ns;

		if ((cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B) {
			/*
			 * Pretend this is a data transfer and rely on the
			 * host driver to compute timeout.  When all host
			 * drivers support cmd.cmd_timeout for R1B, this
			 * can be changed to:
			 *
			 *     mrq.data = NULL;
			 *     cmd.cmd_timeout = idata->ic.cmd_timeout_ms;
			 */
			data.timeout_ns = idata->ic.cmd_timeout_ms * 1000000;
		}

		mrq.data = &data;
	}

	mrq.cmd = &cmd;

541
	err = mmc_blk_part_switch(card, target_part);
542
	if (err)
543
		return err;
544

545 546 547
	if (idata->ic.is_acmd) {
		err = mmc_app_cmd(card->host, card);
		if (err)
548
			return err;
549 550
	}

551
	if (idata->rpmb) {
552 553 554
		err = mmc_set_blockcount(card, data.blocks,
			idata->ic.write_flag & (1 << 31));
		if (err)
555
			return err;
556 557
	}

558 559
	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) &&
	    (cmd.opcode == MMC_SWITCH)) {
560 561 562 563 564 565
		err = ioctl_do_sanitize(card);

		if (err)
			pr_err("%s: ioctl_do_sanitize() failed. err = %d",
			       __func__, err);

566
		return err;
567 568
	}

569 570 571 572 573
	mmc_wait_for_req(card->host, &mrq);

	if (cmd.error) {
		dev_err(mmc_dev(card->host), "%s: cmd error %d\n",
						__func__, cmd.error);
574
		return cmd.error;
575 576 577 578
	}
	if (data.error) {
		dev_err(mmc_dev(card->host), "%s: data error %d\n",
						__func__, data.error);
579
		return data.error;
580 581 582 583 584 585 586 587 588
	}

	/*
	 * According to the SD specs, some commands require a delay after
	 * issuing the command.
	 */
	if (idata->ic.postsleep_min_us)
		usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us);

589
	memcpy(&(idata->ic.response), cmd.resp, sizeof(cmd.resp));
590

591
	if (idata->rpmb) {
592 593 594 595 596 597 598 599 600 601 602
		/*
		 * Ensure RPMB command has completed by polling CMD13
		 * "Send Status".
		 */
		err = ioctl_rpmb_card_status_poll(card, &status, 5);
		if (err)
			dev_err(mmc_dev(card->host),
					"%s: Card Status=0x%08X, error %d\n",
					__func__, status, err);
	}

603 604 605
	return err;
}

606
static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
607 608
			     struct mmc_ioc_cmd __user *ic_ptr,
			     struct mmc_rpmb_data *rpmb)
609 610
{
	struct mmc_blk_ioc_data *idata;
611
	struct mmc_blk_ioc_data *idatas[1];
612
	struct mmc_queue *mq;
613
	struct mmc_card *card;
614
	int err = 0, ioc_err = 0;
615
	struct request *req;
616 617 618 619

	idata = mmc_blk_ioctl_copy_from_user(ic_ptr);
	if (IS_ERR(idata))
		return PTR_ERR(idata);
620 621
	/* This will be NULL on non-RPMB ioctl():s */
	idata->rpmb = rpmb;
622 623 624 625 626 627 628

	card = md->queue.card;
	if (IS_ERR(card)) {
		err = PTR_ERR(card);
		goto cmd_done;
	}

629 630 631 632 633 634 635
	/*
	 * Dispatch the ioctl() into the block request queue.
	 */
	mq = &md->queue;
	req = blk_get_request(mq->queue,
		idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
		__GFP_RECLAIM);
636 637 638 639
	if (IS_ERR(req)) {
		err = PTR_ERR(req);
		goto cmd_done;
	}
640
	idatas[0] = idata;
641 642
	req_to_mmc_queue_req(req)->drv_op =
		rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
643
	req_to_mmc_queue_req(req)->drv_op_data = idatas;
644
	req_to_mmc_queue_req(req)->ioc_count = 1;
645
	blk_execute_rq(mq->queue, NULL, req, 0);
646
	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
647
	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
648
	blk_put_request(req);
649

650 651 652
cmd_done:
	kfree(idata->buf);
	kfree(idata);
653
	return ioc_err ? ioc_err : err;
654 655
}

656
static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
657 658
				   struct mmc_ioc_multi_cmd __user *user,
				   struct mmc_rpmb_data *rpmb)
659 660 661 662
{
	struct mmc_blk_ioc_data **idata = NULL;
	struct mmc_ioc_cmd __user *cmds = user->cmds;
	struct mmc_card *card;
663
	struct mmc_queue *mq;
664
	int i, err = 0, ioc_err = 0;
665
	__u64 num_of_cmds;
666
	struct request *req;
667 668 669 670 671

	if (copy_from_user(&num_of_cmds, &user->num_of_cmds,
			   sizeof(num_of_cmds)))
		return -EFAULT;

672 673 674
	if (!num_of_cmds)
		return 0;

675 676 677 678 679 680 681 682 683 684 685 686 687 688
	if (num_of_cmds > MMC_IOC_MAX_CMDS)
		return -EINVAL;

	idata = kcalloc(num_of_cmds, sizeof(*idata), GFP_KERNEL);
	if (!idata)
		return -ENOMEM;

	for (i = 0; i < num_of_cmds; i++) {
		idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]);
		if (IS_ERR(idata[i])) {
			err = PTR_ERR(idata[i]);
			num_of_cmds = i;
			goto cmd_err;
		}
689 690
		/* This will be NULL on non-RPMB ioctl():s */
		idata[i]->rpmb = rpmb;
691 692 693 694 695
	}

	card = md->queue.card;
	if (IS_ERR(card)) {
		err = PTR_ERR(card);
696
		goto cmd_err;
697 698 699
	}


700 701 702 703 704 705 706
	/*
	 * Dispatch the ioctl()s into the block request queue.
	 */
	mq = &md->queue;
	req = blk_get_request(mq->queue,
		idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
		__GFP_RECLAIM);
707 708 709 710
	if (IS_ERR(req)) {
		err = PTR_ERR(req);
		goto cmd_err;
	}
711 712
	req_to_mmc_queue_req(req)->drv_op =
		rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
713
	req_to_mmc_queue_req(req)->drv_op_data = idata;
714 715
	req_to_mmc_queue_req(req)->ioc_count = num_of_cmds;
	blk_execute_rq(mq->queue, NULL, req, 0);
716
	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
717 718

	/* copy to user if data and response */
719
	for (i = 0; i < num_of_cmds && !err; i++)
720 721
		err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]);

722 723
	blk_put_request(req);

724 725 726 727 728 729
cmd_err:
	for (i = 0; i < num_of_cmds; i++) {
		kfree(idata[i]->buf);
		kfree(idata[i]);
	}
	kfree(idata);
730
	return ioc_err ? ioc_err : err;
731 732
}

L
Linus Walleij 已提交
733 734 735 736 737 738 739 740 741 742 743 744
static int mmc_blk_check_blkdev(struct block_device *bdev)
{
	/*
	 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
	 * whole block device, not on a partition.  This prevents overspray
	 * between sibling partitions.
	 */
	if ((!capable(CAP_SYS_RAWIO)) || (bdev != bdev->bd_contains))
		return -EPERM;
	return 0;
}

745 746 747
static int mmc_blk_ioctl(struct block_device *bdev, fmode_t mode,
	unsigned int cmd, unsigned long arg)
{
748
	struct mmc_blk_data *md;
L
Linus Walleij 已提交
749 750
	int ret;

751 752
	switch (cmd) {
	case MMC_IOC_CMD:
L
Linus Walleij 已提交
753 754 755
		ret = mmc_blk_check_blkdev(bdev);
		if (ret)
			return ret;
756 757 758 759
		md = mmc_blk_get(bdev->bd_disk);
		if (!md)
			return -EINVAL;
		ret = mmc_blk_ioctl_cmd(md,
760 761
					(struct mmc_ioc_cmd __user *)arg,
					NULL);
762 763
		mmc_blk_put(md);
		return ret;
764
	case MMC_IOC_MULTI_CMD:
L
Linus Walleij 已提交
765 766 767
		ret = mmc_blk_check_blkdev(bdev);
		if (ret)
			return ret;
768 769 770 771
		md = mmc_blk_get(bdev->bd_disk);
		if (!md)
			return -EINVAL;
		ret = mmc_blk_ioctl_multi_cmd(md,
772 773
					(struct mmc_ioc_multi_cmd __user *)arg,
					NULL);
774 775
		mmc_blk_put(md);
		return ret;
776 777 778
	default:
		return -EINVAL;
	}
779 780 781 782 783 784 785 786 787 788
}

#ifdef CONFIG_COMPAT
static int mmc_blk_compat_ioctl(struct block_device *bdev, fmode_t mode,
	unsigned int cmd, unsigned long arg)
{
	return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg));
}
#endif

789
static const struct block_device_operations mmc_bdops = {
A
Al Viro 已提交
790 791
	.open			= mmc_blk_open,
	.release		= mmc_blk_release,
792
	.getgeo			= mmc_blk_getgeo,
L
Linus Torvalds 已提交
793
	.owner			= THIS_MODULE,
794 795 796 797
	.ioctl			= mmc_blk_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl		= mmc_blk_compat_ioctl,
#endif
L
Linus Torvalds 已提交
798 799
};

800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
static int mmc_blk_part_switch_pre(struct mmc_card *card,
				   unsigned int part_type)
{
	int ret = 0;

	if (part_type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
		if (card->ext_csd.cmdq_en) {
			ret = mmc_cmdq_disable(card);
			if (ret)
				return ret;
		}
		mmc_retune_pause(card->host);
	}

	return ret;
}

static int mmc_blk_part_switch_post(struct mmc_card *card,
				    unsigned int part_type)
{
	int ret = 0;

	if (part_type == EXT_CSD_PART_CONFIG_ACC_RPMB) {
		mmc_retune_unpause(card->host);
		if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
			ret = mmc_cmdq_enable(card);
	}

	return ret;
}

831
static inline int mmc_blk_part_switch(struct mmc_card *card,
832
				      unsigned int part_type)
833
{
834
	int ret = 0;
835
	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
836

837
	if (main_md->part_curr == part_type)
838 839 840
		return 0;

	if (mmc_card_mmc(card)) {
841 842
		u8 part_config = card->ext_csd.part_config;

843
		ret = mmc_blk_part_switch_pre(card, part_type);
844 845
		if (ret)
			return ret;
846

847
		part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
848
		part_config |= part_type;
849 850

		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
851
				 EXT_CSD_PART_CONFIG, part_config,
852
				 card->ext_csd.part_time);
853
		if (ret) {
854
			mmc_blk_part_switch_post(card, part_type);
855
			return ret;
856
		}
857 858

		card->ext_csd.part_config = part_config;
859

860
		ret = mmc_blk_part_switch_post(card, main_md->part_curr);
861
	}
862

863
	main_md->part_curr = part_type;
864
	return ret;
865 866
}

867
static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks)
868 869
{
	int err;
B
Ben Dooks 已提交
870 871
	u32 result;
	__be32 *blocks;
872

873 874 875
	struct mmc_request mrq = {};
	struct mmc_command cmd = {};
	struct mmc_data data = {};
876 877 878 879 880

	struct scatterlist sg;

	cmd.opcode = MMC_APP_CMD;
	cmd.arg = card->rca << 16;
D
David Brownell 已提交
881
	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
882 883

	err = mmc_wait_for_cmd(card->host, &cmd, 0);
D
David Brownell 已提交
884
	if (err)
885
		return err;
D
David Brownell 已提交
886
	if (!mmc_host_is_spi(card->host) && !(cmd.resp[0] & R1_APP_CMD))
887
		return -EIO;
888 889 890 891 892

	memset(&cmd, 0, sizeof(struct mmc_command));

	cmd.opcode = SD_APP_SEND_NUM_WR_BLKS;
	cmd.arg = 0;
D
David Brownell 已提交
893
	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
894 895 896 897 898 899

	data.blksz = 4;
	data.blocks = 1;
	data.flags = MMC_DATA_READ;
	data.sg = &sg;
	data.sg_len = 1;
900
	mmc_set_data_timeout(&data, card);
901 902 903 904

	mrq.cmd = &cmd;
	mrq.data = &data;

B
Ben Dooks 已提交
905 906
	blocks = kmalloc(4, GFP_KERNEL);
	if (!blocks)
907
		return -ENOMEM;
B
Ben Dooks 已提交
908 909

	sg_init_one(&sg, blocks, 4);
910 911 912

	mmc_wait_for_req(card->host, &mrq);

B
Ben Dooks 已提交
913 914 915
	result = ntohl(*blocks);
	kfree(blocks);

P
Pierre Ossman 已提交
916
	if (cmd.error || data.error)
917 918 919
		return -EIO;

	*written_blocks = result;
920

921
	return 0;
922 923
}

924
static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
925
		bool hw_busy_detect, struct request *req, bool *gen_err)
926 927 928 929 930 931
{
	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
	int err = 0;
	u32 status;

	do {
932
		err = __mmc_send_status(card, &status, 5);
933 934 935 936 937 938 939 940 941
		if (err) {
			pr_err("%s: error %d requesting status\n",
			       req->rq_disk->disk_name, err);
			return err;
		}

		if (status & R1_ERROR) {
			pr_err("%s: %s: error sending status cmd, status %#x\n",
				req->rq_disk->disk_name, __func__, status);
942
			*gen_err = true;
943 944
		}

945 946 947 948 949
		/* We may rely on the host hw to handle busy detection.*/
		if ((card->host->caps & MMC_CAP_WAIT_WHILE_BUSY) &&
			hw_busy_detect)
			break;

950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971
		/*
		 * Timeout if the device never becomes ready for data and never
		 * leaves the program state.
		 */
		if (time_after(jiffies, timeout)) {
			pr_err("%s: Card stuck in programming state! %s %s\n",
				mmc_hostname(card->host),
				req->rq_disk->disk_name, __func__);
			return -ETIMEDOUT;
		}

		/*
		 * Some cards mishandle the status bits,
		 * so make sure to check both the busy
		 * indication and the card state.
		 */
	} while (!(status & R1_READY_FOR_DATA) ||
		 (R1_CURRENT_STATE(status) == R1_STATE_PRG));

	return err;
}

972
static int send_stop(struct mmc_card *card, unsigned int timeout_ms,
973
		struct request *req, bool *gen_err, u32 *stop_status)
974 975
{
	struct mmc_host *host = card->host;
976
	struct mmc_command cmd = {};
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
	int err;
	bool use_r1b_resp = rq_data_dir(req) == WRITE;

	/*
	 * Normally we use R1B responses for WRITE, but in cases where the host
	 * has specified a max_busy_timeout we need to validate it. A failure
	 * means we need to prevent the host from doing hw busy detection, which
	 * is done by converting to a R1 response instead.
	 */
	if (host->max_busy_timeout && (timeout_ms > host->max_busy_timeout))
		use_r1b_resp = false;

	cmd.opcode = MMC_STOP_TRANSMISSION;
	if (use_r1b_resp) {
		cmd.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
		cmd.busy_timeout = timeout_ms;
	} else {
		cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
	}

	err = mmc_wait_for_cmd(host, &cmd, 5);
	if (err)
		return err;

	*stop_status = cmd.resp[0];

	/* No need to check card status in case of READ. */
	if (rq_data_dir(req) == READ)
		return 0;

	if (!mmc_host_is_spi(host) &&
		(*stop_status & R1_ERROR)) {
		pr_err("%s: %s: general error sending stop command, resp %#x\n",
			req->rq_disk->disk_name, __func__, *stop_status);
1011
		*gen_err = true;
1012 1013 1014 1015 1016
	}

	return card_busy_detect(card, timeout_ms, use_r1b_resp, req, gen_err);
}

1017
#define ERR_NOMEDIUM	3
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
#define ERR_RETRY	2
#define ERR_ABORT	1
#define ERR_CONTINUE	0

static int mmc_blk_cmd_error(struct request *req, const char *name, int error,
	bool status_valid, u32 status)
{
	switch (error) {
	case -EILSEQ:
		/* response crc error, retry the r/w cmd */
		pr_err("%s: %s sending %s command, card status %#x\n",
			req->rq_disk->disk_name, "response CRC error",
			name, status);
		return ERR_RETRY;

	case -ETIMEDOUT:
		pr_err("%s: %s sending %s command, card status %#x\n",
			req->rq_disk->disk_name, "timed out", name, status);

		/* If the status cmd initially failed, retry the r/w cmd */
1038 1039 1040
		if (!status_valid) {
			pr_err("%s: status not valid, retrying timeout\n",
				req->rq_disk->disk_name);
1041
			return ERR_RETRY;
1042
		}
1043 1044 1045 1046 1047 1048

		/*
		 * If it was a r/w cmd crc error, or illegal command
		 * (eg, issued in wrong state) then retry - we should
		 * have corrected the state problem above.
		 */
1049 1050 1051
		if (status & (R1_COM_CRC_ERROR | R1_ILLEGAL_COMMAND)) {
			pr_err("%s: command error, retrying timeout\n",
				req->rq_disk->disk_name);
1052
			return ERR_RETRY;
1053
		}
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084

		/* Otherwise abort the command */
		return ERR_ABORT;

	default:
		/* We don't understand the error code the driver gave us */
		pr_err("%s: unknown error %d sending read/write command, card status %#x\n",
		       req->rq_disk->disk_name, error, status);
		return ERR_ABORT;
	}
}

/*
 * Initial r/w and stop cmd error recovery.
 * We don't know whether the card received the r/w cmd or not, so try to
 * restore things back to a sane state.  Essentially, we do this as follows:
 * - Obtain card status.  If the first attempt to obtain card status fails,
 *   the status word will reflect the failed status cmd, not the failed
 *   r/w cmd.  If we fail to obtain card status, it suggests we can no
 *   longer communicate with the card.
 * - Check the card state.  If the card received the cmd but there was a
 *   transient problem with the response, it might still be in a data transfer
 *   mode.  Try to send it a stop command.  If this fails, we can't recover.
 * - If the r/w cmd failed due to a response CRC error, it was probably
 *   transient, so retry the cmd.
 * - If the r/w cmd timed out, but we didn't get the r/w cmd status, retry.
 * - If the r/w cmd timed out, and the r/w cmd failed due to CRC error or
 *   illegal cmd, retry.
 * Otherwise we don't understand what happened, so abort.
 */
static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
1085
	struct mmc_blk_request *brq, bool *ecc_err, bool *gen_err)
1086 1087 1088 1089 1090
{
	bool prev_cmd_status_valid = true;
	u32 status, stop_status = 0;
	int err, retry;

1091 1092 1093
	if (mmc_card_removed(card))
		return ERR_NOMEDIUM;

1094 1095 1096 1097 1098 1099
	/*
	 * Try to get card status which indicates both the card state
	 * and why there was no response.  If the first attempt fails,
	 * we can't be sure the returned status is for the r/w command.
	 */
	for (retry = 2; retry >= 0; retry--) {
1100
		err = __mmc_send_status(card, &status, 0);
1101 1102 1103
		if (!err)
			break;

1104 1105 1106
		/* Re-tune if needed */
		mmc_retune_recheck(card->host);

1107 1108 1109 1110 1111 1112
		prev_cmd_status_valid = false;
		pr_err("%s: error %d sending status command, %sing\n",
		       req->rq_disk->disk_name, err, retry ? "retry" : "abort");
	}

	/* We couldn't get a response from the card.  Give up. */
1113 1114 1115 1116
	if (err) {
		/* Check if the card is removed */
		if (mmc_detect_card_removed(card->host))
			return ERR_NOMEDIUM;
1117
		return ERR_ABORT;
1118
	}
1119

1120 1121 1122 1123
	/* Flag ECC errors */
	if ((status & R1_CARD_ECC_FAILED) ||
	    (brq->stop.resp[0] & R1_CARD_ECC_FAILED) ||
	    (brq->cmd.resp[0] & R1_CARD_ECC_FAILED))
1124
		*ecc_err = true;
1125

1126 1127 1128 1129 1130 1131 1132
	/* Flag General errors */
	if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ)
		if ((status & R1_ERROR) ||
			(brq->stop.resp[0] & R1_ERROR)) {
			pr_err("%s: %s: general error sending stop or status command, stop cmd response %#x, card status %#x\n",
			       req->rq_disk->disk_name, __func__,
			       brq->stop.resp[0], status);
1133
			*gen_err = true;
1134 1135
		}

1136 1137 1138 1139 1140 1141
	/*
	 * Check the current card state.  If it is in some data transfer
	 * mode, tell it to stop (and hopefully transition back to TRAN.)
	 */
	if (R1_CURRENT_STATE(status) == R1_STATE_DATA ||
	    R1_CURRENT_STATE(status) == R1_STATE_RCV) {
1142 1143 1144 1145
		err = send_stop(card,
			DIV_ROUND_UP(brq->data.timeout_ns, 1000000),
			req, gen_err, &stop_status);
		if (err) {
1146 1147
			pr_err("%s: error %d sending stop command\n",
			       req->rq_disk->disk_name, err);
1148 1149 1150 1151
			/*
			 * If the stop cmd also timed out, the card is probably
			 * not present, so abort. Other errors are bad news too.
			 */
1152
			return ERR_ABORT;
1153 1154
		}

1155
		if (stop_status & R1_CARD_ECC_FAILED)
1156
			*ecc_err = true;
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
	}

	/* Check for set block count errors */
	if (brq->sbc.error)
		return mmc_blk_cmd_error(req, "SET_BLOCK_COUNT", brq->sbc.error,
				prev_cmd_status_valid, status);

	/* Check for r/w command errors */
	if (brq->cmd.error)
		return mmc_blk_cmd_error(req, "r/w cmd", brq->cmd.error,
				prev_cmd_status_valid, status);

1169 1170 1171 1172
	/* Data errors */
	if (!brq->stop.error)
		return ERR_CONTINUE;

1173
	/* Now for stop errors.  These aren't fatal to the transfer. */
1174
	pr_info("%s: error %d sending stop command, original cmd response %#x, card status %#x\n",
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
	       req->rq_disk->disk_name, brq->stop.error,
	       brq->cmd.resp[0], status);

	/*
	 * Subsitute in our own stop status as this will give the error
	 * state which happened during the execution of the r/w command.
	 */
	if (stop_status) {
		brq->stop.resp[0] = stop_status;
		brq->stop.error = 0;
	}
	return ERR_CONTINUE;
}

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
			 int type)
{
	int err;

	if (md->reset_done & type)
		return -EEXIST;

	md->reset_done |= type;
	err = mmc_hw_reset(host);
	/* Ensure we switch back to the correct partition */
	if (err != -EOPNOTSUPP) {
1201 1202
		struct mmc_blk_data *main_md =
			dev_get_drvdata(&host->card->dev);
1203 1204 1205
		int part_err;

		main_md->part_curr = main_md->part_type;
1206
		part_err = mmc_blk_part_switch(host->card, md->part_type);
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
		if (part_err) {
			/*
			 * We have failed to get back into the correct
			 * partition, so we need to abort the whole request.
			 */
			return -ENODEV;
		}
	}
	return err;
}

static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
{
	md->reset_done &= ~type;
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
/*
 * The non-block commands come back from the block layer after it queued it and
 * processed it with all other requests and then they get issued in this
 * function.
 */
static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
{
	struct mmc_queue_req *mq_rq;
	struct mmc_card *card = mq->card;
	struct mmc_blk_data *md = mq->blkdata;
1233
	struct mmc_blk_ioc_data **idata;
1234
	bool rpmb_ioctl;
1235 1236
	u8 **ext_csd;
	u32 status;
1237
	int ret;
1238 1239 1240
	int i;

	mq_rq = req_to_mmc_queue_req(req);
1241
	rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB);
1242 1243 1244

	switch (mq_rq->drv_op) {
	case MMC_DRV_OP_IOCTL:
1245
	case MMC_DRV_OP_IOCTL_RPMB:
1246
		idata = mq_rq->drv_op_data;
1247
		for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
1248
			ret = __mmc_blk_ioctl_cmd(card, md, idata[i]);
1249
			if (ret)
1250 1251 1252
				break;
		}
		/* Always switch back to main area after RPMB access */
1253 1254
		if (rpmb_ioctl)
			mmc_blk_part_switch(card, 0);
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
		break;
	case MMC_DRV_OP_BOOT_WP:
		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
				 card->ext_csd.boot_ro_lock |
				 EXT_CSD_BOOT_WP_B_PWR_WP_EN,
				 card->ext_csd.part_time);
		if (ret)
			pr_err("%s: Locking boot partition ro until next power on failed: %d\n",
			       md->disk->disk_name, ret);
		else
			card->ext_csd.boot_ro_lock |=
				EXT_CSD_BOOT_WP_B_PWR_WP_EN;
1267
		break;
1268 1269 1270 1271 1272 1273 1274 1275 1276
	case MMC_DRV_OP_GET_CARD_STATUS:
		ret = mmc_send_status(card, &status);
		if (!ret)
			ret = status;
		break;
	case MMC_DRV_OP_GET_EXT_CSD:
		ext_csd = mq_rq->drv_op_data;
		ret = mmc_get_ext_csd(card, ext_csd);
		break;
1277
	default:
1278 1279 1280
		pr_err("%s: unknown driver specific operation\n",
		       md->disk->disk_name);
		ret = -EINVAL;
1281 1282
		break;
	}
1283
	mq_rq->drv_op_result = ret;
1284
	blk_end_request_all(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
1285 1286
}

1287
static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
A
Adrian Hunter 已提交
1288
{
L
Linus Walleij 已提交
1289
	struct mmc_blk_data *md = mq->blkdata;
A
Adrian Hunter 已提交
1290 1291
	struct mmc_card *card = md->queue.card;
	unsigned int from, nr, arg;
1292
	int err = 0, type = MMC_BLK_DISCARD;
1293
	blk_status_t status = BLK_STS_OK;
A
Adrian Hunter 已提交
1294 1295

	if (!mmc_can_erase(card)) {
1296
		status = BLK_STS_NOTSUPP;
1297
		goto fail;
A
Adrian Hunter 已提交
1298 1299 1300 1301 1302
	}

	from = blk_rq_pos(req);
	nr = blk_rq_sectors(req);

1303 1304 1305
	if (mmc_can_discard(card))
		arg = MMC_DISCARD_ARG;
	else if (mmc_can_trim(card))
A
Adrian Hunter 已提交
1306 1307 1308
		arg = MMC_TRIM_ARG;
	else
		arg = MMC_ERASE_ARG;
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
	do {
		err = 0;
		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
			err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
					 INAND_CMD38_ARG_EXT_CSD,
					 arg == MMC_TRIM_ARG ?
					 INAND_CMD38_ARG_TRIM :
					 INAND_CMD38_ARG_ERASE,
					 0);
		}
		if (!err)
			err = mmc_erase(card, from, nr, arg);
	} while (err == -EIO && !mmc_blk_reset(md, card->host, type));
1322 1323 1324
	if (err)
		status = BLK_STS_IOERR;
	else
1325
		mmc_blk_reset_success(md, type);
1326
fail:
1327
	blk_end_request(req, status, blk_rq_bytes(req));
A
Adrian Hunter 已提交
1328 1329
}

1330
static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
1331 1332
				       struct request *req)
{
L
Linus Walleij 已提交
1333
	struct mmc_blk_data *md = mq->blkdata;
1334
	struct mmc_card *card = md->queue.card;
1335
	unsigned int from, nr, arg;
1336
	int err = 0, type = MMC_BLK_SECDISCARD;
1337
	blk_status_t status = BLK_STS_OK;
1338

1339
	if (!(mmc_can_secure_erase_trim(card))) {
1340
		status = BLK_STS_NOTSUPP;
1341 1342 1343
		goto out;
	}

1344 1345 1346
	from = blk_rq_pos(req);
	nr = blk_rq_sectors(req);

1347 1348 1349 1350
	if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
		arg = MMC_SECURE_TRIM1_ARG;
	else
		arg = MMC_SECURE_ERASE_ARG;
1351

1352
retry:
1353 1354 1355 1356 1357 1358 1359 1360
	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
		err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
				 INAND_CMD38_ARG_EXT_CSD,
				 arg == MMC_SECURE_TRIM1_ARG ?
				 INAND_CMD38_ARG_SECTRIM1 :
				 INAND_CMD38_ARG_SECERASE,
				 0);
		if (err)
1361
			goto out_retry;
1362
	}
1363

1364
	err = mmc_erase(card, from, nr, arg);
1365 1366
	if (err == -EIO)
		goto out_retry;
1367 1368
	if (err) {
		status = BLK_STS_IOERR;
1369
		goto out;
1370
	}
1371 1372

	if (arg == MMC_SECURE_TRIM1_ARG) {
1373 1374 1375 1376 1377 1378
		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
			err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
					 INAND_CMD38_ARG_EXT_CSD,
					 INAND_CMD38_ARG_SECTRIM2,
					 0);
			if (err)
1379
				goto out_retry;
1380
		}
1381

1382
		err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
1383 1384
		if (err == -EIO)
			goto out_retry;
1385 1386
		if (err) {
			status = BLK_STS_IOERR;
1387
			goto out;
1388
		}
1389
	}
1390 1391 1392

out_retry:
	if (err && !mmc_blk_reset(md, card->host, type))
1393 1394 1395
		goto retry;
	if (!err)
		mmc_blk_reset_success(md, type);
1396
out:
1397
	blk_end_request(req, status, blk_rq_bytes(req));
1398 1399
}

1400
static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
A
Andrei Warkentin 已提交
1401
{
L
Linus Walleij 已提交
1402
	struct mmc_blk_data *md = mq->blkdata;
1403 1404 1405 1406
	struct mmc_card *card = md->queue.card;
	int ret = 0;

	ret = mmc_flush_cache(card);
1407
	blk_end_request_all(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
A
Andrei Warkentin 已提交
1408 1409 1410 1411 1412 1413 1414 1415 1416
}

/*
 * Reformat current write as a reliable write, supporting
 * both legacy and the enhanced reliable write MMC cards.
 * In each transfer we'll handle only as much as a single
 * reliable write can handle, thus finish the request in
 * partial completions.
 */
1417 1418 1419
static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
				    struct mmc_card *card,
				    struct request *req)
A
Andrei Warkentin 已提交
1420 1421 1422
{
	if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) {
		/* Legacy mode imposes restrictions on transfers. */
1423
		if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors))
A
Andrei Warkentin 已提交
1424 1425 1426 1427 1428 1429 1430 1431 1432
			brq->data.blocks = 1;

		if (brq->data.blocks > card->ext_csd.rel_sectors)
			brq->data.blocks = card->ext_csd.rel_sectors;
		else if (brq->data.blocks < card->ext_csd.rel_sectors)
			brq->data.blocks = 1;
	}
}

1433 1434 1435 1436 1437
#define CMD_ERRORS							\
	(R1_OUT_OF_RANGE |	/* Command argument out of range */	\
	 R1_ADDRESS_ERROR |	/* Misaligned address */		\
	 R1_BLOCK_LEN_ERROR |	/* Transferred block length incorrect */\
	 R1_WP_VIOLATION |	/* Tried to write to protected block */	\
1438
	 R1_CARD_ECC_FAILED |	/* Card ECC failed */			\
1439 1440 1441
	 R1_CC_ERROR |		/* Card controller error */		\
	 R1_ERROR)		/* General/unknown error */

1442
static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
1443
{
1444
	u32 val;
1445

1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
	/*
	 * Per the SD specification(physical layer version 4.10)[1],
	 * section 4.3.3, it explicitly states that "When the last
	 * block of user area is read using CMD18, the host should
	 * ignore OUT_OF_RANGE error that may occur even the sequence
	 * is correct". And JESD84-B51 for eMMC also has a similar
	 * statement on section 6.8.3.
	 *
	 * Multiple block read/write could be done by either predefined
	 * method, namely CMD23, or open-ending mode. For open-ending mode,
	 * we should ignore the OUT_OF_RANGE error as it's normal behaviour.
	 *
	 * However the spec[1] doesn't tell us whether we should also
	 * ignore that for predefined method. But per the spec[1], section
	 * 4.15 Set Block Count Command, it says"If illegal block count
	 * is set, out of range error will be indicated during read/write
	 * operation (For example, data transfer is stopped at user area
	 * boundary)." In another word, we could expect a out of range error
	 * in the response for the following CMD18/25. And if argument of
	 * CMD23 + the argument of CMD18/25 exceed the max number of blocks,
	 * we could also expect to get a -ETIMEDOUT or any error number from
	 * the host drivers due to missing data response(for write)/data(for
	 * read), as the cards will stop the data transfer by itself per the
	 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
	 */

	if (!brq->stop.error) {
		bool oor_with_open_end;
		/* If there is no error yet, check R1 response */

		val = brq->stop.resp[0] & CMD_ERRORS;
		oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;

		if (val && !oor_with_open_end)
			brq->stop.error = -EIO;
	}
1482 1483
}

1484 1485
static enum mmc_blk_status mmc_blk_err_check(struct mmc_card *card,
					     struct mmc_async_req *areq)
1486
{
1487
	struct mmc_queue_req *mq_mrq = container_of(areq, struct mmc_queue_req,
1488
						    areq);
1489
	struct mmc_blk_request *brq = &mq_mrq->brq;
1490
	struct request *req = mmc_queue_req_to_req(mq_mrq);
1491
	int need_retune = card->host->need_retune;
1492
	bool ecc_err = false;
1493
	bool gen_err = false;
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504

	/*
	 * sbc.error indicates a problem with the set block count
	 * command.  No data will have been transferred.
	 *
	 * cmd.error indicates a problem with the r/w command.  No
	 * data will have been transferred.
	 *
	 * stop.error indicates a problem with the stop command.  Data
	 * may have been transferred, or may still be transferring.
	 */
1505 1506 1507 1508 1509

	mmc_blk_eval_resp_error(brq);

	if (brq->sbc.error || brq->cmd.error ||
	    brq->stop.error || brq->data.error) {
1510
		switch (mmc_blk_cmd_recovery(card, req, brq, &ecc_err, &gen_err)) {
1511 1512 1513 1514
		case ERR_RETRY:
			return MMC_BLK_RETRY;
		case ERR_ABORT:
			return MMC_BLK_ABORT;
1515 1516
		case ERR_NOMEDIUM:
			return MMC_BLK_NOMEDIUM;
1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
		case ERR_CONTINUE:
			break;
		}
	}

	/*
	 * Check for errors relating to the execution of the
	 * initial command - such as address errors.  No data
	 * has been transferred.
	 */
	if (brq->cmd.resp[0] & CMD_ERRORS) {
		pr_err("%s: r/w command failed, status = %#x\n",
		       req->rq_disk->disk_name, brq->cmd.resp[0]);
		return MMC_BLK_ABORT;
	}

	/*
	 * Everything else is either success, or a data error of some
	 * kind.  If it was a write, we may have transitioned to
	 * program mode, which we have to wait for it to complete.
	 */
	if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ) {
1539
		int err;
1540

1541 1542 1543 1544 1545
		/* Check stop command response */
		if (brq->stop.resp[0] & R1_ERROR) {
			pr_err("%s: %s: general error sending stop command, stop cmd response %#x\n",
			       req->rq_disk->disk_name, __func__,
			       brq->stop.resp[0]);
1546
			gen_err = true;
1547 1548
		}

1549 1550
		err = card_busy_detect(card, MMC_BLK_TIMEOUT_MS, false, req,
					&gen_err);
1551 1552
		if (err)
			return MMC_BLK_CMD_ERR;
1553 1554
	}

1555 1556 1557 1558 1559 1560 1561
	/* if general error occurs, retry the write operation. */
	if (gen_err) {
		pr_warn("%s: retrying write for general error\n",
				req->rq_disk->disk_name);
		return MMC_BLK_RETRY;
	}

1562 1563
	/* Some errors (ECC) are flagged on the next commmand, so check stop, too */
	if (brq->data.error || brq->stop.error) {
1564
		if (need_retune && !brq->retune_retry_done) {
1565 1566
			pr_debug("%s: retrying because a re-tune was needed\n",
				 req->rq_disk->disk_name);
1567 1568 1569
			brq->retune_retry_done = 1;
			return MMC_BLK_RETRY;
		}
1570
		pr_err("%s: error %d transferring data, sector %u, nr %u, cmd response %#x, card status %#x\n",
1571
		       req->rq_disk->disk_name, brq->data.error ?: brq->stop.error,
1572 1573 1574 1575 1576
		       (unsigned)blk_rq_pos(req),
		       (unsigned)blk_rq_sectors(req),
		       brq->cmd.resp[0], brq->stop.resp[0]);

		if (rq_data_dir(req) == READ) {
1577 1578
			if (ecc_err)
				return MMC_BLK_ECC_ERR;
1579 1580 1581 1582 1583 1584
			return MMC_BLK_DATA_ERR;
		} else {
			return MMC_BLK_CMD_ERR;
		}
	}

1585 1586
	if (!brq->data.bytes_xfered)
		return MMC_BLK_RETRY;
1587

1588 1589 1590 1591
	if (blk_rq_bytes(req) != brq->data.bytes_xfered)
		return MMC_BLK_PARTIAL;

	return MMC_BLK_SUCCESS;
1592 1593
}

1594
static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
1595 1596
			      int disable_multi, bool *do_rel_wr_p,
			      bool *do_data_tag_p)
L
Linus Torvalds 已提交
1597
{
1598 1599
	struct mmc_blk_data *md = mq->blkdata;
	struct mmc_card *card = md->queue.card;
1600
	struct mmc_blk_request *brq = &mqrq->brq;
1601
	struct request *req = mmc_queue_req_to_req(mqrq);
1602
	bool do_rel_wr, do_data_tag;
L
Linus Torvalds 已提交
1603

A
Andrei Warkentin 已提交
1604 1605
	/*
	 * Reliable writes are used to implement Forced Unit Access and
1606
	 * are supported only on MMCs.
A
Andrei Warkentin 已提交
1607
	 */
1608 1609 1610
	do_rel_wr = (req->cmd_flags & REQ_FUA) &&
		    rq_data_dir(req) == WRITE &&
		    (md->flags & MMC_BLK_REL_WR);
A
Andrei Warkentin 已提交
1611

1612
	memset(brq, 0, sizeof(struct mmc_blk_request));
1613

1614
	brq->mrq.data = &brq->data;
A
Adrian Hunter 已提交
1615
	brq->mrq.tag = req->tag;
L
Linus Torvalds 已提交
1616

1617 1618
	brq->stop.opcode = MMC_STOP_TRANSMISSION;
	brq->stop.arg = 0;
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628

	if (rq_data_dir(req) == READ) {
		brq->data.flags = MMC_DATA_READ;
		brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
	} else {
		brq->data.flags = MMC_DATA_WRITE;
		brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
	}

	brq->data.blksz = 512;
1629
	brq->data.blocks = blk_rq_sectors(req);
A
Adrian Hunter 已提交
1630 1631 1632 1633 1634 1635 1636 1637
	brq->data.blk_addr = blk_rq_pos(req);

	/*
	 * The command queue supports 2 priorities: "high" (1) and "simple" (0).
	 * The eMMC will give "high" priority tasks priority over "simple"
	 * priority tasks. Here we always set "simple" priority by not setting
	 * MMC_DATA_PRIO.
	 */
1638

1639 1640 1641 1642 1643 1644 1645
	/*
	 * The block layer doesn't support all sector count
	 * restrictions, so we need to be prepared for too big
	 * requests.
	 */
	if (brq->data.blocks > card->host->max_blk_count)
		brq->data.blocks = card->host->max_blk_count;
L
Linus Torvalds 已提交
1646

1647 1648 1649 1650 1651 1652 1653 1654 1655
	if (brq->data.blocks > 1) {
		/*
		 * After a read error, we redo the request one sector
		 * at a time in order to accurately determine which
		 * sectors can be read successfully.
		 */
		if (disable_multi)
			brq->data.blocks = 1;

1656 1657 1658 1659 1660 1661 1662 1663 1664
		/*
		 * Some controllers have HW issues while operating
		 * in multiple I/O mode
		 */
		if (card->host->ops->multi_io_quirk)
			brq->data.blocks = card->host->ops->multi_io_quirk(card,
						(rq_data_dir(req) == READ) ?
						MMC_DATA_READ : MMC_DATA_WRITE,
						brq->data.blocks);
1665
	}
1666

A
Adrian Hunter 已提交
1667
	if (do_rel_wr) {
1668
		mmc_apply_rel_rw(brq, card, req);
A
Adrian Hunter 已提交
1669 1670
		brq->data.flags |= MMC_DATA_REL_WR;
	}
1671 1672 1673 1674 1675

	/*
	 * Data tag is used only during writing meta data to speed
	 * up write and any subsequent read of this meta data
	 */
1676 1677 1678 1679 1680
	do_data_tag = card->ext_csd.data_tag_unit_size &&
		      (req->cmd_flags & REQ_META) &&
		      (rq_data_dir(req) == WRITE) &&
		      ((brq->data.blocks * brq->data.blksz) >=
		       card->ext_csd.data_tag_unit_size);
1681

A
Adrian Hunter 已提交
1682 1683 1684
	if (do_data_tag)
		brq->data.flags |= MMC_DATA_DAT_TAG;

1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
	mmc_set_data_timeout(&brq->data, card);

	brq->data.sg = mqrq->sg;
	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);

	/*
	 * Adjust the sg list so it is the same size as the
	 * request.
	 */
	if (brq->data.blocks != blk_rq_sectors(req)) {
		int i, data_size = brq->data.blocks << 9;
		struct scatterlist *sg;

		for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) {
			data_size -= sg->length;
			if (data_size <= 0) {
				sg->length += data_size;
				i++;
				break;
			}
		}
		brq->data.sg_len = i;
	}

	mqrq->areq.mrq = &brq->mrq;
1710 1711 1712 1713 1714 1715

	if (do_rel_wr_p)
		*do_rel_wr_p = do_rel_wr;

	if (do_data_tag_p)
		*do_data_tag_p = do_data_tag;
1716 1717 1718 1719 1720 1721 1722 1723 1724
}

static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
			       struct mmc_card *card,
			       int disable_multi,
			       struct mmc_queue *mq)
{
	u32 readcmd, writecmd;
	struct mmc_blk_request *brq = &mqrq->brq;
1725
	struct request *req = mmc_queue_req_to_req(mqrq);
1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
	struct mmc_blk_data *md = mq->blkdata;
	bool do_rel_wr, do_data_tag;

	mmc_blk_data_prep(mq, mqrq, disable_multi, &do_rel_wr, &do_data_tag);

	brq->mrq.cmd = &brq->cmd;

	brq->cmd.arg = blk_rq_pos(req);
	if (!mmc_card_blockaddr(card))
		brq->cmd.arg <<= 9;
	brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;

1738 1739 1740
	if (brq->data.blocks > 1 || do_rel_wr) {
		/* SPI multiblock writes terminate using a special
		 * token, not a STOP_TRANSMISSION request.
1741
		 */
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751
		if (!mmc_host_is_spi(card->host) ||
		    rq_data_dir(req) == READ)
			brq->mrq.stop = &brq->stop;
		readcmd = MMC_READ_MULTIPLE_BLOCK;
		writecmd = MMC_WRITE_MULTIPLE_BLOCK;
	} else {
		brq->mrq.stop = NULL;
		readcmd = MMC_READ_SINGLE_BLOCK;
		writecmd = MMC_WRITE_BLOCK;
	}
1752
	brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd;
S
Saugata Das 已提交
1753

1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
	/*
	 * Pre-defined multi-block transfers are preferable to
	 * open ended-ones (and necessary for reliable writes).
	 * However, it is not sufficient to just send CMD23,
	 * and avoid the final CMD12, as on an error condition
	 * CMD12 (stop) needs to be sent anyway. This, coupled
	 * with Auto-CMD23 enhancements provided by some
	 * hosts, means that the complexity of dealing
	 * with this is best left to the host. If CMD23 is
	 * supported by card and host, we'll fill sbc in and let
	 * the host deal with handling it correctly. This means
	 * that for hosts that don't expose MMC_CAP_CMD23, no
	 * change of behavior will be observed.
	 *
	 * N.B: Some MMC cards experience perf degradation.
	 * We'll avoid using CMD23-bounded multiblock writes for
	 * these, while retaining features like reliable writes.
	 */
S
Saugata Das 已提交
1772 1773 1774
	if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) &&
	    (do_rel_wr || !(card->quirks & MMC_QUIRK_BLK_NO_CMD23) ||
	     do_data_tag)) {
1775 1776
		brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
		brq->sbc.arg = brq->data.blocks |
S
Saugata Das 已提交
1777 1778
			(do_rel_wr ? (1 << 31) : 0) |
			(do_data_tag ? (1 << 29) : 0);
1779 1780 1781
		brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
		brq->mrq.sbc = &brq->sbc;
	}
1782

1783
	mqrq->areq.err_check = mmc_blk_err_check;
1784
}
1785

1786 1787 1788
static bool mmc_blk_rw_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
			       struct mmc_blk_request *brq, struct request *req,
			       bool old_req_pending)
1789
{
1790 1791
	bool req_pending;

1792 1793 1794 1795 1796 1797 1798 1799 1800 1801
	/*
	 * If this is an SD card and we're writing, we can first
	 * mark the known good sectors as ok.
	 *
	 * If the card is not SD, we can still ok written sectors
	 * as reported by the controller (which might be less than
	 * the real number of written sectors, but never more).
	 */
	if (mmc_card_sd(card)) {
		u32 blocks;
1802
		int err;
1803

1804
		err = mmc_sd_num_wr_blocks(card, &blocks);
1805 1806 1807
		if (err)
			req_pending = old_req_pending;
		else
1808
			req_pending = blk_end_request(req, BLK_STS_OK, blocks << 9);
1809
	} else {
1810
		req_pending = blk_end_request(req, BLK_STS_OK, brq->data.bytes_xfered);
1811
	}
1812
	return req_pending;
1813 1814
}

1815 1816 1817
static void mmc_blk_rw_cmd_abort(struct mmc_queue *mq, struct mmc_card *card,
				 struct request *req,
				 struct mmc_queue_req *mqrq)
1818 1819 1820
{
	if (mmc_card_removed(card))
		req->rq_flags |= RQF_QUIET;
1821
	while (blk_end_request(req, BLK_STS_IOERR, blk_rq_cur_bytes(req)));
1822
	mq->qcnt--;
1823 1824
}

1825 1826 1827 1828 1829
/**
 * mmc_blk_rw_try_restart() - tries to restart the current async request
 * @mq: the queue with the card and host to restart
 * @req: a new request that want to be started after the current one
 */
1830 1831
static void mmc_blk_rw_try_restart(struct mmc_queue *mq, struct request *req,
				   struct mmc_queue_req *mqrq)
1832 1833 1834 1835
{
	if (!req)
		return;

1836 1837 1838 1839
	/*
	 * If the card was removed, just cancel everything and return.
	 */
	if (mmc_card_removed(mq->card)) {
1840
		req->rq_flags |= RQF_QUIET;
1841
		blk_end_request_all(req, BLK_STS_IOERR);
1842
		mq->qcnt--; /* FIXME: just set to 0? */
1843
		return;
1844
	}
1845
	/* Else proceed and try to restart the current async request */
1846 1847
	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
	mmc_start_areq(mq->card->host, &mqrq->areq, NULL);
1848 1849
}

L
Linus Walleij 已提交
1850
static void mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *new_req)
1851
{
L
Linus Walleij 已提交
1852
	struct mmc_blk_data *md = mq->blkdata;
1853
	struct mmc_card *card = md->queue.card;
1854
	struct mmc_blk_request *brq;
1855
	int disable_multi = 0, retry = 0, type, retune_retry_done = 0;
1856
	enum mmc_blk_status status;
1857
	struct mmc_queue_req *mqrq_cur = NULL;
1858
	struct mmc_queue_req *mq_rq;
L
Linus Walleij 已提交
1859
	struct request *old_req;
1860 1861
	struct mmc_async_req *new_areq;
	struct mmc_async_req *old_areq;
1862
	bool req_pending = true;
L
Linus Torvalds 已提交
1863

1864
	if (new_req) {
1865 1866
		mqrq_cur = req_to_mmc_queue_req(new_req);
		mq->qcnt++;
1867 1868 1869
	}

	if (!mq->qcnt)
1870
		return;
1871

1872
	do {
L
Linus Walleij 已提交
1873
		if (new_req) {
1874 1875 1876 1877
			/*
			 * When 4KB native sector is enabled, only 8 blocks
			 * multiple read or write is allowed
			 */
1878
			if (mmc_large_sector(card) &&
L
Linus Walleij 已提交
1879
				!IS_ALIGNED(blk_rq_sectors(new_req), 8)) {
1880
				pr_err("%s: Transfer size is not 4KB sector size aligned\n",
L
Linus Walleij 已提交
1881
					new_req->rq_disk->disk_name);
1882
				mmc_blk_rw_cmd_abort(mq, card, new_req, mqrq_cur);
1883
				return;
1884
			}
1885

1886 1887
			mmc_blk_rw_rq_prep(mqrq_cur, card, 0, mq);
			new_areq = &mqrq_cur->areq;
1888
		} else
1889 1890
			new_areq = NULL;

1891
		old_areq = mmc_start_areq(card->host, new_areq, &status);
1892
		if (!old_areq) {
1893 1894 1895 1896 1897
			/*
			 * We have just put the first request into the pipeline
			 * and there is nothing more to do until it is
			 * complete.
			 */
1898
			return;
1899
		}
1900

1901 1902 1903 1904
		/*
		 * An asynchronous request has been completed and we proceed
		 * to handle the result of it.
		 */
1905
		mq_rq =	container_of(old_areq, struct mmc_queue_req, areq);
1906
		brq = &mq_rq->brq;
1907
		old_req = mmc_queue_req_to_req(mq_rq);
L
Linus Walleij 已提交
1908
		type = rq_data_dir(old_req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1909

1910 1911 1912 1913 1914 1915
		switch (status) {
		case MMC_BLK_SUCCESS:
		case MMC_BLK_PARTIAL:
			/*
			 * A block was successfully transferred.
			 */
1916
			mmc_blk_reset_success(md, type);
1917

1918
			req_pending = blk_end_request(old_req, BLK_STS_OK,
1919
						      brq->data.bytes_xfered);
1920 1921 1922 1923 1924
			/*
			 * If the blk_end_request function returns non-zero even
			 * though all data has been transferred and no errors
			 * were returned by the host controller, it's a bug.
			 */
1925
			if (status == MMC_BLK_SUCCESS && req_pending) {
1926
				pr_err("%s BUG rq_tot %d d_xfer %d\n",
L
Linus Walleij 已提交
1927
				       __func__, blk_rq_bytes(old_req),
1928
				       brq->data.bytes_xfered);
1929
				mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
1930
				return;
1931
			}
1932 1933
			break;
		case MMC_BLK_CMD_ERR:
1934
			req_pending = mmc_blk_rw_cmd_err(md, card, brq, old_req, req_pending);
1935
			if (mmc_blk_reset(md, card->host, type)) {
1936
				if (req_pending)
1937 1938
					mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
				else
1939
					mq->qcnt--;
1940
				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1941 1942
				return;
			}
1943
			if (!req_pending) {
1944
				mq->qcnt--;
1945
				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1946 1947
				return;
			}
1948
			break;
1949
		case MMC_BLK_RETRY:
1950
			retune_retry_done = brq->retune_retry_done;
1951
			if (retry++ < 5)
1952
				break;
1953
			/* Fall through */
1954
		case MMC_BLK_ABORT:
1955 1956
			if (!mmc_blk_reset(md, card->host, type))
				break;
1957
			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
1958
			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1959
			return;
1960 1961 1962 1963 1964 1965
		case MMC_BLK_DATA_ERR: {
			int err;

			err = mmc_blk_reset(md, card->host, type);
			if (!err)
				break;
1966
			if (err == -ENODEV) {
1967
				mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
1968
				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1969 1970
				return;
			}
1971 1972 1973 1974 1975
			/* Fall through */
		}
		case MMC_BLK_ECC_ERR:
			if (brq->data.blocks > 1) {
				/* Redo read one sector at a time */
J
Joe Perches 已提交
1976
				pr_warn("%s: retrying using single block read\n",
L
Linus Walleij 已提交
1977
					old_req->rq_disk->disk_name);
1978 1979 1980
				disable_multi = 1;
				break;
			}
1981 1982 1983 1984 1985
			/*
			 * After an error, we redo I/O one sector at a
			 * time, so we only reach here after trying to
			 * read a single sector.
			 */
1986
			req_pending = blk_end_request(old_req, BLK_STS_IOERR,
1987 1988
						      brq->data.blksz);
			if (!req_pending) {
1989
				mq->qcnt--;
1990
				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1991 1992
				return;
			}
1993
			break;
1994
		case MMC_BLK_NOMEDIUM:
1995
			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
1996
			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
1997
			return;
1998 1999
		default:
			pr_err("%s: Unhandled return value (%d)",
L
Linus Walleij 已提交
2000
					old_req->rq_disk->disk_name, status);
2001
			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
2002
			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
2003
			return;
2004 2005
		}

2006
		if (req_pending) {
2007 2008 2009 2010 2011 2012
			/*
			 * In case of a incomplete request
			 * prepare it again and resend.
			 */
			mmc_blk_rw_rq_prep(mq_rq, card,
					disable_multi, mq);
2013
			mmc_start_areq(card->host,
2014
					&mq_rq->areq, NULL);
2015
			mq_rq->brq.retune_retry_done = retune_retry_done;
2016
		}
2017
	} while (req_pending);
2018

2019
	mq->qcnt--;
L
Linus Torvalds 已提交
2020 2021
}

2022
void mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
A
Adrian Hunter 已提交
2023
{
2024
	int ret;
L
Linus Walleij 已提交
2025
	struct mmc_blk_data *md = mq->blkdata;
2026 2027
	struct mmc_card *card = md->queue.card;

2028
	if (req && !mq->qcnt)
2029
		/* claim host only for the first request */
2030
		mmc_get_card(card, NULL);
2031

2032
	ret = mmc_blk_part_switch(card, md->part_type);
2033
	if (ret) {
2034
		if (req) {
2035
			blk_end_request_all(req, BLK_STS_IOERR);
2036
		}
2037 2038
		goto out;
	}
2039

2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
	if (req) {
		switch (req_op(req)) {
		case REQ_OP_DRV_IN:
		case REQ_OP_DRV_OUT:
			/*
			 * Complete ongoing async transfer before issuing
			 * ioctl()s
			 */
			if (mq->qcnt)
				mmc_blk_issue_rw_rq(mq, NULL);
2050
			mmc_blk_issue_drv_op(mq, req);
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
			break;
		case REQ_OP_DISCARD:
			/*
			 * Complete ongoing async transfer before issuing
			 * discard.
			 */
			if (mq->qcnt)
				mmc_blk_issue_rw_rq(mq, NULL);
			mmc_blk_issue_discard_rq(mq, req);
			break;
		case REQ_OP_SECURE_ERASE:
			/*
			 * Complete ongoing async transfer before issuing
			 * secure erase.
			 */
			if (mq->qcnt)
				mmc_blk_issue_rw_rq(mq, NULL);
			mmc_blk_issue_secdiscard_rq(mq, req);
			break;
		case REQ_OP_FLUSH:
			/*
			 * Complete ongoing async transfer before issuing
			 * flush.
			 */
			if (mq->qcnt)
				mmc_blk_issue_rw_rq(mq, NULL);
			mmc_blk_issue_flush(mq, req);
			break;
		default:
			/* Normal request, just issue it */
			mmc_blk_issue_rw_rq(mq, req);
			card->host->context_info.is_waiting_last_req = false;
			break;
2084
		}
2085
	} else {
2086 2087
		/* No request, flushing the pipeline with NULL */
		mmc_blk_issue_rw_rq(mq, NULL);
2088
		card->host->context_info.is_waiting_last_req = false;
2089
	}
2090

2091
out:
2092
	if (!mq->qcnt)
2093
		mmc_put_card(card, NULL);
A
Adrian Hunter 已提交
2094
}
L
Linus Torvalds 已提交
2095

2096 2097 2098 2099 2100 2101
static inline int mmc_blk_readonly(struct mmc_card *card)
{
	return mmc_card_readonly(card) ||
	       !(card->csd.cmdclass & CCC_BLOCK_WRITE);
}

2102 2103 2104 2105
static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
					      struct device *parent,
					      sector_t size,
					      bool default_ro,
2106 2107
					      const char *subname,
					      int area_type)
L
Linus Torvalds 已提交
2108 2109 2110 2111
{
	struct mmc_blk_data *md;
	int devidx, ret;

2112
	devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL);
2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	if (devidx < 0) {
		/*
		 * We get -ENOSPC because there are no more any available
		 * devidx. The reason may be that, either userspace haven't yet
		 * unmounted the partitions, which postpones mmc_blk_release()
		 * from being called, or the device has more partitions than
		 * what we support.
		 */
		if (devidx == -ENOSPC)
			dev_err(mmc_dev(card->host),
				"no more device IDs available\n");

2125
		return ERR_PTR(devidx);
2126
	}
L
Linus Torvalds 已提交
2127

2128
	md = kzalloc(sizeof(struct mmc_blk_data), GFP_KERNEL);
2129 2130 2131 2132
	if (!md) {
		ret = -ENOMEM;
		goto out;
	}
L
Linus Torvalds 已提交
2133

2134 2135
	md->area_type = area_type;

2136 2137 2138 2139 2140
	/*
	 * Set the read-only status based on the supported commands
	 * and the write protect switch.
	 */
	md->read_only = mmc_blk_readonly(card);
L
Linus Torvalds 已提交
2141

2142
	md->disk = alloc_disk(perdev_minors);
2143 2144 2145 2146
	if (md->disk == NULL) {
		ret = -ENOMEM;
		goto err_kfree;
	}
L
Linus Torvalds 已提交
2147

2148
	spin_lock_init(&md->lock);
2149
	INIT_LIST_HEAD(&md->part);
2150
	INIT_LIST_HEAD(&md->rpmbs);
2151
	md->usage = 1;
L
Linus Torvalds 已提交
2152

2153
	ret = mmc_init_queue(&md->queue, card, &md->lock, subname);
2154 2155
	if (ret)
		goto err_putdisk;
L
Linus Torvalds 已提交
2156

L
Linus Walleij 已提交
2157
	md->queue.blkdata = md;
2158

2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
	/*
	 * Keep an extra reference to the queue so that we can shutdown the
	 * queue (i.e. call blk_cleanup_queue()) while there are still
	 * references to the 'md'. The corresponding blk_put_queue() is in
	 * mmc_blk_put().
	 */
	if (!blk_get_queue(md->queue.queue)) {
		mmc_cleanup_queue(&md->queue);
		goto err_putdisk;
	}

2170
	md->disk->major	= MMC_BLOCK_MAJOR;
2171
	md->disk->first_minor = devidx * perdev_minors;
2172 2173 2174
	md->disk->fops = &mmc_bdops;
	md->disk->private_data = md;
	md->disk->queue = md->queue.queue;
2175
	md->parent = parent;
2176
	set_disk_ro(md->disk, md->read_only || default_ro);
2177
	md->disk->flags = GENHD_FL_EXT_DEVT;
2178
	if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT))
2179
		md->disk->flags |= GENHD_FL_NO_PART_SCAN;
2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192

	/*
	 * As discussed on lkml, GENHD_FL_REMOVABLE should:
	 *
	 * - be set for removable media with permanent block devices
	 * - be unset for removable block devices with permanent media
	 *
	 * Since MMC block devices clearly fall under the second
	 * case, we do not set GENHD_FL_REMOVABLE.  Userspace
	 * should use the block device creation/destruction hotplug
	 * messages to tell when the card is present.
	 */

2193
	snprintf(md->disk->disk_name, sizeof(md->disk->disk_name),
2194
		 "mmcblk%u%s", card->host->index, subname ? subname : "");
2195

2196 2197 2198 2199 2200 2201
	if (mmc_card_mmc(card))
		blk_queue_logical_block_size(md->queue.queue,
					     card->ext_csd.data_sector_size);
	else
		blk_queue_logical_block_size(md->queue.queue, 512);

2202
	set_capacity(md->disk, size);
2203

2204
	if (mmc_host_cmd23(card->host)) {
2205 2206
		if ((mmc_card_mmc(card) &&
		     card->csd.mmca_vsn >= CSD_SPEC_VER_3) ||
2207 2208 2209 2210
		    (mmc_card_sd(card) &&
		     card->scr.cmds & SD_SCR_CMD23_SUPPORT))
			md->flags |= MMC_BLK_CMD23;
	}
2211 2212 2213 2214 2215 2216

	if (mmc_card_mmc(card) &&
	    md->flags & MMC_BLK_CMD23 &&
	    ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) ||
	     card->ext_csd.rel_sectors)) {
		md->flags |= MMC_BLK_REL_WR;
2217
		blk_queue_write_cache(md->queue.queue, true, true);
2218 2219
	}

2220 2221 2222 2223 2224 2225 2226
	return md;

 err_putdisk:
	put_disk(md->disk);
 err_kfree:
	kfree(md);
 out:
2227
	ida_simple_remove(&mmc_blk_ida, devidx);
2228 2229 2230 2231 2232 2233
	return ERR_PTR(ret);
}

static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card)
{
	sector_t size;
2234

P
Pierre Ossman 已提交
2235 2236 2237 2238 2239
	if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
		/*
		 * The EXT_CSD sector count is in number or 512 byte
		 * sectors.
		 */
2240
		size = card->ext_csd.sectors;
P
Pierre Ossman 已提交
2241 2242 2243 2244 2245
	} else {
		/*
		 * The CSD capacity field is in units of read_blkbits.
		 * set_capacity takes units of 512 bytes.
		 */
2246 2247
		size = (typeof(sector_t))card->csd.capacity
			<< (card->csd.read_blkbits - 9);
P
Pierre Ossman 已提交
2248
	}
2249

2250
	return mmc_blk_alloc_req(card, &card->dev, size, false, NULL,
2251
					MMC_BLK_DATA_AREA_MAIN);
2252
}
2253

2254 2255 2256 2257 2258
static int mmc_blk_alloc_part(struct mmc_card *card,
			      struct mmc_blk_data *md,
			      unsigned int part_type,
			      sector_t size,
			      bool default_ro,
2259 2260
			      const char *subname,
			      int area_type)
2261 2262 2263 2264 2265
{
	char cap_str[10];
	struct mmc_blk_data *part_md;

	part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro,
2266
				    subname, area_type);
2267 2268 2269 2270 2271
	if (IS_ERR(part_md))
		return PTR_ERR(part_md);
	part_md->part_type = part_type;
	list_add(&part_md->part, &md->part);

2272
	string_get_size((u64)get_capacity(part_md->disk), 512, STRING_UNITS_2,
2273
			cap_str, sizeof(cap_str));
2274
	pr_info("%s: %s %s partition %u %s\n",
2275 2276 2277 2278 2279
	       part_md->disk->disk_name, mmc_card_id(card),
	       mmc_card_name(card), part_md->part_type, cap_str);
	return 0;
}

2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
/**
 * mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev
 * @filp: the character device file
 * @cmd: the ioctl() command
 * @arg: the argument from userspace
 *
 * This will essentially just redirect the ioctl()s coming in over to
 * the main block device spawning the RPMB character device.
 */
static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd,
			   unsigned long arg)
{
	struct mmc_rpmb_data *rpmb = filp->private_data;
	int ret;

	switch (cmd) {
	case MMC_IOC_CMD:
		ret = mmc_blk_ioctl_cmd(rpmb->md,
					(struct mmc_ioc_cmd __user *)arg,
					rpmb);
		break;
	case MMC_IOC_MULTI_CMD:
		ret = mmc_blk_ioctl_multi_cmd(rpmb->md,
					(struct mmc_ioc_multi_cmd __user *)arg,
					rpmb);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	return 0;
}

#ifdef CONFIG_COMPAT
static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd,
			      unsigned long arg)
{
	return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
}
#endif

static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp)
{
	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
						  struct mmc_rpmb_data, chrdev);

	get_device(&rpmb->dev);
	filp->private_data = rpmb;
2329
	mmc_blk_get(rpmb->md->disk);
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339

	return nonseekable_open(inode, filp);
}

static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp)
{
	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
						  struct mmc_rpmb_data, chrdev);

	put_device(&rpmb->dev);
2340
	mmc_blk_put(rpmb->md);
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355

	return 0;
}

static const struct file_operations mmc_rpmb_fileops = {
	.release = mmc_rpmb_chrdev_release,
	.open = mmc_rpmb_chrdev_open,
	.owner = THIS_MODULE,
	.llseek = no_llseek,
	.unlocked_ioctl = mmc_rpmb_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl = mmc_rpmb_ioctl_compat,
#endif
};

2356 2357 2358 2359 2360 2361 2362
static void mmc_blk_rpmb_device_release(struct device *dev)
{
	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);

	ida_simple_remove(&mmc_rpmb_ida, rpmb->id);
	kfree(rpmb);
}
2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380

static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
				   struct mmc_blk_data *md,
				   unsigned int part_index,
				   sector_t size,
				   const char *subname)
{
	int devidx, ret;
	char rpmb_name[DISK_NAME_LEN];
	char cap_str[10];
	struct mmc_rpmb_data *rpmb;

	/* This creates the minor number for the RPMB char device */
	devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL);
	if (devidx < 0)
		return devidx;

	rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL);
2381 2382
	if (!rpmb) {
		ida_simple_remove(&mmc_rpmb_ida, devidx);
2383
		return -ENOMEM;
2384
	}
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394

	snprintf(rpmb_name, sizeof(rpmb_name),
		 "mmcblk%u%s", card->host->index, subname ? subname : "");

	rpmb->id = devidx;
	rpmb->part_index = part_index;
	rpmb->dev.init_name = rpmb_name;
	rpmb->dev.bus = &mmc_rpmb_bus_type;
	rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id);
	rpmb->dev.parent = &card->dev;
2395
	rpmb->dev.release = mmc_blk_rpmb_device_release;
2396 2397 2398 2399 2400 2401 2402 2403 2404
	device_initialize(&rpmb->dev);
	dev_set_drvdata(&rpmb->dev, rpmb);
	rpmb->md = md;

	cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops);
	rpmb->chrdev.owner = THIS_MODULE;
	ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev);
	if (ret) {
		pr_err("%s: could not add character device\n", rpmb_name);
2405
		goto out_put_device;
2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419
	}

	list_add(&rpmb->node, &md->rpmbs);

	string_get_size((u64)size, 512, STRING_UNITS_2,
			cap_str, sizeof(cap_str));

	pr_info("%s: %s %s partition %u %s, chardev (%d:%d)\n",
		rpmb_name, mmc_card_id(card),
		mmc_card_name(card), EXT_CSD_PART_CONFIG_ACC_RPMB, cap_str,
		MAJOR(mmc_rpmb_devt), rpmb->id);

	return 0;

2420 2421
out_put_device:
	put_device(&rpmb->dev);
2422 2423 2424 2425
	return ret;
}

static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
2426

2427 2428
{
	cdev_device_del(&rpmb->chrdev, &rpmb->dev);
2429
	put_device(&rpmb->dev);
2430 2431
}

2432 2433 2434 2435 2436 2437
/* MMC Physical partitions consist of two boot partitions and
 * up to four general purpose partitions.
 * For each partition enabled in EXT_CSD a block device will be allocatedi
 * to provide access to the partition.
 */

2438 2439
static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md)
{
2440
	int idx, ret;
2441 2442 2443 2444

	if (!mmc_card_mmc(card))
		return 0;

2445
	for (idx = 0; idx < card->nr_parts; idx++) {
2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
		if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) {
			/*
			 * RPMB partitions does not provide block access, they
			 * are only accessed using ioctl():s. Thus create
			 * special RPMB block devices that do not have a
			 * backing block queue for these.
			 */
			ret = mmc_blk_alloc_rpmb_part(card, md,
				card->part[idx].part_cfg,
				card->part[idx].size >> 9,
				card->part[idx].name);
			if (ret)
				return ret;
		} else if (card->part[idx].size) {
2460 2461 2462 2463
			ret = mmc_blk_alloc_part(card, md,
				card->part[idx].part_cfg,
				card->part[idx].size >> 9,
				card->part[idx].force_ro,
2464 2465
				card->part[idx].name,
				card->part[idx].area_type);
2466 2467 2468
			if (ret)
				return ret;
		}
2469 2470
	}

2471
	return 0;
L
Linus Torvalds 已提交
2472 2473
}

2474 2475
static void mmc_blk_remove_req(struct mmc_blk_data *md)
{
2476 2477
	struct mmc_card *card;

2478
	if (md) {
2479 2480 2481 2482 2483
		/*
		 * Flush remaining requests and free queues. It
		 * is freeing the queue that stops new requests
		 * from being accepted.
		 */
2484
		card = md->queue.card;
2485
		mmc_cleanup_queue(&md->queue);
2486 2487
		if (md->disk->flags & GENHD_FL_UP) {
			device_remove_file(disk_to_dev(md->disk), &md->force_ro);
2488 2489 2490 2491
			if ((md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
					card->ext_csd.boot_ro_lockable)
				device_remove_file(disk_to_dev(md->disk),
					&md->power_ro_lock);
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503

			del_gendisk(md->disk);
		}
		mmc_blk_put(md);
	}
}

static void mmc_blk_remove_parts(struct mmc_card *card,
				 struct mmc_blk_data *md)
{
	struct list_head *pos, *q;
	struct mmc_blk_data *part_md;
2504
	struct mmc_rpmb_data *rpmb;
2505

2506 2507 2508 2509 2510 2511 2512
	/* Remove RPMB partitions */
	list_for_each_safe(pos, q, &md->rpmbs) {
		rpmb = list_entry(pos, struct mmc_rpmb_data, node);
		list_del(pos);
		mmc_blk_remove_rpmb_part(rpmb);
	}
	/* Remove block partitions */
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522
	list_for_each_safe(pos, q, &md->part) {
		part_md = list_entry(pos, struct mmc_blk_data, part);
		list_del(pos);
		mmc_blk_remove_req(part_md);
	}
}

static int mmc_add_disk(struct mmc_blk_data *md)
{
	int ret;
2523
	struct mmc_card *card = md->queue.card;
2524

2525
	device_add_disk(md->parent, md->disk);
2526 2527
	md->force_ro.show = force_ro_show;
	md->force_ro.store = force_ro_store;
2528
	sysfs_attr_init(&md->force_ro.attr);
2529 2530 2531 2532
	md->force_ro.attr.name = "force_ro";
	md->force_ro.attr.mode = S_IRUGO | S_IWUSR;
	ret = device_create_file(disk_to_dev(md->disk), &md->force_ro);
	if (ret)
2533 2534 2535 2536
		goto force_ro_fail;

	if ((md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
	     card->ext_csd.boot_ro_lockable) {
A
Al Viro 已提交
2537
		umode_t mode;
2538 2539 2540 2541 2542 2543 2544 2545

		if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_DIS)
			mode = S_IRUGO;
		else
			mode = S_IRUGO | S_IWUSR;

		md->power_ro_lock.show = power_ro_lock_show;
		md->power_ro_lock.store = power_ro_lock_store;
2546
		sysfs_attr_init(&md->power_ro_lock.attr);
2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
		md->power_ro_lock.attr.mode = mode;
		md->power_ro_lock.attr.name =
					"ro_lock_until_next_power_on";
		ret = device_create_file(disk_to_dev(md->disk),
				&md->power_ro_lock);
		if (ret)
			goto power_ro_lock_fail;
	}
	return ret;

power_ro_lock_fail:
	device_remove_file(disk_to_dev(md->disk), &md->force_ro);
force_ro_fail:
	del_gendisk(md->disk);
2561 2562 2563 2564

	return ret;
}

2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576
#ifdef CONFIG_DEBUG_FS

static int mmc_dbg_card_status_get(void *data, u64 *val)
{
	struct mmc_card *card = data;
	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
	struct mmc_queue *mq = &md->queue;
	struct request *req;
	int ret;

	/* Ask the block layer about the card status */
	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
2577 2578
	if (IS_ERR(req))
		return PTR_ERR(req);
2579 2580 2581 2582 2583 2584 2585
	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
	blk_execute_rq(mq->queue, NULL, req, 0);
	ret = req_to_mmc_queue_req(req)->drv_op_result;
	if (ret >= 0) {
		*val = ret;
		ret = 0;
	}
2586
	blk_put_request(req);
2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612

	return ret;
}
DEFINE_SIMPLE_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
		NULL, "%08llx\n");

/* That is two digits * 512 + 1 for newline */
#define EXT_CSD_STR_LEN 1025

static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
{
	struct mmc_card *card = inode->i_private;
	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
	struct mmc_queue *mq = &md->queue;
	struct request *req;
	char *buf;
	ssize_t n = 0;
	u8 *ext_csd;
	int err, i;

	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/* Ask the block layer for the EXT CSD */
	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
2613 2614 2615 2616
	if (IS_ERR(req)) {
		err = PTR_ERR(req);
		goto out_free;
	}
2617 2618 2619 2620
	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
	blk_execute_rq(mq->queue, NULL, req, 0);
	err = req_to_mmc_queue_req(req)->drv_op_result;
2621
	blk_put_request(req);
2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666
	if (err) {
		pr_err("FAILED %d\n", err);
		goto out_free;
	}

	for (i = 0; i < 512; i++)
		n += sprintf(buf + n, "%02x", ext_csd[i]);
	n += sprintf(buf + n, "\n");

	if (n != EXT_CSD_STR_LEN) {
		err = -EINVAL;
		goto out_free;
	}

	filp->private_data = buf;
	kfree(ext_csd);
	return 0;

out_free:
	kfree(buf);
	return err;
}

static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
				size_t cnt, loff_t *ppos)
{
	char *buf = filp->private_data;

	return simple_read_from_buffer(ubuf, cnt, ppos,
				       buf, EXT_CSD_STR_LEN);
}

static int mmc_ext_csd_release(struct inode *inode, struct file *file)
{
	kfree(file->private_data);
	return 0;
}

static const struct file_operations mmc_dbg_ext_csd_fops = {
	.open		= mmc_ext_csd_open,
	.read		= mmc_ext_csd_read,
	.release	= mmc_ext_csd_release,
	.llseek		= default_llseek,
};

2667
static int mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
2668 2669 2670 2671 2672 2673 2674 2675 2676
{
	struct dentry *root;

	if (!card->debugfs_root)
		return 0;

	root = card->debugfs_root;

	if (mmc_card_mmc(card) || mmc_card_sd(card)) {
2677 2678 2679 2680
		md->status_dentry =
			debugfs_create_file("status", S_IRUSR, root, card,
					    &mmc_dbg_card_status_fops);
		if (!md->status_dentry)
2681 2682 2683 2684
			return -EIO;
	}

	if (mmc_card_mmc(card)) {
2685 2686 2687 2688
		md->ext_csd_dentry =
			debugfs_create_file("ext_csd", S_IRUSR, root, card,
					    &mmc_dbg_ext_csd_fops);
		if (!md->ext_csd_dentry)
2689 2690 2691 2692 2693 2694
			return -EIO;
	}

	return 0;
}

2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710
static void mmc_blk_remove_debugfs(struct mmc_card *card,
				   struct mmc_blk_data *md)
{
	if (!card->debugfs_root)
		return;

	if (!IS_ERR_OR_NULL(md->status_dentry)) {
		debugfs_remove(md->status_dentry);
		md->status_dentry = NULL;
	}

	if (!IS_ERR_OR_NULL(md->ext_csd_dentry)) {
		debugfs_remove(md->ext_csd_dentry);
		md->ext_csd_dentry = NULL;
	}
}
2711 2712 2713

#else

2714
static int mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md)
2715 2716 2717 2718
{
	return 0;
}

2719 2720 2721 2722 2723
static void mmc_blk_remove_debugfs(struct mmc_card *card,
				   struct mmc_blk_data *md)
{
}

2724 2725
#endif /* CONFIG_DEBUG_FS */

2726
static int mmc_blk_probe(struct mmc_card *card)
L
Linus Torvalds 已提交
2727
{
2728
	struct mmc_blk_data *md, *part_md;
2729 2730
	char cap_str[10];

2731 2732 2733 2734
	/*
	 * Check that the card supports the command class(es) we need.
	 */
	if (!(card->csd.cmdclass & CCC_BLOCK_READ))
L
Linus Torvalds 已提交
2735 2736
		return -ENODEV;

2737
	mmc_fixup_device(card, mmc_blk_fixups);
2738

L
Linus Torvalds 已提交
2739
	md = mmc_blk_alloc(card);
2740
	if (IS_ERR(md))
L
Linus Torvalds 已提交
2741 2742
		return PTR_ERR(md);

2743
	string_get_size((u64)get_capacity(md->disk), 512, STRING_UNITS_2,
2744
			cap_str, sizeof(cap_str));
2745
	pr_info("%s: %s %s %s %s\n",
L
Linus Torvalds 已提交
2746
		md->disk->disk_name, mmc_card_id(card), mmc_card_name(card),
2747
		cap_str, md->read_only ? "(ro)" : "");
L
Linus Torvalds 已提交
2748

2749 2750 2751
	if (mmc_blk_alloc_parts(card, md))
		goto out;

2752
	dev_set_drvdata(&card->dev, md);
2753

2754 2755 2756 2757 2758 2759 2760
	if (mmc_add_disk(md))
		goto out;

	list_for_each_entry(part_md, &md->part, part) {
		if (mmc_add_disk(part_md))
			goto out;
	}
2761

2762
	/* Add two debugfs entries */
2763
	mmc_blk_add_debugfs(card, md);
2764

2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776
	pm_runtime_set_autosuspend_delay(&card->dev, 3000);
	pm_runtime_use_autosuspend(&card->dev);

	/*
	 * Don't enable runtime PM for SD-combo cards here. Leave that
	 * decision to be taken during the SDIO init sequence instead.
	 */
	if (card->type != MMC_TYPE_SD_COMBO) {
		pm_runtime_set_active(&card->dev);
		pm_runtime_enable(&card->dev);
	}

L
Linus Torvalds 已提交
2777 2778 2779
	return 0;

 out:
2780 2781
	mmc_blk_remove_parts(card, md);
	mmc_blk_remove_req(md);
2782
	return 0;
L
Linus Torvalds 已提交
2783 2784
}

2785
static void mmc_blk_remove(struct mmc_card *card)
L
Linus Torvalds 已提交
2786
{
2787
	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
L
Linus Torvalds 已提交
2788

2789
	mmc_blk_remove_debugfs(card, md);
2790
	mmc_blk_remove_parts(card, md);
2791
	pm_runtime_get_sync(&card->dev);
2792
	mmc_claim_host(card->host);
2793
	mmc_blk_part_switch(card, md->part_type);
2794
	mmc_release_host(card->host);
2795 2796 2797
	if (card->type != MMC_TYPE_SD_COMBO)
		pm_runtime_disable(&card->dev);
	pm_runtime_put_noidle(&card->dev);
2798
	mmc_blk_remove_req(md);
2799
	dev_set_drvdata(&card->dev, NULL);
L
Linus Torvalds 已提交
2800 2801
}

2802
static int _mmc_blk_suspend(struct mmc_card *card)
L
Linus Torvalds 已提交
2803
{
2804
	struct mmc_blk_data *part_md;
2805
	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
L
Linus Torvalds 已提交
2806 2807 2808

	if (md) {
		mmc_queue_suspend(&md->queue);
2809 2810 2811
		list_for_each_entry(part_md, &md->part, part) {
			mmc_queue_suspend(&part_md->queue);
		}
L
Linus Torvalds 已提交
2812 2813 2814 2815
	}
	return 0;
}

2816
static void mmc_blk_shutdown(struct mmc_card *card)
2817
{
2818
	_mmc_blk_suspend(card);
2819 2820
}

2821 2822
#ifdef CONFIG_PM_SLEEP
static int mmc_blk_suspend(struct device *dev)
2823
{
2824 2825 2826
	struct mmc_card *card = mmc_dev_to_card(dev);

	return _mmc_blk_suspend(card);
2827 2828
}

2829
static int mmc_blk_resume(struct device *dev)
L
Linus Torvalds 已提交
2830
{
2831
	struct mmc_blk_data *part_md;
2832
	struct mmc_blk_data *md = dev_get_drvdata(dev);
L
Linus Torvalds 已提交
2833 2834

	if (md) {
2835 2836 2837 2838 2839
		/*
		 * Resume involves the card going into idle state,
		 * so current partition is always the main one.
		 */
		md->part_curr = md->part_type;
L
Linus Torvalds 已提交
2840
		mmc_queue_resume(&md->queue);
2841 2842 2843
		list_for_each_entry(part_md, &md->part, part) {
			mmc_queue_resume(&part_md->queue);
		}
L
Linus Torvalds 已提交
2844 2845 2846 2847 2848
	}
	return 0;
}
#endif

2849 2850
static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume);

2851 2852 2853 2854 2855
static struct mmc_driver mmc_driver = {
	.drv		= {
		.name	= "mmcblk",
		.pm	= &mmc_blk_pm_ops,
	},
L
Linus Torvalds 已提交
2856 2857
	.probe		= mmc_blk_probe,
	.remove		= mmc_blk_remove,
2858
	.shutdown	= mmc_blk_shutdown,
L
Linus Torvalds 已提交
2859 2860 2861 2862
};

static int __init mmc_blk_init(void)
{
2863
	int res;
L
Linus Torvalds 已提交
2864

2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875
	res  = bus_register(&mmc_rpmb_bus_type);
	if (res < 0) {
		pr_err("mmcblk: could not register RPMB bus type\n");
		return res;
	}
	res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb");
	if (res < 0) {
		pr_err("mmcblk: failed to allocate rpmb chrdev region\n");
		goto out_bus_unreg;
	}

2876 2877 2878
	if (perdev_minors != CONFIG_MMC_BLOCK_MINORS)
		pr_info("mmcblk: using %d minors per device\n", perdev_minors);

B
Ben Hutchings 已提交
2879
	max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors);
2880

2881 2882
	res = register_blkdev(MMC_BLOCK_MAJOR, "mmc");
	if (res)
2883
		goto out_chrdev_unreg;
L
Linus Torvalds 已提交
2884

2885 2886
	res = mmc_register_driver(&mmc_driver);
	if (res)
2887
		goto out_blkdev_unreg;
L
Linus Torvalds 已提交
2888

2889
	return 0;
2890 2891

out_blkdev_unreg:
2892
	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
2893 2894 2895 2896
out_chrdev_unreg:
	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
out_bus_unreg:
	bus_unregister(&mmc_rpmb_bus_type);
L
Linus Torvalds 已提交
2897 2898 2899 2900 2901 2902
	return res;
}

static void __exit mmc_blk_exit(void)
{
	mmc_unregister_driver(&mmc_driver);
2903
	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
2904
	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
L
Linus Torvalds 已提交
2905 2906 2907 2908 2909 2910 2911 2912
}

module_init(mmc_blk_init);
module_exit(mmc_blk_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");