bsg.c 23.4 KB
Newer Older
1
/*
2
 * bsg.c - block layer implementation of the sg v4 interface
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 * Copyright (C) 2004 Jens Axboe <axboe@suse.de> SUSE Labs
 * Copyright (C) 2004 Peter M. Jones <pjones@redhat.com>
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License version 2.  See the file "COPYING" in the main directory of this
 *  archive for more details.
 *
 */
#include <linux/module.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/blkdev.h>
#include <linux/poll.h>
#include <linux/cdev.h>
#include <linux/percpu.h>
#include <linux/uio.h>
20
#include <linux/idr.h>
21
#include <linux/bsg.h>
22
#include <linux/smp_lock.h>
23 24 25 26

#include <scsi/scsi.h>
#include <scsi/scsi_ioctl.h>
#include <scsi/scsi_cmnd.h>
F
FUJITA Tomonori 已提交
27 28
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
29 30
#include <scsi/sg.h>

F
FUJITA Tomonori 已提交
31 32
#define BSG_DESCRIPTION	"Block layer SCSI generic (bsg) driver"
#define BSG_VERSION	"0.4"
33 34

struct bsg_device {
35
	struct request_queue *queue;
36 37 38 39 40 41 42 43 44
	spinlock_t lock;
	struct list_head busy_list;
	struct list_head done_list;
	struct hlist_node dev_list;
	atomic_t ref_count;
	int queued_cmds;
	int done_cmds;
	wait_queue_head_t wq_done;
	wait_queue_head_t wq_free;
45
	char name[20];
46 47 48 49 50 51 52 53
	int max_queue;
	unsigned long flags;
};

enum {
	BSG_F_BLOCK		= 1,
};

J
Jens Axboe 已提交
54
#define BSG_DEFAULT_CMDS	64
F
FUJITA Tomonori 已提交
55
#define BSG_MAX_DEVS		32768
56 57 58 59

#undef BSG_DEBUG

#ifdef BSG_DEBUG
60
#define dprintk(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ##args)
61 62 63 64 65
#else
#define dprintk(fmt, args...)
#endif

static DEFINE_MUTEX(bsg_mutex);
66
static DEFINE_IDR(bsg_minor_idr);
67

68 69
#define BSG_LIST_ARRAY_SIZE	8
static struct hlist_head bsg_device_list[BSG_LIST_ARRAY_SIZE];
70 71

static struct class *bsg_class;
J
Jens Axboe 已提交
72
static int bsg_major;
73

J
Jens Axboe 已提交
74 75
static struct kmem_cache *bsg_cmd_cachep;

76 77 78 79 80 81 82 83
/*
 * our internal command type
 */
struct bsg_command {
	struct bsg_device *bd;
	struct list_head list;
	struct request *rq;
	struct bio *bio;
F
FUJITA Tomonori 已提交
84
	struct bio *bidi_bio;
85
	int err;
F
FUJITA Tomonori 已提交
86
	struct sg_io_v4 hdr;
87 88 89 90 91 92 93 94
	char sense[SCSI_SENSE_BUFFERSIZE];
};

static void bsg_free_command(struct bsg_command *bc)
{
	struct bsg_device *bd = bc->bd;
	unsigned long flags;

J
Jens Axboe 已提交
95
	kmem_cache_free(bsg_cmd_cachep, bc);
96 97 98 99 100 101 102 103

	spin_lock_irqsave(&bd->lock, flags);
	bd->queued_cmds--;
	spin_unlock_irqrestore(&bd->lock, flags);

	wake_up(&bd->wq_free);
}

F
FUJITA Tomonori 已提交
104
static struct bsg_command *bsg_alloc_command(struct bsg_device *bd)
105
{
F
FUJITA Tomonori 已提交
106
	struct bsg_command *bc = ERR_PTR(-EINVAL);
107 108 109 110 111 112 113 114 115

	spin_lock_irq(&bd->lock);

	if (bd->queued_cmds >= bd->max_queue)
		goto out;

	bd->queued_cmds++;
	spin_unlock_irq(&bd->lock);

116
	bc = kmem_cache_zalloc(bsg_cmd_cachep, GFP_KERNEL);
J
Jens Axboe 已提交
117 118
	if (unlikely(!bc)) {
		spin_lock_irq(&bd->lock);
119
		bd->queued_cmds--;
F
FUJITA Tomonori 已提交
120
		bc = ERR_PTR(-ENOMEM);
121
		goto out;
J
Jens Axboe 已提交
122 123
	}

124 125
	bc->bd = bd;
	INIT_LIST_HEAD(&bc->list);
J
Jens Axboe 已提交
126
	dprintk("%s: returning free cmd %p\n", bd->name, bc);
127 128 129 130 131 132
	return bc;
out:
	spin_unlock_irq(&bd->lock);
	return bc;
}

F
FUJITA Tomonori 已提交
133
static inline struct hlist_head *bsg_dev_idx_hash(int index)
134
{
F
FUJITA Tomonori 已提交
135
	return &bsg_device_list[index & (BSG_LIST_ARRAY_SIZE - 1)];
136 137
}

138
static int bsg_io_schedule(struct bsg_device *bd)
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
{
	DEFINE_WAIT(wait);
	int ret = 0;

	spin_lock_irq(&bd->lock);

	BUG_ON(bd->done_cmds > bd->queued_cmds);

	/*
	 * -ENOSPC or -ENODATA?  I'm going for -ENODATA, meaning "I have no
	 * work to do", even though we return -ENOSPC after this same test
	 * during bsg_write() -- there, it means our buffer can't have more
	 * bsg_commands added to it, thus has no space left.
	 */
	if (bd->done_cmds == bd->queued_cmds) {
		ret = -ENODATA;
		goto unlock;
	}

	if (!test_bit(BSG_F_BLOCK, &bd->flags)) {
		ret = -EAGAIN;
		goto unlock;
	}

163
	prepare_to_wait(&bd->wq_done, &wait, TASK_UNINTERRUPTIBLE);
164 165 166 167 168 169 170 171 172 173
	spin_unlock_irq(&bd->lock);
	io_schedule();
	finish_wait(&bd->wq_done, &wait);

	return ret;
unlock:
	spin_unlock_irq(&bd->lock);
	return ret;
}

174
static int blk_fill_sgv4_hdr_rq(struct request_queue *q, struct request *rq,
175
				struct sg_io_v4 *hdr, struct bsg_device *bd,
176
				fmode_t has_write_perm)
F
FUJITA Tomonori 已提交
177
{
178 179 180 181 182
	if (hdr->request_len > BLK_MAX_CDB) {
		rq->cmd = kzalloc(hdr->request_len, GFP_KERNEL);
		if (!rq->cmd)
			return -ENOMEM;
	}
F
FUJITA Tomonori 已提交
183 184 185 186

	if (copy_from_user(rq->cmd, (void *)(unsigned long)hdr->request,
			   hdr->request_len))
		return -EFAULT;
187 188

	if (hdr->subprotocol == BSG_SUB_PROTOCOL_SCSI_CMD) {
189
		if (blk_verify_command(rq->cmd, has_write_perm))
190 191
			return -EPERM;
	} else if (!capable(CAP_SYS_RAWIO))
F
FUJITA Tomonori 已提交
192 193 194 195 196 197 198 199 200 201 202 203 204
		return -EPERM;

	/*
	 * fill in request structure
	 */
	rq->cmd_len = hdr->request_len;
	rq->cmd_type = REQ_TYPE_BLOCK_PC;

	rq->timeout = (hdr->timeout * HZ) / 1000;
	if (!rq->timeout)
		rq->timeout = q->sg_timeout;
	if (!rq->timeout)
		rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
L
Linus Torvalds 已提交
205 206
	if (rq->timeout < BLK_MIN_SG_TIMEOUT)
		rq->timeout = BLK_MIN_SG_TIMEOUT;
F
FUJITA Tomonori 已提交
207 208 209 210

	return 0;
}

211
/*
F
FUJITA Tomonori 已提交
212
 * Check if sg_io_v4 from user is allowed and valid
213 214
 */
static int
215
bsg_validate_sgv4_hdr(struct request_queue *q, struct sg_io_v4 *hdr, int *rw)
216
{
217 218
	int ret = 0;

F
FUJITA Tomonori 已提交
219
	if (hdr->guard != 'Q')
220 221
		return -EINVAL;

222 223 224 225 226 227 228 229 230 231 232 233 234
	switch (hdr->protocol) {
	case BSG_PROTOCOL_SCSI:
		switch (hdr->subprotocol) {
		case BSG_SUB_PROTOCOL_SCSI_CMD:
		case BSG_SUB_PROTOCOL_SCSI_TRANSPORT:
			break;
		default:
			ret = -EINVAL;
		}
		break;
	default:
		ret = -EINVAL;
	}
F
FUJITA Tomonori 已提交
235 236

	*rw = hdr->dout_xfer_len ? WRITE : READ;
237
	return ret;
238 239 240
}

/*
F
FUJITA Tomonori 已提交
241
 * map sg_io_v4 to a request.
242 243
 */
static struct request *
244 245
bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm,
	    u8 *sense)
246
{
247
	struct request_queue *q = bd->queue;
F
FUJITA Tomonori 已提交
248
	struct request *rq, *next_rq = NULL;
249
	int ret, rw;
F
FUJITA Tomonori 已提交
250 251
	unsigned int dxfer_len;
	void *dxferp = NULL;
252

F
FUJITA Tomonori 已提交
253 254 255
	dprintk("map hdr %llx/%u %llx/%u\n", (unsigned long long) hdr->dout_xferp,
		hdr->dout_xfer_len, (unsigned long long) hdr->din_xferp,
		hdr->din_xfer_len);
256

F
FUJITA Tomonori 已提交
257
	ret = bsg_validate_sgv4_hdr(q, hdr, &rw);
258 259 260 261 262 263 264
	if (ret)
		return ERR_PTR(ret);

	/*
	 * map scatter-gather elements seperately and string them to request
	 */
	rq = blk_get_request(q, rw, GFP_KERNEL);
F
FUJITA Tomonori 已提交
265 266
	if (!rq)
		return ERR_PTR(-ENOMEM);
267
	ret = blk_fill_sgv4_hdr_rq(q, rq, hdr, bd, has_write_perm);
F
FUJITA Tomonori 已提交
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
	if (ret)
		goto out;

	if (rw == WRITE && hdr->din_xfer_len) {
		if (!test_bit(QUEUE_FLAG_BIDI, &q->queue_flags)) {
			ret = -EOPNOTSUPP;
			goto out;
		}

		next_rq = blk_get_request(q, READ, GFP_KERNEL);
		if (!next_rq) {
			ret = -ENOMEM;
			goto out;
		}
		rq->next_rq = next_rq;
283
		next_rq->cmd_type = rq->cmd_type;
F
FUJITA Tomonori 已提交
284 285

		dxferp = (void*)(unsigned long)hdr->din_xferp;
286 287
		ret =  blk_rq_map_user(q, next_rq, NULL, dxferp,
				       hdr->din_xfer_len, GFP_KERNEL);
F
FUJITA Tomonori 已提交
288 289
		if (ret)
			goto out;
290 291
	}

F
FUJITA Tomonori 已提交
292 293 294 295 296 297 298 299 300 301
	if (hdr->dout_xfer_len) {
		dxfer_len = hdr->dout_xfer_len;
		dxferp = (void*)(unsigned long)hdr->dout_xferp;
	} else if (hdr->din_xfer_len) {
		dxfer_len = hdr->din_xfer_len;
		dxferp = (void*)(unsigned long)hdr->din_xferp;
	} else
		dxfer_len = 0;

	if (dxfer_len) {
302 303
		ret = blk_rq_map_user(q, rq, NULL, dxferp, dxfer_len,
				      GFP_KERNEL);
F
FUJITA Tomonori 已提交
304 305
		if (ret)
			goto out;
306
	}
307 308 309 310

	rq->sense = sense;
	rq->sense_len = 0;

311
	return rq;
F
FUJITA Tomonori 已提交
312
out:
313 314
	if (rq->cmd != rq->__cmd)
		kfree(rq->cmd);
F
FUJITA Tomonori 已提交
315 316 317 318 319 320
	blk_put_request(rq);
	if (next_rq) {
		blk_rq_unmap_user(next_rq->bio);
		blk_put_request(next_rq);
	}
	return ERR_PTR(ret);
321 322 323 324 325 326 327 328 329 330 331 332
}

/*
 * async completion call-back from the block layer, when scsi/ide/whatever
 * calls end_that_request_last() on a request
 */
static void bsg_rq_end_io(struct request *rq, int uptodate)
{
	struct bsg_command *bc = rq->end_io_data;
	struct bsg_device *bd = bc->bd;
	unsigned long flags;

J
Jens Axboe 已提交
333 334
	dprintk("%s: finished rq %p bc %p, bio %p stat %d\n",
		bd->name, rq, bc, bc->bio, uptodate);
335 336 337 338

	bc->hdr.duration = jiffies_to_msecs(jiffies - bc->hdr.duration);

	spin_lock_irqsave(&bd->lock, flags);
339 340
	list_move_tail(&bc->list, &bd->done_list);
	bd->done_cmds++;
341
	spin_unlock_irqrestore(&bd->lock, flags);
342 343

	wake_up(&bd->wq_done);
344 345 346 347 348 349
}

/*
 * do final setup of a 'bc' and submit the matching 'rq' to the block
 * layer for io
 */
350
static void bsg_add_command(struct bsg_device *bd, struct request_queue *q,
351 352
			    struct bsg_command *bc, struct request *rq)
{
B
Boaz Harrosh 已提交
353 354
	int at_head = (0 == (bc->hdr.flags & BSG_FLAG_Q_AT_TAIL));

355 356 357 358 359
	/*
	 * add bc command to busy queue and submit rq for io
	 */
	bc->rq = rq;
	bc->bio = rq->bio;
F
FUJITA Tomonori 已提交
360 361
	if (rq->next_rq)
		bc->bidi_bio = rq->next_rq->bio;
362 363 364 365 366 367 368 369
	bc->hdr.duration = jiffies;
	spin_lock_irq(&bd->lock);
	list_add_tail(&bc->list, &bd->busy_list);
	spin_unlock_irq(&bd->lock);

	dprintk("%s: queueing rq %p, bc %p\n", bd->name, rq, bc);

	rq->end_io_data = bc;
B
Boaz Harrosh 已提交
370
	blk_execute_rq_nowait(q, NULL, rq, at_head, bsg_rq_end_io);
371 372
}

373
static struct bsg_command *bsg_next_done_cmd(struct bsg_device *bd)
374 375 376 377 378
{
	struct bsg_command *bc = NULL;

	spin_lock_irq(&bd->lock);
	if (bd->done_cmds) {
379
		bc = list_first_entry(&bd->done_list, struct bsg_command, list);
380 381
		list_del(&bc->list);
		bd->done_cmds--;
382 383 384 385 386 387 388 389 390
	}
	spin_unlock_irq(&bd->lock);

	return bc;
}

/*
 * Get a finished command from the done list
 */
F
FUJITA Tomonori 已提交
391
static struct bsg_command *bsg_get_done_cmd(struct bsg_device *bd)
392 393 394 395 396 397 398 399 400
{
	struct bsg_command *bc;
	int ret;

	do {
		bc = bsg_next_done_cmd(bd);
		if (bc)
			break;

F
FUJITA Tomonori 已提交
401 402 403 404 405 406
		if (!test_bit(BSG_F_BLOCK, &bd->flags)) {
			bc = ERR_PTR(-EAGAIN);
			break;
		}

		ret = wait_event_interruptible(bd->wq_done, bd->done_cmds);
407
		if (ret) {
F
FUJITA Tomonori 已提交
408
			bc = ERR_PTR(-ERESTARTSYS);
409 410 411 412 413 414 415 416 417
			break;
		}
	} while (1);

	dprintk("%s: returning done %p\n", bd->name, bc);

	return bc;
}

F
FUJITA Tomonori 已提交
418
static int blk_complete_sgv4_hdr_rq(struct request *rq, struct sg_io_v4 *hdr,
F
FUJITA Tomonori 已提交
419
				    struct bio *bio, struct bio *bidi_bio)
F
FUJITA Tomonori 已提交
420 421 422
{
	int ret = 0;

423
	dprintk("rq %p bio %p 0x%x\n", rq, bio, rq->errors);
F
FUJITA Tomonori 已提交
424 425 426 427 428 429 430 431 432 433 434 435
	/*
	 * fill in all the output members
	 */
	hdr->device_status = status_byte(rq->errors);
	hdr->transport_status = host_byte(rq->errors);
	hdr->driver_status = driver_byte(rq->errors);
	hdr->info = 0;
	if (hdr->device_status || hdr->transport_status || hdr->driver_status)
		hdr->info |= SG_INFO_CHECK;
	hdr->response_len = 0;

	if (rq->sense_len && hdr->response) {
436 437
		int len = min_t(unsigned int, hdr->max_response_len,
					rq->sense_len);
F
FUJITA Tomonori 已提交
438 439 440 441 442 443 444 445 446

		ret = copy_to_user((void*)(unsigned long)hdr->response,
				   rq->sense, len);
		if (!ret)
			hdr->response_len = len;
		else
			ret = -EFAULT;
	}

F
FUJITA Tomonori 已提交
447
	if (rq->next_rq) {
T
Tejun Heo 已提交
448 449
		hdr->dout_resid = rq->resid_len;
		hdr->din_resid = rq->next_rq->resid_len;
F
FUJITA Tomonori 已提交
450 451
		blk_rq_unmap_user(bidi_bio);
		blk_put_request(rq->next_rq);
452
	} else if (rq_data_dir(rq) == READ)
T
Tejun Heo 已提交
453
		hdr->din_resid = rq->resid_len;
454
	else
T
Tejun Heo 已提交
455
		hdr->dout_resid = rq->resid_len;
F
FUJITA Tomonori 已提交
456

457 458 459 460 461 462 463 464 465
	/*
	 * If the request generated a negative error number, return it
	 * (providing we aren't already returning an error); if it's
	 * just a protocol response (i.e. non negative), that gets
	 * processed above.
	 */
	if (!ret && rq->errors < 0)
		ret = rq->errors;

F
FUJITA Tomonori 已提交
466
	blk_rq_unmap_user(bio);
467 468
	if (rq->cmd != rq->__cmd)
		kfree(rq->cmd);
F
FUJITA Tomonori 已提交
469 470 471 472 473
	blk_put_request(rq);

	return ret;
}

474 475 476 477 478 479 480 481 482 483 484 485
static int bsg_complete_all_commands(struct bsg_device *bd)
{
	struct bsg_command *bc;
	int ret, tret;

	dprintk("%s: entered\n", bd->name);

	/*
	 * wait for all commands to complete
	 */
	ret = 0;
	do {
486
		ret = bsg_io_schedule(bd);
487 488 489 490 491 492 493 494 495 496 497 498 499 500
		/*
		 * look for -ENODATA specifically -- we'll sometimes get
		 * -ERESTARTSYS when we've taken a signal, but we can't
		 * return until we're done freeing the queue, so ignore
		 * it.  The signal will get handled when we're done freeing
		 * the bsg_device.
		 */
	} while (ret != -ENODATA);

	/*
	 * discard done commands
	 */
	ret = 0;
	do {
F
FUJITA Tomonori 已提交
501 502 503
		spin_lock_irq(&bd->lock);
		if (!bd->queued_cmds) {
			spin_unlock_irq(&bd->lock);
504 505
			break;
		}
506
		spin_unlock_irq(&bd->lock);
507

F
FUJITA Tomonori 已提交
508 509 510 511
		bc = bsg_get_done_cmd(bd);
		if (IS_ERR(bc))
			break;

F
FUJITA Tomonori 已提交
512 513
		tret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
						bc->bidi_bio);
514 515 516 517 518 519 520 521 522
		if (!ret)
			ret = tret;

		bsg_free_command(bc);
	} while (1);

	return ret;
}

523
static int
F
FUJITA Tomonori 已提交
524 525
__bsg_read(char __user *buf, size_t count, struct bsg_device *bd,
	   const struct iovec *iov, ssize_t *bytes_read)
526 527 528 529
{
	struct bsg_command *bc;
	int nr_commands, ret;

F
FUJITA Tomonori 已提交
530
	if (count % sizeof(struct sg_io_v4))
531 532 533
		return -EINVAL;

	ret = 0;
F
FUJITA Tomonori 已提交
534
	nr_commands = count / sizeof(struct sg_io_v4);
535
	while (nr_commands) {
F
FUJITA Tomonori 已提交
536
		bc = bsg_get_done_cmd(bd);
537 538 539 540 541 542 543 544 545 546
		if (IS_ERR(bc)) {
			ret = PTR_ERR(bc);
			break;
		}

		/*
		 * this is the only case where we need to copy data back
		 * after completing the request. so do that here,
		 * bsg_complete_work() cannot do that for us
		 */
F
FUJITA Tomonori 已提交
547 548
		ret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
					       bc->bidi_bio);
549

550
		if (copy_to_user(buf, &bc->hdr, sizeof(bc->hdr)))
551 552 553 554 555 556 557
			ret = -EFAULT;

		bsg_free_command(bc);

		if (ret)
			break;

F
FUJITA Tomonori 已提交
558 559
		buf += sizeof(struct sg_io_v4);
		*bytes_read += sizeof(struct sg_io_v4);
560 561 562 563 564 565 566 567 568 569 570 571 572 573
		nr_commands--;
	}

	return ret;
}

static inline void bsg_set_block(struct bsg_device *bd, struct file *file)
{
	if (file->f_flags & O_NONBLOCK)
		clear_bit(BSG_F_BLOCK, &bd->flags);
	else
		set_bit(BSG_F_BLOCK, &bd->flags);
}

574 575 576
/*
 * Check if the error is a "real" error that we should return.
 */
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
static inline int err_block_err(int ret)
{
	if (ret && ret != -ENOSPC && ret != -ENODATA && ret != -EAGAIN)
		return 1;

	return 0;
}

static ssize_t
bsg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
	struct bsg_device *bd = file->private_data;
	int ret;
	ssize_t bytes_read;

F
FUJITA Tomonori 已提交
592
	dprintk("%s: read %Zd bytes\n", bd->name, count);
593 594

	bsg_set_block(bd, file);
595

596
	bytes_read = 0;
F
FUJITA Tomonori 已提交
597
	ret = __bsg_read(buf, count, bd, NULL, &bytes_read);
598 599 600 601 602 603 604 605
	*ppos = bytes_read;

	if (!bytes_read || (bytes_read && err_block_err(ret)))
		bytes_read = ret;

	return bytes_read;
}

606
static int __bsg_write(struct bsg_device *bd, const char __user *buf,
607 608
		       size_t count, ssize_t *bytes_written,
		       fmode_t has_write_perm)
609 610 611 612 613
{
	struct bsg_command *bc;
	struct request *rq;
	int ret, nr_commands;

F
FUJITA Tomonori 已提交
614
	if (count % sizeof(struct sg_io_v4))
615 616
		return -EINVAL;

F
FUJITA Tomonori 已提交
617
	nr_commands = count / sizeof(struct sg_io_v4);
618 619 620 621
	rq = NULL;
	bc = NULL;
	ret = 0;
	while (nr_commands) {
622
		struct request_queue *q = bd->queue;
623

F
FUJITA Tomonori 已提交
624
		bc = bsg_alloc_command(bd);
625 626 627 628 629 630 631 632 633 634 635 636 637 638
		if (IS_ERR(bc)) {
			ret = PTR_ERR(bc);
			bc = NULL;
			break;
		}

		if (copy_from_user(&bc->hdr, buf, sizeof(bc->hdr))) {
			ret = -EFAULT;
			break;
		}

		/*
		 * get a request, fill in the blanks, and add to request queue
		 */
639
		rq = bsg_map_hdr(bd, &bc->hdr, has_write_perm, bc->sense);
640 641 642 643 644 645 646 647 648 649
		if (IS_ERR(rq)) {
			ret = PTR_ERR(rq);
			rq = NULL;
			break;
		}

		bsg_add_command(bd, q, bc, rq);
		bc = NULL;
		rq = NULL;
		nr_commands--;
F
FUJITA Tomonori 已提交
650
		buf += sizeof(struct sg_io_v4);
651
		*bytes_written += sizeof(struct sg_io_v4);
652 653 654 655 656 657 658 659 660 661 662 663
	}

	if (bc)
		bsg_free_command(bc);

	return ret;
}

static ssize_t
bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
{
	struct bsg_device *bd = file->private_data;
664
	ssize_t bytes_written;
665 666
	int ret;

F
FUJITA Tomonori 已提交
667
	dprintk("%s: write %Zd bytes\n", bd->name, count);
668 669 670

	bsg_set_block(bd, file);

671
	bytes_written = 0;
672 673 674
	ret = __bsg_write(bd, buf, count, &bytes_written,
			  file->f_mode & FMODE_WRITE);

675
	*ppos = bytes_written;
676 677 678 679

	/*
	 * return bytes written on non-fatal errors
	 */
680 681
	if (!bytes_written || (bytes_written && err_block_err(ret)))
		bytes_written = ret;
682

683 684
	dprintk("%s: returning %Zd\n", bd->name, bytes_written);
	return bytes_written;
685 686 687 688 689 690 691 692 693 694 695 696
}

static struct bsg_device *bsg_alloc_device(void)
{
	struct bsg_device *bd;

	bd = kzalloc(sizeof(struct bsg_device), GFP_KERNEL);
	if (unlikely(!bd))
		return NULL;

	spin_lock_init(&bd->lock);

J
Jens Axboe 已提交
697
	bd->max_queue = BSG_DEFAULT_CMDS;
698 699 700 701 702 703 704 705 706 707

	INIT_LIST_HEAD(&bd->busy_list);
	INIT_LIST_HEAD(&bd->done_list);
	INIT_HLIST_NODE(&bd->dev_list);

	init_waitqueue_head(&bd->wq_free);
	init_waitqueue_head(&bd->wq_done);
	return bd;
}

708 709 710 711
static void bsg_kref_release_function(struct kref *kref)
{
	struct bsg_class_device *bcd =
		container_of(kref, struct bsg_class_device, ref);
J
James Bottomley 已提交
712
	struct device *parent = bcd->parent;
713 714 715 716

	if (bcd->release)
		bcd->release(bcd->parent);

J
James Bottomley 已提交
717
	put_device(parent);
718 719
}

720 721
static int bsg_put_device(struct bsg_device *bd)
{
722 723
	int ret = 0, do_free;
	struct request_queue *q = bd->queue;
724 725 726

	mutex_lock(&bsg_mutex);

727
	do_free = atomic_dec_and_test(&bd->ref_count);
728 729
	if (!do_free) {
		mutex_unlock(&bsg_mutex);
730
		goto out;
731 732 733 734
	}

	hlist_del(&bd->dev_list);
	mutex_unlock(&bsg_mutex);
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749

	dprintk("%s: tearing down\n", bd->name);

	/*
	 * close can always block
	 */
	set_bit(BSG_F_BLOCK, &bd->flags);

	/*
	 * correct error detection baddies here again. it's the responsibility
	 * of the app to properly reap commands before close() if it wants
	 * fool-proof error detection
	 */
	ret = bsg_complete_all_commands(bd);

J
Jens Axboe 已提交
750
	kfree(bd);
751
out:
752 753 754
	kref_put(&q->bsg_dev.ref, bsg_kref_release_function);
	if (do_free)
		blk_put_queue(q);
755 756 757 758
	return ret;
}

static struct bsg_device *bsg_add_device(struct inode *inode,
759
					 struct request_queue *rq,
760 761
					 struct file *file)
{
762
	struct bsg_device *bd;
763
	int ret;
764 765 766
#ifdef BSG_DEBUG
	unsigned char buf[32];
#endif
767 768 769
	ret = blk_get_queue(rq);
	if (ret)
		return ERR_PTR(-ENXIO);
770 771

	bd = bsg_alloc_device();
772 773
	if (!bd) {
		blk_put_queue(rq);
774
		return ERR_PTR(-ENOMEM);
775
	}
776

777
	bd->queue = rq;
778

779 780 781 782
	bsg_set_block(bd, file);

	atomic_set(&bd->ref_count, 1);
	mutex_lock(&bsg_mutex);
783
	hlist_add_head(&bd->dev_list, bsg_dev_idx_hash(iminor(inode)));
784

785
	strncpy(bd->name, dev_name(rq->bsg_dev.class_dev), sizeof(bd->name) - 1);
786
	dprintk("bound to <%s>, max queue %d\n",
F
FUJITA Tomonori 已提交
787
		format_dev_t(buf, inode->i_rdev), bd->max_queue);
788 789 790 791 792

	mutex_unlock(&bsg_mutex);
	return bd;
}

793
static struct bsg_device *__bsg_get_device(int minor, struct request_queue *q)
794
{
795
	struct bsg_device *bd;
796 797 798 799
	struct hlist_node *entry;

	mutex_lock(&bsg_mutex);

800
	hlist_for_each_entry(bd, entry, bsg_dev_idx_hash(minor), dev_list) {
801
		if (bd->queue == q) {
802
			atomic_inc(&bd->ref_count);
803
			goto found;
804 805
		}
	}
806 807
	bd = NULL;
found:
808 809 810 811 812 813
	mutex_unlock(&bsg_mutex);
	return bd;
}

static struct bsg_device *bsg_get_device(struct inode *inode, struct file *file)
{
814 815
	struct bsg_device *bd;
	struct bsg_class_device *bcd;
816 817 818 819 820

	/*
	 * find the class device
	 */
	mutex_lock(&bsg_mutex);
821
	bcd = idr_find(&bsg_minor_idr, iminor(inode));
822
	if (bcd)
823
		kref_get(&bcd->ref);
824 825 826 827 828
	mutex_unlock(&bsg_mutex);

	if (!bcd)
		return ERR_PTR(-ENODEV);

829
	bd = __bsg_get_device(iminor(inode), bcd->queue);
830 831 832 833 834
	if (bd)
		return bd;

	bd = bsg_add_device(inode, bcd->queue, file);
	if (IS_ERR(bd))
835
		kref_put(&bcd->ref, bsg_kref_release_function);
836 837

	return bd;
838 839 840 841
}

static int bsg_open(struct inode *inode, struct file *file)
{
842 843 844 845 846
	struct bsg_device *bd;

	lock_kernel();
	bd = bsg_get_device(inode, file);
	unlock_kernel();
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880

	if (IS_ERR(bd))
		return PTR_ERR(bd);

	file->private_data = bd;
	return 0;
}

static int bsg_release(struct inode *inode, struct file *file)
{
	struct bsg_device *bd = file->private_data;

	file->private_data = NULL;
	return bsg_put_device(bd);
}

static unsigned int bsg_poll(struct file *file, poll_table *wait)
{
	struct bsg_device *bd = file->private_data;
	unsigned int mask = 0;

	poll_wait(file, &bd->wq_done, wait);
	poll_wait(file, &bd->wq_free, wait);

	spin_lock_irq(&bd->lock);
	if (!list_empty(&bd->done_list))
		mask |= POLLIN | POLLRDNORM;
	if (bd->queued_cmds >= bd->max_queue)
		mask |= POLLOUT;
	spin_unlock_irq(&bd->lock);

	return mask;
}

881
static long bsg_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
882 883 884
{
	struct bsg_device *bd = file->private_data;
	int __user *uarg = (int __user *) arg;
885
	int ret;
886 887 888 889 890 891 892

	switch (cmd) {
		/*
		 * our own ioctls
		 */
	case SG_GET_COMMAND_Q:
		return put_user(bd->max_queue, uarg);
J
Jens Axboe 已提交
893
	case SG_SET_COMMAND_Q: {
894 895 896 897
		int queue;

		if (get_user(queue, uarg))
			return -EFAULT;
J
Jens Axboe 已提交
898
		if (queue < 1)
899 900
			return -EINVAL;

J
Jens Axboe 已提交
901
		spin_lock_irq(&bd->lock);
902
		bd->max_queue = queue;
J
Jens Axboe 已提交
903
		spin_unlock_irq(&bd->lock);
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
		return 0;
	}

	/*
	 * SCSI/sg ioctls
	 */
	case SG_GET_VERSION_NUM:
	case SCSI_IOCTL_GET_IDLUN:
	case SCSI_IOCTL_GET_BUS_NUMBER:
	case SG_SET_TIMEOUT:
	case SG_GET_TIMEOUT:
	case SG_GET_RESERVED_SIZE:
	case SG_SET_RESERVED_SIZE:
	case SG_EMULATED_HOST:
	case SCSI_IOCTL_SEND_COMMAND: {
		void __user *uarg = (void __user *) arg;
920
		return scsi_cmd_ioctl(bd->queue, NULL, file->f_mode, cmd, uarg);
921
	}
F
FUJITA Tomonori 已提交
922 923
	case SG_IO: {
		struct request *rq;
F
FUJITA Tomonori 已提交
924
		struct bio *bio, *bidi_bio = NULL;
F
FUJITA Tomonori 已提交
925
		struct sg_io_v4 hdr;
B
Boaz Harrosh 已提交
926
		int at_head;
927
		u8 sense[SCSI_SENSE_BUFFERSIZE];
F
FUJITA Tomonori 已提交
928 929 930 931

		if (copy_from_user(&hdr, uarg, sizeof(hdr)))
			return -EFAULT;

932
		rq = bsg_map_hdr(bd, &hdr, file->f_mode & FMODE_WRITE, sense);
F
FUJITA Tomonori 已提交
933 934 935 936
		if (IS_ERR(rq))
			return PTR_ERR(rq);

		bio = rq->bio;
F
FUJITA Tomonori 已提交
937 938
		if (rq->next_rq)
			bidi_bio = rq->next_rq->bio;
B
Boaz Harrosh 已提交
939 940 941

		at_head = (0 == (hdr.flags & BSG_FLAG_Q_AT_TAIL));
		blk_execute_rq(bd->queue, NULL, rq, at_head);
942
		ret = blk_complete_sgv4_hdr_rq(rq, &hdr, bio, bidi_bio);
F
FUJITA Tomonori 已提交
943 944 945

		if (copy_to_user(uarg, &hdr, sizeof(hdr)))
			return -EFAULT;
J
Jens Axboe 已提交
946

947
		return ret;
F
FUJITA Tomonori 已提交
948
	}
949 950 951 952 953 954 955 956 957 958 959 960
	/*
	 * block device ioctls
	 */
	default:
#if 0
		return ioctl_by_bdev(bd->bdev, cmd, arg);
#else
		return -ENOTTY;
#endif
	}
}

961
static const struct file_operations bsg_fops = {
962 963 964 965 966
	.read		=	bsg_read,
	.write		=	bsg_write,
	.poll		=	bsg_poll,
	.open		=	bsg_open,
	.release	=	bsg_release,
967
	.unlocked_ioctl	=	bsg_ioctl,
968 969 970
	.owner		=	THIS_MODULE,
};

971
void bsg_unregister_queue(struct request_queue *q)
972
{
973
	struct bsg_class_device *bcd = &q->bsg_dev;
974

975 976
	if (!bcd->class_dev)
		return;
977 978

	mutex_lock(&bsg_mutex);
979
	idr_remove(&bsg_minor_idr, bcd->minor);
980
	sysfs_remove_link(&q->kobj, "bsg");
981
	device_unregister(bcd->class_dev);
982
	bcd->class_dev = NULL;
983
	kref_put(&bcd->ref, bsg_kref_release_function);
984 985
	mutex_unlock(&bsg_mutex);
}
F
FUJITA Tomonori 已提交
986
EXPORT_SYMBOL_GPL(bsg_unregister_queue);
987

988 989
int bsg_register_queue(struct request_queue *q, struct device *parent,
		       const char *name, void (*release)(struct device *))
990
{
991
	struct bsg_class_device *bcd;
992
	dev_t dev;
993
	int ret, minor;
994
	struct device *class_dev = NULL;
995 996 997 998 999
	const char *devname;

	if (name)
		devname = name;
	else
1000
		devname = dev_name(parent);
1001 1002 1003 1004 1005 1006 1007

	/*
	 * we need a proper transport to send commands, not a stacked device
	 */
	if (!q->request_fn)
		return 0;

1008
	bcd = &q->bsg_dev;
1009 1010 1011
	memset(bcd, 0, sizeof(*bcd));

	mutex_lock(&bsg_mutex);
F
FUJITA Tomonori 已提交
1012

1013 1014 1015 1016
	ret = idr_pre_get(&bsg_minor_idr, GFP_KERNEL);
	if (!ret) {
		ret = -ENOMEM;
		goto unlock;
F
FUJITA Tomonori 已提交
1017 1018
	}

1019 1020 1021
	ret = idr_get_new(&bsg_minor_idr, bcd, &minor);
	if (ret < 0)
		goto unlock;
F
FUJITA Tomonori 已提交
1022

1023 1024 1025 1026 1027 1028 1029
	if (minor >= BSG_MAX_DEVS) {
		printk(KERN_ERR "bsg: too many bsg devices\n");
		ret = -EINVAL;
		goto remove_idr;
	}

	bcd->minor = minor;
1030
	bcd->queue = q;
1031 1032 1033
	bcd->parent = get_device(parent);
	bcd->release = release;
	kref_init(&bcd->ref);
J
Jens Axboe 已提交
1034
	dev = MKDEV(bsg_major, bcd->minor);
1035
	class_dev = device_create(bsg_class, parent, dev, NULL, "%s", devname);
F
FUJITA Tomonori 已提交
1036 1037
	if (IS_ERR(class_dev)) {
		ret = PTR_ERR(class_dev);
1038
		goto put_dev;
F
FUJITA Tomonori 已提交
1039 1040 1041
	}
	bcd->class_dev = class_dev;

1042
	if (q->kobj.sd) {
F
FUJITA Tomonori 已提交
1043 1044
		ret = sysfs_create_link(&q->kobj, &bcd->class_dev->kobj, "bsg");
		if (ret)
1045
			goto unregister_class_dev;
F
FUJITA Tomonori 已提交
1046 1047
	}

1048 1049
	mutex_unlock(&bsg_mutex);
	return 0;
1050

1051
unregister_class_dev:
1052
	device_unregister(class_dev);
1053
put_dev:
1054
	put_device(parent);
1055 1056 1057
remove_idr:
	idr_remove(&bsg_minor_idr, minor);
unlock:
1058
	mutex_unlock(&bsg_mutex);
F
FUJITA Tomonori 已提交
1059 1060
	return ret;
}
F
FUJITA Tomonori 已提交
1061
EXPORT_SYMBOL_GPL(bsg_register_queue);
F
FUJITA Tomonori 已提交
1062

1063
static struct cdev bsg_cdev;
F
FUJITA Tomonori 已提交
1064

1065
static char *bsg_devnode(struct device *dev, mode_t *mode)
1066 1067 1068 1069
{
	return kasprintf(GFP_KERNEL, "bsg/%s", dev_name(dev));
}

1070 1071 1072
static int __init bsg_init(void)
{
	int ret, i;
J
Jens Axboe 已提交
1073
	dev_t devid;
1074

J
Jens Axboe 已提交
1075
	bsg_cmd_cachep = kmem_cache_create("bsg_cmd",
1076
				sizeof(struct bsg_command), 0, 0, NULL);
J
Jens Axboe 已提交
1077 1078 1079 1080 1081
	if (!bsg_cmd_cachep) {
		printk(KERN_ERR "bsg: failed creating slab cache\n");
		return -ENOMEM;
	}

1082
	for (i = 0; i < BSG_LIST_ARRAY_SIZE; i++)
1083 1084 1085
		INIT_HLIST_HEAD(&bsg_device_list[i]);

	bsg_class = class_create(THIS_MODULE, "bsg");
J
Jens Axboe 已提交
1086
	if (IS_ERR(bsg_class)) {
1087 1088
		ret = PTR_ERR(bsg_class);
		goto destroy_kmemcache;
J
Jens Axboe 已提交
1089
	}
1090
	bsg_class->devnode = bsg_devnode;
1091

J
Jens Axboe 已提交
1092
	ret = alloc_chrdev_region(&devid, 0, BSG_MAX_DEVS, "bsg");
1093 1094
	if (ret)
		goto destroy_bsg_class;
F
FUJITA Tomonori 已提交
1095

J
Jens Axboe 已提交
1096 1097
	bsg_major = MAJOR(devid);

F
FUJITA Tomonori 已提交
1098
	cdev_init(&bsg_cdev, &bsg_fops);
J
Jens Axboe 已提交
1099
	ret = cdev_add(&bsg_cdev, MKDEV(bsg_major, 0), BSG_MAX_DEVS);
1100 1101
	if (ret)
		goto unregister_chrdev;
1102

1103
	printk(KERN_INFO BSG_DESCRIPTION " version " BSG_VERSION
F
FUJITA Tomonori 已提交
1104
	       " loaded (major %d)\n", bsg_major);
1105
	return 0;
1106 1107 1108 1109 1110 1111 1112
unregister_chrdev:
	unregister_chrdev_region(MKDEV(bsg_major, 0), BSG_MAX_DEVS);
destroy_bsg_class:
	class_destroy(bsg_class);
destroy_kmemcache:
	kmem_cache_destroy(bsg_cmd_cachep);
	return ret;
1113 1114 1115
}

MODULE_AUTHOR("Jens Axboe");
F
FUJITA Tomonori 已提交
1116
MODULE_DESCRIPTION(BSG_DESCRIPTION);
1117 1118
MODULE_LICENSE("GPL");

F
FUJITA Tomonori 已提交
1119
device_initcall(bsg_init);