bsg.c 23.5 KB
Newer Older
1
/*
2
 * bsg.c - block layer implementation of the sg v4 interface
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
 *
 * Copyright (C) 2004 Jens Axboe <axboe@suse.de> SUSE Labs
 * Copyright (C) 2004 Peter M. Jones <pjones@redhat.com>
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License version 2.  See the file "COPYING" in the main directory of this
 *  archive for more details.
 *
 */
#include <linux/module.h>
#include <linux/init.h>
#include <linux/file.h>
#include <linux/blkdev.h>
#include <linux/poll.h>
#include <linux/cdev.h>
R
Randy Dunlap 已提交
18
#include <linux/jiffies.h>
19 20
#include <linux/percpu.h>
#include <linux/uio.h>
21
#include <linux/idr.h>
22
#include <linux/bsg.h>
23
#include <linux/smp_lock.h>
24
#include <linux/slab.h>
25 26 27 28

#include <scsi/scsi.h>
#include <scsi/scsi_ioctl.h>
#include <scsi/scsi_cmnd.h>
F
FUJITA Tomonori 已提交
29 30
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
31 32
#include <scsi/sg.h>

F
FUJITA Tomonori 已提交
33 34
#define BSG_DESCRIPTION	"Block layer SCSI generic (bsg) driver"
#define BSG_VERSION	"0.4"
35 36

struct bsg_device {
37
	struct request_queue *queue;
38 39 40 41 42 43 44 45 46
	spinlock_t lock;
	struct list_head busy_list;
	struct list_head done_list;
	struct hlist_node dev_list;
	atomic_t ref_count;
	int queued_cmds;
	int done_cmds;
	wait_queue_head_t wq_done;
	wait_queue_head_t wq_free;
47
	char name[20];
48 49 50 51 52 53 54 55
	int max_queue;
	unsigned long flags;
};

enum {
	BSG_F_BLOCK		= 1,
};

J
Jens Axboe 已提交
56
#define BSG_DEFAULT_CMDS	64
F
FUJITA Tomonori 已提交
57
#define BSG_MAX_DEVS		32768
58 59 60 61

#undef BSG_DEBUG

#ifdef BSG_DEBUG
62
#define dprintk(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ##args)
63 64 65 66 67
#else
#define dprintk(fmt, args...)
#endif

static DEFINE_MUTEX(bsg_mutex);
68
static DEFINE_IDR(bsg_minor_idr);
69

70 71
#define BSG_LIST_ARRAY_SIZE	8
static struct hlist_head bsg_device_list[BSG_LIST_ARRAY_SIZE];
72 73

static struct class *bsg_class;
J
Jens Axboe 已提交
74
static int bsg_major;
75

J
Jens Axboe 已提交
76 77
static struct kmem_cache *bsg_cmd_cachep;

78 79 80 81 82 83 84 85
/*
 * our internal command type
 */
struct bsg_command {
	struct bsg_device *bd;
	struct list_head list;
	struct request *rq;
	struct bio *bio;
F
FUJITA Tomonori 已提交
86
	struct bio *bidi_bio;
87
	int err;
F
FUJITA Tomonori 已提交
88
	struct sg_io_v4 hdr;
89 90 91 92 93 94 95 96
	char sense[SCSI_SENSE_BUFFERSIZE];
};

static void bsg_free_command(struct bsg_command *bc)
{
	struct bsg_device *bd = bc->bd;
	unsigned long flags;

J
Jens Axboe 已提交
97
	kmem_cache_free(bsg_cmd_cachep, bc);
98 99 100 101 102 103 104 105

	spin_lock_irqsave(&bd->lock, flags);
	bd->queued_cmds--;
	spin_unlock_irqrestore(&bd->lock, flags);

	wake_up(&bd->wq_free);
}

F
FUJITA Tomonori 已提交
106
static struct bsg_command *bsg_alloc_command(struct bsg_device *bd)
107
{
F
FUJITA Tomonori 已提交
108
	struct bsg_command *bc = ERR_PTR(-EINVAL);
109 110 111 112 113 114 115 116 117

	spin_lock_irq(&bd->lock);

	if (bd->queued_cmds >= bd->max_queue)
		goto out;

	bd->queued_cmds++;
	spin_unlock_irq(&bd->lock);

118
	bc = kmem_cache_zalloc(bsg_cmd_cachep, GFP_KERNEL);
J
Jens Axboe 已提交
119 120
	if (unlikely(!bc)) {
		spin_lock_irq(&bd->lock);
121
		bd->queued_cmds--;
F
FUJITA Tomonori 已提交
122
		bc = ERR_PTR(-ENOMEM);
123
		goto out;
J
Jens Axboe 已提交
124 125
	}

126 127
	bc->bd = bd;
	INIT_LIST_HEAD(&bc->list);
J
Jens Axboe 已提交
128
	dprintk("%s: returning free cmd %p\n", bd->name, bc);
129 130 131 132 133 134
	return bc;
out:
	spin_unlock_irq(&bd->lock);
	return bc;
}

F
FUJITA Tomonori 已提交
135
static inline struct hlist_head *bsg_dev_idx_hash(int index)
136
{
F
FUJITA Tomonori 已提交
137
	return &bsg_device_list[index & (BSG_LIST_ARRAY_SIZE - 1)];
138 139
}

140
static int bsg_io_schedule(struct bsg_device *bd)
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
{
	DEFINE_WAIT(wait);
	int ret = 0;

	spin_lock_irq(&bd->lock);

	BUG_ON(bd->done_cmds > bd->queued_cmds);

	/*
	 * -ENOSPC or -ENODATA?  I'm going for -ENODATA, meaning "I have no
	 * work to do", even though we return -ENOSPC after this same test
	 * during bsg_write() -- there, it means our buffer can't have more
	 * bsg_commands added to it, thus has no space left.
	 */
	if (bd->done_cmds == bd->queued_cmds) {
		ret = -ENODATA;
		goto unlock;
	}

	if (!test_bit(BSG_F_BLOCK, &bd->flags)) {
		ret = -EAGAIN;
		goto unlock;
	}

165
	prepare_to_wait(&bd->wq_done, &wait, TASK_UNINTERRUPTIBLE);
166 167 168 169 170 171 172 173 174 175
	spin_unlock_irq(&bd->lock);
	io_schedule();
	finish_wait(&bd->wq_done, &wait);

	return ret;
unlock:
	spin_unlock_irq(&bd->lock);
	return ret;
}

176
static int blk_fill_sgv4_hdr_rq(struct request_queue *q, struct request *rq,
177
				struct sg_io_v4 *hdr, struct bsg_device *bd,
178
				fmode_t has_write_perm)
F
FUJITA Tomonori 已提交
179
{
180 181 182 183 184
	if (hdr->request_len > BLK_MAX_CDB) {
		rq->cmd = kzalloc(hdr->request_len, GFP_KERNEL);
		if (!rq->cmd)
			return -ENOMEM;
	}
F
FUJITA Tomonori 已提交
185 186 187 188

	if (copy_from_user(rq->cmd, (void *)(unsigned long)hdr->request,
			   hdr->request_len))
		return -EFAULT;
189 190

	if (hdr->subprotocol == BSG_SUB_PROTOCOL_SCSI_CMD) {
191
		if (blk_verify_command(rq->cmd, has_write_perm))
192 193
			return -EPERM;
	} else if (!capable(CAP_SYS_RAWIO))
F
FUJITA Tomonori 已提交
194 195 196 197 198 199 200 201
		return -EPERM;

	/*
	 * fill in request structure
	 */
	rq->cmd_len = hdr->request_len;
	rq->cmd_type = REQ_TYPE_BLOCK_PC;

R
Randy Dunlap 已提交
202
	rq->timeout = msecs_to_jiffies(hdr->timeout);
F
FUJITA Tomonori 已提交
203 204 205 206
	if (!rq->timeout)
		rq->timeout = q->sg_timeout;
	if (!rq->timeout)
		rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
L
Linus Torvalds 已提交
207 208
	if (rq->timeout < BLK_MIN_SG_TIMEOUT)
		rq->timeout = BLK_MIN_SG_TIMEOUT;
F
FUJITA Tomonori 已提交
209 210 211 212

	return 0;
}

213
/*
F
FUJITA Tomonori 已提交
214
 * Check if sg_io_v4 from user is allowed and valid
215 216
 */
static int
217
bsg_validate_sgv4_hdr(struct request_queue *q, struct sg_io_v4 *hdr, int *rw)
218
{
219 220
	int ret = 0;

F
FUJITA Tomonori 已提交
221
	if (hdr->guard != 'Q')
222 223
		return -EINVAL;

224 225 226 227 228 229 230 231 232 233 234 235 236
	switch (hdr->protocol) {
	case BSG_PROTOCOL_SCSI:
		switch (hdr->subprotocol) {
		case BSG_SUB_PROTOCOL_SCSI_CMD:
		case BSG_SUB_PROTOCOL_SCSI_TRANSPORT:
			break;
		default:
			ret = -EINVAL;
		}
		break;
	default:
		ret = -EINVAL;
	}
F
FUJITA Tomonori 已提交
237 238

	*rw = hdr->dout_xfer_len ? WRITE : READ;
239
	return ret;
240 241 242
}

/*
F
FUJITA Tomonori 已提交
243
 * map sg_io_v4 to a request.
244 245
 */
static struct request *
246 247
bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm,
	    u8 *sense)
248
{
249
	struct request_queue *q = bd->queue;
F
FUJITA Tomonori 已提交
250
	struct request *rq, *next_rq = NULL;
251
	int ret, rw;
F
FUJITA Tomonori 已提交
252 253
	unsigned int dxfer_len;
	void *dxferp = NULL;
254

F
FUJITA Tomonori 已提交
255 256 257
	dprintk("map hdr %llx/%u %llx/%u\n", (unsigned long long) hdr->dout_xferp,
		hdr->dout_xfer_len, (unsigned long long) hdr->din_xferp,
		hdr->din_xfer_len);
258

F
FUJITA Tomonori 已提交
259
	ret = bsg_validate_sgv4_hdr(q, hdr, &rw);
260 261 262 263
	if (ret)
		return ERR_PTR(ret);

	/*
D
Daniel Mack 已提交
264
	 * map scatter-gather elements separately and string them to request
265 266
	 */
	rq = blk_get_request(q, rw, GFP_KERNEL);
F
FUJITA Tomonori 已提交
267 268
	if (!rq)
		return ERR_PTR(-ENOMEM);
269
	ret = blk_fill_sgv4_hdr_rq(q, rq, hdr, bd, has_write_perm);
F
FUJITA Tomonori 已提交
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
	if (ret)
		goto out;

	if (rw == WRITE && hdr->din_xfer_len) {
		if (!test_bit(QUEUE_FLAG_BIDI, &q->queue_flags)) {
			ret = -EOPNOTSUPP;
			goto out;
		}

		next_rq = blk_get_request(q, READ, GFP_KERNEL);
		if (!next_rq) {
			ret = -ENOMEM;
			goto out;
		}
		rq->next_rq = next_rq;
285
		next_rq->cmd_type = rq->cmd_type;
F
FUJITA Tomonori 已提交
286 287

		dxferp = (void*)(unsigned long)hdr->din_xferp;
288 289
		ret =  blk_rq_map_user(q, next_rq, NULL, dxferp,
				       hdr->din_xfer_len, GFP_KERNEL);
F
FUJITA Tomonori 已提交
290 291
		if (ret)
			goto out;
292 293
	}

F
FUJITA Tomonori 已提交
294 295 296 297 298 299 300 301 302 303
	if (hdr->dout_xfer_len) {
		dxfer_len = hdr->dout_xfer_len;
		dxferp = (void*)(unsigned long)hdr->dout_xferp;
	} else if (hdr->din_xfer_len) {
		dxfer_len = hdr->din_xfer_len;
		dxferp = (void*)(unsigned long)hdr->din_xferp;
	} else
		dxfer_len = 0;

	if (dxfer_len) {
304 305
		ret = blk_rq_map_user(q, rq, NULL, dxferp, dxfer_len,
				      GFP_KERNEL);
F
FUJITA Tomonori 已提交
306 307
		if (ret)
			goto out;
308
	}
309 310 311 312

	rq->sense = sense;
	rq->sense_len = 0;

313
	return rq;
F
FUJITA Tomonori 已提交
314
out:
315 316
	if (rq->cmd != rq->__cmd)
		kfree(rq->cmd);
F
FUJITA Tomonori 已提交
317 318 319 320 321 322
	blk_put_request(rq);
	if (next_rq) {
		blk_rq_unmap_user(next_rq->bio);
		blk_put_request(next_rq);
	}
	return ERR_PTR(ret);
323 324 325 326 327 328 329 330 331 332 333 334
}

/*
 * async completion call-back from the block layer, when scsi/ide/whatever
 * calls end_that_request_last() on a request
 */
static void bsg_rq_end_io(struct request *rq, int uptodate)
{
	struct bsg_command *bc = rq->end_io_data;
	struct bsg_device *bd = bc->bd;
	unsigned long flags;

J
Jens Axboe 已提交
335 336
	dprintk("%s: finished rq %p bc %p, bio %p stat %d\n",
		bd->name, rq, bc, bc->bio, uptodate);
337 338 339 340

	bc->hdr.duration = jiffies_to_msecs(jiffies - bc->hdr.duration);

	spin_lock_irqsave(&bd->lock, flags);
341 342
	list_move_tail(&bc->list, &bd->done_list);
	bd->done_cmds++;
343
	spin_unlock_irqrestore(&bd->lock, flags);
344 345

	wake_up(&bd->wq_done);
346 347 348 349 350 351
}

/*
 * do final setup of a 'bc' and submit the matching 'rq' to the block
 * layer for io
 */
352
static void bsg_add_command(struct bsg_device *bd, struct request_queue *q,
353 354
			    struct bsg_command *bc, struct request *rq)
{
B
Boaz Harrosh 已提交
355 356
	int at_head = (0 == (bc->hdr.flags & BSG_FLAG_Q_AT_TAIL));

357 358 359 360 361
	/*
	 * add bc command to busy queue and submit rq for io
	 */
	bc->rq = rq;
	bc->bio = rq->bio;
F
FUJITA Tomonori 已提交
362 363
	if (rq->next_rq)
		bc->bidi_bio = rq->next_rq->bio;
364 365 366 367 368 369 370 371
	bc->hdr.duration = jiffies;
	spin_lock_irq(&bd->lock);
	list_add_tail(&bc->list, &bd->busy_list);
	spin_unlock_irq(&bd->lock);

	dprintk("%s: queueing rq %p, bc %p\n", bd->name, rq, bc);

	rq->end_io_data = bc;
B
Boaz Harrosh 已提交
372
	blk_execute_rq_nowait(q, NULL, rq, at_head, bsg_rq_end_io);
373 374
}

375
static struct bsg_command *bsg_next_done_cmd(struct bsg_device *bd)
376 377 378 379 380
{
	struct bsg_command *bc = NULL;

	spin_lock_irq(&bd->lock);
	if (bd->done_cmds) {
381
		bc = list_first_entry(&bd->done_list, struct bsg_command, list);
382 383
		list_del(&bc->list);
		bd->done_cmds--;
384 385 386 387 388 389 390 391 392
	}
	spin_unlock_irq(&bd->lock);

	return bc;
}

/*
 * Get a finished command from the done list
 */
F
FUJITA Tomonori 已提交
393
static struct bsg_command *bsg_get_done_cmd(struct bsg_device *bd)
394 395 396 397 398 399 400 401 402
{
	struct bsg_command *bc;
	int ret;

	do {
		bc = bsg_next_done_cmd(bd);
		if (bc)
			break;

F
FUJITA Tomonori 已提交
403 404 405 406 407 408
		if (!test_bit(BSG_F_BLOCK, &bd->flags)) {
			bc = ERR_PTR(-EAGAIN);
			break;
		}

		ret = wait_event_interruptible(bd->wq_done, bd->done_cmds);
409
		if (ret) {
F
FUJITA Tomonori 已提交
410
			bc = ERR_PTR(-ERESTARTSYS);
411 412 413 414 415 416 417 418 419
			break;
		}
	} while (1);

	dprintk("%s: returning done %p\n", bd->name, bc);

	return bc;
}

F
FUJITA Tomonori 已提交
420
static int blk_complete_sgv4_hdr_rq(struct request *rq, struct sg_io_v4 *hdr,
F
FUJITA Tomonori 已提交
421
				    struct bio *bio, struct bio *bidi_bio)
F
FUJITA Tomonori 已提交
422 423 424
{
	int ret = 0;

425
	dprintk("rq %p bio %p 0x%x\n", rq, bio, rq->errors);
F
FUJITA Tomonori 已提交
426 427 428 429 430 431 432 433 434 435 436 437
	/*
	 * fill in all the output members
	 */
	hdr->device_status = status_byte(rq->errors);
	hdr->transport_status = host_byte(rq->errors);
	hdr->driver_status = driver_byte(rq->errors);
	hdr->info = 0;
	if (hdr->device_status || hdr->transport_status || hdr->driver_status)
		hdr->info |= SG_INFO_CHECK;
	hdr->response_len = 0;

	if (rq->sense_len && hdr->response) {
438 439
		int len = min_t(unsigned int, hdr->max_response_len,
					rq->sense_len);
F
FUJITA Tomonori 已提交
440 441 442 443 444 445 446 447 448

		ret = copy_to_user((void*)(unsigned long)hdr->response,
				   rq->sense, len);
		if (!ret)
			hdr->response_len = len;
		else
			ret = -EFAULT;
	}

F
FUJITA Tomonori 已提交
449
	if (rq->next_rq) {
T
Tejun Heo 已提交
450 451
		hdr->dout_resid = rq->resid_len;
		hdr->din_resid = rq->next_rq->resid_len;
F
FUJITA Tomonori 已提交
452 453
		blk_rq_unmap_user(bidi_bio);
		blk_put_request(rq->next_rq);
454
	} else if (rq_data_dir(rq) == READ)
T
Tejun Heo 已提交
455
		hdr->din_resid = rq->resid_len;
456
	else
T
Tejun Heo 已提交
457
		hdr->dout_resid = rq->resid_len;
F
FUJITA Tomonori 已提交
458

459 460 461 462 463 464 465 466 467
	/*
	 * If the request generated a negative error number, return it
	 * (providing we aren't already returning an error); if it's
	 * just a protocol response (i.e. non negative), that gets
	 * processed above.
	 */
	if (!ret && rq->errors < 0)
		ret = rq->errors;

F
FUJITA Tomonori 已提交
468
	blk_rq_unmap_user(bio);
469 470
	if (rq->cmd != rq->__cmd)
		kfree(rq->cmd);
F
FUJITA Tomonori 已提交
471 472 473 474 475
	blk_put_request(rq);

	return ret;
}

476 477 478 479 480 481 482 483 484 485 486 487
static int bsg_complete_all_commands(struct bsg_device *bd)
{
	struct bsg_command *bc;
	int ret, tret;

	dprintk("%s: entered\n", bd->name);

	/*
	 * wait for all commands to complete
	 */
	ret = 0;
	do {
488
		ret = bsg_io_schedule(bd);
489 490 491 492 493 494 495 496 497 498 499 500 501 502
		/*
		 * look for -ENODATA specifically -- we'll sometimes get
		 * -ERESTARTSYS when we've taken a signal, but we can't
		 * return until we're done freeing the queue, so ignore
		 * it.  The signal will get handled when we're done freeing
		 * the bsg_device.
		 */
	} while (ret != -ENODATA);

	/*
	 * discard done commands
	 */
	ret = 0;
	do {
F
FUJITA Tomonori 已提交
503 504 505
		spin_lock_irq(&bd->lock);
		if (!bd->queued_cmds) {
			spin_unlock_irq(&bd->lock);
506 507
			break;
		}
508
		spin_unlock_irq(&bd->lock);
509

F
FUJITA Tomonori 已提交
510 511 512 513
		bc = bsg_get_done_cmd(bd);
		if (IS_ERR(bc))
			break;

F
FUJITA Tomonori 已提交
514 515
		tret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
						bc->bidi_bio);
516 517 518 519 520 521 522 523 524
		if (!ret)
			ret = tret;

		bsg_free_command(bc);
	} while (1);

	return ret;
}

525
static int
F
FUJITA Tomonori 已提交
526 527
__bsg_read(char __user *buf, size_t count, struct bsg_device *bd,
	   const struct iovec *iov, ssize_t *bytes_read)
528 529 530 531
{
	struct bsg_command *bc;
	int nr_commands, ret;

F
FUJITA Tomonori 已提交
532
	if (count % sizeof(struct sg_io_v4))
533 534 535
		return -EINVAL;

	ret = 0;
F
FUJITA Tomonori 已提交
536
	nr_commands = count / sizeof(struct sg_io_v4);
537
	while (nr_commands) {
F
FUJITA Tomonori 已提交
538
		bc = bsg_get_done_cmd(bd);
539 540 541 542 543 544 545 546 547 548
		if (IS_ERR(bc)) {
			ret = PTR_ERR(bc);
			break;
		}

		/*
		 * this is the only case where we need to copy data back
		 * after completing the request. so do that here,
		 * bsg_complete_work() cannot do that for us
		 */
F
FUJITA Tomonori 已提交
549 550
		ret = blk_complete_sgv4_hdr_rq(bc->rq, &bc->hdr, bc->bio,
					       bc->bidi_bio);
551

552
		if (copy_to_user(buf, &bc->hdr, sizeof(bc->hdr)))
553 554 555 556 557 558 559
			ret = -EFAULT;

		bsg_free_command(bc);

		if (ret)
			break;

F
FUJITA Tomonori 已提交
560 561
		buf += sizeof(struct sg_io_v4);
		*bytes_read += sizeof(struct sg_io_v4);
562 563 564 565 566 567 568 569 570 571 572 573 574 575
		nr_commands--;
	}

	return ret;
}

static inline void bsg_set_block(struct bsg_device *bd, struct file *file)
{
	if (file->f_flags & O_NONBLOCK)
		clear_bit(BSG_F_BLOCK, &bd->flags);
	else
		set_bit(BSG_F_BLOCK, &bd->flags);
}

576 577 578
/*
 * Check if the error is a "real" error that we should return.
 */
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static inline int err_block_err(int ret)
{
	if (ret && ret != -ENOSPC && ret != -ENODATA && ret != -EAGAIN)
		return 1;

	return 0;
}

static ssize_t
bsg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
	struct bsg_device *bd = file->private_data;
	int ret;
	ssize_t bytes_read;

F
FUJITA Tomonori 已提交
594
	dprintk("%s: read %Zd bytes\n", bd->name, count);
595 596

	bsg_set_block(bd, file);
597

598
	bytes_read = 0;
F
FUJITA Tomonori 已提交
599
	ret = __bsg_read(buf, count, bd, NULL, &bytes_read);
600 601 602 603 604 605 606 607
	*ppos = bytes_read;

	if (!bytes_read || (bytes_read && err_block_err(ret)))
		bytes_read = ret;

	return bytes_read;
}

608
static int __bsg_write(struct bsg_device *bd, const char __user *buf,
609 610
		       size_t count, ssize_t *bytes_written,
		       fmode_t has_write_perm)
611 612 613 614 615
{
	struct bsg_command *bc;
	struct request *rq;
	int ret, nr_commands;

F
FUJITA Tomonori 已提交
616
	if (count % sizeof(struct sg_io_v4))
617 618
		return -EINVAL;

F
FUJITA Tomonori 已提交
619
	nr_commands = count / sizeof(struct sg_io_v4);
620 621 622 623
	rq = NULL;
	bc = NULL;
	ret = 0;
	while (nr_commands) {
624
		struct request_queue *q = bd->queue;
625

F
FUJITA Tomonori 已提交
626
		bc = bsg_alloc_command(bd);
627 628 629 630 631 632 633 634 635 636 637 638 639 640
		if (IS_ERR(bc)) {
			ret = PTR_ERR(bc);
			bc = NULL;
			break;
		}

		if (copy_from_user(&bc->hdr, buf, sizeof(bc->hdr))) {
			ret = -EFAULT;
			break;
		}

		/*
		 * get a request, fill in the blanks, and add to request queue
		 */
641
		rq = bsg_map_hdr(bd, &bc->hdr, has_write_perm, bc->sense);
642 643 644 645 646 647 648 649 650 651
		if (IS_ERR(rq)) {
			ret = PTR_ERR(rq);
			rq = NULL;
			break;
		}

		bsg_add_command(bd, q, bc, rq);
		bc = NULL;
		rq = NULL;
		nr_commands--;
F
FUJITA Tomonori 已提交
652
		buf += sizeof(struct sg_io_v4);
653
		*bytes_written += sizeof(struct sg_io_v4);
654 655 656 657 658 659 660 661 662 663 664 665
	}

	if (bc)
		bsg_free_command(bc);

	return ret;
}

static ssize_t
bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
{
	struct bsg_device *bd = file->private_data;
666
	ssize_t bytes_written;
667 668
	int ret;

F
FUJITA Tomonori 已提交
669
	dprintk("%s: write %Zd bytes\n", bd->name, count);
670 671 672

	bsg_set_block(bd, file);

673
	bytes_written = 0;
674 675 676
	ret = __bsg_write(bd, buf, count, &bytes_written,
			  file->f_mode & FMODE_WRITE);

677
	*ppos = bytes_written;
678 679 680 681

	/*
	 * return bytes written on non-fatal errors
	 */
682 683
	if (!bytes_written || (bytes_written && err_block_err(ret)))
		bytes_written = ret;
684

685 686
	dprintk("%s: returning %Zd\n", bd->name, bytes_written);
	return bytes_written;
687 688 689 690 691 692 693 694 695 696 697 698
}

static struct bsg_device *bsg_alloc_device(void)
{
	struct bsg_device *bd;

	bd = kzalloc(sizeof(struct bsg_device), GFP_KERNEL);
	if (unlikely(!bd))
		return NULL;

	spin_lock_init(&bd->lock);

J
Jens Axboe 已提交
699
	bd->max_queue = BSG_DEFAULT_CMDS;
700 701 702 703 704 705 706 707 708 709

	INIT_LIST_HEAD(&bd->busy_list);
	INIT_LIST_HEAD(&bd->done_list);
	INIT_HLIST_NODE(&bd->dev_list);

	init_waitqueue_head(&bd->wq_free);
	init_waitqueue_head(&bd->wq_done);
	return bd;
}

710 711 712 713
static void bsg_kref_release_function(struct kref *kref)
{
	struct bsg_class_device *bcd =
		container_of(kref, struct bsg_class_device, ref);
J
James Bottomley 已提交
714
	struct device *parent = bcd->parent;
715 716 717 718

	if (bcd->release)
		bcd->release(bcd->parent);

J
James Bottomley 已提交
719
	put_device(parent);
720 721
}

722 723
static int bsg_put_device(struct bsg_device *bd)
{
724 725
	int ret = 0, do_free;
	struct request_queue *q = bd->queue;
726 727 728

	mutex_lock(&bsg_mutex);

729
	do_free = atomic_dec_and_test(&bd->ref_count);
730 731
	if (!do_free) {
		mutex_unlock(&bsg_mutex);
732
		goto out;
733 734 735 736
	}

	hlist_del(&bd->dev_list);
	mutex_unlock(&bsg_mutex);
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751

	dprintk("%s: tearing down\n", bd->name);

	/*
	 * close can always block
	 */
	set_bit(BSG_F_BLOCK, &bd->flags);

	/*
	 * correct error detection baddies here again. it's the responsibility
	 * of the app to properly reap commands before close() if it wants
	 * fool-proof error detection
	 */
	ret = bsg_complete_all_commands(bd);

J
Jens Axboe 已提交
752
	kfree(bd);
753
out:
754 755 756
	kref_put(&q->bsg_dev.ref, bsg_kref_release_function);
	if (do_free)
		blk_put_queue(q);
757 758 759 760
	return ret;
}

static struct bsg_device *bsg_add_device(struct inode *inode,
761
					 struct request_queue *rq,
762 763
					 struct file *file)
{
764
	struct bsg_device *bd;
765
	int ret;
766 767 768
#ifdef BSG_DEBUG
	unsigned char buf[32];
#endif
769 770 771
	ret = blk_get_queue(rq);
	if (ret)
		return ERR_PTR(-ENXIO);
772 773

	bd = bsg_alloc_device();
774 775
	if (!bd) {
		blk_put_queue(rq);
776
		return ERR_PTR(-ENOMEM);
777
	}
778

779
	bd->queue = rq;
780

781 782 783 784
	bsg_set_block(bd, file);

	atomic_set(&bd->ref_count, 1);
	mutex_lock(&bsg_mutex);
785
	hlist_add_head(&bd->dev_list, bsg_dev_idx_hash(iminor(inode)));
786

787
	strncpy(bd->name, dev_name(rq->bsg_dev.class_dev), sizeof(bd->name) - 1);
788
	dprintk("bound to <%s>, max queue %d\n",
F
FUJITA Tomonori 已提交
789
		format_dev_t(buf, inode->i_rdev), bd->max_queue);
790 791 792 793 794

	mutex_unlock(&bsg_mutex);
	return bd;
}

795
static struct bsg_device *__bsg_get_device(int minor, struct request_queue *q)
796
{
797
	struct bsg_device *bd;
798 799 800 801
	struct hlist_node *entry;

	mutex_lock(&bsg_mutex);

802
	hlist_for_each_entry(bd, entry, bsg_dev_idx_hash(minor), dev_list) {
803
		if (bd->queue == q) {
804
			atomic_inc(&bd->ref_count);
805
			goto found;
806 807
		}
	}
808 809
	bd = NULL;
found:
810 811 812 813 814 815
	mutex_unlock(&bsg_mutex);
	return bd;
}

static struct bsg_device *bsg_get_device(struct inode *inode, struct file *file)
{
816 817
	struct bsg_device *bd;
	struct bsg_class_device *bcd;
818 819 820 821 822

	/*
	 * find the class device
	 */
	mutex_lock(&bsg_mutex);
823
	bcd = idr_find(&bsg_minor_idr, iminor(inode));
824
	if (bcd)
825
		kref_get(&bcd->ref);
826 827 828 829 830
	mutex_unlock(&bsg_mutex);

	if (!bcd)
		return ERR_PTR(-ENODEV);

831
	bd = __bsg_get_device(iminor(inode), bcd->queue);
832 833 834 835 836
	if (bd)
		return bd;

	bd = bsg_add_device(inode, bcd->queue, file);
	if (IS_ERR(bd))
837
		kref_put(&bcd->ref, bsg_kref_release_function);
838 839

	return bd;
840 841 842 843
}

static int bsg_open(struct inode *inode, struct file *file)
{
844 845 846 847 848
	struct bsg_device *bd;

	lock_kernel();
	bd = bsg_get_device(inode, file);
	unlock_kernel();
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882

	if (IS_ERR(bd))
		return PTR_ERR(bd);

	file->private_data = bd;
	return 0;
}

static int bsg_release(struct inode *inode, struct file *file)
{
	struct bsg_device *bd = file->private_data;

	file->private_data = NULL;
	return bsg_put_device(bd);
}

static unsigned int bsg_poll(struct file *file, poll_table *wait)
{
	struct bsg_device *bd = file->private_data;
	unsigned int mask = 0;

	poll_wait(file, &bd->wq_done, wait);
	poll_wait(file, &bd->wq_free, wait);

	spin_lock_irq(&bd->lock);
	if (!list_empty(&bd->done_list))
		mask |= POLLIN | POLLRDNORM;
	if (bd->queued_cmds >= bd->max_queue)
		mask |= POLLOUT;
	spin_unlock_irq(&bd->lock);

	return mask;
}

883
static long bsg_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
884 885 886
{
	struct bsg_device *bd = file->private_data;
	int __user *uarg = (int __user *) arg;
887
	int ret;
888 889 890 891 892 893 894

	switch (cmd) {
		/*
		 * our own ioctls
		 */
	case SG_GET_COMMAND_Q:
		return put_user(bd->max_queue, uarg);
J
Jens Axboe 已提交
895
	case SG_SET_COMMAND_Q: {
896 897 898 899
		int queue;

		if (get_user(queue, uarg))
			return -EFAULT;
J
Jens Axboe 已提交
900
		if (queue < 1)
901 902
			return -EINVAL;

J
Jens Axboe 已提交
903
		spin_lock_irq(&bd->lock);
904
		bd->max_queue = queue;
J
Jens Axboe 已提交
905
		spin_unlock_irq(&bd->lock);
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
		return 0;
	}

	/*
	 * SCSI/sg ioctls
	 */
	case SG_GET_VERSION_NUM:
	case SCSI_IOCTL_GET_IDLUN:
	case SCSI_IOCTL_GET_BUS_NUMBER:
	case SG_SET_TIMEOUT:
	case SG_GET_TIMEOUT:
	case SG_GET_RESERVED_SIZE:
	case SG_SET_RESERVED_SIZE:
	case SG_EMULATED_HOST:
	case SCSI_IOCTL_SEND_COMMAND: {
		void __user *uarg = (void __user *) arg;
922
		return scsi_cmd_ioctl(bd->queue, NULL, file->f_mode, cmd, uarg);
923
	}
F
FUJITA Tomonori 已提交
924 925
	case SG_IO: {
		struct request *rq;
F
FUJITA Tomonori 已提交
926
		struct bio *bio, *bidi_bio = NULL;
F
FUJITA Tomonori 已提交
927
		struct sg_io_v4 hdr;
B
Boaz Harrosh 已提交
928
		int at_head;
929
		u8 sense[SCSI_SENSE_BUFFERSIZE];
F
FUJITA Tomonori 已提交
930 931 932 933

		if (copy_from_user(&hdr, uarg, sizeof(hdr)))
			return -EFAULT;

934
		rq = bsg_map_hdr(bd, &hdr, file->f_mode & FMODE_WRITE, sense);
F
FUJITA Tomonori 已提交
935 936 937 938
		if (IS_ERR(rq))
			return PTR_ERR(rq);

		bio = rq->bio;
F
FUJITA Tomonori 已提交
939 940
		if (rq->next_rq)
			bidi_bio = rq->next_rq->bio;
B
Boaz Harrosh 已提交
941 942 943

		at_head = (0 == (hdr.flags & BSG_FLAG_Q_AT_TAIL));
		blk_execute_rq(bd->queue, NULL, rq, at_head);
944
		ret = blk_complete_sgv4_hdr_rq(rq, &hdr, bio, bidi_bio);
F
FUJITA Tomonori 已提交
945 946 947

		if (copy_to_user(uarg, &hdr, sizeof(hdr)))
			return -EFAULT;
J
Jens Axboe 已提交
948

949
		return ret;
F
FUJITA Tomonori 已提交
950
	}
951 952 953 954 955 956 957 958 959 960 961 962
	/*
	 * block device ioctls
	 */
	default:
#if 0
		return ioctl_by_bdev(bd->bdev, cmd, arg);
#else
		return -ENOTTY;
#endif
	}
}

963
static const struct file_operations bsg_fops = {
964 965 966 967 968
	.read		=	bsg_read,
	.write		=	bsg_write,
	.poll		=	bsg_poll,
	.open		=	bsg_open,
	.release	=	bsg_release,
969
	.unlocked_ioctl	=	bsg_ioctl,
970 971 972
	.owner		=	THIS_MODULE,
};

973
void bsg_unregister_queue(struct request_queue *q)
974
{
975
	struct bsg_class_device *bcd = &q->bsg_dev;
976

977 978
	if (!bcd->class_dev)
		return;
979 980

	mutex_lock(&bsg_mutex);
981
	idr_remove(&bsg_minor_idr, bcd->minor);
982
	sysfs_remove_link(&q->kobj, "bsg");
983
	device_unregister(bcd->class_dev);
984
	bcd->class_dev = NULL;
985
	kref_put(&bcd->ref, bsg_kref_release_function);
986 987
	mutex_unlock(&bsg_mutex);
}
F
FUJITA Tomonori 已提交
988
EXPORT_SYMBOL_GPL(bsg_unregister_queue);
989

990 991
int bsg_register_queue(struct request_queue *q, struct device *parent,
		       const char *name, void (*release)(struct device *))
992
{
993
	struct bsg_class_device *bcd;
994
	dev_t dev;
995
	int ret, minor;
996
	struct device *class_dev = NULL;
997 998 999 1000 1001
	const char *devname;

	if (name)
		devname = name;
	else
1002
		devname = dev_name(parent);
1003 1004 1005 1006 1007 1008 1009

	/*
	 * we need a proper transport to send commands, not a stacked device
	 */
	if (!q->request_fn)
		return 0;

1010
	bcd = &q->bsg_dev;
1011 1012 1013
	memset(bcd, 0, sizeof(*bcd));

	mutex_lock(&bsg_mutex);
F
FUJITA Tomonori 已提交
1014

1015 1016 1017 1018
	ret = idr_pre_get(&bsg_minor_idr, GFP_KERNEL);
	if (!ret) {
		ret = -ENOMEM;
		goto unlock;
F
FUJITA Tomonori 已提交
1019 1020
	}

1021 1022 1023
	ret = idr_get_new(&bsg_minor_idr, bcd, &minor);
	if (ret < 0)
		goto unlock;
F
FUJITA Tomonori 已提交
1024

1025 1026 1027 1028 1029 1030 1031
	if (minor >= BSG_MAX_DEVS) {
		printk(KERN_ERR "bsg: too many bsg devices\n");
		ret = -EINVAL;
		goto remove_idr;
	}

	bcd->minor = minor;
1032
	bcd->queue = q;
1033 1034 1035
	bcd->parent = get_device(parent);
	bcd->release = release;
	kref_init(&bcd->ref);
J
Jens Axboe 已提交
1036
	dev = MKDEV(bsg_major, bcd->minor);
1037
	class_dev = device_create(bsg_class, parent, dev, NULL, "%s", devname);
F
FUJITA Tomonori 已提交
1038 1039
	if (IS_ERR(class_dev)) {
		ret = PTR_ERR(class_dev);
1040
		goto put_dev;
F
FUJITA Tomonori 已提交
1041 1042 1043
	}
	bcd->class_dev = class_dev;

1044
	if (q->kobj.sd) {
F
FUJITA Tomonori 已提交
1045 1046
		ret = sysfs_create_link(&q->kobj, &bcd->class_dev->kobj, "bsg");
		if (ret)
1047
			goto unregister_class_dev;
F
FUJITA Tomonori 已提交
1048 1049
	}

1050 1051
	mutex_unlock(&bsg_mutex);
	return 0;
1052

1053
unregister_class_dev:
1054
	device_unregister(class_dev);
1055
put_dev:
1056
	put_device(parent);
1057 1058 1059
remove_idr:
	idr_remove(&bsg_minor_idr, minor);
unlock:
1060
	mutex_unlock(&bsg_mutex);
F
FUJITA Tomonori 已提交
1061 1062
	return ret;
}
F
FUJITA Tomonori 已提交
1063
EXPORT_SYMBOL_GPL(bsg_register_queue);
F
FUJITA Tomonori 已提交
1064

1065
static struct cdev bsg_cdev;
F
FUJITA Tomonori 已提交
1066

1067
static char *bsg_devnode(struct device *dev, mode_t *mode)
1068 1069 1070 1071
{
	return kasprintf(GFP_KERNEL, "bsg/%s", dev_name(dev));
}

1072 1073 1074
static int __init bsg_init(void)
{
	int ret, i;
J
Jens Axboe 已提交
1075
	dev_t devid;
1076

J
Jens Axboe 已提交
1077
	bsg_cmd_cachep = kmem_cache_create("bsg_cmd",
1078
				sizeof(struct bsg_command), 0, 0, NULL);
J
Jens Axboe 已提交
1079 1080 1081 1082 1083
	if (!bsg_cmd_cachep) {
		printk(KERN_ERR "bsg: failed creating slab cache\n");
		return -ENOMEM;
	}

1084
	for (i = 0; i < BSG_LIST_ARRAY_SIZE; i++)
1085 1086 1087
		INIT_HLIST_HEAD(&bsg_device_list[i]);

	bsg_class = class_create(THIS_MODULE, "bsg");
J
Jens Axboe 已提交
1088
	if (IS_ERR(bsg_class)) {
1089 1090
		ret = PTR_ERR(bsg_class);
		goto destroy_kmemcache;
J
Jens Axboe 已提交
1091
	}
1092
	bsg_class->devnode = bsg_devnode;
1093

J
Jens Axboe 已提交
1094
	ret = alloc_chrdev_region(&devid, 0, BSG_MAX_DEVS, "bsg");
1095 1096
	if (ret)
		goto destroy_bsg_class;
F
FUJITA Tomonori 已提交
1097

J
Jens Axboe 已提交
1098 1099
	bsg_major = MAJOR(devid);

F
FUJITA Tomonori 已提交
1100
	cdev_init(&bsg_cdev, &bsg_fops);
J
Jens Axboe 已提交
1101
	ret = cdev_add(&bsg_cdev, MKDEV(bsg_major, 0), BSG_MAX_DEVS);
1102 1103
	if (ret)
		goto unregister_chrdev;
1104

1105
	printk(KERN_INFO BSG_DESCRIPTION " version " BSG_VERSION
F
FUJITA Tomonori 已提交
1106
	       " loaded (major %d)\n", bsg_major);
1107
	return 0;
1108 1109 1110 1111 1112 1113 1114
unregister_chrdev:
	unregister_chrdev_region(MKDEV(bsg_major, 0), BSG_MAX_DEVS);
destroy_bsg_class:
	class_destroy(bsg_class);
destroy_kmemcache:
	kmem_cache_destroy(bsg_cmd_cachep);
	return ret;
1115 1116 1117
}

MODULE_AUTHOR("Jens Axboe");
F
FUJITA Tomonori 已提交
1118
MODULE_DESCRIPTION(BSG_DESCRIPTION);
1119 1120
MODULE_LICENSE("GPL");

F
FUJITA Tomonori 已提交
1121
device_initcall(bsg_init);