drbd_main.c 108.2 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

#include <linux/module.h>
#include <linux/drbd.h>
#include <asm/uaccess.h>
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
35
#include <linux/mutex.h>
P
Philipp Reisner 已提交
36 37 38 39 40 41 42 43 44 45 46 47
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
48
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
49 50 51 52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>

#include <linux/drbd_limits.h>
#include "drbd_int.h"
55
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
56 57 58 59
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */

#include "drbd_vli.h"

60
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
61
static int drbd_open(struct block_device *bdev, fmode_t mode);
62
static void drbd_release(struct gendisk *gd, fmode_t mode);
63
static int w_md_sync(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
64
static void md_sync_timer_fn(unsigned long data);
65
static int w_bitmap_io(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
66 67 68 69 70 71

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
72
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
73
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
102
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
103 104
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
105 106 107 108 109 110 111 112 113 114 115
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
char usermode_helper[80] = "/sbin/drbdadm";

module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
116
struct idr drbd_devices;
117
struct list_head drbd_resources;
P
Philipp Reisner 已提交
118 119

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
120
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
121 122 123 124
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
125
mempool_t *drbd_md_io_page_pool;
126
struct bio_set *drbd_md_io_bio_set;
P
Philipp Reisner 已提交
127 128 129 130 131 132 133 134 135 136 137 138 139 140

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

141
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
142 143 144 145
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
146

147 148 149
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
150

151 152
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
153

154 155 156 157
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
158 159
}

P
Philipp Reisner 已提交
160 161 162 163
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
164
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
165 166 167
{
	int io_allowed;

168 169
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
170
	if (!io_allowed) {
171 172
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
173 174 175 176 177
	}
	return io_allowed;
}

#endif
178

P
Philipp Reisner 已提交
179
/**
180
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
181
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
182 183 184 185
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
186 187
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
188
 */
189
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
190
		unsigned int set_size)
P
Philipp Reisner 已提交
191 192
{
	struct drbd_request *r;
193 194 195
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
196

197
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
198

199
	/* find oldest not yet barrier-acked write request,
200
	 * count writes in its epoch. */
201
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
202
		const unsigned s = r->rq_state;
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
221
		}
P
Philipp Reisner 已提交
222
	}
223

P
Philipp Reisner 已提交
224
	/* first some paranoia code */
225
	if (req == NULL) {
226
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
227
			 barrier_nr);
P
Philipp Reisner 已提交
228 229
		goto bail;
	}
230
	if (expect_epoch != barrier_nr) {
231
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
232
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
233
		goto bail;
234 235
	}

236
	if (expect_size != set_size) {
237
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
238
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
239 240 241
		goto bail;
	}

242 243 244 245
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
246
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
247 248
		if (req->epoch == expect_epoch)
			break;
249
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
250 251 252
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
253
	}
254
	spin_unlock_irq(&connection->resource->req_lock);
255

P
Philipp Reisner 已提交
256 257 258
	return;

bail:
259
	spin_unlock_irq(&connection->resource->req_lock);
260
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
261 262 263 264
}


/**
265
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
266
 * @device:	DRBD device.
267
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
268
 *
269 270
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
271
 */
272
/* must hold resource->req_lock */
273
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
274
{
275
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
276

277
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
278 279
		_req_mod(req, what);
}
280

281
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
282
{
283
	spin_lock_irq(&connection->resource->req_lock);
284
	_tl_restart(connection, what);
285
	spin_unlock_irq(&connection->resource->req_lock);
286
}
P
Philipp Reisner 已提交
287 288 289

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
290
 * @device:	DRBD device.
P
Philipp Reisner 已提交
291 292 293 294 295
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
296
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
297
{
298
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
299
}
300

301
/**
302 303
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
304
 */
305
void tl_abort_disk_io(struct drbd_device *device)
306
{
307
	struct drbd_connection *connection = first_peer_device(device)->connection;
308
	struct drbd_request *req, *r;
309

310
	spin_lock_irq(&connection->resource->req_lock);
311
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
312 313
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
314
		if (req->device != device)
315 316
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
317
	}
318
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
319 320 321 322 323
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
324
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
325 326 327
	unsigned long flags;
	int retval;

328
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
329
		 thi->name[0],
330
		 resource->name);
331

P
Philipp Reisner 已提交
332 333 334 335 336
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

337
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
338 339 340
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
341
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
342
	 * t_state check and assignment needs to be within the same spinlock,
343 344
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
345 346
	 */

347
	if (thi->t_state == RESTARTING) {
348
		drbd_info(resource, "Restarting %s thread\n", thi->name);
349
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
350 351 352 353 354
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
355
	thi->t_state = NONE;
P
Philipp Reisner 已提交
356
	smp_mb();
L
Lars Ellenberg 已提交
357
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
358 359
	spin_unlock_irqrestore(&thi->t_lock, flags);

360
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
361 362

	/* Release mod reference taken when thread was started */
363

364 365 366
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
367 368 369 370
	module_put(THIS_MODULE);
	return retval;
}

371
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
372
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
373 374 375
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
376
	thi->t_state = NONE;
P
Philipp Reisner 已提交
377
	thi->function = func;
378 379
	thi->resource = resource;
	thi->connection = NULL;
380
	thi->name = name;
P
Philipp Reisner 已提交
381 382 383 384
}

int drbd_thread_start(struct drbd_thread *thi)
{
385
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
386 387 388 389 390 391 392 393
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
394
	case NONE:
395
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
396
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
397 398 399

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
400
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
401
			spin_unlock_irqrestore(&thi->t_lock, flags);
402
			return false;
P
Philipp Reisner 已提交
403 404
		}

405 406 407
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
408

P
Philipp Reisner 已提交
409 410
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
411
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
412 413 414 415
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
416
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
417 418

		if (IS_ERR(nt)) {
419
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
420

421 422 423
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
424
			module_put(THIS_MODULE);
425
			return false;
P
Philipp Reisner 已提交
426 427 428
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
429
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
430 431 432
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
433 434
	case EXITING:
		thi->t_state = RESTARTING;
435
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
436
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
437
		/* fall through */
438 439
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
440 441 442 443 444
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

445
	return true;
P
Philipp Reisner 已提交
446 447 448 449 450 451 452
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

453
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
454 455 456 457

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

458
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

484
int conn_lowest_minor(struct drbd_connection *connection)
485
{
486 487
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
488

489
	rcu_read_lock();
490 491 492
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
493 494
	rcu_read_unlock();

495
	return minor;
496
}
497

P
Philipp Reisner 已提交
498 499 500 501
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
502
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
503 504
 * DRBD's performance. May be overwritten by user's configuration.
 */
505
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
506
{
507
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
508

509 510 511 512
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
513

514 515 516 517
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
518
		}
519 520 521 522 523 524 525 526 527 528 529 530
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
531
	}
532
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
533 534 535 536
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
537
 * @device:	DRBD device.
538
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
539 540 541 542
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
543
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
544
{
545
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
546
	struct task_struct *p = current;
547

P
Philipp Reisner 已提交
548 549 550
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
551
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
552
}
553 554
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
555 556
#endif

557 558 559 560 561 562 563
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
564
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
565
{
566
	if (connection->agreed_pro_version >= 100) {
567 568 569 570 571 572 573 574
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
575
}
P
Philipp Reisner 已提交
576

577
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
578 579 580 581
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
582
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
583
}
P
Philipp Reisner 已提交
584

585
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
586 587
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
588
	h->command = cpu_to_be16(cmd);
589
	h->length = cpu_to_be32(size);
590
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
591
}
P
Philipp Reisner 已提交
592

593 594 595 596 597 598 599 600 601 602
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
603

604
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
605
				   void *buffer, enum drbd_packet cmd, int size)
606
{
607
	if (connection->agreed_pro_version >= 100)
608
		return prepare_header100(buffer, cmd, size, vnr);
609
	else if (connection->agreed_pro_version >= 95 &&
610
		 size > DRBD_MAX_SIZE_H80_PACKET)
611
		return prepare_header95(buffer, cmd, size);
612
	else
613
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
614 615
}

616
static void *__conn_prepare_command(struct drbd_connection *connection,
617
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
618
{
619 620
	if (!sock->socket)
		return NULL;
621
	return sock->sbuf + drbd_header_size(connection);
622
}
P
Philipp Reisner 已提交
623

624
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
625
{
626
	void *p;
P
Philipp Reisner 已提交
627

628
	mutex_lock(&sock->mutex);
629
	p = __conn_prepare_command(connection, sock);
630
	if (!p)
631
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
632

633
	return p;
P
Philipp Reisner 已提交
634 635
}

636
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
637
{
638
	return conn_prepare_command(peer_device->connection, sock);
639
}
P
Philipp Reisner 已提交
640

641
static int __send_command(struct drbd_connection *connection, int vnr,
642 643 644 645 646 647
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
648

649 650 651 652 653 654 655 656 657
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

658
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
659
				      header_size + size);
660
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
661 662
			    msg_flags);
	if (data && !err)
663
		err = drbd_send_all(connection, sock->socket, data, size, 0);
664 665 666 667 668
	/* DRBD protocol "pings" are latency critical.
	 * This is supposed to trigger tcp_push_pending_frames() */
	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
		drbd_tcp_nodelay(sock->socket);

669 670 671
	return err;
}

672
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
673 674 675
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
676
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
677 678
}

679
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
680 681 682 683
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
684

685
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
686 687 688 689
	mutex_unlock(&sock->mutex);
	return err;
}

690
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
691 692 693 694 695
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

696 697
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
698 699 700
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
701

702
int drbd_send_ping(struct drbd_connection *connection)
703
{
704 705
	struct drbd_socket *sock;

706 707
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
708
		return -EIO;
709
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
710
}
P
Philipp Reisner 已提交
711

712
int drbd_send_ping_ack(struct drbd_connection *connection)
713
{
714 715
	struct drbd_socket *sock;

716 717
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
718
		return -EIO;
719
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
720 721
}

722
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
723
{
724
	struct drbd_socket *sock;
725
	struct p_rs_param_95 *p;
726
	int size;
727
	const int apv = peer_device->connection->agreed_pro_version;
728
	enum drbd_packet cmd;
729
	struct net_conf *nc;
P
Philipp Reisner 已提交
730
	struct disk_conf *dc;
731

732 733
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
734 735
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
736

737
	rcu_read_lock();
738
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
739 740 741

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
742
			+ strlen(nc->verify_alg) + 1
743 744
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
745

746
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
747

748 749
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
750

751 752
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
753
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
754 755 756 757
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
758
		put_ldev(peer_device->device);
759
	} else {
760
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
761 762 763 764 765
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
766

767
	if (apv >= 88)
768
		strcpy(p->verify_alg, nc->verify_alg);
769
	if (apv >= 89)
770 771
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
772

773
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
774 775
}

776
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
777
{
778
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
779
	struct p_protocol *p;
780
	struct net_conf *nc;
781
	int size, cf;
P
Philipp Reisner 已提交
782

783 784
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
785 786
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
787

788
	rcu_read_lock();
789
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
790

791
	if (nc->tentative && connection->agreed_pro_version < 92) {
792 793
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
794
		drbd_err(connection, "--dry-run is not supported by peer");
795 796
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
797

798
	size = sizeof(*p);
799
	if (connection->agreed_pro_version >= 87)
800
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
801

802 803 804 805 806
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
807
	cf = 0;
808 809
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
810
	if (nc->tentative)
811
		cf |= CF_DRY_RUN;
812 813
	p->conn_flags    = cpu_to_be32(cf);

814
	if (connection->agreed_pro_version >= 87)
815 816
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
817

818
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
819 820
}

821
int drbd_send_protocol(struct drbd_connection *connection)
822 823 824
{
	int err;

825 826 827
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
828 829

	return err;
P
Philipp Reisner 已提交
830 831
}

832
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
833
{
834
	struct drbd_device *device = peer_device->device;
835 836
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
837 838
	int i;

839
	if (!get_ldev_if_state(device, D_NEGOTIATING))
840
		return 0;
P
Philipp Reisner 已提交
841

842 843
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
844
	if (!p) {
845
		put_ldev(device);
846 847
		return -EIO;
	}
848
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
849
	for (i = UI_CURRENT; i < UI_SIZE; i++)
850 851
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
852

853 854
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
855
	rcu_read_lock();
856
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
857
	rcu_read_unlock();
858 859
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
860
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
861

862
	put_ldev(device);
863
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
864 865
}

866
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
867
{
868
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
869 870
}

871
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
872
{
873
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
874 875
}

876
void drbd_print_uuids(struct drbd_device *device, const char *text)
877
{
878 879
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
880
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
881 882 883 884 885
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
886
		put_ldev(device);
887
	} else {
888
		drbd_info(device, "%s effective data uuid: %016llX\n",
889
				text,
890
				(unsigned long long)device->ed_uuid);
891 892 893
	}
}

894
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
895
{
896
	struct drbd_device *device = peer_device->device;
897 898
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
899 900
	u64 uuid;

901
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
902

903
	uuid = device->ldev->md.uuid[UI_BITMAP];
904 905 906 907
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
908 909 910
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
911

912 913
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
914 915
	if (p) {
		p->uuid = cpu_to_be64(uuid);
916
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
917
	}
P
Philipp Reisner 已提交
918 919
}

920
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
921
{
922
	struct drbd_device *device = peer_device->device;
923 924
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
925
	sector_t d_size, u_size;
926 927
	int q_order_type;
	unsigned int max_bio_size;
P
Philipp Reisner 已提交
928

929
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
930
		D_ASSERT(device, device->ldev->backing_bdev);
931
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
932
		rcu_read_lock();
933
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
934
		rcu_read_unlock();
935 936
		q_order_type = drbd_queue_order_type(device);
		max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9;
937
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
938
		put_ldev(device);
P
Philipp Reisner 已提交
939 940 941 942
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
943
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
P
Philipp Reisner 已提交
944 945
	}

946 947
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
948 949
	if (!p)
		return -EIO;
950

951
	if (peer_device->connection->agreed_pro_version <= 94)
952
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
953
	else if (peer_device->connection->agreed_pro_version < 100)
954
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
955

956 957
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
958
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
959 960 961
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
962
	return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
963 964 965
}

/**
966
 * drbd_send_current_state() - Sends the drbd state to the peer
967
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
968
 */
969
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
970
{
971
	struct drbd_socket *sock;
972
	struct p_state *p;
P
Philipp Reisner 已提交
973

974 975
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
976 977
	if (!p)
		return -EIO;
978 979
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
980 981
}

982 983
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
984
 * @peer_device:      DRBD peer device.
985
 * @state:     the state to send, not necessarily the current state.
986 987 988 989 990 991
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
992
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
993
{
994 995
	struct drbd_socket *sock;
	struct p_state *p;
996

997 998
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
999 1000 1001
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
1002
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1003
}
1004

1005
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1006 1007 1008
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1009

1010 1011
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1012 1013 1014 1015
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1016
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1017
}
1018

1019
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1020
{
1021 1022 1023
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1024

1025 1026 1027
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1028 1029 1030 1031
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1032
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1033 1034
}

1035
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1036
{
1037 1038
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1039

1040 1041
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1042 1043
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1044
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1045
	}
P
Philipp Reisner 已提交
1046 1047
}

1048
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1049
{
1050 1051
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1052
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1053

1054 1055
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1056 1057
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1058
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1059
	}
P
Philipp Reisner 已提交
1060 1061
}

1062
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1063
{
1064 1065 1066
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1067

1068 1069 1070 1071
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1072

1073 1074 1075 1076
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1077 1078
}

1079
static int fill_bitmap_rle_bits(struct drbd_device *device,
1080 1081 1082
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1083 1084 1085 1086 1087 1088 1089
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1090
	int bits, use_rle;
P
Philipp Reisner 已提交
1091 1092

	/* may we use this feature? */
1093
	rcu_read_lock();
1094
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1095
	rcu_read_unlock();
1096
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1097
		return 0;
P
Philipp Reisner 已提交
1098 1099 1100 1101 1102

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1103 1104
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1116 1117
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1118 1119 1120 1121 1122 1123 1124 1125
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1126
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1127 1128 1129 1130
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1131
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1132 1133 1134 1135 1136
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1137
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1138 1139 1140 1141 1142 1143 1144 1145
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1146
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1171
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1172 1173 1174 1175

	return len;
}

1176 1177 1178 1179 1180 1181 1182
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1183
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1184
{
1185 1186
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1187
	struct p_compressed_bm *p = sock->sbuf + header_size;
1188
	int len, err;
P
Philipp Reisner 已提交
1189

1190
	len = fill_bitmap_rle_bits(device, p,
1191
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1192
	if (len < 0)
1193
		return -EIO;
P
Philipp Reisner 已提交
1194 1195

	if (len) {
1196
		dcbp_set_code(p, RLE_VLI_Bits);
1197
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1198 1199
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1200
		c->packets[0]++;
1201
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1202 1203 1204 1205 1206 1207

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1208 1209
		unsigned int data_size;
		unsigned long num_words;
1210
		unsigned long *p = sock->sbuf + header_size;
1211 1212

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1213
		num_words = min_t(size_t, data_size / sizeof(*p),
1214
				  c->bm_words - c->word_offset);
1215
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1216
		if (len)
1217
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1218
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1219 1220 1221 1222
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1223
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1224 1225 1226 1227

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1228
	if (!err) {
1229
		if (len == 0) {
1230
			INFO_bm_xfer_stats(device, "send", c);
1231 1232 1233 1234 1235
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1236 1237 1238
}

/* See the comment at receive_bitmap() */
1239
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1240 1241
{
	struct bm_xfer_ctx c;
1242
	int err;
P
Philipp Reisner 已提交
1243

1244
	if (!expect(device->bitmap))
1245
		return false;
P
Philipp Reisner 已提交
1246

1247 1248
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1249
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1250 1251
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1252 1253 1254
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1255
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1256
			} else {
1257 1258
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1259 1260
			}
		}
1261
		put_ldev(device);
P
Philipp Reisner 已提交
1262 1263 1264
	}

	c = (struct bm_xfer_ctx) {
1265 1266
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1267 1268 1269
	};

	do {
1270
		err = send_bitmap_rle_or_plain(device, &c);
1271
	} while (err > 0);
P
Philipp Reisner 已提交
1272

1273
	return err == 0;
P
Philipp Reisner 已提交
1274 1275
}

1276
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1277
{
1278
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1279
	int err = -1;
P
Philipp Reisner 已提交
1280

1281 1282
	mutex_lock(&sock->mutex);
	if (sock->socket)
1283
		err = !_drbd_send_bitmap(device);
1284
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1285 1286 1287
	return err;
}

1288
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1289
{
1290 1291
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1292

1293
	if (connection->cstate < C_WF_REPORT_PARAMS)
1294
		return;
P
Philipp Reisner 已提交
1295

1296 1297
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1298 1299 1300 1301
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1302
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1303 1304 1305 1306
}

/**
 * _drbd_send_ack() - Sends an ack packet
1307
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1308 1309 1310 1311 1312
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1313
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1314
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1315
{
1316 1317
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1318

1319
	if (peer_device->device->state.conn < C_CONNECTED)
1320
		return -EIO;
P
Philipp Reisner 已提交
1321

1322 1323
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1324
	if (!p)
1325
		return -EIO;
1326 1327 1328
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1329 1330
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1331 1332
}

1333 1334 1335
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1336
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1337
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1338
{
1339 1340 1341
	if (peer_device->connection->peer_integrity_tfm)
		data_size -= crypto_hash_digestsize(peer_device->connection->peer_integrity_tfm);
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1342
		       dp->block_id);
P
Philipp Reisner 已提交
1343 1344
}

1345
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1346
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1347
{
1348
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1349 1350 1351 1352
}

/**
 * drbd_send_ack() - Sends an ack packet
1353
 * @device:	DRBD device
1354 1355
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1356
 */
1357
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1358
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1359
{
1360
	return _drbd_send_ack(peer_device, cmd,
1361 1362 1363
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1364 1365 1366 1367
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1368
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1369 1370
		     sector_t sector, int blksize, u64 block_id)
{
1371
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1372 1373 1374 1375 1376
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1377
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1378 1379
		       sector_t sector, int size, u64 block_id)
{
1380 1381
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1382

1383 1384
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1385 1386 1387 1388 1389
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1390
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1391 1392
}

1393
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1394
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1395
{
1396 1397
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1398

1399
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1400

1401 1402
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1403 1404 1405 1406 1407
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1408
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1409 1410
}

1411
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1412
{
1413 1414
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1415

1416 1417
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1418 1419 1420 1421 1422
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1423
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1424 1425 1426
}

/* called on sndtimeo
1427 1428
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1429
 */
1430
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1431 1432
{
	int drop_it;
1433
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1434

1435 1436 1437 1438
	drop_it =   connection->meta.socket == sock
		|| !connection->asender.task
		|| get_t_state(&connection->asender) != RUNNING
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1439 1440

	if (drop_it)
1441
		return true;
P
Philipp Reisner 已提交
1442

1443
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1444
	if (!drop_it) {
1445
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1446 1447
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1448 1449
	}

1450
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1451 1452
}

1453
static void drbd_update_congested(struct drbd_connection *connection)
1454
{
1455
	struct sock *sk = connection->data.socket->sk;
1456
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1457
		set_bit(NET_CONGESTED, &connection->flags);
1458 1459
}

P
Philipp Reisner 已提交
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1481
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1482
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1483
{
1484 1485 1486 1487
	struct socket *socket;
	void *addr;
	int err;

1488
	socket = peer_device->connection->data.socket;
1489
	addr = kmap(page) + offset;
1490
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1491
	kunmap(page);
1492
	if (!err)
1493
		peer_device->device->send_cnt += size >> 9;
1494
	return err;
P
Philipp Reisner 已提交
1495 1496
}

1497
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1498
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1499
{
1500
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1501 1502
	mm_segment_t oldfs = get_fs();
	int len = size;
1503
	int err = -EIO;
P
Philipp Reisner 已提交
1504 1505 1506 1507 1508 1509 1510 1511

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1512
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1513

L
Lars Ellenberg 已提交
1514
	msg_flags |= MSG_NOSIGNAL;
1515
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1516 1517
	set_fs(KERNEL_DS);
	do {
1518 1519 1520
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1521
		if (sent <= 0) {
1522
			if (sent == -EAGAIN) {
1523
				if (we_should_drop_the_connection(peer_device->connection, socket))
1524 1525 1526
					break;
				continue;
			}
1527
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1528
			     __func__, (int)size, len, sent);
1529 1530
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1531 1532 1533 1534
			break;
		}
		len    -= sent;
		offset += sent;
1535
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
P
Philipp Reisner 已提交
1536
	set_fs(oldfs);
1537
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1538

1539 1540
	if (len == 0) {
		err = 0;
1541
		peer_device->device->send_cnt += size >> 9;
1542 1543
	}
	return err;
P
Philipp Reisner 已提交
1544 1545
}

1546
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1547
{
1548 1549 1550
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1551
	/* hint all but last page with MSG_MORE */
1552
	bio_for_each_segment(bvec, bio, iter) {
1553 1554
		int err;

1555
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1556
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1557
					 bio_iter_last(bvec, iter)
1558
					 ? 0 : MSG_MORE);
1559 1560
		if (err)
			return err;
P
Philipp Reisner 已提交
1561
	}
1562
	return 0;
P
Philipp Reisner 已提交
1563 1564
}

1565
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1566
{
1567 1568 1569
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1570
	/* hint all but last page with MSG_MORE */
1571
	bio_for_each_segment(bvec, bio, iter) {
1572 1573
		int err;

1574
		err = _drbd_send_page(peer_device, bvec.bv_page,
1575
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1576
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1577 1578
		if (err)
			return err;
P
Philipp Reisner 已提交
1579
	}
1580
	return 0;
P
Philipp Reisner 已提交
1581 1582
}

1583
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1584
			    struct drbd_peer_request *peer_req)
1585
{
1586 1587
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1588
	int err;
1589

L
Lars Ellenberg 已提交
1590
	/* hint all but last page with MSG_MORE */
1591 1592
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1593

1594
		err = _drbd_send_page(peer_device, page, 0, l,
1595 1596 1597
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1598 1599
		len -= l;
	}
1600
	return 0;
1601 1602
}

1603
static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw)
1604
{
1605
	if (connection->agreed_pro_version >= 95)
1606 1607 1608 1609 1610
		return  (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bi_rw & REQ_FUA ? DP_FUA : 0) |
			(bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
			(bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
	else
J
Jens Axboe 已提交
1611
		return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
1612 1613
}

1614 1615
/* Used to send write or TRIM aka REQ_DISCARD requests
 * R_PRIMARY -> Peer	(P_DATA, P_TRIM)
P
Philipp Reisner 已提交
1616
 */
1617
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1618
{
1619
	struct drbd_device *device = peer_device->device;
1620 1621
	struct drbd_socket *sock;
	struct p_data *p;
P
Philipp Reisner 已提交
1622 1623
	unsigned int dp_flags = 0;
	int dgs;
1624
	int err;
P
Philipp Reisner 已提交
1625

1626 1627 1628 1629
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1630

1631 1632 1633 1634
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1635
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
1636
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
1637 1638
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1639
		dp_flags |= DP_MAY_SET_IN_SYNC;
1640
	if (peer_device->connection->agreed_pro_version >= 100) {
1641 1642 1643 1644 1645
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
		if (req->rq_state & RQ_EXP_WRITE_ACK)
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1646
	p->dp_flags = cpu_to_be32(dp_flags);
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656

	if (dp_flags & DP_DISCARD) {
		struct p_trim *t = (struct p_trim*)p;
		t->size = cpu_to_be32(req->i.size);
		err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0);
		goto out;
	}

	/* our digest is still only over the payload.
	 * TRIM does not carry any payload. */
1657
	if (dgs)
1658 1659
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + dgs, NULL, req->i.size);
1660
	if (!err) {
1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1672
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || dgs)
1673
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1674
		else
1675
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1676 1677 1678

		/* double check digest, sometimes buffers have been modified in flight. */
		if (dgs > 0 && dgs <= 64) {
B
Bart Van Assche 已提交
1679
			/* 64 byte, 512 bit, is the largest digest size
1680 1681
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1682
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1683
			if (memcmp(p + 1, digest, dgs)) {
1684
				drbd_warn(device,
1685
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1686
					(unsigned long long)req->i.sector, req->i.size);
1687 1688 1689 1690
			}
		} /* else if (dgs > 64) {
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1691
	}
1692
out:
1693
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1694

1695
	return err;
P
Philipp Reisner 已提交
1696 1697 1698 1699 1700 1701
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1702
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1703
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1704
{
1705
	struct drbd_device *device = peer_device->device;
1706 1707
	struct drbd_socket *sock;
	struct p_data *p;
1708
	int err;
P
Philipp Reisner 已提交
1709 1710
	int dgs;

1711 1712
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1713

1714 1715
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1716

1717 1718 1719 1720 1721
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1722
	p->dp_flags = 0;
1723
	if (dgs)
1724 1725
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + dgs, NULL, peer_req->i.size);
1726
	if (!err)
1727
		err = _drbd_send_zc_ee(peer_device, peer_req);
1728
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1729

1730
	return err;
P
Philipp Reisner 已提交
1731 1732
}

1733
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1734
{
1735 1736
	struct drbd_socket *sock;
	struct p_block_desc *p;
1737

1738 1739
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1740 1741 1742 1743
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1744
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1745 1746
}

P
Philipp Reisner 已提交
1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1763
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1764 1765 1766 1767 1768 1769 1770
	      void *buf, size_t size, unsigned msg_flags)
{
	struct kvec iov;
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1771
		return -EBADR;
P
Philipp Reisner 已提交
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783

	/* THINK  if (signal_pending) return ... ? */

	iov.iov_base = buf;
	iov.iov_len  = size;

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

1784
	if (sock == connection->data.socket) {
1785
		rcu_read_lock();
1786
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1787
		rcu_read_unlock();
1788
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801
	}
	do {
		/* STRANGE
		 * tcp_sendmsg does _not_ use its size parameter at all ?
		 *
		 * -EAGAIN on timeout, -EINTR on signal.
		 */
/* THINK
 * do we need to block DRBD_SIG if sock == &meta.socket ??
 * otherwise wake_asender() might interrupt some send_*Ack !
 */
		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
		if (rv == -EAGAIN) {
1802
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
		iov.iov_base += rv;
		iov.iov_len  -= rv;
	} while (sent < size);

1818 1819
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1820 1821 1822

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1823
			drbd_err(connection, "%s_sendmsg returned %d\n",
1824
				 sock == connection->meta.socket ? "msock" : "sock",
1825
				 rv);
1826
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1827
		} else
1828
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1829 1830 1831 1832 1833
	}

	return sent;
}

1834 1835 1836 1837 1838
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1839
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1840 1841 1842 1843
		  size_t size, unsigned msg_flags)
{
	int err;

1844
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1845 1846 1847 1848 1849 1850 1851
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1852 1853
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1854
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1855 1856 1857
	unsigned long flags;
	int rv = 0;

1858
	mutex_lock(&drbd_main_mutex);
1859
	spin_lock_irqsave(&device->resource->req_lock, flags);
1860
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1861 1862
	 * and no race with updating open_cnt */

1863
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1864 1865 1866 1867 1868 1869 1870
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1871
		device->open_cnt++;
1872
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1873
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1874 1875 1876 1877

	return rv;
}

1878
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1879
{
1880
	struct drbd_device *device = gd->private_data;
1881
	mutex_lock(&drbd_main_mutex);
1882
	device->open_cnt--;
1883
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1884 1885
}

1886
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1887
{
1888 1889
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1890
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1891 1892 1893 1894 1895 1896 1897 1898
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1899
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1900 1901 1902 1903
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
	atomic_set(&device->md_io_in_use, 0);

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->md_sync_work.list);
	INIT_LIST_HEAD(&device->start_resync_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->md_sync_work.cb = w_md_sync;
	device->bm_io_work.w.cb = w_bitmap_io;
	device->start_resync_work.cb = w_start_resync;

	init_timer(&device->resync_timer);
	init_timer(&device->md_sync_timer);
	init_timer(&device->start_resync_timer);
	init_timer(&device->request_timer);
	device->resync_timer.function = resync_timer_fn;
	device->resync_timer.data = (unsigned long) device;
	device->md_sync_timer.function = md_sync_timer_fn;
	device->md_sync_timer.data = (unsigned long) device;
	device->start_resync_timer.function = start_resync_timer_fn;
	device->start_resync_timer.data = (unsigned long) device;
	device->request_timer.function = request_timer_fn;
	device->request_timer.data = (unsigned long) device;

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
1966
{
1967
	int i;
1968
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
1969
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1970
				first_peer_device(device)->connection->receiver.t_state);
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
1984
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1985 1986
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
1987
	}
1988
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
1989

1990 1991
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
1992
		/* maybe never allocated. */
1993 1994
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
1995 1996
	}

1997
	drbd_free_ldev(device->ldev);
1998
	device->ldev = NULL;
1999

2000
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
2001

2002 2003 2004 2005 2006 2007 2008 2009 2010
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
2011

2012
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2027
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2028

2029 2030
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2031 2032
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2046
	drbd_md_io_bio_set   = NULL;
2047
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2061
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2071
	drbd_md_io_page_pool = NULL;
2072
	drbd_md_io_bio_set   = NULL;
P
Philipp Reisner 已提交
2073 2074 2075 2076 2077 2078 2079 2080

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2081
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2096 2097 2098 2099
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2100 2101 2102 2103
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

P
Philipp Reisner 已提交
2104 2105 2106 2107 2108 2109 2110
	drbd_request_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
	if (drbd_request_mempool == NULL)
		goto Enomem;

	drbd_ee_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2111
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

static int drbd_notify_sys(struct notifier_block *this, unsigned long code,
	void *unused)
{
	/* just so we have it.  you never know what interesting things we
	 * might want to do here some day...
	 */

	return NOTIFY_DONE;
}

static struct notifier_block drbd_notifier = {
	.notifier_call = drbd_notify_sys,
};

2147
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2148 2149 2150
{
	int rr;

2151
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2152
	if (rr)
2153
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2154

2155
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2156
	if (rr)
2157
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2158

2159
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2160
	if (rr)
2161
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2162

2163
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2164
	if (rr)
2165
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2166

2167
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2168
	if (rr)
2169
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2170 2171
}

2172
/* caution. no locking. */
2173
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2174
{
2175
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2176 2177
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
P
Philipp Reisner 已提交
2178

2179
	del_timer_sync(&device->request_timer);
2180

P
Philipp Reisner 已提交
2181
	/* paranoia asserts */
2182
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2183 2184 2185 2186 2187
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2188 2189
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2190

2191
	drbd_free_ldev(device->ldev);
2192
	device->ldev = NULL;
P
Philipp Reisner 已提交
2193

2194
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2195

2196 2197
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2198

2199 2200
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2201

2202 2203 2204 2205 2206 2207
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
	__free_page(device->md_io_page);
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2208
	kfree(first_peer_device(device));
2209
	kfree(device);
2210

2211 2212 2213
	for_each_connection(connection, resource)
		kref_put(&connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2214 2215
}

2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2238
		struct drbd_device *device = req->device;
2239 2240
		struct bio *bio = req->master_bio;
		unsigned long start_time = req->start_time;
2241 2242
		bool expected;

2243
		expected =
2244 2245 2246 2247 2248 2249
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2250
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2251 2252 2253 2254 2255 2256 2257
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2258
		 */
2259
		kref_put(&req->kref, drbd_req_destroy);
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2274 2275
		inc_ap_bio(device);
		__drbd_make_request(device, bio, start_time);
2276 2277 2278
	}
}

2279
void drbd_restart_request(struct drbd_request *req)
2280 2281 2282 2283 2284 2285 2286 2287 2288
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2289
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2290

2291
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2292 2293
}

2294 2295 2296 2297 2298
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2299
	idr_destroy(&resource->devices);
2300
	free_cpumask_var(resource->cpu_mask);
2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
	kfree(resource->name);
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
		kref_put(&connection->kref, drbd_destroy_connection);
	}
	kref_put(&resource->kref, drbd_destroy_resource);
}
2315

P
Philipp Reisner 已提交
2316 2317 2318
static void drbd_cleanup(void)
{
	unsigned int i;
2319
	struct drbd_device *device;
2320
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2321 2322 2323

	unregister_reboot_notifier(&drbd_notifier);

2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2335 2336
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2337

2338
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2339

2340
	idr_for_each_entry(&drbd_devices, device, i)
2341
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2342

P
Philipp Reisner 已提交
2343
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2344 2345 2346
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2347
	}
P
Philipp Reisner 已提交
2348

2349
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2350 2351
	unregister_blkdev(DRBD_MAJOR, "drbd");

2352
	idr_destroy(&drbd_devices);
2353

P
Philipp Reisner 已提交
2354 2355 2356 2357
	printk(KERN_INFO "drbd: module cleanup done.\n");
}

/**
2358
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2359
 * @congested_data:	User data
2360
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2361 2362 2363 2364 2365
 *
 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2366
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2367 2368 2369 2370
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2371
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2372 2373 2374 2375 2376 2377
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2378
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2379 2380 2381 2382 2383 2384
		r |= (1 << BDI_async_congested);
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2385
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2386 2387
			r |= (1 << BDI_sync_congested);
		else
2388
			put_ldev(device);
2389 2390 2391 2392 2393
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2394 2395
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
P
Philipp Reisner 已提交
2396
		r = bdi_congested(&q->backing_dev_info, bdi_bits);
2397
		put_ldev(device);
P
Philipp Reisner 已提交
2398 2399 2400 2401
		if (r)
			reason = 'b';
	}

2402 2403
	if (bdi_bits & (1 << BDI_async_congested) &&
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
P
Philipp Reisner 已提交
2404 2405 2406 2407 2408
		r |= (1 << BDI_async_congested);
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2409
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2410 2411 2412
	return r;
}

2413 2414 2415 2416
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2417
	init_waitqueue_head(&wq->q_wait);
2418 2419
}

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2444
struct drbd_resource *drbd_find_resource(const char *name)
2445
{
2446
	struct drbd_resource *resource;
2447

2448 2449 2450
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2451
	rcu_read_lock();
2452 2453
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2454
			kref_get(&resource->kref);
2455
			goto found;
2456
		}
2457
	}
2458
	resource = NULL;
2459
found:
P
Philipp Reisner 已提交
2460
	rcu_read_unlock();
2461
	return resource;
2462 2463
}

2464
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2465 2466
				     void *peer_addr, int peer_addr_len)
{
2467
	struct drbd_resource *resource;
2468
	struct drbd_connection *connection;
2469 2470

	rcu_read_lock();
2471 2472 2473 2474 2475 2476 2477 2478 2479
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2480 2481
		}
	}
2482
	connection = NULL;
2483 2484
found:
	rcu_read_unlock();
2485
	return connection;
2486 2487
}

2488 2489 2490 2491 2492
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2493 2494 2495
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2496 2497 2498 2499 2500
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2501
	free_page((unsigned long) socket->sbuf);
2502 2503 2504
	free_page((unsigned long) socket->rbuf);
}

2505
void conn_free_crypto(struct drbd_connection *connection)
2506
{
2507
	drbd_free_sock(connection);
2508

2509 2510 2511 2512 2513 2514 2515
	crypto_free_hash(connection->csums_tfm);
	crypto_free_hash(connection->verify_tfm);
	crypto_free_hash(connection->cram_hmac_tfm);
	crypto_free_hash(connection->integrity_tfm);
	crypto_free_hash(connection->peer_integrity_tfm);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2516

2517 2518 2519 2520 2521 2522 2523
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2524 2525
}

2526
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2527
{
2528
	struct drbd_connection *connection;
2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;
		/*
		retcode = ERR_NOMEM;
		drbd_msg_put_info("unable to allocate cpumask");
		*/

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2541
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2542
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2543
		if (err) {
2544
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2545 2546 2547 2548
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2549
	resource->res_opts = *res_opts;
2550 2551 2552 2553 2554
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2555 2556 2557 2558
			connection->receiver.reset_cpu_mask = 1;
			connection->asender.reset_cpu_mask = 1;
			connection->worker.reset_cpu_mask = 1;
		}
2559 2560 2561 2562 2563 2564 2565 2566 2567
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2568 2569 2570 2571
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2572
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2573
	if (!resource)
2574
		goto fail;
2575
	resource->name = kstrdup(name, GFP_KERNEL);
2576 2577 2578 2579
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2580
	kref_init(&resource->kref);
2581
	idr_init(&resource->devices);
2582
	INIT_LIST_HEAD(&resource->connections);
2583
	resource->write_ordering = WO_bdev_flush;
2584
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2585
	mutex_init(&resource->conf_update);
2586
	mutex_init(&resource->adm_mutex);
2587
	spin_lock_init(&resource->req_lock);
2588
	return resource;
2589 2590 2591 2592 2593 2594 2595

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2596 2597
}

2598
/* caller must be under genl_lock() */
2599
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2600
{
2601
	struct drbd_resource *resource;
2602
	struct drbd_connection *connection;
2603

2604 2605
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2606 2607
		return NULL;

2608
	if (drbd_alloc_socket(&connection->data))
2609
		goto fail;
2610
	if (drbd_alloc_socket(&connection->meta))
2611 2612
		goto fail;

2613 2614
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2615
		goto fail;
2616

2617
	INIT_LIST_HEAD(&connection->transfer_log);
2618

2619 2620 2621
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
2622

2623 2624 2625
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2626

2627 2628 2629 2630
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2631 2632 2633
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2634
	idr_init(&connection->peer_devices);
2635

2636 2637 2638
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2639

2640 2641 2642 2643 2644 2645
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
	drbd_thread_init(resource, &connection->asender, drbd_asender, "asender");
	connection->asender.connection = connection;
2646

2647
	kref_init(&connection->kref);
2648 2649

	connection->resource = resource;
2650

2651 2652 2653 2654 2655
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2656
	return connection;
2657

2658 2659 2660
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2661
fail:
2662 2663 2664 2665
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2666 2667 2668
	return NULL;
}

2669
void drbd_destroy_connection(struct kref *kref)
2670
{
2671
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2672
	struct drbd_resource *resource = connection->resource;
2673

2674
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2675
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2676
	kfree(connection->current_epoch);
2677

2678
	idr_destroy(&connection->peer_devices);
2679

2680 2681 2682 2683 2684
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
	kfree(connection);
2685
	kref_put(&resource->kref, drbd_destroy_resource);
2686 2687
}

2688
static int init_submitter(struct drbd_device *device)
2689 2690 2691
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2692 2693 2694
	device->submit.wq = alloc_workqueue("drbd%u_submit",
			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
	if (!device->submit.wq)
2695 2696
		return -ENOMEM;

2697 2698 2699
	INIT_WORK(&device->submit.worker, do_submit);
	spin_lock_init(&device->submit.lock);
	INIT_LIST_HEAD(&device->submit.writes);
2700 2701 2702
	return 0;
}

2703
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2704
{
2705
	struct drbd_resource *resource = adm_ctx->resource;
2706
	struct drbd_connection *connection;
2707
	struct drbd_device *device;
2708
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2709 2710
	struct gendisk *disk;
	struct request_queue *q;
2711
	int id;
2712
	int vnr = adm_ctx->volume;
2713
	enum drbd_ret_code err = ERR_NOMEM;
2714

2715 2716
	device = minor_to_device(minor);
	if (device)
2717
		return ERR_MINOR_EXISTS;
P
Philipp Reisner 已提交
2718 2719

	/* GFP_KERNEL, we are outside of all write-out paths */
2720 2721
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2722
		return ERR_NOMEM;
2723 2724 2725 2726
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2727 2728
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2729

2730
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2731 2732 2733 2734

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2735 2736
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2737 2738 2739 2740

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2741
	device->vdisk = disk;
P
Philipp Reisner 已提交
2742

2743
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2744 2745 2746 2747 2748 2749

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2750
	disk->private_data = device;
P
Philipp Reisner 已提交
2751

2752
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2753
	/* we have no partitions. we contain only ourselves. */
2754
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2755 2756

	q->backing_dev_info.congested_fn = drbd_congested;
2757
	q->backing_dev_info.congested_data = device;
P
Philipp Reisner 已提交
2758

2759
	blk_queue_make_request(q, drbd_make_request);
2760
	blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
2761 2762 2763
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
P
Philipp Reisner 已提交
2764 2765
	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
	blk_queue_merge_bvec(q, drbd_merge_bvec);
2766
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2767

2768 2769
	device->md_io_page = alloc_page(GFP_KERNEL);
	if (!device->md_io_page)
P
Philipp Reisner 已提交
2770 2771
		goto out_no_io_page;

2772
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2773
		goto out_no_bitmap;
2774 2775
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2776

2777 2778 2779
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2780
			err = ERR_MINOR_EXISTS;
2781
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
T
Tejun Heo 已提交
2782
		}
2783
		goto out_no_minor_idr;
2784
	}
2785 2786 2787 2788 2789 2790
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
			err = ERR_MINOR_EXISTS;
2791
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
2792 2793 2794 2795
		}
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2796

2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
	INIT_LIST_HEAD(&device->peer_devices);
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
			if (id == -ENOSPC) {
				err = ERR_INVALID_REQUEST;
2812
				drbd_msg_put_info(adm_ctx->reply_skb, "requested volume exists already");
2813 2814
			}
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2815
		}
2816
		kref_get(&connection->kref);
2817
	}
T
Tejun Heo 已提交
2818

2819
	if (init_submitter(device)) {
2820
		err = ERR_NOMEM;
2821
		drbd_msg_put_info(adm_ctx->reply_skb, "unable to create submit workqueue");
2822 2823 2824
		goto out_idr_remove_vol;
	}

2825 2826
	add_disk(disk);

2827
	/* inherit the connection state */
2828
	device->state.conn = first_connection(resource)->cstate;
2829 2830 2831 2832
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2833

2834
	return NO_ERROR;
P
Philipp Reisner 已提交
2835

2836
out_idr_remove_vol:
2837
	idr_remove(&connection->peer_devices, vnr);
2838
out_idr_remove_from_resource:
2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849
	for_each_connection(connection, resource) {
		peer_device = idr_find(&connection->peer_devices, vnr);
		if (peer_device) {
			idr_remove(&connection->peer_devices, vnr);
			kref_put(&connection->kref, drbd_destroy_connection);
		}
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2850
	idr_remove(&resource->devices, vnr);
2851
out_idr_remove_minor:
2852
	idr_remove(&drbd_devices, minor);
2853
	synchronize_rcu();
2854
out_no_minor_idr:
2855
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2856
out_no_bitmap:
2857
	__free_page(device->md_io_page);
P
Philipp Reisner 已提交
2858 2859 2860 2861 2862
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2863
	kref_put(&resource->kref, drbd_destroy_resource);
2864
	kfree(device);
2865
	return err;
P
Philipp Reisner 已提交
2866 2867
}

2868
void drbd_delete_device(struct drbd_device *device)
2869 2870 2871 2872 2873 2874
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
	int refs = 3;

	for_each_connection(connection, resource) {
2875
		idr_remove(&connection->peer_devices, device->vnr);
2876 2877 2878 2879 2880 2881 2882 2883 2884
		refs++;
	}
	idr_remove(&resource->devices, device->vnr);
	idr_remove(&drbd_devices, device_to_minor(device));
	del_gendisk(device->vdisk);
	synchronize_rcu();
	kref_sub(&device->kref, refs, drbd_destroy_device);
}

P
Philipp Reisner 已提交
2885 2886 2887 2888
int __init drbd_init(void)
{
	int err;

2889
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
P
Philipp Reisner 已提交
2890
		printk(KERN_ERR
2891
		       "drbd: invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2892 2893 2894
#ifdef MODULE
		return -EINVAL;
#else
2895
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
		printk(KERN_ERR
		       "drbd: unable to register block device major %d\n",
		       DRBD_MAJOR);
		return err;
	}

	register_reboot_notifier(&drbd_notifier);

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
2915
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
2916

2917
	rwlock_init(&global_state_lock);
2918
	INIT_LIST_HEAD(&drbd_resources);
2919 2920 2921 2922 2923 2924 2925

	err = drbd_genl_register();
	if (err) {
		printk(KERN_ERR "drbd: unable to register generic netlink family\n");
		goto fail;
	}

P
Philipp Reisner 已提交
2926 2927
	err = drbd_create_mempools();
	if (err)
2928
		goto fail;
P
Philipp Reisner 已提交
2929

2930
	err = -ENOMEM;
2931
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
2932 2933
	if (!drbd_proc)	{
		printk(KERN_ERR "drbd: unable to register proc file\n");
2934
		goto fail;
P
Philipp Reisner 已提交
2935 2936
	}

2937 2938 2939 2940 2941 2942 2943 2944
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
		printk(KERN_ERR "drbd: unable to create retry workqueue\n");
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
2945 2946 2947 2948 2949 2950 2951 2952 2953 2954

	printk(KERN_INFO "drbd: initialized. "
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
	printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
	printk(KERN_INFO "drbd: registered as block device major %d\n",
		DRBD_MAJOR);

	return 0; /* Success! */

2955
fail:
P
Philipp Reisner 已提交
2956 2957 2958 2959 2960 2961 2962 2963
	drbd_cleanup();
	if (err == -ENOMEM)
		printk(KERN_ERR "drbd: ran out of memory\n");
	else
		printk(KERN_ERR "drbd: initialization failure\n");
	return err;
}

2964
void drbd_free_ldev(struct drbd_backing_dev *ldev)
P
Philipp Reisner 已提交
2965 2966 2967 2968
{
	if (ldev == NULL)
		return;

2969 2970
	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
P
Philipp Reisner 已提交
2971

L
Lars Ellenberg 已提交
2972
	kfree(ldev->disk_conf);
P
Philipp Reisner 已提交
2973 2974 2975
	kfree(ldev);
}

2976
void drbd_free_sock(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2977
{
2978 2979 2980 2981 2982 2983
	if (connection->data.socket) {
		mutex_lock(&connection->data.mutex);
		kernel_sock_shutdown(connection->data.socket, SHUT_RDWR);
		sock_release(connection->data.socket);
		connection->data.socket = NULL;
		mutex_unlock(&connection->data.mutex);
P
Philipp Reisner 已提交
2984
	}
2985 2986 2987 2988 2989 2990
	if (connection->meta.socket) {
		mutex_lock(&connection->meta.mutex);
		kernel_sock_shutdown(connection->meta.socket, SHUT_RDWR);
		sock_release(connection->meta.socket);
		connection->meta.socket = NULL;
		mutex_unlock(&connection->meta.mutex);
P
Philipp Reisner 已提交
2991 2992 2993 2994 2995
	}
}

/* meta data management */

2996
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2997
{
2998
	struct drbd_peer_device *peer_device;
2999
	int vnr;
P
Philipp Reisner 已提交
3000

3001
	rcu_read_lock();
3002 3003 3004
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

3005
		kref_get(&device->kref);
3006
		rcu_read_unlock();
3007
		drbd_md_sync(device);
3008
		kref_put(&device->kref, drbd_destroy_device);
3009 3010 3011
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
3012 3013
}

3014
/* aligned 4kByte */
P
Philipp Reisner 已提交
3015
struct meta_data_on_disk {
3016
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
3017 3018 3019 3020 3021 3022 3023
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3024
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3025
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3026 3027
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3028
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3029

3030 3031 3032 3033 3034
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3035 3036
} __packed;

3037 3038


3039
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3040
{
3041
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3042 3043 3044
	sector_t sector;
	int i;

3045
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3046

3047
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3048
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3049 3050
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3051
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3052

3053 3054 3055
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3056
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3057
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3058

3059 3060
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3061

3062 3063
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3064

3065
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3066
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3067

3068
	if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
P
Philipp Reisner 已提交
3069
		/* this was a try anyways ... */
3070
		drbd_err(device, "meta data update failed!\n");
3071
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3072
	}
3073 3074 3075 3076
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3077
 * @device:	DRBD device.
3078
 */
3079
void drbd_md_sync(struct drbd_device *device)
3080 3081 3082 3083 3084 3085 3086
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3087
	del_timer(&device->md_sync_timer);
3088
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3089
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3090 3091 3092 3093
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3094
	if (!get_ldev_if_state(device, D_FAILED))
3095 3096
		return;

3097
	buffer = drbd_md_get_buffer(device);
3098 3099 3100
	if (!buffer)
		goto out;

3101
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3102

3103
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3104
	 * since we updated it on metadata. */
3105
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3106

3107
	drbd_md_put_buffer(device);
3108
out:
3109
	put_ldev(device);
P
Philipp Reisner 已提交
3110 3111
}

3112
static int check_activity_log_stripe_size(struct drbd_device *device,
3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3152
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3153 3154 3155 3156
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3157
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3224
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3237 3238
/**
 * drbd_md_read() - Reads in the meta data super block
3239
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3240 3241
 * @bdev:	Device from which the meta data should be read in.
 *
3242
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3243
 * something goes wrong.
3244
 *
3245
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3246
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3247
 */
3248
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3249 3250
{
	struct meta_data_on_disk *buffer;
3251
	u32 magic, flags;
P
Philipp Reisner 已提交
3252 3253
	int i, rv = NO_ERROR;

3254
	if (device->state.disk != D_DISKLESS)
3255
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3256

3257
	buffer = drbd_md_get_buffer(device);
3258
	if (!buffer)
3259
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3260

3261 3262
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3263 3264
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
P
Philipp Reisner 已提交
3265

3266
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
L
Lucas De Marchi 已提交
3267
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3268
		   called BEFORE disk is attached */
3269
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3270 3271 3272 3273
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3274 3275 3276 3277 3278
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3279
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3280 3281 3282
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3283 3284

	rv = ERR_MD_INVALID;
3285
	if (magic != DRBD_MD_MAGIC_08) {
3286
		if (magic == DRBD_MD_MAGIC_07)
3287
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3288
		else
3289
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3290 3291
		goto err;
	}
3292

3293
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3294
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3295
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3296 3297
		goto err;
	}
3298

3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3311
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3312
		goto err;
3313
	if (check_offsets_and_sizes(device, bdev))
3314 3315
		goto err;

P
Philipp Reisner 已提交
3316
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3317
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3318 3319 3320 3321
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3322
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3323 3324 3325 3326
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3327
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3328

3329
	spin_lock_irq(&device->resource->req_lock);
3330
	if (device->state.conn < C_CONNECTED) {
3331
		unsigned int peer;
3332
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3333
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3334
		device->peer_max_bio_size = peer;
3335
	}
3336
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3337 3338

 err:
3339
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3340 3341 3342 3343 3344 3345

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3346
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3347 3348 3349 3350 3351
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3352
#ifdef DEBUG
3353
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3354
{
3355 3356 3357 3358
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3359 3360 3361
	}
}
#else
3362
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3363
{
3364 3365
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3366
}
3367
#endif
P
Philipp Reisner 已提交
3368

3369
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3370 3371 3372
{
	int i;

3373
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3374
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3375 3376
}

3377
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3378 3379
{
	if (idx == UI_CURRENT) {
3380
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3381 3382 3383 3384
			val |= 1;
		else
			val &= ~((u64)1);

3385
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3386 3387
	}

3388 3389
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3390 3391
}

3392
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3393 3394
{
	unsigned long flags;
3395 3396 3397
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3398
}
P
Philipp Reisner 已提交
3399

3400
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3401
{
3402
	unsigned long flags;
3403 3404 3405 3406
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3407
	}
3408 3409
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3410 3411 3412 3413
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3414
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3415 3416 3417 3418
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3419
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3420 3421
{
	u64 val;
3422 3423 3424 3425
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3426 3427
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3428 3429

	if (bm_uuid)
3430
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3431

3432 3433 3434
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3435

3436
	drbd_print_uuids(device, "new current UUID");
3437
	/* get it to stable storage _now_ */
3438
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3439 3440
}

3441
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3442
{
3443
	unsigned long flags;
3444
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3445 3446
		return;

3447
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3448
	if (val == 0) {
3449 3450 3451
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3452
	} else {
3453
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3454
		if (bm_uuid)
3455
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3456

3457
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3458
	}
3459
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3460

3461
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3462 3463 3464 3465
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3466
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3467 3468 3469
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3470
int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3471 3472 3473
{
	int rv = -EIO;

3474 3475 3476
	drbd_md_set_flag(device, MDF_FULL_SYNC);
	drbd_md_sync(device);
	drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3477

3478
	rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3479

3480 3481 3482
	if (!rv) {
		drbd_md_clear_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
P
Philipp Reisner 已提交
3483 3484 3485 3486 3487 3488 3489
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3490
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3491 3492 3493
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3494
int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3495
{
3496
	drbd_resume_al(device);
3497 3498
	drbd_bm_clear_all(device);
	return drbd_bm_write(device);
P
Philipp Reisner 已提交
3499 3500
}

3501
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3502
{
3503 3504 3505
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3506
	int rv = -EIO;
P
Philipp Reisner 已提交
3507

3508
	D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0);
P
Philipp Reisner 已提交
3509

3510 3511 3512 3513 3514
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3515
	}
P
Philipp Reisner 已提交
3516

3517 3518
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3519 3520

	if (work->done)
3521
		work->done(device, rv);
P
Philipp Reisner 已提交
3522

3523
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3524
	work->why = NULL;
3525
	work->flags = 0;
P
Philipp Reisner 已提交
3526

3527
	return 0;
P
Philipp Reisner 已提交
3528 3529 3530 3531
}

/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3532
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3533 3534 3535 3536 3537 3538 3539 3540
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
3541 3542 3543
 *
 * Its worker function encloses the call of io_fn() by get_ldev() and
 * put_ldev().
P
Philipp Reisner 已提交
3544
 */
3545
void drbd_queue_bitmap_io(struct drbd_device *device,
3546 3547
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3548
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3549
{
3550
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3551

3552 3553 3554
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3555
	if (device->bm_io_work.why)
3556
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3557
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3558

3559 3560 3561 3562
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3563

3564
	spin_lock_irq(&device->resource->req_lock);
3565 3566 3567
	set_bit(BITMAP_IO, &device->flags);
	if (atomic_read(&device->ap_bio_cnt) == 0) {
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3568 3569
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3570
	}
3571
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3572 3573 3574 3575
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3576
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3577 3578 3579 3580 3581 3582
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3583
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3584
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3585 3586 3587
{
	int rv;

3588
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3589

3590
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3591
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3592

3593 3594 3595
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3596

3597
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3598
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3599 3600 3601 3602

	return rv;
}

3603
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3604
{
3605 3606 3607
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3608 3609 3610
	}
}

3611
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3612
{
3613 3614 3615
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3616 3617 3618 3619 3620 3621 3622 3623 3624
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3625
	struct drbd_device *device = (struct drbd_device *) data;
P
Philipp Reisner 已提交
3626

3627
	/* must not double-queue! */
3628
	if (list_empty(&device->md_sync_work.list))
3629 3630
		drbd_queue_work_front(&first_peer_device(device)->connection->sender_work,
				      &device->md_sync_work);
P
Philipp Reisner 已提交
3631 3632
}

3633
static int w_md_sync(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3634
{
3635 3636
	struct drbd_device *device =
		container_of(w, struct drbd_device, md_sync_work);
3637

3638
	drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3639
#ifdef DEBUG
3640
	drbd_warn(device, "last md_mark_dirty: %s:%u\n",
3641
		device->last_md_mark_dirty.func, device->last_md_mark_dirty.line);
3642
#endif
3643
	drbd_md_sync(device);
3644
	return 0;
P
Philipp Reisner 已提交
3645 3646
}

3647
const char *cmdname(enum drbd_packet cmd)
3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3677
		[P_SUPERSEDED]          = "Superseded",
3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3692
		[P_RETRY_WRITE]		= "RetryWrite",
3693 3694 3695
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3696 3697
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3698 3699 3700 3701 3702

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3703 3704
	};

3705
	/* too big for the array: 0xfffX */
3706 3707 3708 3709
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3710 3711
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3712
	if (cmd >= ARRAY_SIZE(cmdnames))
3713 3714 3715 3716
		return "Unknown";
	return cmdnames[cmd];
}

3717 3718
/**
 * drbd_wait_misc  -  wait for a request to make progress
3719
 * @device:	device associated with the request
3720 3721 3722
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3723
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3724
{
3725
	struct net_conf *nc;
3726 3727 3728
	DEFINE_WAIT(wait);
	long timeout;

3729
	rcu_read_lock();
3730
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3731 3732
	if (!nc) {
		rcu_read_unlock();
3733
		return -ETIMEDOUT;
3734 3735 3736
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3737

3738
	/* Indicate to wake up device->misc_wait on progress.  */
3739
	i->waiting = true;
3740
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3741
	spin_unlock_irq(&device->resource->req_lock);
3742
	timeout = schedule_timeout(timeout);
3743
	finish_wait(&device->misc_wait, &wait);
3744
	spin_lock_irq(&device->resource->req_lock);
3745
	if (!timeout || device->state.conn < C_CONNECTED)
3746 3747 3748 3749
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772
}

#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3773
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3793 3794
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3795 3796 3797 3798 3799 3800
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3801
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3802 3803 3804 3805 3806
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3807
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3808 3809 3810 3811 3812
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3813
		if (__ratelimit(&drbd_ratelimit_state))
3814
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3830 3831 3832 3833
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);