drbd_main.c 107.1 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

#include <linux/module.h>
#include <linux/drbd.h>
#include <asm/uaccess.h>
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
35
#include <linux/mutex.h>
P
Philipp Reisner 已提交
36 37 38 39 40 41 42 43 44 45 46 47
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
48
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
49 50 51 52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>

#include <linux/drbd_limits.h>
#include "drbd_int.h"
55
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
56 57 58 59
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */

#include "drbd_vli.h"

60
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
61 62 63 64
int drbd_worker(struct drbd_thread *);

int drbd_init(void);
static int drbd_open(struct block_device *bdev, fmode_t mode);
65
static void drbd_release(struct gendisk *gd, fmode_t mode);
66
static int w_md_sync(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
67
static void md_sync_timer_fn(unsigned long data);
68 69
static int w_bitmap_io(struct drbd_work *w, int unused);
static int w_go_diskless(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
70 71 72 73 74 75

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
76
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
77
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
106
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
107 108
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
109 110 111 112 113 114 115 116 117 118 119
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
char usermode_helper[80] = "/sbin/drbdadm";

module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
120
struct idr drbd_devices;
121
struct list_head drbd_resources;
P
Philipp Reisner 已提交
122 123

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
124
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
125 126 127 128
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
129
mempool_t *drbd_md_io_page_pool;
130
struct bio_set *drbd_md_io_bio_set;
P
Philipp Reisner 已提交
131 132 133 134 135 136 137 138 139 140 141 142 143 144

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

145
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
146 147 148 149
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
150

151 152 153
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
154

155 156
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
157

158 159 160 161
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
162 163
}

P
Philipp Reisner 已提交
164 165 166 167
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
168
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
169 170 171
{
	int io_allowed;

172 173
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
174
	if (!io_allowed) {
175 176
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
177 178 179 180 181
	}
	return io_allowed;
}

#endif
182

P
Philipp Reisner 已提交
183
/**
184
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
185
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
186 187 188 189
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
190 191
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
192
 */
193
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
194
		unsigned int set_size)
P
Philipp Reisner 已提交
195 196
{
	struct drbd_request *r;
197 198 199
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
200

201
	spin_lock_irq(&connection->req_lock);
P
Philipp Reisner 已提交
202

203
	/* find oldest not yet barrier-acked write request,
204
	 * count writes in its epoch. */
205
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
206
		const unsigned s = r->rq_state;
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
225
		}
P
Philipp Reisner 已提交
226
	}
227

P
Philipp Reisner 已提交
228
	/* first some paranoia code */
229
	if (req == NULL) {
230
		conn_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
231
			 barrier_nr);
P
Philipp Reisner 已提交
232 233
		goto bail;
	}
234
	if (expect_epoch != barrier_nr) {
235
		conn_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
236
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
237
		goto bail;
238 239
	}

240
	if (expect_size != set_size) {
241
		conn_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
242
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
243 244 245
		goto bail;
	}

246 247 248 249
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
250
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
251 252
		if (req->epoch == expect_epoch)
			break;
253
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
254 255 256
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
257
	}
258
	spin_unlock_irq(&connection->req_lock);
259

P
Philipp Reisner 已提交
260 261 262
	return;

bail:
263 264
	spin_unlock_irq(&connection->req_lock);
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
265 266 267 268
}


/**
269
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
270
 * @device:	DRBD device.
271
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
272
 *
273 274
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
275
 */
276
/* must hold resource->req_lock */
277
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
278
{
279
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
280

281
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
282 283
		_req_mod(req, what);
}
284

285
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
286
{
287 288 289
	spin_lock_irq(&connection->req_lock);
	_tl_restart(connection, what);
	spin_unlock_irq(&connection->req_lock);
290
}
P
Philipp Reisner 已提交
291 292 293

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
294
 * @device:	DRBD device.
P
Philipp Reisner 已提交
295 296 297 298 299
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
300
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
301
{
302
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
303
}
304

305
/**
306 307
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
308
 */
309
void tl_abort_disk_io(struct drbd_device *device)
310
{
311
	struct drbd_connection *connection = first_peer_device(device)->connection;
312
	struct drbd_request *req, *r;
313

314 315
	spin_lock_irq(&connection->req_lock);
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
316 317
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
318
		if (req->w.device != device)
319 320
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
321
	}
322
	spin_unlock_irq(&connection->req_lock);
P
Philipp Reisner 已提交
323 324 325 326 327
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
328
	struct drbd_connection *connection = thi->connection;
P
Philipp Reisner 已提交
329 330 331
	unsigned long flags;
	int retval;

332
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
333 334
		 thi->name[0],
		 thi->connection->resource->name);
335

P
Philipp Reisner 已提交
336 337 338 339 340
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

341
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
342 343 344
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
345
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
346
	 * t_state check and assignment needs to be within the same spinlock,
347 348
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
349 350
	 */

351
	if (thi->t_state == RESTARTING) {
352
		conn_info(connection, "Restarting %s thread\n", thi->name);
353
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
354 355 356 357 358
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
359
	thi->t_state = NONE;
P
Philipp Reisner 已提交
360
	smp_mb();
L
Lars Ellenberg 已提交
361
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
362 363
	spin_unlock_irqrestore(&thi->t_lock, flags);

364
	conn_info(connection, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
365 366

	/* Release mod reference taken when thread was started */
367

368
	kref_put(&connection->kref, drbd_destroy_connection);
P
Philipp Reisner 已提交
369 370 371 372
	module_put(THIS_MODULE);
	return retval;
}

373
static void drbd_thread_init(struct drbd_connection *connection, struct drbd_thread *thi,
374
			     int (*func) (struct drbd_thread *), char *name)
P
Philipp Reisner 已提交
375 376 377
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
378
	thi->t_state = NONE;
P
Philipp Reisner 已提交
379
	thi->function = func;
380
	thi->connection = connection;
381
	strncpy(thi->name, name, ARRAY_SIZE(thi->name));
P
Philipp Reisner 已提交
382 383 384 385
}

int drbd_thread_start(struct drbd_thread *thi)
{
386
	struct drbd_connection *connection = thi->connection;
P
Philipp Reisner 已提交
387 388 389 390 391 392 393 394
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
395
	case NONE:
396
		conn_info(connection, "Starting %s thread (from %s [%d])\n",
397
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
398 399 400

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
401
			conn_err(connection, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
402
			spin_unlock_irqrestore(&thi->t_lock, flags);
403
			return false;
P
Philipp Reisner 已提交
404 405
		}

406
		kref_get(&thi->connection->kref);
407

P
Philipp Reisner 已提交
408 409
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
410
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
411 412 413 414
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
415
				    "drbd_%c_%s", thi->name[0], thi->connection->resource->name);
P
Philipp Reisner 已提交
416 417

		if (IS_ERR(nt)) {
418
			conn_err(connection, "Couldn't start thread\n");
P
Philipp Reisner 已提交
419

420
			kref_put(&connection->kref, drbd_destroy_connection);
P
Philipp Reisner 已提交
421
			module_put(THIS_MODULE);
422
			return false;
P
Philipp Reisner 已提交
423 424 425
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
426
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
427 428 429
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
430 431
	case EXITING:
		thi->t_state = RESTARTING;
432
		conn_info(connection, "Restarting %s thread (from %s [%d])\n",
433
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
434
		/* fall through */
435 436
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
437 438 439 440 441
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

442
	return true;
P
Philipp Reisner 已提交
443 444 445 446 447 448 449
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

450
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
451 452 453 454

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

455
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

481
static struct drbd_thread *drbd_task_to_thread(struct drbd_connection *connection, struct task_struct *task)
482 483
{
	struct drbd_thread *thi =
484 485 486
		task == connection->receiver.task ? &connection->receiver :
		task == connection->asender.task  ? &connection->asender :
		task == connection->worker.task   ? &connection->worker : NULL;
487 488 489 490

	return thi;
}

491
char *drbd_task_to_thread_name(struct drbd_connection *connection, struct task_struct *task)
492
{
493
	struct drbd_thread *thi = drbd_task_to_thread(connection, task);
494 495 496
	return thi ? thi->name : task->comm;
}

497
int conn_lowest_minor(struct drbd_connection *connection)
498
{
499
	struct drbd_device *device;
500
	int vnr = 0, m;
501

502
	rcu_read_lock();
503
	device = idr_get_next(&connection->volumes, &vnr);
504
	m = device ? device_to_minor(device) : -1;
505 506 507
	rcu_read_unlock();

	return m;
508
}
509

P
Philipp Reisner 已提交
510 511 512
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
513
 * @device:	DRBD device.
P
Philipp Reisner 已提交
514 515 516 517
 *
 * Forces all threads of a device onto the same CPU. This is beneficial for
 * DRBD's performance. May be overwritten by user's configuration.
 */
518
void drbd_calc_cpu_mask(struct drbd_connection *connection)
P
Philipp Reisner 已提交
519 520 521 522
{
	int ord, cpu;

	/* user override. */
523
	if (cpumask_weight(connection->cpu_mask))
P
Philipp Reisner 已提交
524 525
		return;

526
	ord = conn_lowest_minor(connection) % cpumask_weight(cpu_online_mask);
P
Philipp Reisner 已提交
527 528
	for_each_online_cpu(cpu) {
		if (ord-- == 0) {
529
			cpumask_set_cpu(cpu, connection->cpu_mask);
P
Philipp Reisner 已提交
530 531 532 533
			return;
		}
	}
	/* should not be reached */
534
	cpumask_setall(connection->cpu_mask);
P
Philipp Reisner 已提交
535 536 537 538
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
539
 * @device:	DRBD device.
540
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
541 542 543 544
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
545
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
546 547
{
	struct task_struct *p = current;
548

P
Philipp Reisner 已提交
549 550 551
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
552
	set_cpus_allowed_ptr(p, thi->connection->cpu_mask);
P
Philipp Reisner 已提交
553 554 555
}
#endif

556 557 558 559 560 561 562
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
563
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
564
{
565
	if (connection->agreed_pro_version >= 100) {
566 567 568 569 570 571 572 573
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
574
}
P
Philipp Reisner 已提交
575

576
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
577 578 579 580
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
581
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
582
}
P
Philipp Reisner 已提交
583

584
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
585 586
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
587
	h->command = cpu_to_be16(cmd);
588
	h->length = cpu_to_be32(size);
589
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
590
}
P
Philipp Reisner 已提交
591

592 593 594 595 596 597 598 599 600 601
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
602

603
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
604
				   void *buffer, enum drbd_packet cmd, int size)
605
{
606
	if (connection->agreed_pro_version >= 100)
607
		return prepare_header100(buffer, cmd, size, vnr);
608
	else if (connection->agreed_pro_version >= 95 &&
609
		 size > DRBD_MAX_SIZE_H80_PACKET)
610
		return prepare_header95(buffer, cmd, size);
611
	else
612
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
613 614
}

615
static void *__conn_prepare_command(struct drbd_connection *connection,
616
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
617
{
618 619
	if (!sock->socket)
		return NULL;
620
	return sock->sbuf + drbd_header_size(connection);
621
}
P
Philipp Reisner 已提交
622

623
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
624
{
625
	void *p;
P
Philipp Reisner 已提交
626

627
	mutex_lock(&sock->mutex);
628
	p = __conn_prepare_command(connection, sock);
629
	if (!p)
630
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
631

632
	return p;
P
Philipp Reisner 已提交
633 634
}

635
void *drbd_prepare_command(struct drbd_device *device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
636
{
637
	return conn_prepare_command(first_peer_device(device)->connection, sock);
638
}
P
Philipp Reisner 已提交
639

640
static int __send_command(struct drbd_connection *connection, int vnr,
641 642 643 644 645 646
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
647

648 649 650 651 652 653 654 655 656
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

657
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
658
				      header_size + size);
659
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
660 661
			    msg_flags);
	if (data && !err)
662
		err = drbd_send_all(connection, sock->socket, data, size, 0);
663 664 665
	return err;
}

666
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
667 668 669
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
670
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
671 672
}

673
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
674 675 676 677
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
678

679
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
680 681 682 683
	mutex_unlock(&sock->mutex);
	return err;
}

684
int drbd_send_command(struct drbd_device *device, struct drbd_socket *sock,
685 686 687 688 689
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

690
	err = __send_command(first_peer_device(device)->connection, device->vnr, sock, cmd, header_size,
691 692 693 694
			     data, size);
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
695

696
int drbd_send_ping(struct drbd_connection *connection)
697
{
698 699
	struct drbd_socket *sock;

700 701
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
702
		return -EIO;
703
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
704
}
P
Philipp Reisner 已提交
705

706
int drbd_send_ping_ack(struct drbd_connection *connection)
707
{
708 709
	struct drbd_socket *sock;

710 711
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
712
		return -EIO;
713
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
714 715
}

716
int drbd_send_sync_param(struct drbd_device *device)
P
Philipp Reisner 已提交
717
{
718
	struct drbd_socket *sock;
719
	struct p_rs_param_95 *p;
720
	int size;
721
	const int apv = first_peer_device(device)->connection->agreed_pro_version;
722
	enum drbd_packet cmd;
723
	struct net_conf *nc;
P
Philipp Reisner 已提交
724
	struct disk_conf *dc;
725

726
	sock = &first_peer_device(device)->connection->data;
727
	p = drbd_prepare_command(device, sock);
728 729
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
730

731
	rcu_read_lock();
732
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
P
Philipp Reisner 已提交
733 734 735

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
736
			+ strlen(nc->verify_alg) + 1
737 738
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
739

740
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
741

742 743
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
744

745 746
	if (get_ldev(device)) {
		dc = rcu_dereference(device->ldev->disk_conf);
747
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
748 749 750 751
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
752
		put_ldev(device);
753
	} else {
754
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
755 756 757 758 759
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
760

761
	if (apv >= 88)
762
		strcpy(p->verify_alg, nc->verify_alg);
763
	if (apv >= 89)
764 765
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
766

767
	return drbd_send_command(device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
768 769
}

770
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
771
{
772
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
773
	struct p_protocol *p;
774
	struct net_conf *nc;
775
	int size, cf;
P
Philipp Reisner 已提交
776

777 778
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
779 780
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
781

782
	rcu_read_lock();
783
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
784

785
	if (nc->tentative && connection->agreed_pro_version < 92) {
786 787
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
788
		conn_err(connection, "--dry-run is not supported by peer");
789 790
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
791

792
	size = sizeof(*p);
793
	if (connection->agreed_pro_version >= 87)
794
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
795

796 797 798 799 800
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
801
	cf = 0;
802 803
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
804
	if (nc->tentative)
805
		cf |= CF_DRY_RUN;
806 807
	p->conn_flags    = cpu_to_be32(cf);

808
	if (connection->agreed_pro_version >= 87)
809 810
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
811

812
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
813 814
}

815
int drbd_send_protocol(struct drbd_connection *connection)
816 817 818
{
	int err;

819 820 821
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
822 823

	return err;
P
Philipp Reisner 已提交
824 825
}

826
static int _drbd_send_uuids(struct drbd_device *device, u64 uuid_flags)
P
Philipp Reisner 已提交
827
{
828 829
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
830 831
	int i;

832
	if (!get_ldev_if_state(device, D_NEGOTIATING))
833
		return 0;
P
Philipp Reisner 已提交
834

835
	sock = &first_peer_device(device)->connection->data;
836
	p = drbd_prepare_command(device, sock);
837
	if (!p) {
838
		put_ldev(device);
839 840
		return -EIO;
	}
841
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
842
	for (i = UI_CURRENT; i < UI_SIZE; i++)
843 844
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
845

846 847
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
848
	rcu_read_lock();
849
	uuid_flags |= rcu_dereference(first_peer_device(device)->connection->net_conf)->discard_my_data ? 1 : 0;
850
	rcu_read_unlock();
851 852
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
853
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
854

855 856
	put_ldev(device);
	return drbd_send_command(device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
857 858
}

859
int drbd_send_uuids(struct drbd_device *device)
P
Philipp Reisner 已提交
860
{
861
	return _drbd_send_uuids(device, 0);
P
Philipp Reisner 已提交
862 863
}

864
int drbd_send_uuids_skip_initial_sync(struct drbd_device *device)
P
Philipp Reisner 已提交
865
{
866
	return _drbd_send_uuids(device, 8);
P
Philipp Reisner 已提交
867 868
}

869
void drbd_print_uuids(struct drbd_device *device, const char *text)
870
{
871 872
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
873 874 875 876 877 878
		dev_info(DEV, "%s %016llX:%016llX:%016llX:%016llX\n",
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
879
		put_ldev(device);
880 881 882
	} else {
		dev_info(DEV, "%s effective data uuid: %016llX\n",
				text,
883
				(unsigned long long)device->ed_uuid);
884 885 886
	}
}

887
void drbd_gen_and_send_sync_uuid(struct drbd_device *device)
P
Philipp Reisner 已提交
888
{
889 890
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
891 892
	u64 uuid;

893
	D_ASSERT(device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
894

895
	uuid = device->ldev->md.uuid[UI_BITMAP];
896 897 898 899
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
900 901 902
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
903

904
	sock = &first_peer_device(device)->connection->data;
905
	p = drbd_prepare_command(device, sock);
906 907
	if (p) {
		p->uuid = cpu_to_be64(uuid);
908
		drbd_send_command(device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
909
	}
P
Philipp Reisner 已提交
910 911
}

912
int drbd_send_sizes(struct drbd_device *device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
913
{
914 915
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
916
	sector_t d_size, u_size;
917 918
	int q_order_type;
	unsigned int max_bio_size;
P
Philipp Reisner 已提交
919

920 921 922
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		D_ASSERT(device->ldev->backing_bdev);
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
923
		rcu_read_lock();
924
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
925
		rcu_read_unlock();
926 927
		q_order_type = drbd_queue_order_type(device);
		max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9;
928
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
929
		put_ldev(device);
P
Philipp Reisner 已提交
930 931 932 933
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
934
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
P
Philipp Reisner 已提交
935 936
	}

937
	sock = &first_peer_device(device)->connection->data;
938
	p = drbd_prepare_command(device, sock);
939 940
	if (!p)
		return -EIO;
941

942
	if (first_peer_device(device)->connection->agreed_pro_version <= 94)
943
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
944
	else if (first_peer_device(device)->connection->agreed_pro_version < 100)
945
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
946

947 948
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
949
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
950 951 952
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
953
	return drbd_send_command(device, sock, P_SIZES, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
954 955 956
}

/**
957
 * drbd_send_current_state() - Sends the drbd state to the peer
958
 * @device:	DRBD device.
P
Philipp Reisner 已提交
959
 */
960
int drbd_send_current_state(struct drbd_device *device)
P
Philipp Reisner 已提交
961
{
962
	struct drbd_socket *sock;
963
	struct p_state *p;
P
Philipp Reisner 已提交
964

965
	sock = &first_peer_device(device)->connection->data;
966
	p = drbd_prepare_command(device, sock);
967 968
	if (!p)
		return -EIO;
969 970
	p->state = cpu_to_be32(device->state.i); /* Within the send mutex */
	return drbd_send_command(device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
971 972
}

973 974
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
975
 * @device:      DRBD device.
976
 * @state:     the state to send, not necessarily the current state.
977 978 979 980 981 982
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
983
int drbd_send_state(struct drbd_device *device, union drbd_state state)
984
{
985 986
	struct drbd_socket *sock;
	struct p_state *p;
987

988
	sock = &first_peer_device(device)->connection->data;
989
	p = drbd_prepare_command(device, sock);
990 991 992
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
993
	return drbd_send_command(device, sock, P_STATE, sizeof(*p), NULL, 0);
994
}
995

996
int drbd_send_state_req(struct drbd_device *device, union drbd_state mask, union drbd_state val)
997 998 999
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1000

1001
	sock = &first_peer_device(device)->connection->data;
1002
	p = drbd_prepare_command(device, sock);
1003 1004 1005 1006
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1007
	return drbd_send_command(device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1008
}
1009

1010
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1011
{
1012 1013 1014
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1015

1016 1017 1018
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1019 1020 1021 1022
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1023
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1024 1025
}

1026
void drbd_send_sr_reply(struct drbd_device *device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1027
{
1028 1029
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1030

1031
	sock = &first_peer_device(device)->connection->meta;
1032
	p = drbd_prepare_command(device, sock);
1033 1034
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1035
		drbd_send_command(device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1036
	}
P
Philipp Reisner 已提交
1037 1038
}

1039
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1040
{
1041 1042
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1043
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1044

1045 1046
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1047 1048
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1049
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1050
	}
P
Philipp Reisner 已提交
1051 1052
}

1053
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1054
{
1055 1056 1057
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1058

1059 1060 1061 1062
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1063

1064 1065 1066 1067
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1068 1069
}

1070
static int fill_bitmap_rle_bits(struct drbd_device *device,
1071 1072 1073
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1074 1075 1076 1077 1078 1079 1080
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1081
	int bits, use_rle;
P
Philipp Reisner 已提交
1082 1083

	/* may we use this feature? */
1084
	rcu_read_lock();
1085
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1086
	rcu_read_unlock();
1087
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1088
		return 0;
P
Philipp Reisner 已提交
1089 1090 1091 1092 1093

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1094 1095
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1107 1108
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1109 1110 1111 1112 1113 1114 1115 1116
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1117
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1118 1119 1120 1121
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1122
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
			dev_err(DEV, "unexpected zero runlength while encoding bitmap "
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
			dev_err(DEV, "error while encoding bitmap: %d\n", bits);
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1162
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1163 1164 1165 1166

	return len;
}

1167 1168 1169 1170 1171 1172 1173
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1174
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1175
{
1176 1177
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1178
	struct p_compressed_bm *p = sock->sbuf + header_size;
1179
	int len, err;
P
Philipp Reisner 已提交
1180

1181
	len = fill_bitmap_rle_bits(device, p,
1182
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1183
	if (len < 0)
1184
		return -EIO;
P
Philipp Reisner 已提交
1185 1186

	if (len) {
1187
		dcbp_set_code(p, RLE_VLI_Bits);
1188
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1189 1190
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1191
		c->packets[0]++;
1192
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1193 1194 1195 1196 1197 1198

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1199 1200
		unsigned int data_size;
		unsigned long num_words;
1201
		unsigned long *p = sock->sbuf + header_size;
1202 1203

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1204
		num_words = min_t(size_t, data_size / sizeof(*p),
1205
				  c->bm_words - c->word_offset);
1206
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1207
		if (len)
1208
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1209
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1210 1211 1212 1213
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1214
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1215 1216 1217 1218

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1219
	if (!err) {
1220
		if (len == 0) {
1221
			INFO_bm_xfer_stats(device, "send", c);
1222 1223 1224 1225 1226
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1227 1228 1229
}

/* See the comment at receive_bitmap() */
1230
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1231 1232
{
	struct bm_xfer_ctx c;
1233
	int err;
P
Philipp Reisner 已提交
1234

1235
	if (!expect(device->bitmap))
1236
		return false;
P
Philipp Reisner 已提交
1237

1238 1239
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
P
Philipp Reisner 已提交
1240
			dev_info(DEV, "Writing the whole bitmap, MDF_FullSync was set.\n");
1241 1242
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1243 1244 1245 1246 1247
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
				dev_err(DEV, "Failed to write bitmap to disk!\n");
			} else {
1248 1249
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1250 1251
			}
		}
1252
		put_ldev(device);
P
Philipp Reisner 已提交
1253 1254 1255
	}

	c = (struct bm_xfer_ctx) {
1256 1257
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1258 1259 1260
	};

	do {
1261
		err = send_bitmap_rle_or_plain(device, &c);
1262
	} while (err > 0);
P
Philipp Reisner 已提交
1263

1264
	return err == 0;
P
Philipp Reisner 已提交
1265 1266
}

1267
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1268
{
1269
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1270
	int err = -1;
P
Philipp Reisner 已提交
1271

1272 1273
	mutex_lock(&sock->mutex);
	if (sock->socket)
1274
		err = !_drbd_send_bitmap(device);
1275
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1276 1277 1278
	return err;
}

1279
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1280
{
1281 1282
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1283

1284
	if (connection->cstate < C_WF_REPORT_PARAMS)
1285
		return;
P
Philipp Reisner 已提交
1286

1287 1288
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1289 1290 1291 1292
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1293
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1294 1295 1296 1297
}

/**
 * _drbd_send_ack() - Sends an ack packet
1298
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1299 1300 1301 1302 1303
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1304
static int _drbd_send_ack(struct drbd_device *device, enum drbd_packet cmd,
1305
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1306
{
1307 1308
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1309

1310
	if (device->state.conn < C_CONNECTED)
1311
		return -EIO;
P
Philipp Reisner 已提交
1312

1313
	sock = &first_peer_device(device)->connection->meta;
1314
	p = drbd_prepare_command(device, sock);
1315
	if (!p)
1316
		return -EIO;
1317 1318 1319
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1320 1321
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
	return drbd_send_command(device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1322 1323
}

1324 1325 1326
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1327
void drbd_send_ack_dp(struct drbd_device *device, enum drbd_packet cmd,
1328
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1329
{
1330 1331
	if (first_peer_device(device)->connection->peer_integrity_tfm)
		data_size -= crypto_hash_digestsize(first_peer_device(device)->connection->peer_integrity_tfm);
1332
	_drbd_send_ack(device, cmd, dp->sector, cpu_to_be32(data_size),
1333
		       dp->block_id);
P
Philipp Reisner 已提交
1334 1335
}

1336
void drbd_send_ack_rp(struct drbd_device *device, enum drbd_packet cmd,
1337
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1338
{
1339
	_drbd_send_ack(device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1340 1341 1342 1343
}

/**
 * drbd_send_ack() - Sends an ack packet
1344
 * @device:	DRBD device
1345 1346
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1347
 */
1348
int drbd_send_ack(struct drbd_device *device, enum drbd_packet cmd,
1349
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1350
{
1351
	return _drbd_send_ack(device, cmd,
1352 1353 1354
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1355 1356 1357 1358
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1359
int drbd_send_ack_ex(struct drbd_device *device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1360 1361
		     sector_t sector, int blksize, u64 block_id)
{
1362
	return _drbd_send_ack(device, cmd,
P
Philipp Reisner 已提交
1363 1364 1365 1366 1367
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1368
int drbd_send_drequest(struct drbd_device *device, int cmd,
P
Philipp Reisner 已提交
1369 1370
		       sector_t sector, int size, u64 block_id)
{
1371 1372
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1373

1374
	sock = &first_peer_device(device)->connection->data;
1375
	p = drbd_prepare_command(device, sock);
1376 1377 1378 1379 1380
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1381
	return drbd_send_command(device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1382 1383
}

1384
int drbd_send_drequest_csum(struct drbd_device *device, sector_t sector, int size,
1385
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1386
{
1387 1388
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1389

1390
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1391

1392
	sock = &first_peer_device(device)->connection->data;
1393
	p = drbd_prepare_command(device, sock);
1394 1395 1396 1397 1398
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1399
	return drbd_send_command(device, sock, cmd, sizeof(*p),
1400
				 digest, digest_size);
P
Philipp Reisner 已提交
1401 1402
}

1403
int drbd_send_ov_request(struct drbd_device *device, sector_t sector, int size)
P
Philipp Reisner 已提交
1404
{
1405 1406
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1407

1408
	sock = &first_peer_device(device)->connection->data;
1409
	p = drbd_prepare_command(device, sock);
1410 1411 1412 1413 1414
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1415
	return drbd_send_command(device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1416 1417 1418
}

/* called on sndtimeo
1419 1420
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1421
 */
1422
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1423 1424
{
	int drop_it;
1425
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1426

1427 1428 1429 1430
	drop_it =   connection->meta.socket == sock
		|| !connection->asender.task
		|| get_t_state(&connection->asender) != RUNNING
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1431 1432

	if (drop_it)
1433
		return true;
P
Philipp Reisner 已提交
1434

1435
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1436
	if (!drop_it) {
1437 1438 1439
		conn_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1440 1441
	}

1442
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1443 1444
}

1445
static void drbd_update_congested(struct drbd_connection *connection)
1446
{
1447
	struct sock *sk = connection->data.socket->sk;
1448
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1449
		set_bit(NET_CONGESTED, &connection->flags);
1450 1451
}

P
Philipp Reisner 已提交
1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1473
static int _drbd_no_send_page(struct drbd_device *device, struct page *page,
1474
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1475
{
1476 1477 1478 1479
	struct socket *socket;
	void *addr;
	int err;

1480
	socket = first_peer_device(device)->connection->data.socket;
1481
	addr = kmap(page) + offset;
1482
	err = drbd_send_all(first_peer_device(device)->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1483
	kunmap(page);
1484
	if (!err)
1485
		device->send_cnt += size >> 9;
1486
	return err;
P
Philipp Reisner 已提交
1487 1488
}

1489
static int _drbd_send_page(struct drbd_device *device, struct page *page,
L
Lars Ellenberg 已提交
1490
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1491
{
1492
	struct socket *socket = first_peer_device(device)->connection->data.socket;
P
Philipp Reisner 已提交
1493 1494
	mm_segment_t oldfs = get_fs();
	int len = size;
1495
	int err = -EIO;
P
Philipp Reisner 已提交
1496 1497 1498 1499 1500 1501 1502 1503

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1504
		return _drbd_no_send_page(device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1505

L
Lars Ellenberg 已提交
1506
	msg_flags |= MSG_NOSIGNAL;
1507
	drbd_update_congested(first_peer_device(device)->connection);
P
Philipp Reisner 已提交
1508 1509
	set_fs(KERNEL_DS);
	do {
1510 1511 1512
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1513
		if (sent <= 0) {
1514
			if (sent == -EAGAIN) {
1515
				if (we_should_drop_the_connection(first_peer_device(device)->connection, socket))
1516 1517 1518
					break;
				continue;
			}
P
Philipp Reisner 已提交
1519 1520
			dev_warn(DEV, "%s: size=%d len=%d sent=%d\n",
			     __func__, (int)size, len, sent);
1521 1522
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1523 1524 1525 1526
			break;
		}
		len    -= sent;
		offset += sent;
1527
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
P
Philipp Reisner 已提交
1528
	set_fs(oldfs);
1529
	clear_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags);
P
Philipp Reisner 已提交
1530

1531 1532
	if (len == 0) {
		err = 0;
1533
		device->send_cnt += size >> 9;
1534 1535
	}
	return err;
P
Philipp Reisner 已提交
1536 1537
}

1538
static int _drbd_send_bio(struct drbd_device *device, struct bio *bio)
P
Philipp Reisner 已提交
1539
{
1540 1541 1542
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1543
	/* hint all but last page with MSG_MORE */
1544
	bio_for_each_segment(bvec, bio, iter) {
1545 1546
		int err;

1547
		err = _drbd_no_send_page(device, bvec.bv_page,
1548
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1549
					 bio_iter_last(bvec, iter)
1550
					 ? 0 : MSG_MORE);
1551 1552
		if (err)
			return err;
P
Philipp Reisner 已提交
1553
	}
1554
	return 0;
P
Philipp Reisner 已提交
1555 1556
}

1557
static int _drbd_send_zc_bio(struct drbd_device *device, struct bio *bio)
P
Philipp Reisner 已提交
1558
{
1559 1560 1561
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1562
	/* hint all but last page with MSG_MORE */
1563
	bio_for_each_segment(bvec, bio, iter) {
1564 1565
		int err;

1566
		err = _drbd_send_page(device, bvec.bv_page,
1567
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1568
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1569 1570
		if (err)
			return err;
P
Philipp Reisner 已提交
1571
	}
1572
	return 0;
P
Philipp Reisner 已提交
1573 1574
}

1575
static int _drbd_send_zc_ee(struct drbd_device *device,
1576
			    struct drbd_peer_request *peer_req)
1577
{
1578 1579
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1580
	int err;
1581

L
Lars Ellenberg 已提交
1582
	/* hint all but last page with MSG_MORE */
1583 1584
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1585

1586
		err = _drbd_send_page(device, page, 0, l,
1587 1588 1589
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1590 1591
		len -= l;
	}
1592
	return 0;
1593 1594
}

1595
static u32 bio_flags_to_wire(struct drbd_device *device, unsigned long bi_rw)
1596
{
1597
	if (first_peer_device(device)->connection->agreed_pro_version >= 95)
1598 1599 1600 1601 1602
		return  (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bi_rw & REQ_FUA ? DP_FUA : 0) |
			(bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
			(bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
	else
J
Jens Axboe 已提交
1603
		return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
1604 1605
}

P
Philipp Reisner 已提交
1606 1607 1608
/* Used to send write requests
 * R_PRIMARY -> Peer	(P_DATA)
 */
1609
int drbd_send_dblock(struct drbd_device *device, struct drbd_request *req)
P
Philipp Reisner 已提交
1610
{
1611 1612
	struct drbd_socket *sock;
	struct p_data *p;
P
Philipp Reisner 已提交
1613 1614
	unsigned int dp_flags = 0;
	int dgs;
1615
	int err;
P
Philipp Reisner 已提交
1616

1617
	sock = &first_peer_device(device)->connection->data;
1618
	p = drbd_prepare_command(device, sock);
1619 1620
	dgs = first_peer_device(device)->connection->integrity_tfm ?
	      crypto_hash_digestsize(first_peer_device(device)->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1621

1622 1623 1624 1625
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1626 1627 1628 1629
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
	dp_flags = bio_flags_to_wire(device, req->master_bio->bi_rw);
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1630
		dp_flags |= DP_MAY_SET_IN_SYNC;
1631
	if (first_peer_device(device)->connection->agreed_pro_version >= 100) {
1632 1633 1634 1635 1636
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
		if (req->rq_state & RQ_EXP_WRITE_ACK)
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1637 1638
	p->dp_flags = cpu_to_be32(dp_flags);
	if (dgs)
1639 1640
		drbd_csum_bio(device, first_peer_device(device)->connection->integrity_tfm, req->master_bio, p + 1);
	err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_DATA, sizeof(*p) + dgs, NULL, req->i.size);
1641
	if (!err) {
1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1653
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || dgs)
1654
			err = _drbd_send_bio(device, req->master_bio);
P
Philipp Reisner 已提交
1655
		else
1656
			err = _drbd_send_zc_bio(device, req->master_bio);
1657 1658 1659

		/* double check digest, sometimes buffers have been modified in flight. */
		if (dgs > 0 && dgs <= 64) {
B
Bart Van Assche 已提交
1660
			/* 64 byte, 512 bit, is the largest digest size
1661 1662
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1663
			drbd_csum_bio(device, first_peer_device(device)->connection->integrity_tfm, req->master_bio, digest);
1664
			if (memcmp(p + 1, digest, dgs)) {
1665 1666
				dev_warn(DEV,
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1667
					(unsigned long long)req->i.sector, req->i.size);
1668 1669 1670 1671
			}
		} /* else if (dgs > 64) {
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1672
	}
1673
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1674

1675
	return err;
P
Philipp Reisner 已提交
1676 1677 1678 1679 1680 1681
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1682
int drbd_send_block(struct drbd_device *device, enum drbd_packet cmd,
1683
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1684
{
1685 1686
	struct drbd_socket *sock;
	struct p_data *p;
1687
	int err;
P
Philipp Reisner 已提交
1688 1689
	int dgs;

1690
	sock = &first_peer_device(device)->connection->data;
1691
	p = drbd_prepare_command(device, sock);
P
Philipp Reisner 已提交
1692

1693 1694
	dgs = first_peer_device(device)->connection->integrity_tfm ?
	      crypto_hash_digestsize(first_peer_device(device)->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1695

1696 1697 1698 1699 1700
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1701
	p->dp_flags = 0;
1702
	if (dgs)
1703 1704
		drbd_csum_ee(device, first_peer_device(device)->connection->integrity_tfm, peer_req, p + 1);
	err = __send_command(first_peer_device(device)->connection, device->vnr, sock, cmd, sizeof(*p) + dgs, NULL, peer_req->i.size);
1705
	if (!err)
1706
		err = _drbd_send_zc_ee(device, peer_req);
1707
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1708

1709
	return err;
P
Philipp Reisner 已提交
1710 1711
}

1712
int drbd_send_out_of_sync(struct drbd_device *device, struct drbd_request *req)
1713
{
1714 1715
	struct drbd_socket *sock;
	struct p_block_desc *p;
1716

1717
	sock = &first_peer_device(device)->connection->data;
1718
	p = drbd_prepare_command(device, sock);
1719 1720 1721 1722
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1723
	return drbd_send_command(device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1724 1725
}

P
Philipp Reisner 已提交
1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1742
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1743 1744 1745 1746 1747 1748 1749
	      void *buf, size_t size, unsigned msg_flags)
{
	struct kvec iov;
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1750
		return -EBADR;
P
Philipp Reisner 已提交
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762

	/* THINK  if (signal_pending) return ... ? */

	iov.iov_base = buf;
	iov.iov_len  = size;

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

1763
	if (sock == connection->data.socket) {
1764
		rcu_read_lock();
1765
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1766
		rcu_read_unlock();
1767
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
	}
	do {
		/* STRANGE
		 * tcp_sendmsg does _not_ use its size parameter at all ?
		 *
		 * -EAGAIN on timeout, -EINTR on signal.
		 */
/* THINK
 * do we need to block DRBD_SIG if sock == &meta.socket ??
 * otherwise wake_asender() might interrupt some send_*Ack !
 */
		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
		if (rv == -EAGAIN) {
1781
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
		iov.iov_base += rv;
		iov.iov_len  -= rv;
	} while (sent < size);

1797 1798
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1799 1800 1801

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1802 1803
			conn_err(connection, "%s_sendmsg returned %d\n",
				 sock == connection->meta.socket ? "msock" : "sock",
1804
				 rv);
1805
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1806
		} else
1807
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1808 1809 1810 1811 1812
	}

	return sent;
}

1813 1814 1815 1816 1817
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1818
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1819 1820 1821 1822
		  size_t size, unsigned msg_flags)
{
	int err;

1823
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1824 1825 1826 1827 1828 1829 1830
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1831 1832
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1833
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1834 1835 1836
	unsigned long flags;
	int rv = 0;

1837
	mutex_lock(&drbd_main_mutex);
1838
	spin_lock_irqsave(&first_peer_device(device)->connection->req_lock, flags);
1839
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1840 1841
	 * and no race with updating open_cnt */

1842
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1843 1844 1845 1846 1847 1848 1849
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1850
		device->open_cnt++;
1851
	spin_unlock_irqrestore(&first_peer_device(device)->connection->req_lock, flags);
1852
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1853 1854 1855 1856

	return rv;
}

1857
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1858
{
1859
	struct drbd_device *device = gd->private_data;
1860
	mutex_lock(&drbd_main_mutex);
1861
	device->open_cnt--;
1862
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1863 1864
}

1865
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1866
{
1867 1868
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1869
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1870 1871 1872 1873 1874 1875 1876 1877
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1878
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1879 1880 1881 1882
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
	atomic_set(&device->md_io_in_use, 0);

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->go_diskless.list);
	INIT_LIST_HEAD(&device->md_sync_work.list);
	INIT_LIST_HEAD(&device->start_resync_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->go_diskless.cb  = w_go_diskless;
	device->md_sync_work.cb = w_md_sync;
	device->bm_io_work.w.cb = w_bitmap_io;
	device->start_resync_work.cb = w_start_resync;

	device->resync_work.device  = device;
	device->unplug_work.device  = device;
	device->go_diskless.device  = device;
	device->md_sync_work.device = device;
	device->bm_io_work.w.device = device;
	device->start_resync_work.device = device;

	init_timer(&device->resync_timer);
	init_timer(&device->md_sync_timer);
	init_timer(&device->start_resync_timer);
	init_timer(&device->request_timer);
	device->resync_timer.function = resync_timer_fn;
	device->resync_timer.data = (unsigned long) device;
	device->md_sync_timer.function = md_sync_timer_fn;
	device->md_sync_timer.data = (unsigned long) device;
	device->start_resync_timer.function = start_resync_timer_fn;
	device->start_resync_timer.data = (unsigned long) device;
	device->request_timer.function = request_timer_fn;
	device->request_timer.data = (unsigned long) device;

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
1954
{
1955
	int i;
1956
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
P
Philipp Reisner 已提交
1957
		dev_err(DEV, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1958
				first_peer_device(device)->connection->receiver.t_state);
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
1972
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1973 1974
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
1975
	}
1976
	D_ASSERT(first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
1977

1978 1979
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
1980
		/* maybe never allocated. */
1981 1982
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
1983 1984
	}

1985 1986
	drbd_free_bc(device->ldev);
	device->ldev = NULL;
1987

1988
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
1989

1990 1991 1992 1993 1994 1995
	D_ASSERT(list_empty(&device->active_ee));
	D_ASSERT(list_empty(&device->sync_ee));
	D_ASSERT(list_empty(&device->done_ee));
	D_ASSERT(list_empty(&device->read_ee));
	D_ASSERT(list_empty(&device->net_ee));
	D_ASSERT(list_empty(&device->resync_reads));
1996
	D_ASSERT(list_empty(&first_peer_device(device)->connection->sender_work.q));
1997 1998 1999
	D_ASSERT(list_empty(&device->resync_work.list));
	D_ASSERT(list_empty(&device->unplug_work.list));
	D_ASSERT(list_empty(&device->go_diskless.list));
2000

2001
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

	/* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */

2018 2019
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2020 2021
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2035
	drbd_md_io_bio_set   = NULL;
2036
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2050
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2051 2052 2053 2054 2055 2056 2057 2058 2059
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2060
	drbd_md_io_page_pool = NULL;
2061
	drbd_md_io_bio_set   = NULL;
P
Philipp Reisner 已提交
2062 2063 2064 2065 2066 2067 2068 2069

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2070
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2085 2086 2087 2088
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2089 2090 2091 2092
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

P
Philipp Reisner 已提交
2093 2094 2095 2096 2097 2098 2099
	drbd_request_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
	if (drbd_request_mempool == NULL)
		goto Enomem;

	drbd_ee_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2100
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

static int drbd_notify_sys(struct notifier_block *this, unsigned long code,
	void *unused)
{
	/* just so we have it.  you never know what interesting things we
	 * might want to do here some day...
	 */

	return NOTIFY_DONE;
}

static struct notifier_block drbd_notifier = {
	.notifier_call = drbd_notify_sys,
};

2136
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2137 2138 2139
{
	int rr;

2140
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2141 2142 2143
	if (rr)
		dev_err(DEV, "%d EEs in active list found!\n", rr);

2144
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2145 2146 2147
	if (rr)
		dev_err(DEV, "%d EEs in sync list found!\n", rr);

2148
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2149 2150 2151
	if (rr)
		dev_err(DEV, "%d EEs in read list found!\n", rr);

2152
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2153 2154 2155
	if (rr)
		dev_err(DEV, "%d EEs in done list found!\n", rr);

2156
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2157 2158 2159 2160
	if (rr)
		dev_err(DEV, "%d EEs in net list found!\n", rr);
}

2161
/* caution. no locking. */
2162
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2163
{
2164
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2165
	struct drbd_connection *connection = first_peer_device(device)->connection;
P
Philipp Reisner 已提交
2166

2167
	del_timer_sync(&device->request_timer);
2168

P
Philipp Reisner 已提交
2169
	/* paranoia asserts */
2170
	D_ASSERT(device->open_cnt == 0);
P
Philipp Reisner 已提交
2171 2172 2173 2174 2175
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2176 2177
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2178

2179 2180
	drbd_free_bc(device->ldev);
	device->ldev = NULL;
P
Philipp Reisner 已提交
2181

2182
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2183

2184 2185
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2186

2187 2188
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2189

2190 2191 2192 2193 2194 2195
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
	__free_page(device->md_io_page);
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2196
	kfree(first_peer_device(device));
2197
	kfree(device);
2198

2199
	kref_put(&connection->kref, drbd_destroy_connection);
P
Philipp Reisner 已提交
2200 2201
}

2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2224
		struct drbd_device *device = req->w.device;
2225 2226
		struct bio *bio = req->master_bio;
		unsigned long start_time = req->start_time;
2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243
		bool expected;

		expected = 
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
			dev_err(DEV, "req=%p completion_ref=%d rq_state=%x\n",
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2244
		 */
2245
		kref_put(&req->kref, drbd_req_destroy);
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2260 2261
		inc_ap_bio(device);
		__drbd_make_request(device, bio, start_time);
2262 2263 2264
	}
}

2265
void drbd_restart_request(struct drbd_request *req)
2266 2267 2268 2269 2270 2271 2272 2273 2274
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2275
	dec_ap_bio(req->w.device);
P
Philipp Reisner 已提交
2276

2277
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2278 2279
}

2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

	kfree(resource->name);
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
		kref_put(&connection->kref, drbd_destroy_connection);
	}
	kref_put(&resource->kref, drbd_destroy_resource);
}
2299

P
Philipp Reisner 已提交
2300 2301 2302
static void drbd_cleanup(void)
{
	unsigned int i;
2303
	struct drbd_device *device;
2304
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2305 2306 2307

	unregister_reboot_notifier(&drbd_notifier);

2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2319 2320
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2321

2322
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2323

2324 2325
	idr_for_each_entry(&drbd_devices, device, i) {
		idr_remove(&drbd_devices, device_to_minor(device));
2326
		idr_remove(&first_peer_device(device)->connection->volumes, device->vnr);
2327 2328
		destroy_workqueue(device->submit.wq);
		del_gendisk(device->vdisk);
P
Philipp Reisner 已提交
2329
		/* synchronize_rcu(); No other threads running at this point */
2330
		kref_put(&device->kref, drbd_destroy_device);
P
Philipp Reisner 已提交
2331 2332
	}

P
Philipp Reisner 已提交
2333
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2334 2335 2336
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2337
	}
P
Philipp Reisner 已提交
2338

2339
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2340 2341
	unregister_blkdev(DRBD_MAJOR, "drbd");

2342
	idr_destroy(&drbd_devices);
2343

P
Philipp Reisner 已提交
2344 2345 2346 2347
	printk(KERN_INFO "drbd: module cleanup done.\n");
}

/**
2348
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2349
 * @congested_data:	User data
2350
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2351 2352 2353 2354 2355
 *
 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2356
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2357 2358 2359 2360
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2361
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2362 2363 2364 2365 2366 2367
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2368
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2369 2370 2371 2372 2373 2374
		r |= (1 << BDI_async_congested);
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2375
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2376 2377
			r |= (1 << BDI_sync_congested);
		else
2378
			put_ldev(device);
2379 2380 2381 2382 2383
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2384 2385
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
P
Philipp Reisner 已提交
2386
		r = bdi_congested(&q->backing_dev_info, bdi_bits);
2387
		put_ldev(device);
P
Philipp Reisner 已提交
2388 2389 2390 2391
		if (r)
			reason = 'b';
	}

2392 2393
	if (bdi_bits & (1 << BDI_async_congested) &&
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
P
Philipp Reisner 已提交
2394 2395 2396 2397 2398
		r |= (1 << BDI_async_congested);
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2399
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2400 2401 2402
	return r;
}

2403 2404 2405 2406
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2407
	init_waitqueue_head(&wq->q_wait);
2408 2409
}

2410
struct drbd_connection *conn_get_by_name(const char *name)
2411
{
2412
	struct drbd_connection *connection;
2413
	struct drbd_resource *resource;
2414

2415 2416 2417
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2418
	rcu_read_lock();
2419 2420 2421
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
			connection = first_connection(resource);
2422
			kref_get(&connection->kref);
2423
			goto found;
2424
		}
2425
	}
2426
	connection = NULL;
2427
found:
P
Philipp Reisner 已提交
2428
	rcu_read_unlock();
2429
	return connection;
2430 2431
}

2432
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2433 2434
				     void *peer_addr, int peer_addr_len)
{
2435
	struct drbd_resource *resource;
2436
	struct drbd_connection *connection;
2437 2438

	rcu_read_lock();
2439 2440 2441 2442 2443 2444 2445 2446 2447
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2448 2449
		}
	}
2450
	connection = NULL;
2451 2452
found:
	rcu_read_unlock();
2453
	return connection;
2454 2455
}

2456 2457 2458 2459 2460
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2461 2462 2463
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2464 2465 2466 2467 2468
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2469
	free_page((unsigned long) socket->sbuf);
2470 2471 2472
	free_page((unsigned long) socket->rbuf);
}

2473
void conn_free_crypto(struct drbd_connection *connection)
2474
{
2475
	drbd_free_sock(connection);
2476

2477 2478 2479 2480 2481 2482 2483
	crypto_free_hash(connection->csums_tfm);
	crypto_free_hash(connection->verify_tfm);
	crypto_free_hash(connection->cram_hmac_tfm);
	crypto_free_hash(connection->integrity_tfm);
	crypto_free_hash(connection->peer_integrity_tfm);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2484

2485 2486 2487 2488 2489 2490 2491
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2492 2493
}

2494
int set_resource_options(struct drbd_connection *connection, struct res_opts *res_opts)
2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508
{
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;
		/*
		retcode = ERR_NOMEM;
		drbd_msg_put_info("unable to allocate cpumask");
		*/

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
		/* FIXME: Get rid of constant 32 here */
2509 2510
		err = bitmap_parse(res_opts->cpu_mask, 32,
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2511
		if (err) {
2512
			conn_warn(connection, "bitmap_parse() failed with %d\n", err);
2513 2514 2515 2516
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2517 2518 2519 2520 2521 2522 2523
	connection->res_opts = *res_opts;
	if (!cpumask_equal(connection->cpu_mask, new_cpu_mask)) {
		cpumask_copy(connection->cpu_mask, new_cpu_mask);
		drbd_calc_cpu_mask(connection);
		connection->receiver.reset_cpu_mask = 1;
		connection->asender.reset_cpu_mask = 1;
		connection->worker.reset_cpu_mask = 1;
2524 2525 2526 2527 2528 2529 2530 2531 2532
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

	resource = kmalloc(sizeof(struct drbd_resource), GFP_KERNEL);
	if (!resource)
		return NULL;
	resource->name = kstrdup(name, GFP_KERNEL);
	if (!resource->name) {
		kfree(resource);
		return NULL;
	}
	kref_init(&resource->kref);
	INIT_LIST_HEAD(&resource->connections);
	list_add_tail_rcu(&resource->resources, &drbd_resources);
	return resource;
}

2551
/* caller must be under genl_lock() */
2552
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2553
{
2554
	struct drbd_resource *resource;
2555
	struct drbd_connection *connection;
2556

2557 2558
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2559 2560
		return NULL;

2561
	if (drbd_alloc_socket(&connection->data))
2562
		goto fail;
2563
	if (drbd_alloc_socket(&connection->meta))
2564 2565
		goto fail;

2566
	if (!zalloc_cpumask_var(&connection->cpu_mask, GFP_KERNEL))
2567 2568
		goto fail;

2569
	if (set_resource_options(connection, res_opts))
2570 2571
		goto fail;

2572 2573
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2574
		goto fail;
2575

2576
	INIT_LIST_HEAD(&connection->transfer_log);
2577

2578 2579 2580 2581
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
	connection->write_ordering = WO_bdev_flush;
2582

2583 2584 2585
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2586

2587 2588 2589 2590
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2591 2592 2593 2594 2595 2596
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	spin_lock_init(&connection->req_lock);
	mutex_init(&connection->conf_update);
	init_waitqueue_head(&connection->ping_wait);
	idr_init(&connection->volumes);
2597

2598 2599 2600
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2601

2602 2603 2604
	drbd_thread_init(connection, &connection->receiver, drbdd_init, "receiver");
	drbd_thread_init(connection, &connection->worker, drbd_worker, "worker");
	drbd_thread_init(connection, &connection->asender, drbd_asender, "asender");
2605

2606
	kref_init(&connection->kref);
2607 2608 2609 2610

	kref_get(&resource->kref);
	connection->resource = resource;
	list_add_tail_rcu(&connection->connections, &resource->connections);
2611

2612
	return connection;
2613 2614

fail:
2615 2616 2617 2618 2619
	kfree(connection->current_epoch);
	free_cpumask_var(connection->cpu_mask);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2620 2621 2622 2623

	return NULL;
}

2624
void drbd_destroy_connection(struct kref *kref)
2625
{
2626
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2627
	struct drbd_resource *resource = connection->resource;
2628

2629 2630 2631
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
		conn_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
	kfree(connection->current_epoch);
2632

2633
	idr_destroy(&connection->volumes);
2634

2635 2636 2637 2638 2639 2640
	free_cpumask_var(connection->cpu_mask);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
	kfree(connection);
2641
	kref_put(&resource->kref, drbd_destroy_resource);
2642 2643
}

2644
static int init_submitter(struct drbd_device *device)
2645 2646 2647
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2648 2649 2650
	device->submit.wq = alloc_workqueue("drbd%u_submit",
			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
	if (!device->submit.wq)
2651 2652
		return -ENOMEM;

2653 2654 2655
	INIT_WORK(&device->submit.worker, do_submit);
	spin_lock_init(&device->submit.lock);
	INIT_LIST_HEAD(&device->submit.writes);
2656 2657 2658
	return 0;
}

2659
enum drbd_ret_code drbd_create_minor(struct drbd_connection *connection, unsigned int minor, int vnr)
P
Philipp Reisner 已提交
2660
{
2661
	struct drbd_device *device;
2662
	struct drbd_peer_device *peer_device;
P
Philipp Reisner 已提交
2663 2664
	struct gendisk *disk;
	struct request_queue *q;
2665
	int id;
2666
	enum drbd_ret_code err = ERR_NOMEM;
2667

2668 2669
	device = minor_to_device(minor);
	if (device)
2670
		return ERR_MINOR_EXISTS;
P
Philipp Reisner 已提交
2671 2672

	/* GFP_KERNEL, we are outside of all write-out paths */
2673 2674
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2675
		return ERR_NOMEM;
2676 2677 2678
	peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
	if (!peer_device)
		goto out_no_peer_device;
2679

2680 2681
	INIT_LIST_HEAD(&device->peer_devices);
	list_add(&peer_device->peer_devices, &device->peer_devices);
2682
	kref_get(&connection->kref);
2683
	device->resource = connection->resource;
2684 2685
	peer_device->connection = connection;
	peer_device->device = device;
P
Philipp Reisner 已提交
2686

2687 2688
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2689

2690
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2691 2692 2693 2694

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2695 2696
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2697 2698 2699 2700

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2701
	device->vdisk = disk;
P
Philipp Reisner 已提交
2702

2703
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2704 2705 2706 2707 2708 2709

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2710
	disk->private_data = device;
P
Philipp Reisner 已提交
2711

2712
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2713
	/* we have no partitions. we contain only ourselves. */
2714
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2715 2716

	q->backing_dev_info.congested_fn = drbd_congested;
2717
	q->backing_dev_info.congested_data = device;
P
Philipp Reisner 已提交
2718

2719
	blk_queue_make_request(q, drbd_make_request);
2720
	blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
2721 2722 2723
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
P
Philipp Reisner 已提交
2724 2725
	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
	blk_queue_merge_bvec(q, drbd_merge_bvec);
2726
	q->queue_lock = &first_peer_device(device)->connection->req_lock; /* needed since we use */
P
Philipp Reisner 已提交
2727

2728 2729
	device->md_io_page = alloc_page(GFP_KERNEL);
	if (!device->md_io_page)
P
Philipp Reisner 已提交
2730 2731
		goto out_no_io_page;

2732
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2733
		goto out_no_bitmap;
2734 2735
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2736

2737 2738 2739
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2740 2741 2742
			err = ERR_MINOR_EXISTS;
			drbd_msg_put_info("requested minor exists already");
		}
2743
		goto out_no_minor_idr;
2744
	}
2745

2746 2747 2748
	id = idr_alloc(&connection->volumes, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2749 2750 2751
			err = ERR_INVALID_REQUEST;
			drbd_msg_put_info("requested volume exists already");
		}
2752 2753
		goto out_idr_remove_minor;
	}
T
Tejun Heo 已提交
2754

2755
	if (init_submitter(device)) {
2756 2757 2758 2759 2760
		err = ERR_NOMEM;
		drbd_msg_put_info("unable to create submit workqueue");
		goto out_idr_remove_vol;
	}

2761
	add_disk(disk);
2762
	kref_init(&device->kref); /* one ref for both idrs and the the add_disk */
2763

2764
	/* inherit the connection state */
2765
	device->state.conn = connection->cstate;
2766 2767
	if (device->state.conn == C_WF_REPORT_PARAMS)
		drbd_connected(device);
2768

2769
	return NO_ERROR;
P
Philipp Reisner 已提交
2770

2771
out_idr_remove_vol:
2772
	idr_remove(&connection->volumes, vnr);
2773
out_idr_remove_minor:
2774
	idr_remove(&drbd_devices, minor);
2775
	synchronize_rcu();
2776
out_no_minor_idr:
2777
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2778
out_no_bitmap:
2779
	__free_page(device->md_io_page);
P
Philipp Reisner 已提交
2780 2781 2782 2783 2784
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2785
	kref_put(&connection->kref, drbd_destroy_connection);
2786 2787
out_no_peer_device:
	kfree(device);
2788
	return err;
P
Philipp Reisner 已提交
2789 2790 2791 2792 2793 2794
}

int __init drbd_init(void)
{
	int err;

2795
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
P
Philipp Reisner 已提交
2796
		printk(KERN_ERR
2797
		       "drbd: invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2798 2799 2800
#ifdef MODULE
		return -EINVAL;
#else
2801
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
		printk(KERN_ERR
		       "drbd: unable to register block device major %d\n",
		       DRBD_MAJOR);
		return err;
	}

	register_reboot_notifier(&drbd_notifier);

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
2821
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
2822

2823
	rwlock_init(&global_state_lock);
2824
	INIT_LIST_HEAD(&drbd_resources);
2825 2826 2827 2828 2829 2830 2831

	err = drbd_genl_register();
	if (err) {
		printk(KERN_ERR "drbd: unable to register generic netlink family\n");
		goto fail;
	}

P
Philipp Reisner 已提交
2832 2833
	err = drbd_create_mempools();
	if (err)
2834
		goto fail;
P
Philipp Reisner 已提交
2835

2836
	err = -ENOMEM;
2837
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
2838 2839
	if (!drbd_proc)	{
		printk(KERN_ERR "drbd: unable to register proc file\n");
2840
		goto fail;
P
Philipp Reisner 已提交
2841 2842
	}

2843 2844 2845 2846 2847 2848 2849 2850
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
		printk(KERN_ERR "drbd: unable to create retry workqueue\n");
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860

	printk(KERN_INFO "drbd: initialized. "
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
	printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
	printk(KERN_INFO "drbd: registered as block device major %d\n",
		DRBD_MAJOR);

	return 0; /* Success! */

2861
fail:
P
Philipp Reisner 已提交
2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874
	drbd_cleanup();
	if (err == -ENOMEM)
		printk(KERN_ERR "drbd: ran out of memory\n");
	else
		printk(KERN_ERR "drbd: initialization failure\n");
	return err;
}

void drbd_free_bc(struct drbd_backing_dev *ldev)
{
	if (ldev == NULL)
		return;

2875 2876
	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
P
Philipp Reisner 已提交
2877

L
Lars Ellenberg 已提交
2878
	kfree(ldev->disk_conf);
P
Philipp Reisner 已提交
2879 2880 2881
	kfree(ldev);
}

2882
void drbd_free_sock(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2883
{
2884 2885 2886 2887 2888 2889
	if (connection->data.socket) {
		mutex_lock(&connection->data.mutex);
		kernel_sock_shutdown(connection->data.socket, SHUT_RDWR);
		sock_release(connection->data.socket);
		connection->data.socket = NULL;
		mutex_unlock(&connection->data.mutex);
P
Philipp Reisner 已提交
2890
	}
2891 2892 2893 2894 2895 2896
	if (connection->meta.socket) {
		mutex_lock(&connection->meta.mutex);
		kernel_sock_shutdown(connection->meta.socket, SHUT_RDWR);
		sock_release(connection->meta.socket);
		connection->meta.socket = NULL;
		mutex_unlock(&connection->meta.mutex);
P
Philipp Reisner 已提交
2897 2898 2899 2900 2901
	}
}

/* meta data management */

2902
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2903
{
2904
	struct drbd_device *device;
2905
	int vnr;
P
Philipp Reisner 已提交
2906

2907
	rcu_read_lock();
2908
	idr_for_each_entry(&connection->volumes, device, vnr) {
2909
		kref_get(&device->kref);
2910
		rcu_read_unlock();
2911
		drbd_md_sync(device);
2912
		kref_put(&device->kref, drbd_destroy_device);
2913 2914 2915
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
2916 2917
}

2918
/* aligned 4kByte */
P
Philipp Reisner 已提交
2919
struct meta_data_on_disk {
2920
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
2921 2922 2923 2924 2925 2926 2927
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
2928
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
2929
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
2930 2931
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
2932
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
2933

2934 2935 2936 2937 2938
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
2939 2940
} __packed;

2941 2942


2943
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
2944
{
2945
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
2946 2947 2948
	sector_t sector;
	int i;

2949
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
2950

2951
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
2952
	for (i = UI_CURRENT; i < UI_SIZE; i++)
2953 2954
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
2955
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
2956

2957 2958 2959
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
2960
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
2961
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
2962

2963 2964
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
2965

2966 2967
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
2968

2969 2970
	D_ASSERT(drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
2971

2972
	if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
P
Philipp Reisner 已提交
2973 2974
		/* this was a try anyways ... */
		dev_err(DEV, "meta data update failed!\n");
2975
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
2976
	}
2977 2978 2979 2980
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2981
 * @device:	DRBD device.
2982
 */
2983
void drbd_md_sync(struct drbd_device *device)
2984 2985 2986 2987 2988 2989 2990
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

2991
	del_timer(&device->md_sync_timer);
2992
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
2993
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
2994 2995 2996 2997
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
2998
	if (!get_ldev_if_state(device, D_FAILED))
2999 3000
		return;

3001
	buffer = drbd_md_get_buffer(device);
3002 3003 3004
	if (!buffer)
		goto out;

3005
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3006

3007
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3008
	 * since we updated it on metadata. */
3009
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3010

3011
	drbd_md_put_buffer(device);
3012
out:
3013
	put_ldev(device);
P
Philipp Reisner 已提交
3014 3015
}

3016
static int check_activity_log_stripe_size(struct drbd_device *device,
3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
	dev_err(DEV, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3061
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
	dev_err(DEV, "meta data offsets don't make sense: idx=%d "
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3141 3142
/**
 * drbd_md_read() - Reads in the meta data super block
3143
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3144 3145
 * @bdev:	Device from which the meta data should be read in.
 *
3146
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3147
 * something goes wrong.
3148
 *
3149
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3150
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3151
 */
3152
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3153 3154
{
	struct meta_data_on_disk *buffer;
3155
	u32 magic, flags;
P
Philipp Reisner 已提交
3156 3157
	int i, rv = NO_ERROR;

3158
	if (device->state.disk != D_DISKLESS)
3159
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3160

3161
	buffer = drbd_md_get_buffer(device);
3162
	if (!buffer)
3163
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3164

3165 3166
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3167 3168
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
P
Philipp Reisner 已提交
3169

3170
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
L
Lucas De Marchi 已提交
3171
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3172 3173 3174 3175 3176 3177
		   called BEFORE disk is attached */
		dev_err(DEV, "Error while reading metadata.\n");
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3178 3179 3180 3181 3182 3183 3184 3185 3186
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
		dev_err(DEV, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3187 3188

	rv = ERR_MD_INVALID;
3189
	if (magic != DRBD_MD_MAGIC_08) {
3190
		if (magic == DRBD_MD_MAGIC_07)
3191 3192 3193
			dev_err(DEV, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
		else
			dev_err(DEV, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3194 3195
		goto err;
	}
3196

3197 3198 3199
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
		dev_err(DEV, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3200 3201
		goto err;
	}
3202

3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3215
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3216
		goto err;
3217
	if (check_offsets_and_sizes(device, bdev))
3218 3219
		goto err;

P
Philipp Reisner 已提交
3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
		dev_err(DEV, "unexpected bm_offset: %d (expected %d)\n",
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
		dev_err(DEV, "unexpected md_size: %u (expected %u)\n",
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3231
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3232

3233
	spin_lock_irq(&first_peer_device(device)->connection->req_lock);
3234
	if (device->state.conn < C_CONNECTED) {
3235
		unsigned int peer;
3236
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3237
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3238
		device->peer_max_bio_size = peer;
3239
	}
3240
	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
P
Philipp Reisner 已提交
3241 3242

 err:
3243
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3244 3245 3246 3247 3248 3249

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3250
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3251 3252 3253 3254 3255
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3256
#ifdef DEBUG
3257
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3258
{
3259 3260 3261 3262
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3263 3264 3265
	}
}
#else
3266
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3267
{
3268 3269
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3270
}
3271
#endif
P
Philipp Reisner 已提交
3272

3273
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3274 3275 3276
{
	int i;

3277
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3278
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3279 3280
}

3281
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3282 3283
{
	if (idx == UI_CURRENT) {
3284
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3285 3286 3287 3288
			val |= 1;
		else
			val &= ~((u64)1);

3289
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3290 3291
	}

3292 3293
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3294 3295
}

3296
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3297 3298
{
	unsigned long flags;
3299 3300 3301
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3302
}
P
Philipp Reisner 已提交
3303

3304
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3305
{
3306
	unsigned long flags;
3307 3308 3309 3310
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3311
	}
3312 3313
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3314 3315 3316 3317
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3318
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3319 3320 3321 3322
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3323
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3324 3325
{
	u64 val;
3326 3327 3328 3329
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3330 3331
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3332 3333 3334

	if (bm_uuid)
		dev_warn(DEV, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3335

3336 3337 3338
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3339

3340
	drbd_print_uuids(device, "new current UUID");
3341
	/* get it to stable storage _now_ */
3342
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3343 3344
}

3345
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3346
{
3347
	unsigned long flags;
3348
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3349 3350
		return;

3351
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3352
	if (val == 0) {
3353 3354 3355
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3356
	} else {
3357
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3358 3359
		if (bm_uuid)
			dev_warn(DEV, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3360

3361
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3362
	}
3363
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3364

3365
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3366 3367 3368 3369
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3370
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3371 3372 3373
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3374
int drbd_bmio_set_n_write(struct drbd_device *device)
P
Philipp Reisner 已提交
3375 3376 3377
{
	int rv = -EIO;

3378 3379 3380 3381
	if (get_ldev_if_state(device, D_ATTACHING)) {
		drbd_md_set_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
		drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3382

3383
		rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3384 3385

		if (!rv) {
3386 3387
			drbd_md_clear_flag(device, MDF_FULL_SYNC);
			drbd_md_sync(device);
P
Philipp Reisner 已提交
3388 3389
		}

3390
		put_ldev(device);
P
Philipp Reisner 已提交
3391 3392 3393 3394 3395 3396 3397
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3398
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3399 3400 3401
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3402
int drbd_bmio_clear_n_write(struct drbd_device *device)
P
Philipp Reisner 已提交
3403 3404 3405
{
	int rv = -EIO;

3406 3407 3408 3409 3410
	drbd_resume_al(device);
	if (get_ldev_if_state(device, D_ATTACHING)) {
		drbd_bm_clear_all(device);
		rv = drbd_bm_write(device);
		put_ldev(device);
P
Philipp Reisner 已提交
3411 3412 3413 3414 3415
	}

	return rv;
}

3416
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3417 3418
{
	struct bm_io_work *work = container_of(w, struct bm_io_work, w);
3419
	struct drbd_device *device = w->device;
3420
	int rv = -EIO;
P
Philipp Reisner 已提交
3421

3422
	D_ASSERT(atomic_read(&device->ap_bio_cnt) == 0);
P
Philipp Reisner 已提交
3423

3424 3425 3426 3427 3428
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3429
	}
P
Philipp Reisner 已提交
3430

3431 3432
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3433 3434

	if (work->done)
3435
		work->done(device, rv);
P
Philipp Reisner 已提交
3436

3437
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3438
	work->why = NULL;
3439
	work->flags = 0;
P
Philipp Reisner 已提交
3440

3441
	return 0;
P
Philipp Reisner 已提交
3442 3443
}

3444
void drbd_ldev_destroy(struct drbd_device *device)
3445
{
3446 3447 3448 3449
	lc_destroy(device->resync);
	device->resync = NULL;
	lc_destroy(device->act_log);
	device->act_log = NULL;
3450
	__no_warn(local,
3451 3452
		drbd_free_bc(device->ldev);
		device->ldev = NULL;);
3453

3454
	clear_bit(GO_DISKLESS, &device->flags);
3455 3456
}

3457
static int w_go_diskless(struct drbd_work *w, int unused)
3458
{
3459
	struct drbd_device *device = w->device;
3460

3461
	D_ASSERT(device->state.disk == D_FAILED);
3462 3463
	/* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
	 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
3464 3465
	 * the protected members anymore, though, so once put_ldev reaches zero
	 * again, it will be safe to free them. */
3466 3467 3468 3469 3470 3471 3472 3473 3474

	/* Try to write changed bitmap pages, read errors may have just
	 * set some bits outside the area covered by the activity log.
	 *
	 * If we have an IO error during the bitmap writeout,
	 * we will want a full sync next time, just in case.
	 * (Do we want a specific meta data flag for this?)
	 *
	 * If that does not make it to stable storage either,
3475 3476 3477 3478 3479
	 * we cannot do anything about that anymore.
	 *
	 * We still need to check if both bitmap and ldev are present, we may
	 * end up here after a failed attach, before ldev was even assigned.
	 */
3480
	if (device->bitmap && device->ldev) {
3481 3482 3483 3484
		/* An interrupted resync or similar is allowed to recounts bits
		 * while we detach.
		 * Any modifications would not be expected anymore, though.
		 */
3485
		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
3486
					"detach", BM_LOCKED_TEST_ALLOWED)) {
3487 3488 3489
			if (test_bit(WAS_READ_ERROR, &device->flags)) {
				drbd_md_set_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
3490 3491 3492 3493
			}
		}
	}

3494
	drbd_force_state(device, NS(disk, D_DISKLESS));
3495
	return 0;
3496 3497
}

P
Philipp Reisner 已提交
3498 3499
/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3500
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3501 3502 3503 3504 3505 3506 3507 3508 3509
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
 */
3510
void drbd_queue_bitmap_io(struct drbd_device *device,
3511 3512
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3513
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3514
{
3515
	D_ASSERT(current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3516

3517 3518 3519 3520
	D_ASSERT(!test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(!test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(list_empty(&device->bm_io_work.w.list));
	if (device->bm_io_work.why)
P
Philipp Reisner 已提交
3521
		dev_err(DEV, "FIXME going to queue '%s' but '%s' still pending?\n",
3522
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3523

3524 3525 3526 3527
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3528

3529
	spin_lock_irq(&first_peer_device(device)->connection->req_lock);
3530 3531 3532
	set_bit(BITMAP_IO, &device->flags);
	if (atomic_read(&device->ap_bio_cnt) == 0) {
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3533
			drbd_queue_work(&first_peer_device(device)->connection->sender_work, &device->bm_io_work.w);
P
Philipp Reisner 已提交
3534
	}
3535
	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
P
Philipp Reisner 已提交
3536 3537 3538 3539
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3540
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3541 3542 3543 3544 3545 3546
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3547
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3548
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3549 3550 3551
{
	int rv;

3552
	D_ASSERT(current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3553

3554
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3555
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3556

3557 3558 3559
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3560

3561
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3562
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3563 3564 3565 3566

	return rv;
}

3567
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3568
{
3569 3570 3571
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3572 3573 3574
	}
}

3575
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3576
{
3577 3578 3579
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3580 3581 3582 3583 3584 3585 3586 3587 3588
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3589
	struct drbd_device *device = (struct drbd_device *) data;
P
Philipp Reisner 已提交
3590

3591
	/* must not double-queue! */
3592
	if (list_empty(&device->md_sync_work.list))
3593
		drbd_queue_work_front(&first_peer_device(device)->connection->sender_work, &device->md_sync_work);
P
Philipp Reisner 已提交
3594 3595
}

3596
static int w_md_sync(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3597
{
3598
	struct drbd_device *device = w->device;
3599

P
Philipp Reisner 已提交
3600
	dev_warn(DEV, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3601 3602
#ifdef DEBUG
	dev_warn(DEV, "last md_mark_dirty: %s:%u\n",
3603
		device->last_md_mark_dirty.func, device->last_md_mark_dirty.line);
3604
#endif
3605
	drbd_md_sync(device);
3606
	return 0;
P
Philipp Reisner 已提交
3607 3608
}

3609
const char *cmdname(enum drbd_packet cmd)
3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3639
		[P_SUPERSEDED]          = "Superseded",
3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3654
		[P_RETRY_WRITE]		= "RetryWrite",
3655 3656 3657
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3658 3659
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3660 3661 3662 3663 3664

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3665 3666
	};

3667
	/* too big for the array: 0xfffX */
3668 3669 3670 3671
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3672 3673
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3674
	if (cmd >= ARRAY_SIZE(cmdnames))
3675 3676 3677 3678
		return "Unknown";
	return cmdnames[cmd];
}

3679 3680
/**
 * drbd_wait_misc  -  wait for a request to make progress
3681
 * @device:	device associated with the request
3682 3683 3684
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3685
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3686
{
3687
	struct net_conf *nc;
3688 3689 3690
	DEFINE_WAIT(wait);
	long timeout;

3691
	rcu_read_lock();
3692
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3693 3694
	if (!nc) {
		rcu_read_unlock();
3695
		return -ETIMEDOUT;
3696 3697 3698
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3699

3700
	/* Indicate to wake up device->misc_wait on progress.  */
3701
	i->waiting = true;
3702
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3703
	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
3704
	timeout = schedule_timeout(timeout);
3705
	finish_wait(&device->misc_wait, &wait);
3706
	spin_lock_irq(&first_peer_device(device)->connection->req_lock);
3707
	if (!timeout || device->state.conn < C_CONNECTED)
3708 3709 3710 3711
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734
}

#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3735
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3755 3756
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3757 3758 3759 3760 3761 3762
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3763
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3764 3765 3766 3767 3768
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3769
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3770 3771 3772 3773 3774
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3775
		if (__ratelimit(&drbd_ratelimit_state))
P
Philipp Reisner 已提交
3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791
			dev_warn(DEV, "***Simulating %s failure\n",
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3792 3793 3794 3795
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);