drbd_main.c 107.1 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

#include <linux/module.h>
#include <linux/drbd.h>
#include <asm/uaccess.h>
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
35
#include <linux/mutex.h>
P
Philipp Reisner 已提交
36 37 38 39 40 41 42 43 44 45 46 47
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
48
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
49 50 51 52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>

#include <linux/drbd_limits.h>
#include "drbd_int.h"
55
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
56 57 58 59
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */

#include "drbd_vli.h"

60
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
61
static int drbd_open(struct block_device *bdev, fmode_t mode);
62
static void drbd_release(struct gendisk *gd, fmode_t mode);
P
Philipp Reisner 已提交
63
static void md_sync_timer_fn(unsigned long data);
64
static int w_bitmap_io(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
65 66 67 68 69 70

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
71
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
72
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
101
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
102 103
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
104 105 106 107 108 109 110 111 112 113 114
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
char usermode_helper[80] = "/sbin/drbdadm";

module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
115
struct idr drbd_devices;
116
struct list_head drbd_resources;
P
Philipp Reisner 已提交
117 118

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
119
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
120 121 122 123
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
124
mempool_t *drbd_md_io_page_pool;
125
struct bio_set *drbd_md_io_bio_set;
P
Philipp Reisner 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138 139

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

140
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
141 142 143 144
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
145

146 147 148
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
149

150 151
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
152

153 154 155 156
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
157 158
}

P
Philipp Reisner 已提交
159 160 161 162
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
163
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
164 165 166
{
	int io_allowed;

167 168
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
169
	if (!io_allowed) {
170 171
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
172 173 174 175 176
	}
	return io_allowed;
}

#endif
177

P
Philipp Reisner 已提交
178
/**
179
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
180
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
181 182 183 184
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
185 186
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
187
 */
188
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
189
		unsigned int set_size)
P
Philipp Reisner 已提交
190 191
{
	struct drbd_request *r;
192 193 194
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
195

196
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
197

198
	/* find oldest not yet barrier-acked write request,
199
	 * count writes in its epoch. */
200
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
201
		const unsigned s = r->rq_state;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
220
		}
P
Philipp Reisner 已提交
221
	}
222

P
Philipp Reisner 已提交
223
	/* first some paranoia code */
224
	if (req == NULL) {
225
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
226
			 barrier_nr);
P
Philipp Reisner 已提交
227 228
		goto bail;
	}
229
	if (expect_epoch != barrier_nr) {
230
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
231
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
232
		goto bail;
233 234
	}

235
	if (expect_size != set_size) {
236
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
237
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
238 239 240
		goto bail;
	}

241 242 243 244
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
245
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
246 247
		if (req->epoch == expect_epoch)
			break;
248
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
249 250 251
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
252
	}
253
	spin_unlock_irq(&connection->resource->req_lock);
254

P
Philipp Reisner 已提交
255 256 257
	return;

bail:
258
	spin_unlock_irq(&connection->resource->req_lock);
259
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
260 261 262 263
}


/**
264
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
265
 * @device:	DRBD device.
266
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
267
 *
268 269
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
270
 */
271
/* must hold resource->req_lock */
272
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
273
{
274
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
275

276
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
277 278
		_req_mod(req, what);
}
279

280
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
281
{
282
	spin_lock_irq(&connection->resource->req_lock);
283
	_tl_restart(connection, what);
284
	spin_unlock_irq(&connection->resource->req_lock);
285
}
P
Philipp Reisner 已提交
286 287 288

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
289
 * @device:	DRBD device.
P
Philipp Reisner 已提交
290 291 292 293 294
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
295
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
296
{
297
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
298
}
299

300
/**
301 302
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
303
 */
304
void tl_abort_disk_io(struct drbd_device *device)
305
{
306
	struct drbd_connection *connection = first_peer_device(device)->connection;
307
	struct drbd_request *req, *r;
308

309
	spin_lock_irq(&connection->resource->req_lock);
310
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
311 312
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
313
		if (req->device != device)
314 315
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
316
	}
317
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
318 319 320 321 322
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
323
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
324 325 326
	unsigned long flags;
	int retval;

327
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
328
		 thi->name[0],
329
		 resource->name);
330

P
Philipp Reisner 已提交
331 332 333 334 335
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

336
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
337 338 339
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
340
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
341
	 * t_state check and assignment needs to be within the same spinlock,
342 343
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
344 345
	 */

346
	if (thi->t_state == RESTARTING) {
347
		drbd_info(resource, "Restarting %s thread\n", thi->name);
348
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
349 350 351 352 353
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
354
	thi->t_state = NONE;
P
Philipp Reisner 已提交
355
	smp_mb();
L
Lars Ellenberg 已提交
356
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
357 358
	spin_unlock_irqrestore(&thi->t_lock, flags);

359
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
360 361

	/* Release mod reference taken when thread was started */
362

363 364 365
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
366 367 368 369
	module_put(THIS_MODULE);
	return retval;
}

370
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
371
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
372 373 374
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
375
	thi->t_state = NONE;
P
Philipp Reisner 已提交
376
	thi->function = func;
377 378
	thi->resource = resource;
	thi->connection = NULL;
379
	thi->name = name;
P
Philipp Reisner 已提交
380 381 382 383
}

int drbd_thread_start(struct drbd_thread *thi)
{
384
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
385 386 387 388 389 390 391 392
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
393
	case NONE:
394
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
395
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
396 397 398

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
399
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
400
			spin_unlock_irqrestore(&thi->t_lock, flags);
401
			return false;
P
Philipp Reisner 已提交
402 403
		}

404 405 406
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
407

P
Philipp Reisner 已提交
408 409
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
410
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
411 412 413 414
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
415
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
416 417

		if (IS_ERR(nt)) {
418
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
419

420 421 422
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
423
			module_put(THIS_MODULE);
424
			return false;
P
Philipp Reisner 已提交
425 426 427
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
428
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
429 430 431
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
432 433
	case EXITING:
		thi->t_state = RESTARTING;
434
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
435
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
436
		/* fall through */
437 438
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
439 440 441 442 443
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

444
	return true;
P
Philipp Reisner 已提交
445 446 447 448 449 450 451
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

452
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
453 454 455 456

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

457
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

483
int conn_lowest_minor(struct drbd_connection *connection)
484
{
485 486
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
487

488
	rcu_read_lock();
489 490 491
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
492 493
	rcu_read_unlock();

494
	return minor;
495
}
496

P
Philipp Reisner 已提交
497 498 499 500
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
501
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
502 503
 * DRBD's performance. May be overwritten by user's configuration.
 */
504
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
505
{
506
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
507

508 509 510 511
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
512

513 514 515 516
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
517
		}
518 519 520 521 522 523 524 525 526 527 528 529
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
530
	}
531
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
532 533 534 535
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
536
 * @device:	DRBD device.
537
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
538 539 540 541
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
542
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
543
{
544
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
545
	struct task_struct *p = current;
546

P
Philipp Reisner 已提交
547 548 549
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
550
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
551
}
552 553
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
554 555
#endif

556 557 558 559 560 561 562
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
563
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
564
{
565
	if (connection->agreed_pro_version >= 100) {
566 567 568 569 570 571 572 573
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
574
}
P
Philipp Reisner 已提交
575

576
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
577 578 579 580
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
581
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
582
}
P
Philipp Reisner 已提交
583

584
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
585 586
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
587
	h->command = cpu_to_be16(cmd);
588
	h->length = cpu_to_be32(size);
589
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
590
}
P
Philipp Reisner 已提交
591

592 593 594 595 596 597 598 599 600 601
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
602

603
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
604
				   void *buffer, enum drbd_packet cmd, int size)
605
{
606
	if (connection->agreed_pro_version >= 100)
607
		return prepare_header100(buffer, cmd, size, vnr);
608
	else if (connection->agreed_pro_version >= 95 &&
609
		 size > DRBD_MAX_SIZE_H80_PACKET)
610
		return prepare_header95(buffer, cmd, size);
611
	else
612
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
613 614
}

615
static void *__conn_prepare_command(struct drbd_connection *connection,
616
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
617
{
618 619
	if (!sock->socket)
		return NULL;
620
	return sock->sbuf + drbd_header_size(connection);
621
}
P
Philipp Reisner 已提交
622

623
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
624
{
625
	void *p;
P
Philipp Reisner 已提交
626

627
	mutex_lock(&sock->mutex);
628
	p = __conn_prepare_command(connection, sock);
629
	if (!p)
630
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
631

632
	return p;
P
Philipp Reisner 已提交
633 634
}

635
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
636
{
637
	return conn_prepare_command(peer_device->connection, sock);
638
}
P
Philipp Reisner 已提交
639

640
static int __send_command(struct drbd_connection *connection, int vnr,
641 642 643 644 645 646
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
647

648 649 650 651 652 653 654 655 656
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

657
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
658
				      header_size + size);
659
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
660 661
			    msg_flags);
	if (data && !err)
662
		err = drbd_send_all(connection, sock->socket, data, size, 0);
663 664 665 666 667
	/* DRBD protocol "pings" are latency critical.
	 * This is supposed to trigger tcp_push_pending_frames() */
	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
		drbd_tcp_nodelay(sock->socket);

668 669 670
	return err;
}

671
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
672 673 674
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
675
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
676 677
}

678
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
679 680 681 682
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
683

684
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
685 686 687 688
	mutex_unlock(&sock->mutex);
	return err;
}

689
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
690 691 692 693 694
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

695 696
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
697 698 699
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
700

701
int drbd_send_ping(struct drbd_connection *connection)
702
{
703 704
	struct drbd_socket *sock;

705 706
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
707
		return -EIO;
708
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
709
}
P
Philipp Reisner 已提交
710

711
int drbd_send_ping_ack(struct drbd_connection *connection)
712
{
713 714
	struct drbd_socket *sock;

715 716
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
717
		return -EIO;
718
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
719 720
}

721
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
722
{
723
	struct drbd_socket *sock;
724
	struct p_rs_param_95 *p;
725
	int size;
726
	const int apv = peer_device->connection->agreed_pro_version;
727
	enum drbd_packet cmd;
728
	struct net_conf *nc;
P
Philipp Reisner 已提交
729
	struct disk_conf *dc;
730

731 732
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
733 734
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
735

736
	rcu_read_lock();
737
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
738 739 740

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
741
			+ strlen(nc->verify_alg) + 1
742 743
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
744

745
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
746

747 748
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
749

750 751
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
752
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
753 754 755 756
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
757
		put_ldev(peer_device->device);
758
	} else {
759
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
760 761 762 763 764
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
765

766
	if (apv >= 88)
767
		strcpy(p->verify_alg, nc->verify_alg);
768
	if (apv >= 89)
769 770
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
771

772
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
773 774
}

775
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
776
{
777
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
778
	struct p_protocol *p;
779
	struct net_conf *nc;
780
	int size, cf;
P
Philipp Reisner 已提交
781

782 783
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
784 785
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
786

787
	rcu_read_lock();
788
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
789

790
	if (nc->tentative && connection->agreed_pro_version < 92) {
791 792
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
793
		drbd_err(connection, "--dry-run is not supported by peer");
794 795
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
796

797
	size = sizeof(*p);
798
	if (connection->agreed_pro_version >= 87)
799
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
800

801 802 803 804 805
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
806
	cf = 0;
807 808
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
809
	if (nc->tentative)
810
		cf |= CF_DRY_RUN;
811 812
	p->conn_flags    = cpu_to_be32(cf);

813
	if (connection->agreed_pro_version >= 87)
814 815
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
816

817
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
818 819
}

820
int drbd_send_protocol(struct drbd_connection *connection)
821 822 823
{
	int err;

824 825 826
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
827 828

	return err;
P
Philipp Reisner 已提交
829 830
}

831
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
832
{
833
	struct drbd_device *device = peer_device->device;
834 835
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
836 837
	int i;

838
	if (!get_ldev_if_state(device, D_NEGOTIATING))
839
		return 0;
P
Philipp Reisner 已提交
840

841 842
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
843
	if (!p) {
844
		put_ldev(device);
845 846
		return -EIO;
	}
847
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
848
	for (i = UI_CURRENT; i < UI_SIZE; i++)
849 850
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
851

852 853
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
854
	rcu_read_lock();
855
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
856
	rcu_read_unlock();
857 858
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
859
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
860

861
	put_ldev(device);
862
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
863 864
}

865
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
866
{
867
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
868 869
}

870
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
871
{
872
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
873 874
}

875
void drbd_print_uuids(struct drbd_device *device, const char *text)
876
{
877 878
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
879
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
880 881 882 883 884
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
885
		put_ldev(device);
886
	} else {
887
		drbd_info(device, "%s effective data uuid: %016llX\n",
888
				text,
889
				(unsigned long long)device->ed_uuid);
890 891 892
	}
}

893
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
894
{
895
	struct drbd_device *device = peer_device->device;
896 897
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
898 899
	u64 uuid;

900
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
901

902
	uuid = device->ldev->md.uuid[UI_BITMAP];
903 904 905 906
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
907 908 909
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
910

911 912
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
913 914
	if (p) {
		p->uuid = cpu_to_be64(uuid);
915
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
916
	}
P
Philipp Reisner 已提交
917 918
}

919
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
920
{
921
	struct drbd_device *device = peer_device->device;
922 923
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
924
	sector_t d_size, u_size;
925 926
	int q_order_type;
	unsigned int max_bio_size;
P
Philipp Reisner 已提交
927

928
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
929
		D_ASSERT(device, device->ldev->backing_bdev);
930
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
931
		rcu_read_lock();
932
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
933
		rcu_read_unlock();
934 935
		q_order_type = drbd_queue_order_type(device);
		max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9;
936
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
937
		put_ldev(device);
P
Philipp Reisner 已提交
938 939 940 941
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
942
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
P
Philipp Reisner 已提交
943 944
	}

945 946
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
947 948
	if (!p)
		return -EIO;
949

950
	if (peer_device->connection->agreed_pro_version <= 94)
951
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
952
	else if (peer_device->connection->agreed_pro_version < 100)
953
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
954

955 956
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
957
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
958 959 960
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
961
	return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
962 963 964
}

/**
965
 * drbd_send_current_state() - Sends the drbd state to the peer
966
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
967
 */
968
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
969
{
970
	struct drbd_socket *sock;
971
	struct p_state *p;
P
Philipp Reisner 已提交
972

973 974
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
975 976
	if (!p)
		return -EIO;
977 978
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
979 980
}

981 982
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
983
 * @peer_device:      DRBD peer device.
984
 * @state:     the state to send, not necessarily the current state.
985 986 987 988 989 990
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
991
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
992
{
993 994
	struct drbd_socket *sock;
	struct p_state *p;
995

996 997
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
998 999 1000
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
1001
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1002
}
1003

1004
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1005 1006 1007
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1008

1009 1010
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1011 1012 1013 1014
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1015
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1016
}
1017

1018
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1019
{
1020 1021 1022
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1023

1024 1025 1026
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1027 1028 1029 1030
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1031
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1032 1033
}

1034
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1035
{
1036 1037
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1038

1039 1040
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1041 1042
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1043
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1044
	}
P
Philipp Reisner 已提交
1045 1046
}

1047
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1048
{
1049 1050
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1051
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1052

1053 1054
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1055 1056
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1057
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1058
	}
P
Philipp Reisner 已提交
1059 1060
}

1061
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1062
{
1063 1064 1065
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1066

1067 1068 1069 1070
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1071

1072 1073 1074 1075
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1076 1077
}

1078
static int fill_bitmap_rle_bits(struct drbd_device *device,
1079 1080 1081
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1082 1083 1084 1085 1086 1087 1088
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1089
	int bits, use_rle;
P
Philipp Reisner 已提交
1090 1091

	/* may we use this feature? */
1092
	rcu_read_lock();
1093
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1094
	rcu_read_unlock();
1095
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1096
		return 0;
P
Philipp Reisner 已提交
1097 1098 1099 1100 1101

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1102 1103
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1115 1116
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1117 1118 1119 1120 1121 1122 1123 1124
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1125
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1126 1127 1128 1129
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1130
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1131 1132 1133 1134 1135
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1136
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1137 1138 1139 1140 1141 1142 1143 1144
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1145
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1170
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1171 1172 1173 1174

	return len;
}

1175 1176 1177 1178 1179 1180 1181
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1182
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1183
{
1184 1185
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1186
	struct p_compressed_bm *p = sock->sbuf + header_size;
1187
	int len, err;
P
Philipp Reisner 已提交
1188

1189
	len = fill_bitmap_rle_bits(device, p,
1190
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1191
	if (len < 0)
1192
		return -EIO;
P
Philipp Reisner 已提交
1193 1194

	if (len) {
1195
		dcbp_set_code(p, RLE_VLI_Bits);
1196
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1197 1198
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1199
		c->packets[0]++;
1200
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1201 1202 1203 1204 1205 1206

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1207 1208
		unsigned int data_size;
		unsigned long num_words;
1209
		unsigned long *p = sock->sbuf + header_size;
1210 1211

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1212
		num_words = min_t(size_t, data_size / sizeof(*p),
1213
				  c->bm_words - c->word_offset);
1214
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1215
		if (len)
1216
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1217
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1218 1219 1220 1221
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1222
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1223 1224 1225 1226

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1227
	if (!err) {
1228
		if (len == 0) {
1229
			INFO_bm_xfer_stats(device, "send", c);
1230 1231 1232 1233 1234
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1235 1236 1237
}

/* See the comment at receive_bitmap() */
1238
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1239 1240
{
	struct bm_xfer_ctx c;
1241
	int err;
P
Philipp Reisner 已提交
1242

1243
	if (!expect(device->bitmap))
1244
		return false;
P
Philipp Reisner 已提交
1245

1246 1247
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1248
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1249 1250
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1251 1252 1253
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1254
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1255
			} else {
1256 1257
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1258 1259
			}
		}
1260
		put_ldev(device);
P
Philipp Reisner 已提交
1261 1262 1263
	}

	c = (struct bm_xfer_ctx) {
1264 1265
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1266 1267 1268
	};

	do {
1269
		err = send_bitmap_rle_or_plain(device, &c);
1270
	} while (err > 0);
P
Philipp Reisner 已提交
1271

1272
	return err == 0;
P
Philipp Reisner 已提交
1273 1274
}

1275
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1276
{
1277
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1278
	int err = -1;
P
Philipp Reisner 已提交
1279

1280 1281
	mutex_lock(&sock->mutex);
	if (sock->socket)
1282
		err = !_drbd_send_bitmap(device);
1283
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1284 1285 1286
	return err;
}

1287
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1288
{
1289 1290
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1291

1292
	if (connection->cstate < C_WF_REPORT_PARAMS)
1293
		return;
P
Philipp Reisner 已提交
1294

1295 1296
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1297 1298 1299 1300
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1301
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1302 1303 1304 1305
}

/**
 * _drbd_send_ack() - Sends an ack packet
1306
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1307 1308 1309 1310 1311
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1312
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1313
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1314
{
1315 1316
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1317

1318
	if (peer_device->device->state.conn < C_CONNECTED)
1319
		return -EIO;
P
Philipp Reisner 已提交
1320

1321 1322
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1323
	if (!p)
1324
		return -EIO;
1325 1326 1327
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1328 1329
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1330 1331
}

1332 1333 1334
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1335
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1336
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1337
{
1338 1339 1340
	if (peer_device->connection->peer_integrity_tfm)
		data_size -= crypto_hash_digestsize(peer_device->connection->peer_integrity_tfm);
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1341
		       dp->block_id);
P
Philipp Reisner 已提交
1342 1343
}

1344
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1345
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1346
{
1347
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1348 1349 1350 1351
}

/**
 * drbd_send_ack() - Sends an ack packet
1352
 * @device:	DRBD device
1353 1354
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1355
 */
1356
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1357
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1358
{
1359
	return _drbd_send_ack(peer_device, cmd,
1360 1361 1362
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1363 1364 1365 1366
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1367
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1368 1369
		     sector_t sector, int blksize, u64 block_id)
{
1370
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1371 1372 1373 1374 1375
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1376
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1377 1378
		       sector_t sector, int size, u64 block_id)
{
1379 1380
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1381

1382 1383
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1384 1385 1386 1387 1388
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1389
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1390 1391
}

1392
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1393
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1394
{
1395 1396
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1397

1398
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1399

1400 1401
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1402 1403 1404 1405 1406
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1407
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1408 1409
}

1410
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1411
{
1412 1413
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1414

1415 1416
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1417 1418 1419 1420 1421
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1422
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1423 1424 1425
}

/* called on sndtimeo
1426 1427
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1428
 */
1429
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1430 1431
{
	int drop_it;
1432
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1433

1434 1435 1436 1437
	drop_it =   connection->meta.socket == sock
		|| !connection->asender.task
		|| get_t_state(&connection->asender) != RUNNING
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1438 1439

	if (drop_it)
1440
		return true;
P
Philipp Reisner 已提交
1441

1442
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1443
	if (!drop_it) {
1444
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1445 1446
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1447 1448
	}

1449
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1450 1451
}

1452
static void drbd_update_congested(struct drbd_connection *connection)
1453
{
1454
	struct sock *sk = connection->data.socket->sk;
1455
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1456
		set_bit(NET_CONGESTED, &connection->flags);
1457 1458
}

P
Philipp Reisner 已提交
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1480
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1481
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1482
{
1483 1484 1485 1486
	struct socket *socket;
	void *addr;
	int err;

1487
	socket = peer_device->connection->data.socket;
1488
	addr = kmap(page) + offset;
1489
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1490
	kunmap(page);
1491
	if (!err)
1492
		peer_device->device->send_cnt += size >> 9;
1493
	return err;
P
Philipp Reisner 已提交
1494 1495
}

1496
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1497
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1498
{
1499
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1500 1501
	mm_segment_t oldfs = get_fs();
	int len = size;
1502
	int err = -EIO;
P
Philipp Reisner 已提交
1503 1504 1505 1506 1507 1508 1509 1510

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1511
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1512

L
Lars Ellenberg 已提交
1513
	msg_flags |= MSG_NOSIGNAL;
1514
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1515 1516
	set_fs(KERNEL_DS);
	do {
1517 1518 1519
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1520
		if (sent <= 0) {
1521
			if (sent == -EAGAIN) {
1522
				if (we_should_drop_the_connection(peer_device->connection, socket))
1523 1524 1525
					break;
				continue;
			}
1526
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1527
			     __func__, (int)size, len, sent);
1528 1529
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1530 1531 1532 1533
			break;
		}
		len    -= sent;
		offset += sent;
1534
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
P
Philipp Reisner 已提交
1535
	set_fs(oldfs);
1536
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1537

1538 1539
	if (len == 0) {
		err = 0;
1540
		peer_device->device->send_cnt += size >> 9;
1541 1542
	}
	return err;
P
Philipp Reisner 已提交
1543 1544
}

1545
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1546
{
1547 1548 1549
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1550
	/* hint all but last page with MSG_MORE */
1551
	bio_for_each_segment(bvec, bio, iter) {
1552 1553
		int err;

1554
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1555
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1556
					 bio_iter_last(bvec, iter)
1557
					 ? 0 : MSG_MORE);
1558 1559
		if (err)
			return err;
P
Philipp Reisner 已提交
1560
	}
1561
	return 0;
P
Philipp Reisner 已提交
1562 1563
}

1564
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1565
{
1566 1567 1568
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1569
	/* hint all but last page with MSG_MORE */
1570
	bio_for_each_segment(bvec, bio, iter) {
1571 1572
		int err;

1573
		err = _drbd_send_page(peer_device, bvec.bv_page,
1574
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1575
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1576 1577
		if (err)
			return err;
P
Philipp Reisner 已提交
1578
	}
1579
	return 0;
P
Philipp Reisner 已提交
1580 1581
}

1582
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1583
			    struct drbd_peer_request *peer_req)
1584
{
1585 1586
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1587
	int err;
1588

L
Lars Ellenberg 已提交
1589
	/* hint all but last page with MSG_MORE */
1590 1591
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1592

1593
		err = _drbd_send_page(peer_device, page, 0, l,
1594 1595 1596
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1597 1598
		len -= l;
	}
1599
	return 0;
1600 1601
}

1602
static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw)
1603
{
1604
	if (connection->agreed_pro_version >= 95)
1605 1606 1607 1608 1609
		return  (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bi_rw & REQ_FUA ? DP_FUA : 0) |
			(bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
			(bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
	else
J
Jens Axboe 已提交
1610
		return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
1611 1612
}

1613 1614
/* Used to send write or TRIM aka REQ_DISCARD requests
 * R_PRIMARY -> Peer	(P_DATA, P_TRIM)
P
Philipp Reisner 已提交
1615
 */
1616
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1617
{
1618
	struct drbd_device *device = peer_device->device;
1619 1620
	struct drbd_socket *sock;
	struct p_data *p;
P
Philipp Reisner 已提交
1621 1622
	unsigned int dp_flags = 0;
	int dgs;
1623
	int err;
P
Philipp Reisner 已提交
1624

1625 1626 1627 1628
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1629

1630 1631 1632 1633
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1634
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
1635
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
1636 1637
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1638
		dp_flags |= DP_MAY_SET_IN_SYNC;
1639
	if (peer_device->connection->agreed_pro_version >= 100) {
1640 1641 1642 1643 1644
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
		if (req->rq_state & RQ_EXP_WRITE_ACK)
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1645
	p->dp_flags = cpu_to_be32(dp_flags);
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655

	if (dp_flags & DP_DISCARD) {
		struct p_trim *t = (struct p_trim*)p;
		t->size = cpu_to_be32(req->i.size);
		err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0);
		goto out;
	}

	/* our digest is still only over the payload.
	 * TRIM does not carry any payload. */
1656
	if (dgs)
1657 1658
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + dgs, NULL, req->i.size);
1659
	if (!err) {
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1671
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || dgs)
1672
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1673
		else
1674
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1675 1676 1677

		/* double check digest, sometimes buffers have been modified in flight. */
		if (dgs > 0 && dgs <= 64) {
B
Bart Van Assche 已提交
1678
			/* 64 byte, 512 bit, is the largest digest size
1679 1680
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1681
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1682
			if (memcmp(p + 1, digest, dgs)) {
1683
				drbd_warn(device,
1684
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1685
					(unsigned long long)req->i.sector, req->i.size);
1686 1687 1688 1689
			}
		} /* else if (dgs > 64) {
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1690
	}
1691
out:
1692
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1693

1694
	return err;
P
Philipp Reisner 已提交
1695 1696 1697 1698 1699 1700
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1701
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1702
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1703
{
1704
	struct drbd_device *device = peer_device->device;
1705 1706
	struct drbd_socket *sock;
	struct p_data *p;
1707
	int err;
P
Philipp Reisner 已提交
1708 1709
	int dgs;

1710 1711
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1712

1713 1714
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1715

1716 1717 1718 1719 1720
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1721
	p->dp_flags = 0;
1722
	if (dgs)
1723 1724
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + dgs, NULL, peer_req->i.size);
1725
	if (!err)
1726
		err = _drbd_send_zc_ee(peer_device, peer_req);
1727
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1728

1729
	return err;
P
Philipp Reisner 已提交
1730 1731
}

1732
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1733
{
1734 1735
	struct drbd_socket *sock;
	struct p_block_desc *p;
1736

1737 1738
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1739 1740 1741 1742
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1743
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1744 1745
}

P
Philipp Reisner 已提交
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1762
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1763 1764 1765 1766 1767 1768 1769
	      void *buf, size_t size, unsigned msg_flags)
{
	struct kvec iov;
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1770
		return -EBADR;
P
Philipp Reisner 已提交
1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782

	/* THINK  if (signal_pending) return ... ? */

	iov.iov_base = buf;
	iov.iov_len  = size;

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

1783
	if (sock == connection->data.socket) {
1784
		rcu_read_lock();
1785
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1786
		rcu_read_unlock();
1787
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
	}
	do {
		/* STRANGE
		 * tcp_sendmsg does _not_ use its size parameter at all ?
		 *
		 * -EAGAIN on timeout, -EINTR on signal.
		 */
/* THINK
 * do we need to block DRBD_SIG if sock == &meta.socket ??
 * otherwise wake_asender() might interrupt some send_*Ack !
 */
		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
		if (rv == -EAGAIN) {
1801
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
		iov.iov_base += rv;
		iov.iov_len  -= rv;
	} while (sent < size);

1817 1818
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1819 1820 1821

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1822
			drbd_err(connection, "%s_sendmsg returned %d\n",
1823
				 sock == connection->meta.socket ? "msock" : "sock",
1824
				 rv);
1825
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1826
		} else
1827
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1828 1829 1830 1831 1832
	}

	return sent;
}

1833 1834 1835 1836 1837
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1838
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1839 1840 1841 1842
		  size_t size, unsigned msg_flags)
{
	int err;

1843
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1844 1845 1846 1847 1848 1849 1850
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1851 1852
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1853
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1854 1855 1856
	unsigned long flags;
	int rv = 0;

1857
	mutex_lock(&drbd_main_mutex);
1858
	spin_lock_irqsave(&device->resource->req_lock, flags);
1859
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1860 1861
	 * and no race with updating open_cnt */

1862
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1863 1864 1865 1866 1867 1868 1869
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1870
		device->open_cnt++;
1871
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1872
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1873 1874 1875 1876

	return rv;
}

1877
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1878
{
1879
	struct drbd_device *device = gd->private_data;
1880
	mutex_lock(&drbd_main_mutex);
1881
	device->open_cnt--;
1882
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1883 1884
}

1885
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1886
{
1887 1888
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1889
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1890 1891 1892 1893 1894 1895 1896 1897
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1898
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1899 1900 1901 1902
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
	atomic_set(&device->md_io_in_use, 0);

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->bm_io_work.w.cb = w_bitmap_io;

	init_timer(&device->resync_timer);
	init_timer(&device->md_sync_timer);
	init_timer(&device->start_resync_timer);
	init_timer(&device->request_timer);
	device->resync_timer.function = resync_timer_fn;
	device->resync_timer.data = (unsigned long) device;
	device->md_sync_timer.function = md_sync_timer_fn;
	device->md_sync_timer.data = (unsigned long) device;
	device->start_resync_timer.function = start_resync_timer_fn;
	device->start_resync_timer.data = (unsigned long) device;
	device->request_timer.function = request_timer_fn;
	device->request_timer.data = (unsigned long) device;

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
1961
{
1962
	int i;
1963
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
1964
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1965
				first_peer_device(device)->connection->receiver.t_state);
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
1979
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1980 1981
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
1982
	}
1983
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
1984

1985 1986
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
1987
		/* maybe never allocated. */
1988 1989
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
1990 1991
	}

1992
	drbd_free_ldev(device->ldev);
1993
	device->ldev = NULL;
1994

1995
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
1996

1997 1998 1999 2000 2001 2002 2003 2004 2005
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
2006

2007
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2022
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2023

2024 2025
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2026 2027
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2041
	drbd_md_io_bio_set   = NULL;
2042
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2056
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2057 2058 2059 2060 2061 2062 2063 2064 2065
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2066
	drbd_md_io_page_pool = NULL;
2067
	drbd_md_io_bio_set   = NULL;
P
Philipp Reisner 已提交
2068 2069 2070 2071 2072 2073 2074 2075

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2076
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2091 2092 2093 2094
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2095 2096 2097 2098
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

P
Philipp Reisner 已提交
2099 2100 2101 2102 2103 2104 2105
	drbd_request_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
	if (drbd_request_mempool == NULL)
		goto Enomem;

	drbd_ee_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2106
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

2128
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2129 2130 2131
{
	int rr;

2132
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2133
	if (rr)
2134
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2135

2136
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2137
	if (rr)
2138
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2139

2140
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2141
	if (rr)
2142
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2143

2144
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2145
	if (rr)
2146
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2147

2148
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2149
	if (rr)
2150
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2151 2152
}

2153
/* caution. no locking. */
2154
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2155
{
2156
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2157 2158
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
P
Philipp Reisner 已提交
2159

2160
	del_timer_sync(&device->request_timer);
2161

P
Philipp Reisner 已提交
2162
	/* paranoia asserts */
2163
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2164 2165 2166 2167 2168
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2169 2170
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2171

2172
	drbd_free_ldev(device->ldev);
2173
	device->ldev = NULL;
P
Philipp Reisner 已提交
2174

2175
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2176

2177 2178
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2179

2180 2181
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2182

2183 2184 2185 2186 2187 2188
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
	__free_page(device->md_io_page);
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2189
	kfree(first_peer_device(device));
2190
	kfree(device);
2191

2192 2193 2194
	for_each_connection(connection, resource)
		kref_put(&connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2195 2196
}

2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2219
		struct drbd_device *device = req->device;
2220 2221
		struct bio *bio = req->master_bio;
		unsigned long start_time = req->start_time;
2222 2223
		bool expected;

2224
		expected =
2225 2226 2227 2228 2229 2230
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2231
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2232 2233 2234 2235 2236 2237 2238
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2239
		 */
2240
		kref_put(&req->kref, drbd_req_destroy);
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2255 2256
		inc_ap_bio(device);
		__drbd_make_request(device, bio, start_time);
2257 2258 2259
	}
}

2260
void drbd_restart_request(struct drbd_request *req)
2261 2262 2263 2264 2265 2266 2267 2268 2269
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2270
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2271

2272
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2273 2274
}

2275 2276 2277 2278 2279
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2280
	idr_destroy(&resource->devices);
2281
	free_cpumask_var(resource->cpu_mask);
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
	kfree(resource->name);
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
		kref_put(&connection->kref, drbd_destroy_connection);
	}
	kref_put(&resource->kref, drbd_destroy_resource);
}
2296

P
Philipp Reisner 已提交
2297 2298 2299
static void drbd_cleanup(void)
{
	unsigned int i;
2300
	struct drbd_device *device;
2301
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2302

2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2314 2315
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2316

2317
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2318

2319
	idr_for_each_entry(&drbd_devices, device, i)
2320
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2321

P
Philipp Reisner 已提交
2322
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2323 2324 2325
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2326
	}
P
Philipp Reisner 已提交
2327

2328
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2329 2330
	unregister_blkdev(DRBD_MAJOR, "drbd");

2331
	idr_destroy(&drbd_devices);
2332

P
Philipp Reisner 已提交
2333 2334 2335 2336
	printk(KERN_INFO "drbd: module cleanup done.\n");
}

/**
2337
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2338
 * @congested_data:	User data
2339
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2340 2341 2342 2343 2344
 *
 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2345
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2346 2347 2348 2349
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2350
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2351 2352 2353 2354 2355 2356
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2357
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2358 2359 2360 2361 2362 2363
		r |= (1 << BDI_async_congested);
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2364
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2365 2366
			r |= (1 << BDI_sync_congested);
		else
2367
			put_ldev(device);
2368 2369 2370 2371 2372
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2373 2374
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
P
Philipp Reisner 已提交
2375
		r = bdi_congested(&q->backing_dev_info, bdi_bits);
2376
		put_ldev(device);
P
Philipp Reisner 已提交
2377 2378 2379 2380
		if (r)
			reason = 'b';
	}

2381 2382
	if (bdi_bits & (1 << BDI_async_congested) &&
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
P
Philipp Reisner 已提交
2383 2384 2385 2386 2387
		r |= (1 << BDI_async_congested);
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2388
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2389 2390 2391
	return r;
}

2392 2393 2394 2395
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2396
	init_waitqueue_head(&wq->q_wait);
2397 2398
}

2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2423
struct drbd_resource *drbd_find_resource(const char *name)
2424
{
2425
	struct drbd_resource *resource;
2426

2427 2428 2429
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2430
	rcu_read_lock();
2431 2432
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2433
			kref_get(&resource->kref);
2434
			goto found;
2435
		}
2436
	}
2437
	resource = NULL;
2438
found:
P
Philipp Reisner 已提交
2439
	rcu_read_unlock();
2440
	return resource;
2441 2442
}

2443
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2444 2445
				     void *peer_addr, int peer_addr_len)
{
2446
	struct drbd_resource *resource;
2447
	struct drbd_connection *connection;
2448 2449

	rcu_read_lock();
2450 2451 2452 2453 2454 2455 2456 2457 2458
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2459 2460
		}
	}
2461
	connection = NULL;
2462 2463
found:
	rcu_read_unlock();
2464
	return connection;
2465 2466
}

2467 2468 2469 2470 2471
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2472 2473 2474
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2475 2476 2477 2478 2479
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2480
	free_page((unsigned long) socket->sbuf);
2481 2482 2483
	free_page((unsigned long) socket->rbuf);
}

2484
void conn_free_crypto(struct drbd_connection *connection)
2485
{
2486
	drbd_free_sock(connection);
2487

2488 2489 2490 2491 2492 2493 2494
	crypto_free_hash(connection->csums_tfm);
	crypto_free_hash(connection->verify_tfm);
	crypto_free_hash(connection->cram_hmac_tfm);
	crypto_free_hash(connection->integrity_tfm);
	crypto_free_hash(connection->peer_integrity_tfm);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2495

2496 2497 2498 2499 2500 2501 2502
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2503 2504
}

2505
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2506
{
2507
	struct drbd_connection *connection;
2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;
		/*
		retcode = ERR_NOMEM;
		drbd_msg_put_info("unable to allocate cpumask");
		*/

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2520
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2521
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2522
		if (err) {
2523
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2524 2525 2526 2527
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2528
	resource->res_opts = *res_opts;
2529 2530 2531 2532 2533
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2534 2535 2536 2537
			connection->receiver.reset_cpu_mask = 1;
			connection->asender.reset_cpu_mask = 1;
			connection->worker.reset_cpu_mask = 1;
		}
2538 2539 2540 2541 2542 2543 2544 2545 2546
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2547 2548 2549 2550
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2551
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2552
	if (!resource)
2553
		goto fail;
2554
	resource->name = kstrdup(name, GFP_KERNEL);
2555 2556 2557 2558
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2559
	kref_init(&resource->kref);
2560
	idr_init(&resource->devices);
2561
	INIT_LIST_HEAD(&resource->connections);
2562
	resource->write_ordering = WO_bdev_flush;
2563
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2564
	mutex_init(&resource->conf_update);
2565
	mutex_init(&resource->adm_mutex);
2566
	spin_lock_init(&resource->req_lock);
2567
	return resource;
2568 2569 2570 2571 2572 2573 2574

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2575 2576
}

2577
/* caller must be under genl_lock() */
2578
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2579
{
2580
	struct drbd_resource *resource;
2581
	struct drbd_connection *connection;
2582

2583 2584
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2585 2586
		return NULL;

2587
	if (drbd_alloc_socket(&connection->data))
2588
		goto fail;
2589
	if (drbd_alloc_socket(&connection->meta))
2590 2591
		goto fail;

2592 2593
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2594
		goto fail;
2595

2596
	INIT_LIST_HEAD(&connection->transfer_log);
2597

2598 2599 2600
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
2601

2602 2603 2604
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2605

2606 2607 2608 2609
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2610 2611 2612
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2613
	idr_init(&connection->peer_devices);
2614

2615 2616 2617
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2618

2619 2620 2621 2622 2623 2624
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
	drbd_thread_init(resource, &connection->asender, drbd_asender, "asender");
	connection->asender.connection = connection;
2625

2626
	kref_init(&connection->kref);
2627 2628

	connection->resource = resource;
2629

2630 2631 2632 2633 2634
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2635
	return connection;
2636

2637 2638 2639
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2640
fail:
2641 2642 2643 2644
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2645 2646 2647
	return NULL;
}

2648
void drbd_destroy_connection(struct kref *kref)
2649
{
2650
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2651
	struct drbd_resource *resource = connection->resource;
2652

2653
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2654
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2655
	kfree(connection->current_epoch);
2656

2657
	idr_destroy(&connection->peer_devices);
2658

2659 2660 2661 2662 2663
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
	kfree(connection);
2664
	kref_put(&resource->kref, drbd_destroy_resource);
2665 2666
}

2667
static int init_submitter(struct drbd_device *device)
2668 2669 2670
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2671 2672 2673
	device->submit.wq = alloc_workqueue("drbd%u_submit",
			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
	if (!device->submit.wq)
2674 2675
		return -ENOMEM;

2676 2677 2678
	INIT_WORK(&device->submit.worker, do_submit);
	spin_lock_init(&device->submit.lock);
	INIT_LIST_HEAD(&device->submit.writes);
2679 2680 2681
	return 0;
}

2682
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2683
{
2684
	struct drbd_resource *resource = adm_ctx->resource;
2685
	struct drbd_connection *connection;
2686
	struct drbd_device *device;
2687
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2688 2689
	struct gendisk *disk;
	struct request_queue *q;
2690
	int id;
2691
	int vnr = adm_ctx->volume;
2692
	enum drbd_ret_code err = ERR_NOMEM;
2693

2694 2695
	device = minor_to_device(minor);
	if (device)
2696
		return ERR_MINOR_EXISTS;
P
Philipp Reisner 已提交
2697 2698

	/* GFP_KERNEL, we are outside of all write-out paths */
2699 2700
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2701
		return ERR_NOMEM;
2702 2703 2704 2705
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2706 2707
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2708

2709
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2710 2711 2712 2713

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2714 2715
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2716 2717 2718 2719

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2720
	device->vdisk = disk;
P
Philipp Reisner 已提交
2721

2722
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2723 2724 2725 2726 2727 2728

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2729
	disk->private_data = device;
P
Philipp Reisner 已提交
2730

2731
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2732
	/* we have no partitions. we contain only ourselves. */
2733
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2734 2735

	q->backing_dev_info.congested_fn = drbd_congested;
2736
	q->backing_dev_info.congested_data = device;
P
Philipp Reisner 已提交
2737

2738
	blk_queue_make_request(q, drbd_make_request);
2739
	blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
2740 2741 2742
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
P
Philipp Reisner 已提交
2743 2744
	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
	blk_queue_merge_bvec(q, drbd_merge_bvec);
2745
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2746

2747 2748
	device->md_io_page = alloc_page(GFP_KERNEL);
	if (!device->md_io_page)
P
Philipp Reisner 已提交
2749 2750
		goto out_no_io_page;

2751
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2752
		goto out_no_bitmap;
2753 2754
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2755

2756 2757 2758
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2759
			err = ERR_MINOR_EXISTS;
2760
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
T
Tejun Heo 已提交
2761
		}
2762
		goto out_no_minor_idr;
2763
	}
2764 2765 2766 2767 2768 2769
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
			err = ERR_MINOR_EXISTS;
2770
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
2771 2772 2773 2774
		}
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2775

2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790
	INIT_LIST_HEAD(&device->peer_devices);
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
			if (id == -ENOSPC) {
				err = ERR_INVALID_REQUEST;
2791
				drbd_msg_put_info(adm_ctx->reply_skb, "requested volume exists already");
2792 2793
			}
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2794
		}
2795
		kref_get(&connection->kref);
2796
	}
T
Tejun Heo 已提交
2797

2798
	if (init_submitter(device)) {
2799
		err = ERR_NOMEM;
2800
		drbd_msg_put_info(adm_ctx->reply_skb, "unable to create submit workqueue");
2801 2802 2803
		goto out_idr_remove_vol;
	}

2804 2805
	add_disk(disk);

2806
	/* inherit the connection state */
2807
	device->state.conn = first_connection(resource)->cstate;
2808 2809 2810 2811
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2812

2813
	return NO_ERROR;
P
Philipp Reisner 已提交
2814

2815
out_idr_remove_vol:
2816
	idr_remove(&connection->peer_devices, vnr);
2817
out_idr_remove_from_resource:
2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828
	for_each_connection(connection, resource) {
		peer_device = idr_find(&connection->peer_devices, vnr);
		if (peer_device) {
			idr_remove(&connection->peer_devices, vnr);
			kref_put(&connection->kref, drbd_destroy_connection);
		}
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2829
	idr_remove(&resource->devices, vnr);
2830
out_idr_remove_minor:
2831
	idr_remove(&drbd_devices, minor);
2832
	synchronize_rcu();
2833
out_no_minor_idr:
2834
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2835
out_no_bitmap:
2836
	__free_page(device->md_io_page);
P
Philipp Reisner 已提交
2837 2838 2839 2840 2841
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2842
	kref_put(&resource->kref, drbd_destroy_resource);
2843
	kfree(device);
2844
	return err;
P
Philipp Reisner 已提交
2845 2846
}

2847
void drbd_delete_device(struct drbd_device *device)
2848 2849 2850 2851 2852 2853
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
	int refs = 3;

	for_each_connection(connection, resource) {
2854
		idr_remove(&connection->peer_devices, device->vnr);
2855 2856 2857 2858 2859 2860 2861 2862 2863
		refs++;
	}
	idr_remove(&resource->devices, device->vnr);
	idr_remove(&drbd_devices, device_to_minor(device));
	del_gendisk(device->vdisk);
	synchronize_rcu();
	kref_sub(&device->kref, refs, drbd_destroy_device);
}

2864
static int __init drbd_init(void)
P
Philipp Reisner 已提交
2865 2866 2867
{
	int err;

2868
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
P
Philipp Reisner 已提交
2869
		printk(KERN_ERR
2870
		       "drbd: invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2871 2872 2873
#ifdef MODULE
		return -EINVAL;
#else
2874
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
		printk(KERN_ERR
		       "drbd: unable to register block device major %d\n",
		       DRBD_MAJOR);
		return err;
	}

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
2892
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
2893

2894
	rwlock_init(&global_state_lock);
2895
	INIT_LIST_HEAD(&drbd_resources);
2896 2897 2898 2899 2900 2901 2902

	err = drbd_genl_register();
	if (err) {
		printk(KERN_ERR "drbd: unable to register generic netlink family\n");
		goto fail;
	}

P
Philipp Reisner 已提交
2903 2904
	err = drbd_create_mempools();
	if (err)
2905
		goto fail;
P
Philipp Reisner 已提交
2906

2907
	err = -ENOMEM;
2908
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
2909 2910
	if (!drbd_proc)	{
		printk(KERN_ERR "drbd: unable to register proc file\n");
2911
		goto fail;
P
Philipp Reisner 已提交
2912 2913
	}

2914 2915 2916 2917 2918 2919 2920 2921
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
		printk(KERN_ERR "drbd: unable to create retry workqueue\n");
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
2922 2923 2924 2925 2926 2927 2928 2929 2930 2931

	printk(KERN_INFO "drbd: initialized. "
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
	printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
	printk(KERN_INFO "drbd: registered as block device major %d\n",
		DRBD_MAJOR);

	return 0; /* Success! */

2932
fail:
P
Philipp Reisner 已提交
2933 2934 2935 2936 2937 2938 2939 2940
	drbd_cleanup();
	if (err == -ENOMEM)
		printk(KERN_ERR "drbd: ran out of memory\n");
	else
		printk(KERN_ERR "drbd: initialization failure\n");
	return err;
}

2941
void drbd_free_ldev(struct drbd_backing_dev *ldev)
P
Philipp Reisner 已提交
2942 2943 2944 2945
{
	if (ldev == NULL)
		return;

2946 2947
	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
P
Philipp Reisner 已提交
2948

L
Lars Ellenberg 已提交
2949
	kfree(ldev->disk_conf);
P
Philipp Reisner 已提交
2950 2951 2952
	kfree(ldev);
}

2953
void drbd_free_sock(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2954
{
2955 2956 2957 2958 2959 2960
	if (connection->data.socket) {
		mutex_lock(&connection->data.mutex);
		kernel_sock_shutdown(connection->data.socket, SHUT_RDWR);
		sock_release(connection->data.socket);
		connection->data.socket = NULL;
		mutex_unlock(&connection->data.mutex);
P
Philipp Reisner 已提交
2961
	}
2962 2963 2964 2965 2966 2967
	if (connection->meta.socket) {
		mutex_lock(&connection->meta.mutex);
		kernel_sock_shutdown(connection->meta.socket, SHUT_RDWR);
		sock_release(connection->meta.socket);
		connection->meta.socket = NULL;
		mutex_unlock(&connection->meta.mutex);
P
Philipp Reisner 已提交
2968 2969 2970 2971 2972
	}
}

/* meta data management */

2973
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2974
{
2975
	struct drbd_peer_device *peer_device;
2976
	int vnr;
P
Philipp Reisner 已提交
2977

2978
	rcu_read_lock();
2979 2980 2981
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

2982
		kref_get(&device->kref);
2983
		rcu_read_unlock();
2984
		drbd_md_sync(device);
2985
		kref_put(&device->kref, drbd_destroy_device);
2986 2987 2988
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
2989 2990
}

2991
/* aligned 4kByte */
P
Philipp Reisner 已提交
2992
struct meta_data_on_disk {
2993
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
2994 2995 2996 2997 2998 2999 3000
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3001
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3002
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3003 3004
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3005
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3006

3007 3008 3009 3010 3011
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3012 3013
} __packed;

3014 3015


3016
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3017
{
3018
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3019 3020 3021
	sector_t sector;
	int i;

3022
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3023

3024
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3025
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3026 3027
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3028
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3029

3030 3031 3032
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3033
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3034
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3035

3036 3037
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3038

3039 3040
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3041

3042
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3043
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3044

3045
	if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
P
Philipp Reisner 已提交
3046
		/* this was a try anyways ... */
3047
		drbd_err(device, "meta data update failed!\n");
3048
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3049
	}
3050 3051 3052 3053
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3054
 * @device:	DRBD device.
3055
 */
3056
void drbd_md_sync(struct drbd_device *device)
3057 3058 3059 3060 3061 3062 3063
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3064
	del_timer(&device->md_sync_timer);
3065
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3066
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3067 3068 3069 3070
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3071
	if (!get_ldev_if_state(device, D_FAILED))
3072 3073
		return;

3074
	buffer = drbd_md_get_buffer(device);
3075 3076 3077
	if (!buffer)
		goto out;

3078
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3079

3080
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3081
	 * since we updated it on metadata. */
3082
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3083

3084
	drbd_md_put_buffer(device);
3085
out:
3086
	put_ldev(device);
P
Philipp Reisner 已提交
3087 3088
}

3089
static int check_activity_log_stripe_size(struct drbd_device *device,
3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3129
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3130 3131 3132 3133
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3134
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3201
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3214 3215
/**
 * drbd_md_read() - Reads in the meta data super block
3216
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3217 3218
 * @bdev:	Device from which the meta data should be read in.
 *
3219
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3220
 * something goes wrong.
3221
 *
3222
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3223
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3224
 */
3225
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3226 3227
{
	struct meta_data_on_disk *buffer;
3228
	u32 magic, flags;
P
Philipp Reisner 已提交
3229 3230
	int i, rv = NO_ERROR;

3231
	if (device->state.disk != D_DISKLESS)
3232
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3233

3234
	buffer = drbd_md_get_buffer(device);
3235
	if (!buffer)
3236
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3237

3238 3239
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3240 3241
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
P
Philipp Reisner 已提交
3242

3243
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
L
Lucas De Marchi 已提交
3244
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3245
		   called BEFORE disk is attached */
3246
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3247 3248 3249 3250
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3251 3252 3253 3254 3255
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3256
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3257 3258 3259
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3260 3261

	rv = ERR_MD_INVALID;
3262
	if (magic != DRBD_MD_MAGIC_08) {
3263
		if (magic == DRBD_MD_MAGIC_07)
3264
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3265
		else
3266
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3267 3268
		goto err;
	}
3269

3270
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3271
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3272
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3273 3274
		goto err;
	}
3275

3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3288
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3289
		goto err;
3290
	if (check_offsets_and_sizes(device, bdev))
3291 3292
		goto err;

P
Philipp Reisner 已提交
3293
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3294
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3295 3296 3297 3298
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3299
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3300 3301 3302 3303
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3304
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3305

3306
	spin_lock_irq(&device->resource->req_lock);
3307
	if (device->state.conn < C_CONNECTED) {
3308
		unsigned int peer;
3309
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3310
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3311
		device->peer_max_bio_size = peer;
3312
	}
3313
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3314 3315

 err:
3316
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3317 3318 3319 3320 3321 3322

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3323
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3324 3325 3326 3327 3328
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3329
#ifdef DEBUG
3330
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3331
{
3332 3333 3334 3335
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3336 3337 3338
	}
}
#else
3339
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3340
{
3341 3342
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3343
}
3344
#endif
P
Philipp Reisner 已提交
3345

3346
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3347 3348 3349
{
	int i;

3350
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3351
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3352 3353
}

3354
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3355 3356
{
	if (idx == UI_CURRENT) {
3357
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3358 3359 3360 3361
			val |= 1;
		else
			val &= ~((u64)1);

3362
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3363 3364
	}

3365 3366
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3367 3368
}

3369
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3370 3371
{
	unsigned long flags;
3372 3373 3374
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3375
}
P
Philipp Reisner 已提交
3376

3377
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3378
{
3379
	unsigned long flags;
3380 3381 3382 3383
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3384
	}
3385 3386
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3387 3388 3389 3390
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3391
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3392 3393 3394 3395
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3396
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3397 3398
{
	u64 val;
3399 3400 3401 3402
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3403 3404
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3405 3406

	if (bm_uuid)
3407
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3408

3409 3410 3411
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3412

3413
	drbd_print_uuids(device, "new current UUID");
3414
	/* get it to stable storage _now_ */
3415
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3416 3417
}

3418
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3419
{
3420
	unsigned long flags;
3421
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3422 3423
		return;

3424
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3425
	if (val == 0) {
3426 3427 3428
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3429
	} else {
3430
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3431
		if (bm_uuid)
3432
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3433

3434
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3435
	}
3436
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3437

3438
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3439 3440 3441 3442
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3443
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3444 3445 3446
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3447
int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3448 3449 3450
{
	int rv = -EIO;

3451 3452 3453
	drbd_md_set_flag(device, MDF_FULL_SYNC);
	drbd_md_sync(device);
	drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3454

3455
	rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3456

3457 3458 3459
	if (!rv) {
		drbd_md_clear_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
P
Philipp Reisner 已提交
3460 3461 3462 3463 3464 3465 3466
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3467
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3468 3469 3470
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3471
int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3472
{
3473
	drbd_resume_al(device);
3474 3475
	drbd_bm_clear_all(device);
	return drbd_bm_write(device);
P
Philipp Reisner 已提交
3476 3477
}

3478
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3479
{
3480 3481 3482
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3483
	int rv = -EIO;
P
Philipp Reisner 已提交
3484

3485
	D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0);
P
Philipp Reisner 已提交
3486

3487 3488 3489 3490 3491
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3492
	}
P
Philipp Reisner 已提交
3493

3494 3495
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3496 3497

	if (work->done)
3498
		work->done(device, rv);
P
Philipp Reisner 已提交
3499

3500
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3501
	work->why = NULL;
3502
	work->flags = 0;
P
Philipp Reisner 已提交
3503

3504
	return 0;
P
Philipp Reisner 已提交
3505 3506 3507 3508
}

/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3509
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3510 3511 3512 3513 3514 3515 3516 3517
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
3518 3519 3520
 *
 * Its worker function encloses the call of io_fn() by get_ldev() and
 * put_ldev().
P
Philipp Reisner 已提交
3521
 */
3522
void drbd_queue_bitmap_io(struct drbd_device *device,
3523 3524
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3525
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3526
{
3527
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3528

3529 3530 3531
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3532
	if (device->bm_io_work.why)
3533
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3534
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3535

3536 3537 3538 3539
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3540

3541
	spin_lock_irq(&device->resource->req_lock);
3542 3543 3544
	set_bit(BITMAP_IO, &device->flags);
	if (atomic_read(&device->ap_bio_cnt) == 0) {
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3545 3546
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3547
	}
3548
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3549 3550 3551 3552
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3553
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3554 3555 3556 3557 3558 3559
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3560
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3561
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3562 3563 3564
{
	int rv;

3565
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3566

3567
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3568
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3569

3570 3571 3572
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3573

3574
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3575
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3576 3577 3578 3579

	return rv;
}

3580
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3581
{
3582 3583 3584
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3585 3586 3587
	}
}

3588
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3589
{
3590 3591 3592
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3593 3594 3595 3596 3597 3598 3599 3600 3601
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3602
	struct drbd_device *device = (struct drbd_device *) data;
3603
	drbd_device_post_work(device, MD_SYNC);
P
Philipp Reisner 已提交
3604 3605
}

3606
const char *cmdname(enum drbd_packet cmd)
3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3636
		[P_SUPERSEDED]          = "Superseded",
3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3651
		[P_RETRY_WRITE]		= "RetryWrite",
3652 3653 3654
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3655 3656
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3657 3658 3659 3660 3661

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3662 3663
	};

3664
	/* too big for the array: 0xfffX */
3665 3666 3667 3668
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3669 3670
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3671
	if (cmd >= ARRAY_SIZE(cmdnames))
3672 3673 3674 3675
		return "Unknown";
	return cmdnames[cmd];
}

3676 3677
/**
 * drbd_wait_misc  -  wait for a request to make progress
3678
 * @device:	device associated with the request
3679 3680 3681
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3682
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3683
{
3684
	struct net_conf *nc;
3685 3686 3687
	DEFINE_WAIT(wait);
	long timeout;

3688
	rcu_read_lock();
3689
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3690 3691
	if (!nc) {
		rcu_read_unlock();
3692
		return -ETIMEDOUT;
3693 3694 3695
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3696

3697
	/* Indicate to wake up device->misc_wait on progress.  */
3698
	i->waiting = true;
3699
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3700
	spin_unlock_irq(&device->resource->req_lock);
3701
	timeout = schedule_timeout(timeout);
3702
	finish_wait(&device->misc_wait, &wait);
3703
	spin_lock_irq(&device->resource->req_lock);
3704
	if (!timeout || device->state.conn < C_CONNECTED)
3705 3706 3707 3708
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731
}

#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3732
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3752 3753
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3754 3755 3756 3757 3758 3759
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3760
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3761 3762 3763 3764 3765
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3766
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3767 3768 3769 3770 3771
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3772
		if (__ratelimit(&drbd_ratelimit_state))
3773
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3789 3790 3791 3792
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);