drbd_main.c 107.4 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

29 30
#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

P
Philipp Reisner 已提交
31 32 33 34 35 36
#include <linux/module.h>
#include <linux/drbd.h>
#include <asm/uaccess.h>
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
37
#include <linux/mutex.h>
P
Philipp Reisner 已提交
38 39 40 41 42 43 44 45 46 47 48 49
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
50
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
51 52 53 54 55 56
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>

#include <linux/drbd_limits.h>
#include "drbd_int.h"
57
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
58 59 60 61
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */

#include "drbd_vli.h"

62
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
63
static int drbd_open(struct block_device *bdev, fmode_t mode);
64
static void drbd_release(struct gendisk *gd, fmode_t mode);
P
Philipp Reisner 已提交
65
static void md_sync_timer_fn(unsigned long data);
66
static int w_bitmap_io(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
67 68 69 70 71 72

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
73
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
74
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
103
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
104 105
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
106 107 108 109 110 111 112 113 114 115 116
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
char usermode_helper[80] = "/sbin/drbdadm";

module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
117
struct idr drbd_devices;
118
struct list_head drbd_resources;
P
Philipp Reisner 已提交
119 120

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
121
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
122 123 124 125
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
126
mempool_t *drbd_md_io_page_pool;
127
struct bio_set *drbd_md_io_bio_set;
P
Philipp Reisner 已提交
128 129 130 131 132 133 134 135 136 137 138 139 140 141

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

142
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
143 144 145 146
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
147

148 149 150
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
151

152 153
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
154

155 156 157 158
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
159 160
}

P
Philipp Reisner 已提交
161 162 163 164
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
165
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
166 167 168
{
	int io_allowed;

169 170
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
171
	if (!io_allowed) {
172 173
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
174 175 176 177 178
	}
	return io_allowed;
}

#endif
179

P
Philipp Reisner 已提交
180
/**
181
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
182
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
183 184 185 186
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
187 188
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
189
 */
190
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
191
		unsigned int set_size)
P
Philipp Reisner 已提交
192 193
{
	struct drbd_request *r;
194 195 196
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
197

198
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
199

200
	/* find oldest not yet barrier-acked write request,
201
	 * count writes in its epoch. */
202
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
203
		const unsigned s = r->rq_state;
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
222
		}
P
Philipp Reisner 已提交
223
	}
224

P
Philipp Reisner 已提交
225
	/* first some paranoia code */
226
	if (req == NULL) {
227
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
228
			 barrier_nr);
P
Philipp Reisner 已提交
229 230
		goto bail;
	}
231
	if (expect_epoch != barrier_nr) {
232
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
233
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
234
		goto bail;
235 236
	}

237
	if (expect_size != set_size) {
238
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
239
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
240 241 242
		goto bail;
	}

243 244 245 246
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
247
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
248 249
		if (req->epoch == expect_epoch)
			break;
250
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
251 252 253
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
254
	}
255
	spin_unlock_irq(&connection->resource->req_lock);
256

P
Philipp Reisner 已提交
257 258 259
	return;

bail:
260
	spin_unlock_irq(&connection->resource->req_lock);
261
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
262 263 264 265
}


/**
266
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
267
 * @device:	DRBD device.
268
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
269
 *
270 271
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
272
 */
273
/* must hold resource->req_lock */
274
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
275
{
276
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
277

278
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
279 280
		_req_mod(req, what);
}
281

282
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
283
{
284
	spin_lock_irq(&connection->resource->req_lock);
285
	_tl_restart(connection, what);
286
	spin_unlock_irq(&connection->resource->req_lock);
287
}
P
Philipp Reisner 已提交
288 289 290

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
291
 * @device:	DRBD device.
P
Philipp Reisner 已提交
292 293 294 295 296
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
297
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
298
{
299
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
300
}
301

302
/**
303 304
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
305
 */
306
void tl_abort_disk_io(struct drbd_device *device)
307
{
308
	struct drbd_connection *connection = first_peer_device(device)->connection;
309
	struct drbd_request *req, *r;
310

311
	spin_lock_irq(&connection->resource->req_lock);
312
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
313 314
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
315
		if (req->device != device)
316 317
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
318
	}
319
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
320 321 322 323 324
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
325
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
326 327 328
	unsigned long flags;
	int retval;

329
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
330
		 thi->name[0],
331
		 resource->name);
332

P
Philipp Reisner 已提交
333 334 335 336 337
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

338
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
339 340 341
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
342
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
343
	 * t_state check and assignment needs to be within the same spinlock,
344 345
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
346 347
	 */

348
	if (thi->t_state == RESTARTING) {
349
		drbd_info(resource, "Restarting %s thread\n", thi->name);
350
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
351 352 353 354 355
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
356
	thi->t_state = NONE;
P
Philipp Reisner 已提交
357
	smp_mb();
L
Lars Ellenberg 已提交
358
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
359 360
	spin_unlock_irqrestore(&thi->t_lock, flags);

361
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
362 363

	/* Release mod reference taken when thread was started */
364

365 366 367
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
368 369 370 371
	module_put(THIS_MODULE);
	return retval;
}

372
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
373
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
374 375 376
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
377
	thi->t_state = NONE;
P
Philipp Reisner 已提交
378
	thi->function = func;
379 380
	thi->resource = resource;
	thi->connection = NULL;
381
	thi->name = name;
P
Philipp Reisner 已提交
382 383 384 385
}

int drbd_thread_start(struct drbd_thread *thi)
{
386
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
387 388 389 390 391 392 393 394
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
395
	case NONE:
396
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
397
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
398 399 400

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
401
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
402
			spin_unlock_irqrestore(&thi->t_lock, flags);
403
			return false;
P
Philipp Reisner 已提交
404 405
		}

406 407 408
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
409

P
Philipp Reisner 已提交
410 411
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
412
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
413 414 415 416
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
417
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
418 419

		if (IS_ERR(nt)) {
420
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
421

422 423 424
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
425
			module_put(THIS_MODULE);
426
			return false;
P
Philipp Reisner 已提交
427 428 429
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
430
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
431 432 433
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
434 435
	case EXITING:
		thi->t_state = RESTARTING;
436
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
437
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
438
		/* fall through */
439 440
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
441 442 443 444 445
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

446
	return true;
P
Philipp Reisner 已提交
447 448 449 450 451 452 453
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

454
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
455 456 457 458

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

459
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

485
int conn_lowest_minor(struct drbd_connection *connection)
486
{
487 488
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
489

490
	rcu_read_lock();
491 492 493
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
494 495
	rcu_read_unlock();

496
	return minor;
497
}
498

P
Philipp Reisner 已提交
499 500 501 502
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
503
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
504 505
 * DRBD's performance. May be overwritten by user's configuration.
 */
506
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
507
{
508
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
509

510 511 512 513
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
514

515 516 517 518
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
519
		}
520 521 522 523 524 525 526 527 528 529 530 531
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
532
	}
533
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
534 535 536 537
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
538
 * @device:	DRBD device.
539
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
540 541 542 543
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
544
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
545
{
546
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
547
	struct task_struct *p = current;
548

P
Philipp Reisner 已提交
549 550 551
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
552
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
553
}
554 555
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
556 557
#endif

558 559 560 561 562 563 564
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
565
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
566
{
567
	if (connection->agreed_pro_version >= 100) {
568 569 570 571 572 573 574 575
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
576
}
P
Philipp Reisner 已提交
577

578
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
579 580 581 582
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
583
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
584
}
P
Philipp Reisner 已提交
585

586
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
587 588
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
589
	h->command = cpu_to_be16(cmd);
590
	h->length = cpu_to_be32(size);
591
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
592
}
P
Philipp Reisner 已提交
593

594 595 596 597 598 599 600 601 602 603
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
604

605
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
606
				   void *buffer, enum drbd_packet cmd, int size)
607
{
608
	if (connection->agreed_pro_version >= 100)
609
		return prepare_header100(buffer, cmd, size, vnr);
610
	else if (connection->agreed_pro_version >= 95 &&
611
		 size > DRBD_MAX_SIZE_H80_PACKET)
612
		return prepare_header95(buffer, cmd, size);
613
	else
614
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
615 616
}

617
static void *__conn_prepare_command(struct drbd_connection *connection,
618
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
619
{
620 621
	if (!sock->socket)
		return NULL;
622
	return sock->sbuf + drbd_header_size(connection);
623
}
P
Philipp Reisner 已提交
624

625
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
626
{
627
	void *p;
P
Philipp Reisner 已提交
628

629
	mutex_lock(&sock->mutex);
630
	p = __conn_prepare_command(connection, sock);
631
	if (!p)
632
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
633

634
	return p;
P
Philipp Reisner 已提交
635 636
}

637
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
638
{
639
	return conn_prepare_command(peer_device->connection, sock);
640
}
P
Philipp Reisner 已提交
641

642
static int __send_command(struct drbd_connection *connection, int vnr,
643 644 645 646 647 648
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
649

650 651 652 653 654 655 656 657 658
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

659
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
660
				      header_size + size);
661
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
662 663
			    msg_flags);
	if (data && !err)
664
		err = drbd_send_all(connection, sock->socket, data, size, 0);
665 666 667 668 669
	/* DRBD protocol "pings" are latency critical.
	 * This is supposed to trigger tcp_push_pending_frames() */
	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
		drbd_tcp_nodelay(sock->socket);

670 671 672
	return err;
}

673
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
674 675 676
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
677
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
678 679
}

680
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
681 682 683 684
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
685

686
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
687 688 689 690
	mutex_unlock(&sock->mutex);
	return err;
}

691
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
692 693 694 695 696
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

697 698
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
699 700 701
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
702

703
int drbd_send_ping(struct drbd_connection *connection)
704
{
705 706
	struct drbd_socket *sock;

707 708
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
709
		return -EIO;
710
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
711
}
P
Philipp Reisner 已提交
712

713
int drbd_send_ping_ack(struct drbd_connection *connection)
714
{
715 716
	struct drbd_socket *sock;

717 718
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
719
		return -EIO;
720
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
721 722
}

723
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
724
{
725
	struct drbd_socket *sock;
726
	struct p_rs_param_95 *p;
727
	int size;
728
	const int apv = peer_device->connection->agreed_pro_version;
729
	enum drbd_packet cmd;
730
	struct net_conf *nc;
P
Philipp Reisner 已提交
731
	struct disk_conf *dc;
732

733 734
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
735 736
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
737

738
	rcu_read_lock();
739
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
740 741 742

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
743
			+ strlen(nc->verify_alg) + 1
744 745
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
746

747
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
748

749 750
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
751

752 753
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
754
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
755 756 757 758
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
759
		put_ldev(peer_device->device);
760
	} else {
761
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
762 763 764 765 766
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
767

768
	if (apv >= 88)
769
		strcpy(p->verify_alg, nc->verify_alg);
770
	if (apv >= 89)
771 772
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
773

774
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
775 776
}

777
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
778
{
779
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
780
	struct p_protocol *p;
781
	struct net_conf *nc;
782
	int size, cf;
P
Philipp Reisner 已提交
783

784 785
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
786 787
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
788

789
	rcu_read_lock();
790
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
791

792
	if (nc->tentative && connection->agreed_pro_version < 92) {
793 794
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
795
		drbd_err(connection, "--dry-run is not supported by peer");
796 797
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
798

799
	size = sizeof(*p);
800
	if (connection->agreed_pro_version >= 87)
801
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
802

803 804 805 806 807
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
808
	cf = 0;
809 810
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
811
	if (nc->tentative)
812
		cf |= CF_DRY_RUN;
813 814
	p->conn_flags    = cpu_to_be32(cf);

815
	if (connection->agreed_pro_version >= 87)
816 817
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
818

819
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
820 821
}

822
int drbd_send_protocol(struct drbd_connection *connection)
823 824 825
{
	int err;

826 827 828
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
829 830

	return err;
P
Philipp Reisner 已提交
831 832
}

833
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
834
{
835
	struct drbd_device *device = peer_device->device;
836 837
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
838 839
	int i;

840
	if (!get_ldev_if_state(device, D_NEGOTIATING))
841
		return 0;
P
Philipp Reisner 已提交
842

843 844
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
845
	if (!p) {
846
		put_ldev(device);
847 848
		return -EIO;
	}
849
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
850
	for (i = UI_CURRENT; i < UI_SIZE; i++)
851 852
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
853

854 855
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
856
	rcu_read_lock();
857
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
858
	rcu_read_unlock();
859 860
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
861
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
862

863
	put_ldev(device);
864
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
865 866
}

867
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
868
{
869
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
870 871
}

872
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
873
{
874
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
875 876
}

877
void drbd_print_uuids(struct drbd_device *device, const char *text)
878
{
879 880
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
881
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
882 883 884 885 886
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
887
		put_ldev(device);
888
	} else {
889
		drbd_info(device, "%s effective data uuid: %016llX\n",
890
				text,
891
				(unsigned long long)device->ed_uuid);
892 893 894
	}
}

895
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
896
{
897
	struct drbd_device *device = peer_device->device;
898 899
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
900 901
	u64 uuid;

902
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
903

904
	uuid = device->ldev->md.uuid[UI_BITMAP];
905 906 907 908
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
909 910 911
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
912

913 914
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
915 916
	if (p) {
		p->uuid = cpu_to_be64(uuid);
917
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
918
	}
P
Philipp Reisner 已提交
919 920
}

921
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
922
{
923
	struct drbd_device *device = peer_device->device;
924 925
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
926
	sector_t d_size, u_size;
927 928
	int q_order_type;
	unsigned int max_bio_size;
P
Philipp Reisner 已提交
929

930
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
931
		D_ASSERT(device, device->ldev->backing_bdev);
932
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
933
		rcu_read_lock();
934
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
935
		rcu_read_unlock();
936 937
		q_order_type = drbd_queue_order_type(device);
		max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9;
938
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
939
		put_ldev(device);
P
Philipp Reisner 已提交
940 941 942 943
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
944
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
P
Philipp Reisner 已提交
945 946
	}

947 948
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
949 950
	if (!p)
		return -EIO;
951

952
	if (peer_device->connection->agreed_pro_version <= 94)
953
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
954
	else if (peer_device->connection->agreed_pro_version < 100)
955
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
956

957 958
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
959
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
960 961 962
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
963
	return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
964 965 966
}

/**
967
 * drbd_send_current_state() - Sends the drbd state to the peer
968
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
969
 */
970
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
971
{
972
	struct drbd_socket *sock;
973
	struct p_state *p;
P
Philipp Reisner 已提交
974

975 976
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
977 978
	if (!p)
		return -EIO;
979 980
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
981 982
}

983 984
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
985
 * @peer_device:      DRBD peer device.
986
 * @state:     the state to send, not necessarily the current state.
987 988 989 990 991 992
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
993
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
994
{
995 996
	struct drbd_socket *sock;
	struct p_state *p;
997

998 999
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1000 1001 1002
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
1003
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1004
}
1005

1006
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1007 1008 1009
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1010

1011 1012
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1013 1014 1015 1016
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1017
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1018
}
1019

1020
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1021
{
1022 1023 1024
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1025

1026 1027 1028
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1029 1030 1031 1032
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1033
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1034 1035
}

1036
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1037
{
1038 1039
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1040

1041 1042
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1043 1044
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1045
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1046
	}
P
Philipp Reisner 已提交
1047 1048
}

1049
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1050
{
1051 1052
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1053
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1054

1055 1056
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1057 1058
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1059
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1060
	}
P
Philipp Reisner 已提交
1061 1062
}

1063
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1064
{
1065 1066 1067
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1068

1069 1070 1071 1072
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1073

1074 1075 1076 1077
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1078 1079
}

1080
static int fill_bitmap_rle_bits(struct drbd_device *device,
1081 1082 1083
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1084 1085 1086 1087 1088 1089 1090
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1091
	int bits, use_rle;
P
Philipp Reisner 已提交
1092 1093

	/* may we use this feature? */
1094
	rcu_read_lock();
1095
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1096
	rcu_read_unlock();
1097
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1098
		return 0;
P
Philipp Reisner 已提交
1099 1100 1101 1102 1103

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1104 1105
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1117 1118
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1119 1120 1121 1122 1123 1124 1125 1126
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1127
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1128 1129 1130 1131
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1132
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1133 1134 1135 1136 1137
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1138
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1139 1140 1141 1142 1143 1144 1145 1146
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1147
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1172
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1173 1174 1175 1176

	return len;
}

1177 1178 1179 1180 1181 1182 1183
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1184
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1185
{
1186 1187
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1188
	struct p_compressed_bm *p = sock->sbuf + header_size;
1189
	int len, err;
P
Philipp Reisner 已提交
1190

1191
	len = fill_bitmap_rle_bits(device, p,
1192
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1193
	if (len < 0)
1194
		return -EIO;
P
Philipp Reisner 已提交
1195 1196

	if (len) {
1197
		dcbp_set_code(p, RLE_VLI_Bits);
1198
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1199 1200
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1201
		c->packets[0]++;
1202
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1203 1204 1205 1206 1207 1208

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1209 1210
		unsigned int data_size;
		unsigned long num_words;
1211
		unsigned long *p = sock->sbuf + header_size;
1212 1213

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1214
		num_words = min_t(size_t, data_size / sizeof(*p),
1215
				  c->bm_words - c->word_offset);
1216
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1217
		if (len)
1218
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1219
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1220 1221 1222 1223
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1224
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1225 1226 1227 1228

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1229
	if (!err) {
1230
		if (len == 0) {
1231
			INFO_bm_xfer_stats(device, "send", c);
1232 1233 1234 1235 1236
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1237 1238 1239
}

/* See the comment at receive_bitmap() */
1240
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1241 1242
{
	struct bm_xfer_ctx c;
1243
	int err;
P
Philipp Reisner 已提交
1244

1245
	if (!expect(device->bitmap))
1246
		return false;
P
Philipp Reisner 已提交
1247

1248 1249
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1250
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1251 1252
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1253 1254 1255
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1256
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1257
			} else {
1258 1259
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1260 1261
			}
		}
1262
		put_ldev(device);
P
Philipp Reisner 已提交
1263 1264 1265
	}

	c = (struct bm_xfer_ctx) {
1266 1267
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1268 1269 1270
	};

	do {
1271
		err = send_bitmap_rle_or_plain(device, &c);
1272
	} while (err > 0);
P
Philipp Reisner 已提交
1273

1274
	return err == 0;
P
Philipp Reisner 已提交
1275 1276
}

1277
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1278
{
1279
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1280
	int err = -1;
P
Philipp Reisner 已提交
1281

1282 1283
	mutex_lock(&sock->mutex);
	if (sock->socket)
1284
		err = !_drbd_send_bitmap(device);
1285
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1286 1287 1288
	return err;
}

1289
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1290
{
1291 1292
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1293

1294
	if (connection->cstate < C_WF_REPORT_PARAMS)
1295
		return;
P
Philipp Reisner 已提交
1296

1297 1298
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1299 1300 1301 1302
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1303
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1304 1305 1306 1307
}

/**
 * _drbd_send_ack() - Sends an ack packet
1308
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1309 1310 1311 1312 1313
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1314
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1315
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1316
{
1317 1318
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1319

1320
	if (peer_device->device->state.conn < C_CONNECTED)
1321
		return -EIO;
P
Philipp Reisner 已提交
1322

1323 1324
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1325
	if (!p)
1326
		return -EIO;
1327 1328 1329
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1330 1331
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1332 1333
}

1334 1335 1336
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1337
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1338
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1339
{
1340 1341 1342
	if (peer_device->connection->peer_integrity_tfm)
		data_size -= crypto_hash_digestsize(peer_device->connection->peer_integrity_tfm);
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1343
		       dp->block_id);
P
Philipp Reisner 已提交
1344 1345
}

1346
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1347
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1348
{
1349
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1350 1351 1352 1353
}

/**
 * drbd_send_ack() - Sends an ack packet
1354
 * @device:	DRBD device
1355 1356
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1357
 */
1358
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1359
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1360
{
1361
	return _drbd_send_ack(peer_device, cmd,
1362 1363 1364
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1365 1366 1367 1368
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1369
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1370 1371
		     sector_t sector, int blksize, u64 block_id)
{
1372
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1373 1374 1375 1376 1377
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1378
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1379 1380
		       sector_t sector, int size, u64 block_id)
{
1381 1382
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1383

1384 1385
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1386 1387 1388 1389 1390
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1391
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1392 1393
}

1394
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1395
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1396
{
1397 1398
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1399

1400
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1401

1402 1403
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1404 1405 1406 1407 1408
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1409
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1410 1411
}

1412
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1413
{
1414 1415
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1416

1417 1418
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1419 1420 1421 1422 1423
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1424
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1425 1426 1427
}

/* called on sndtimeo
1428 1429
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1430
 */
1431
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1432 1433
{
	int drop_it;
1434
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1435

1436 1437 1438 1439
	drop_it =   connection->meta.socket == sock
		|| !connection->asender.task
		|| get_t_state(&connection->asender) != RUNNING
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1440 1441

	if (drop_it)
1442
		return true;
P
Philipp Reisner 已提交
1443

1444
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1445
	if (!drop_it) {
1446
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1447 1448
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1449 1450
	}

1451
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1452 1453
}

1454
static void drbd_update_congested(struct drbd_connection *connection)
1455
{
1456
	struct sock *sk = connection->data.socket->sk;
1457
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1458
		set_bit(NET_CONGESTED, &connection->flags);
1459 1460
}

P
Philipp Reisner 已提交
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1482
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1483
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1484
{
1485 1486 1487 1488
	struct socket *socket;
	void *addr;
	int err;

1489
	socket = peer_device->connection->data.socket;
1490
	addr = kmap(page) + offset;
1491
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1492
	kunmap(page);
1493
	if (!err)
1494
		peer_device->device->send_cnt += size >> 9;
1495
	return err;
P
Philipp Reisner 已提交
1496 1497
}

1498
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1499
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1500
{
1501
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1502 1503
	mm_segment_t oldfs = get_fs();
	int len = size;
1504
	int err = -EIO;
P
Philipp Reisner 已提交
1505 1506 1507 1508 1509 1510 1511 1512

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1513
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1514

L
Lars Ellenberg 已提交
1515
	msg_flags |= MSG_NOSIGNAL;
1516
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1517 1518
	set_fs(KERNEL_DS);
	do {
1519 1520 1521
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1522
		if (sent <= 0) {
1523
			if (sent == -EAGAIN) {
1524
				if (we_should_drop_the_connection(peer_device->connection, socket))
1525 1526 1527
					break;
				continue;
			}
1528
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1529
			     __func__, (int)size, len, sent);
1530 1531
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1532 1533 1534 1535
			break;
		}
		len    -= sent;
		offset += sent;
1536
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
P
Philipp Reisner 已提交
1537
	set_fs(oldfs);
1538
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1539

1540 1541
	if (len == 0) {
		err = 0;
1542
		peer_device->device->send_cnt += size >> 9;
1543 1544
	}
	return err;
P
Philipp Reisner 已提交
1545 1546
}

1547
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1548
{
1549 1550 1551
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1552
	/* hint all but last page with MSG_MORE */
1553
	bio_for_each_segment(bvec, bio, iter) {
1554 1555
		int err;

1556
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1557
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1558
					 bio_iter_last(bvec, iter)
1559
					 ? 0 : MSG_MORE);
1560 1561
		if (err)
			return err;
P
Philipp Reisner 已提交
1562
	}
1563
	return 0;
P
Philipp Reisner 已提交
1564 1565
}

1566
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1567
{
1568 1569 1570
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1571
	/* hint all but last page with MSG_MORE */
1572
	bio_for_each_segment(bvec, bio, iter) {
1573 1574
		int err;

1575
		err = _drbd_send_page(peer_device, bvec.bv_page,
1576
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1577
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1578 1579
		if (err)
			return err;
P
Philipp Reisner 已提交
1580
	}
1581
	return 0;
P
Philipp Reisner 已提交
1582 1583
}

1584
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1585
			    struct drbd_peer_request *peer_req)
1586
{
1587 1588
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1589
	int err;
1590

L
Lars Ellenberg 已提交
1591
	/* hint all but last page with MSG_MORE */
1592 1593
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1594

1595
		err = _drbd_send_page(peer_device, page, 0, l,
1596 1597 1598
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1599 1600
		len -= l;
	}
1601
	return 0;
1602 1603
}

1604
static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw)
1605
{
1606
	if (connection->agreed_pro_version >= 95)
1607 1608 1609 1610 1611
		return  (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bi_rw & REQ_FUA ? DP_FUA : 0) |
			(bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
			(bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
	else
J
Jens Axboe 已提交
1612
		return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
1613 1614
}

1615 1616
/* Used to send write or TRIM aka REQ_DISCARD requests
 * R_PRIMARY -> Peer	(P_DATA, P_TRIM)
P
Philipp Reisner 已提交
1617
 */
1618
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1619
{
1620
	struct drbd_device *device = peer_device->device;
1621 1622
	struct drbd_socket *sock;
	struct p_data *p;
P
Philipp Reisner 已提交
1623 1624
	unsigned int dp_flags = 0;
	int dgs;
1625
	int err;
P
Philipp Reisner 已提交
1626

1627 1628 1629 1630
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1631

1632 1633 1634 1635
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1636
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
1637
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
1638 1639
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1640
		dp_flags |= DP_MAY_SET_IN_SYNC;
1641
	if (peer_device->connection->agreed_pro_version >= 100) {
1642 1643
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
1644 1645 1646 1647
		/* During resync, request an explicit write ack,
		 * even in protocol != C */
		if (req->rq_state & RQ_EXP_WRITE_ACK
		|| (dp_flags & DP_MAY_SET_IN_SYNC))
1648 1649
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1650
	p->dp_flags = cpu_to_be32(dp_flags);
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660

	if (dp_flags & DP_DISCARD) {
		struct p_trim *t = (struct p_trim*)p;
		t->size = cpu_to_be32(req->i.size);
		err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0);
		goto out;
	}

	/* our digest is still only over the payload.
	 * TRIM does not carry any payload. */
1661
	if (dgs)
1662 1663
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + dgs, NULL, req->i.size);
1664
	if (!err) {
1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1676
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || dgs)
1677
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1678
		else
1679
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1680 1681 1682

		/* double check digest, sometimes buffers have been modified in flight. */
		if (dgs > 0 && dgs <= 64) {
B
Bart Van Assche 已提交
1683
			/* 64 byte, 512 bit, is the largest digest size
1684 1685
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1686
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1687
			if (memcmp(p + 1, digest, dgs)) {
1688
				drbd_warn(device,
1689
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1690
					(unsigned long long)req->i.sector, req->i.size);
1691 1692 1693 1694
			}
		} /* else if (dgs > 64) {
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1695
	}
1696
out:
1697
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1698

1699
	return err;
P
Philipp Reisner 已提交
1700 1701 1702 1703 1704 1705
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1706
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1707
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1708
{
1709
	struct drbd_device *device = peer_device->device;
1710 1711
	struct drbd_socket *sock;
	struct p_data *p;
1712
	int err;
P
Philipp Reisner 已提交
1713 1714
	int dgs;

1715 1716
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1717

1718 1719
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1720

1721 1722 1723 1724 1725
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1726
	p->dp_flags = 0;
1727
	if (dgs)
1728 1729
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + dgs, NULL, peer_req->i.size);
1730
	if (!err)
1731
		err = _drbd_send_zc_ee(peer_device, peer_req);
1732
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1733

1734
	return err;
P
Philipp Reisner 已提交
1735 1736
}

1737
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1738
{
1739 1740
	struct drbd_socket *sock;
	struct p_block_desc *p;
1741

1742 1743
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1744 1745 1746 1747
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1748
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1749 1750
}

P
Philipp Reisner 已提交
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1767
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1768 1769 1770 1771 1772 1773 1774
	      void *buf, size_t size, unsigned msg_flags)
{
	struct kvec iov;
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1775
		return -EBADR;
P
Philipp Reisner 已提交
1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787

	/* THINK  if (signal_pending) return ... ? */

	iov.iov_base = buf;
	iov.iov_len  = size;

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

1788
	if (sock == connection->data.socket) {
1789
		rcu_read_lock();
1790
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1791
		rcu_read_unlock();
1792
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
	}
	do {
		/* STRANGE
		 * tcp_sendmsg does _not_ use its size parameter at all ?
		 *
		 * -EAGAIN on timeout, -EINTR on signal.
		 */
/* THINK
 * do we need to block DRBD_SIG if sock == &meta.socket ??
 * otherwise wake_asender() might interrupt some send_*Ack !
 */
		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
		if (rv == -EAGAIN) {
1806
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
		iov.iov_base += rv;
		iov.iov_len  -= rv;
	} while (sent < size);

1822 1823
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1824 1825 1826

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1827
			drbd_err(connection, "%s_sendmsg returned %d\n",
1828
				 sock == connection->meta.socket ? "msock" : "sock",
1829
				 rv);
1830
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1831
		} else
1832
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1833 1834 1835 1836 1837
	}

	return sent;
}

1838 1839 1840 1841 1842
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1843
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1844 1845 1846 1847
		  size_t size, unsigned msg_flags)
{
	int err;

1848
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1849 1850 1851 1852 1853 1854 1855
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1856 1857
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1858
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1859 1860 1861
	unsigned long flags;
	int rv = 0;

1862
	mutex_lock(&drbd_main_mutex);
1863
	spin_lock_irqsave(&device->resource->req_lock, flags);
1864
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1865 1866
	 * and no race with updating open_cnt */

1867
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1868 1869 1870 1871 1872 1873 1874
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1875
		device->open_cnt++;
1876
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1877
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1878 1879 1880 1881

	return rv;
}

1882
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1883
{
1884
	struct drbd_device *device = gd->private_data;
1885
	mutex_lock(&drbd_main_mutex);
1886
	device->open_cnt--;
1887
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1888 1889
}

1890
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1891
{
1892 1893
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1894
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1895 1896 1897 1898 1899 1900 1901 1902
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1903
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1904 1905 1906 1907
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
	atomic_set(&device->md_io_in_use, 0);

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->bm_io_work.w.cb = w_bitmap_io;

	init_timer(&device->resync_timer);
	init_timer(&device->md_sync_timer);
	init_timer(&device->start_resync_timer);
	init_timer(&device->request_timer);
	device->resync_timer.function = resync_timer_fn;
	device->resync_timer.data = (unsigned long) device;
	device->md_sync_timer.function = md_sync_timer_fn;
	device->md_sync_timer.data = (unsigned long) device;
	device->start_resync_timer.function = start_resync_timer_fn;
	device->start_resync_timer.data = (unsigned long) device;
	device->request_timer.function = request_timer_fn;
	device->request_timer.data = (unsigned long) device;

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
1966
{
1967
	int i;
1968
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
1969
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1970
				first_peer_device(device)->connection->receiver.t_state);
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
1984
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1985 1986
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
1987
	}
1988
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
1989

1990 1991
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
1992
		/* maybe never allocated. */
1993 1994
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
1995 1996
	}

1997
	drbd_free_ldev(device->ldev);
1998
	device->ldev = NULL;
1999

2000
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
2001

2002 2003 2004 2005 2006 2007 2008 2009 2010
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
2011

2012
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2027
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2028

2029 2030
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2031 2032
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2046
	drbd_md_io_bio_set   = NULL;
2047
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2061
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2071
	drbd_md_io_page_pool = NULL;
2072
	drbd_md_io_bio_set   = NULL;
P
Philipp Reisner 已提交
2073 2074 2075 2076 2077 2078 2079 2080

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2081
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2096 2097 2098 2099
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2100 2101 2102 2103
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

P
Philipp Reisner 已提交
2104 2105 2106 2107 2108 2109 2110
	drbd_request_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
	if (drbd_request_mempool == NULL)
		goto Enomem;

	drbd_ee_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2111
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

2133
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2134 2135 2136
{
	int rr;

2137
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2138
	if (rr)
2139
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2140

2141
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2142
	if (rr)
2143
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2144

2145
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2146
	if (rr)
2147
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2148

2149
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2150
	if (rr)
2151
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2152

2153
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2154
	if (rr)
2155
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2156 2157
}

2158
/* caution. no locking. */
2159
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2160
{
2161
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2162
	struct drbd_resource *resource = device->resource;
2163
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2164

2165
	del_timer_sync(&device->request_timer);
2166

P
Philipp Reisner 已提交
2167
	/* paranoia asserts */
2168
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2169 2170 2171 2172 2173
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2174 2175
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2176

2177
	drbd_free_ldev(device->ldev);
2178
	device->ldev = NULL;
P
Philipp Reisner 已提交
2179

2180
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2181

2182 2183
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2184

2185 2186
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2187

2188 2189 2190 2191 2192 2193
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
	__free_page(device->md_io_page);
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2194 2195 2196 2197 2198 2199 2200 2201

	/* not for_each_connection(connection, resource):
	 * those may have been cleaned up and disassociated already.
	 */
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
		kfree(peer_device);
	}
2202
	memset(device, 0xfd, sizeof(*device));
2203
	kfree(device);
2204
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2205 2206
}

2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2229
		struct drbd_device *device = req->device;
2230 2231
		struct bio *bio = req->master_bio;
		unsigned long start_time = req->start_time;
2232 2233
		bool expected;

2234
		expected =
2235 2236 2237 2238 2239 2240
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2241
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2242 2243 2244 2245 2246 2247 2248
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2249
		 */
2250
		kref_put(&req->kref, drbd_req_destroy);
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2265 2266
		inc_ap_bio(device);
		__drbd_make_request(device, bio, start_time);
2267 2268 2269
	}
}

2270
void drbd_restart_request(struct drbd_request *req)
2271 2272 2273 2274 2275 2276 2277 2278 2279
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2280
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2281

2282
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2283 2284
}

2285 2286 2287 2288 2289
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2290
	idr_destroy(&resource->devices);
2291
	free_cpumask_var(resource->cpu_mask);
2292
	kfree(resource->name);
2293
	memset(resource, 0xf2, sizeof(*resource));
2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
		kref_put(&connection->kref, drbd_destroy_connection);
	}
	kref_put(&resource->kref, drbd_destroy_resource);
}
2307

P
Philipp Reisner 已提交
2308 2309 2310
static void drbd_cleanup(void)
{
	unsigned int i;
2311
	struct drbd_device *device;
2312
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2313

2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2325 2326
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2327

2328
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2329

2330
	idr_for_each_entry(&drbd_devices, device, i)
2331
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2332

P
Philipp Reisner 已提交
2333
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2334 2335 2336
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2337
	}
P
Philipp Reisner 已提交
2338

2339
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2340 2341
	unregister_blkdev(DRBD_MAJOR, "drbd");

2342
	idr_destroy(&drbd_devices);
2343

2344
	pr_info("module cleanup done.\n");
P
Philipp Reisner 已提交
2345 2346 2347
}

/**
2348
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2349
 * @congested_data:	User data
2350
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2351 2352 2353 2354 2355
 *
 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2356
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2357 2358 2359 2360
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2361
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2362 2363 2364 2365 2366 2367
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2368
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2369 2370 2371 2372 2373 2374
		r |= (1 << BDI_async_congested);
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2375
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2376 2377
			r |= (1 << BDI_sync_congested);
		else
2378
			put_ldev(device);
2379 2380 2381 2382 2383
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2384 2385
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
P
Philipp Reisner 已提交
2386
		r = bdi_congested(&q->backing_dev_info, bdi_bits);
2387
		put_ldev(device);
P
Philipp Reisner 已提交
2388 2389 2390 2391
		if (r)
			reason = 'b';
	}

2392 2393
	if (bdi_bits & (1 << BDI_async_congested) &&
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
P
Philipp Reisner 已提交
2394 2395 2396 2397 2398
		r |= (1 << BDI_async_congested);
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2399
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2400 2401 2402
	return r;
}

2403 2404 2405 2406
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2407
	init_waitqueue_head(&wq->q_wait);
2408 2409
}

2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2434
struct drbd_resource *drbd_find_resource(const char *name)
2435
{
2436
	struct drbd_resource *resource;
2437

2438 2439 2440
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2441
	rcu_read_lock();
2442 2443
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2444
			kref_get(&resource->kref);
2445
			goto found;
2446
		}
2447
	}
2448
	resource = NULL;
2449
found:
P
Philipp Reisner 已提交
2450
	rcu_read_unlock();
2451
	return resource;
2452 2453
}

2454
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2455 2456
				     void *peer_addr, int peer_addr_len)
{
2457
	struct drbd_resource *resource;
2458
	struct drbd_connection *connection;
2459 2460

	rcu_read_lock();
2461 2462 2463 2464 2465 2466 2467 2468 2469
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2470 2471
		}
	}
2472
	connection = NULL;
2473 2474
found:
	rcu_read_unlock();
2475
	return connection;
2476 2477
}

2478 2479 2480 2481 2482
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2483 2484 2485
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2486 2487 2488 2489 2490
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2491
	free_page((unsigned long) socket->sbuf);
2492 2493 2494
	free_page((unsigned long) socket->rbuf);
}

2495
void conn_free_crypto(struct drbd_connection *connection)
2496
{
2497
	drbd_free_sock(connection);
2498

2499 2500 2501 2502 2503 2504 2505
	crypto_free_hash(connection->csums_tfm);
	crypto_free_hash(connection->verify_tfm);
	crypto_free_hash(connection->cram_hmac_tfm);
	crypto_free_hash(connection->integrity_tfm);
	crypto_free_hash(connection->peer_integrity_tfm);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2506

2507 2508 2509 2510 2511 2512 2513
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2514 2515
}

2516
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2517
{
2518
	struct drbd_connection *connection;
2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;
		/*
		retcode = ERR_NOMEM;
		drbd_msg_put_info("unable to allocate cpumask");
		*/

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2531
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2532
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2533
		if (err) {
2534
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2535 2536 2537 2538
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2539
	resource->res_opts = *res_opts;
2540 2541 2542 2543 2544
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2545 2546 2547 2548
			connection->receiver.reset_cpu_mask = 1;
			connection->asender.reset_cpu_mask = 1;
			connection->worker.reset_cpu_mask = 1;
		}
2549 2550 2551 2552 2553 2554 2555 2556 2557
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2558 2559 2560 2561
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2562
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2563
	if (!resource)
2564
		goto fail;
2565
	resource->name = kstrdup(name, GFP_KERNEL);
2566 2567 2568 2569
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2570
	kref_init(&resource->kref);
2571
	idr_init(&resource->devices);
2572
	INIT_LIST_HEAD(&resource->connections);
2573
	resource->write_ordering = WO_bdev_flush;
2574
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2575
	mutex_init(&resource->conf_update);
2576
	mutex_init(&resource->adm_mutex);
2577
	spin_lock_init(&resource->req_lock);
2578
	return resource;
2579 2580 2581 2582 2583 2584 2585

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2586 2587
}

2588
/* caller must be under genl_lock() */
2589
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2590
{
2591
	struct drbd_resource *resource;
2592
	struct drbd_connection *connection;
2593

2594 2595
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2596 2597
		return NULL;

2598
	if (drbd_alloc_socket(&connection->data))
2599
		goto fail;
2600
	if (drbd_alloc_socket(&connection->meta))
2601 2602
		goto fail;

2603 2604
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2605
		goto fail;
2606

2607
	INIT_LIST_HEAD(&connection->transfer_log);
2608

2609 2610 2611
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
2612

2613 2614 2615
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2616

2617 2618 2619 2620
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2621 2622 2623
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2624
	idr_init(&connection->peer_devices);
2625

2626 2627 2628
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2629

2630 2631 2632 2633 2634 2635
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
	drbd_thread_init(resource, &connection->asender, drbd_asender, "asender");
	connection->asender.connection = connection;
2636

2637
	kref_init(&connection->kref);
2638 2639

	connection->resource = resource;
2640

2641 2642 2643 2644 2645
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2646
	return connection;
2647

2648 2649 2650
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2651
fail:
2652 2653 2654 2655
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2656 2657 2658
	return NULL;
}

2659
void drbd_destroy_connection(struct kref *kref)
2660
{
2661
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2662
	struct drbd_resource *resource = connection->resource;
2663

2664
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2665
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2666
	kfree(connection->current_epoch);
2667

2668
	idr_destroy(&connection->peer_devices);
2669

2670 2671 2672 2673
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2674
	memset(connection, 0xfc, sizeof(*connection));
2675
	kfree(connection);
2676
	kref_put(&resource->kref, drbd_destroy_resource);
2677 2678
}

2679
static int init_submitter(struct drbd_device *device)
2680 2681 2682
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2683 2684 2685
	device->submit.wq = alloc_workqueue("drbd%u_submit",
			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
	if (!device->submit.wq)
2686 2687
		return -ENOMEM;

2688 2689 2690
	INIT_WORK(&device->submit.worker, do_submit);
	spin_lock_init(&device->submit.lock);
	INIT_LIST_HEAD(&device->submit.writes);
2691 2692 2693
	return 0;
}

2694
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2695
{
2696
	struct drbd_resource *resource = adm_ctx->resource;
2697
	struct drbd_connection *connection;
2698
	struct drbd_device *device;
2699
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2700 2701
	struct gendisk *disk;
	struct request_queue *q;
2702
	int id;
2703
	int vnr = adm_ctx->volume;
2704
	enum drbd_ret_code err = ERR_NOMEM;
2705

2706 2707
	device = minor_to_device(minor);
	if (device)
2708
		return ERR_MINOR_EXISTS;
P
Philipp Reisner 已提交
2709 2710

	/* GFP_KERNEL, we are outside of all write-out paths */
2711 2712
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2713
		return ERR_NOMEM;
2714 2715 2716 2717
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2718 2719
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2720

2721
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2722 2723 2724 2725

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2726 2727
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2728 2729 2730 2731

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2732
	device->vdisk = disk;
P
Philipp Reisner 已提交
2733

2734
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2735 2736 2737 2738 2739 2740

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2741
	disk->private_data = device;
P
Philipp Reisner 已提交
2742

2743
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2744
	/* we have no partitions. we contain only ourselves. */
2745
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2746 2747

	q->backing_dev_info.congested_fn = drbd_congested;
2748
	q->backing_dev_info.congested_data = device;
P
Philipp Reisner 已提交
2749

2750
	blk_queue_make_request(q, drbd_make_request);
2751
	blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
2752 2753 2754
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
P
Philipp Reisner 已提交
2755 2756
	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
	blk_queue_merge_bvec(q, drbd_merge_bvec);
2757
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2758

2759 2760
	device->md_io_page = alloc_page(GFP_KERNEL);
	if (!device->md_io_page)
P
Philipp Reisner 已提交
2761 2762
		goto out_no_io_page;

2763
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2764
		goto out_no_bitmap;
2765 2766
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2767

2768 2769 2770
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2771
			err = ERR_MINOR_EXISTS;
2772
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
T
Tejun Heo 已提交
2773
		}
2774
		goto out_no_minor_idr;
2775
	}
2776 2777 2778 2779 2780 2781
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
			err = ERR_MINOR_EXISTS;
2782
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
2783 2784 2785 2786
		}
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2787

2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802
	INIT_LIST_HEAD(&device->peer_devices);
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
			if (id == -ENOSPC) {
				err = ERR_INVALID_REQUEST;
2803
				drbd_msg_put_info(adm_ctx->reply_skb, "requested volume exists already");
2804 2805
			}
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2806
		}
2807
		kref_get(&connection->kref);
2808
	}
T
Tejun Heo 已提交
2809

2810
	if (init_submitter(device)) {
2811
		err = ERR_NOMEM;
2812
		drbd_msg_put_info(adm_ctx->reply_skb, "unable to create submit workqueue");
2813 2814 2815
		goto out_idr_remove_vol;
	}

2816 2817
	add_disk(disk);

2818
	/* inherit the connection state */
2819
	device->state.conn = first_connection(resource)->cstate;
2820 2821 2822 2823
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2824

2825
	return NO_ERROR;
P
Philipp Reisner 已提交
2826

2827
out_idr_remove_vol:
2828
	idr_remove(&connection->peer_devices, vnr);
2829
out_idr_remove_from_resource:
2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840
	for_each_connection(connection, resource) {
		peer_device = idr_find(&connection->peer_devices, vnr);
		if (peer_device) {
			idr_remove(&connection->peer_devices, vnr);
			kref_put(&connection->kref, drbd_destroy_connection);
		}
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2841
	idr_remove(&resource->devices, vnr);
2842
out_idr_remove_minor:
2843
	idr_remove(&drbd_devices, minor);
2844
	synchronize_rcu();
2845
out_no_minor_idr:
2846
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2847
out_no_bitmap:
2848
	__free_page(device->md_io_page);
P
Philipp Reisner 已提交
2849 2850 2851 2852 2853
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2854
	kref_put(&resource->kref, drbd_destroy_resource);
2855
	kfree(device);
2856
	return err;
P
Philipp Reisner 已提交
2857 2858
}

2859
void drbd_delete_device(struct drbd_device *device)
2860 2861 2862 2863 2864 2865
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
	int refs = 3;

	for_each_connection(connection, resource) {
2866
		idr_remove(&connection->peer_devices, device->vnr);
2867 2868 2869 2870 2871 2872 2873 2874 2875
		refs++;
	}
	idr_remove(&resource->devices, device->vnr);
	idr_remove(&drbd_devices, device_to_minor(device));
	del_gendisk(device->vdisk);
	synchronize_rcu();
	kref_sub(&device->kref, refs, drbd_destroy_device);
}

2876
static int __init drbd_init(void)
P
Philipp Reisner 已提交
2877 2878 2879
{
	int err;

2880
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
2881
		pr_err("invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2882 2883 2884
#ifdef MODULE
		return -EINVAL;
#else
2885
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2886 2887 2888 2889 2890
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
2891
		pr_err("unable to register block device major %d\n",
P
Philipp Reisner 已提交
2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
		       DRBD_MAJOR);
		return err;
	}

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
2902
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
2903

2904
	rwlock_init(&global_state_lock);
2905
	INIT_LIST_HEAD(&drbd_resources);
2906 2907 2908

	err = drbd_genl_register();
	if (err) {
2909
		pr_err("unable to register generic netlink family\n");
2910 2911 2912
		goto fail;
	}

P
Philipp Reisner 已提交
2913 2914
	err = drbd_create_mempools();
	if (err)
2915
		goto fail;
P
Philipp Reisner 已提交
2916

2917
	err = -ENOMEM;
2918
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
2919
	if (!drbd_proc)	{
2920
		pr_err("unable to register proc file\n");
2921
		goto fail;
P
Philipp Reisner 已提交
2922 2923
	}

2924 2925
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
2926
		pr_err("unable to create retry workqueue\n");
2927 2928 2929 2930 2931
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
2932

2933
	pr_info("initialized. "
P
Philipp Reisner 已提交
2934 2935
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
2936 2937
	pr_info("%s\n", drbd_buildtag());
	pr_info("registered as block device major %d\n", DRBD_MAJOR);
P
Philipp Reisner 已提交
2938 2939
	return 0; /* Success! */

2940
fail:
P
Philipp Reisner 已提交
2941 2942
	drbd_cleanup();
	if (err == -ENOMEM)
2943
		pr_err("ran out of memory\n");
P
Philipp Reisner 已提交
2944
	else
2945
		pr_err("initialization failure\n");
P
Philipp Reisner 已提交
2946 2947 2948
	return err;
}

2949
void drbd_free_ldev(struct drbd_backing_dev *ldev)
P
Philipp Reisner 已提交
2950 2951 2952 2953
{
	if (ldev == NULL)
		return;

2954 2955
	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
P
Philipp Reisner 已提交
2956

L
Lars Ellenberg 已提交
2957
	kfree(ldev->disk_conf);
P
Philipp Reisner 已提交
2958 2959 2960
	kfree(ldev);
}

2961
void drbd_free_sock(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2962
{
2963 2964 2965 2966 2967 2968
	if (connection->data.socket) {
		mutex_lock(&connection->data.mutex);
		kernel_sock_shutdown(connection->data.socket, SHUT_RDWR);
		sock_release(connection->data.socket);
		connection->data.socket = NULL;
		mutex_unlock(&connection->data.mutex);
P
Philipp Reisner 已提交
2969
	}
2970 2971 2972 2973 2974 2975
	if (connection->meta.socket) {
		mutex_lock(&connection->meta.mutex);
		kernel_sock_shutdown(connection->meta.socket, SHUT_RDWR);
		sock_release(connection->meta.socket);
		connection->meta.socket = NULL;
		mutex_unlock(&connection->meta.mutex);
P
Philipp Reisner 已提交
2976 2977 2978 2979 2980
	}
}

/* meta data management */

2981
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2982
{
2983
	struct drbd_peer_device *peer_device;
2984
	int vnr;
P
Philipp Reisner 已提交
2985

2986
	rcu_read_lock();
2987 2988 2989
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

2990
		kref_get(&device->kref);
2991
		rcu_read_unlock();
2992
		drbd_md_sync(device);
2993
		kref_put(&device->kref, drbd_destroy_device);
2994 2995 2996
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
2997 2998
}

2999
/* aligned 4kByte */
P
Philipp Reisner 已提交
3000
struct meta_data_on_disk {
3001
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
3002 3003 3004 3005 3006 3007 3008
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3009
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3010
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3011 3012
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3013
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3014

3015 3016 3017 3018 3019
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3020 3021
} __packed;

3022 3023


3024
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3025
{
3026
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3027 3028 3029
	sector_t sector;
	int i;

3030
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3031

3032
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3033
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3034 3035
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3036
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3037

3038 3039 3040
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3041
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3042
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3043

3044 3045
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3046

3047 3048
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3049

3050
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3051
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3052

3053
	if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
P
Philipp Reisner 已提交
3054
		/* this was a try anyways ... */
3055
		drbd_err(device, "meta data update failed!\n");
3056
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3057
	}
3058 3059 3060 3061
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3062
 * @device:	DRBD device.
3063
 */
3064
void drbd_md_sync(struct drbd_device *device)
3065 3066 3067 3068 3069 3070 3071
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3072
	del_timer(&device->md_sync_timer);
3073
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3074
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3075 3076 3077 3078
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3079
	if (!get_ldev_if_state(device, D_FAILED))
3080 3081
		return;

3082
	buffer = drbd_md_get_buffer(device);
3083 3084 3085
	if (!buffer)
		goto out;

3086
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3087

3088
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3089
	 * since we updated it on metadata. */
3090
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3091

3092
	drbd_md_put_buffer(device);
3093
out:
3094
	put_ldev(device);
P
Philipp Reisner 已提交
3095 3096
}

3097
static int check_activity_log_stripe_size(struct drbd_device *device,
3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3137
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3138 3139 3140 3141
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3142
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3209
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3222 3223
/**
 * drbd_md_read() - Reads in the meta data super block
3224
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3225 3226
 * @bdev:	Device from which the meta data should be read in.
 *
3227
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3228
 * something goes wrong.
3229
 *
3230
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3231
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3232
 */
3233
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3234 3235
{
	struct meta_data_on_disk *buffer;
3236
	u32 magic, flags;
P
Philipp Reisner 已提交
3237 3238
	int i, rv = NO_ERROR;

3239
	if (device->state.disk != D_DISKLESS)
3240
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3241

3242
	buffer = drbd_md_get_buffer(device);
3243
	if (!buffer)
3244
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3245

3246 3247
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3248 3249
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
P
Philipp Reisner 已提交
3250

3251
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
L
Lucas De Marchi 已提交
3252
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3253
		   called BEFORE disk is attached */
3254
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3255 3256 3257 3258
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3259 3260 3261 3262 3263
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3264
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3265 3266 3267
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3268 3269

	rv = ERR_MD_INVALID;
3270
	if (magic != DRBD_MD_MAGIC_08) {
3271
		if (magic == DRBD_MD_MAGIC_07)
3272
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3273
		else
3274
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3275 3276
		goto err;
	}
3277

3278
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3279
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3280
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3281 3282
		goto err;
	}
3283

3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3296
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3297
		goto err;
3298
	if (check_offsets_and_sizes(device, bdev))
3299 3300
		goto err;

P
Philipp Reisner 已提交
3301
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3302
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3303 3304 3305 3306
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3307
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3308 3309 3310 3311
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3312
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3313

3314
	spin_lock_irq(&device->resource->req_lock);
3315
	if (device->state.conn < C_CONNECTED) {
3316
		unsigned int peer;
3317
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3318
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3319
		device->peer_max_bio_size = peer;
3320
	}
3321
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3322 3323

 err:
3324
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3325 3326 3327 3328 3329 3330

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3331
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3332 3333 3334 3335 3336
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3337
#ifdef DEBUG
3338
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3339
{
3340 3341 3342 3343
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3344 3345 3346
	}
}
#else
3347
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3348
{
3349 3350
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3351
}
3352
#endif
P
Philipp Reisner 已提交
3353

3354
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3355 3356 3357
{
	int i;

3358
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3359
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3360 3361
}

3362
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3363 3364
{
	if (idx == UI_CURRENT) {
3365
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3366 3367 3368 3369
			val |= 1;
		else
			val &= ~((u64)1);

3370
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3371 3372
	}

3373 3374
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3375 3376
}

3377
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3378 3379
{
	unsigned long flags;
3380 3381 3382
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3383
}
P
Philipp Reisner 已提交
3384

3385
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3386
{
3387
	unsigned long flags;
3388 3389 3390 3391
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3392
	}
3393 3394
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3395 3396 3397 3398
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3399
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3400 3401 3402 3403
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3404
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3405 3406
{
	u64 val;
3407 3408 3409 3410
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3411 3412
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3413 3414

	if (bm_uuid)
3415
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3416

3417 3418 3419
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3420

3421
	drbd_print_uuids(device, "new current UUID");
3422
	/* get it to stable storage _now_ */
3423
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3424 3425
}

3426
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3427
{
3428
	unsigned long flags;
3429
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3430 3431
		return;

3432
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3433
	if (val == 0) {
3434 3435 3436
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3437
	} else {
3438
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3439
		if (bm_uuid)
3440
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3441

3442
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3443
	}
3444
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3445

3446
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3447 3448 3449 3450
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3451
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3452 3453 3454
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3455
int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3456 3457 3458
{
	int rv = -EIO;

3459 3460 3461
	drbd_md_set_flag(device, MDF_FULL_SYNC);
	drbd_md_sync(device);
	drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3462

3463
	rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3464

3465 3466 3467
	if (!rv) {
		drbd_md_clear_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
P
Philipp Reisner 已提交
3468 3469 3470 3471 3472 3473 3474
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3475
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3476 3477 3478
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3479
int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3480
{
3481
	drbd_resume_al(device);
3482 3483
	drbd_bm_clear_all(device);
	return drbd_bm_write(device);
P
Philipp Reisner 已提交
3484 3485
}

3486
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3487
{
3488 3489 3490
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3491
	int rv = -EIO;
P
Philipp Reisner 已提交
3492

3493
	D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0);
P
Philipp Reisner 已提交
3494

3495 3496 3497 3498 3499
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3500
	}
P
Philipp Reisner 已提交
3501

3502 3503
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3504 3505

	if (work->done)
3506
		work->done(device, rv);
P
Philipp Reisner 已提交
3507

3508
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3509
	work->why = NULL;
3510
	work->flags = 0;
P
Philipp Reisner 已提交
3511

3512
	return 0;
P
Philipp Reisner 已提交
3513 3514 3515 3516
}

/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3517
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3518 3519 3520 3521 3522 3523 3524 3525
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
3526 3527 3528
 *
 * Its worker function encloses the call of io_fn() by get_ldev() and
 * put_ldev().
P
Philipp Reisner 已提交
3529
 */
3530
void drbd_queue_bitmap_io(struct drbd_device *device,
3531 3532
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3533
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3534
{
3535
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3536

3537 3538 3539
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3540
	if (device->bm_io_work.why)
3541
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3542
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3543

3544 3545 3546 3547
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3548

3549
	spin_lock_irq(&device->resource->req_lock);
3550 3551 3552
	set_bit(BITMAP_IO, &device->flags);
	if (atomic_read(&device->ap_bio_cnt) == 0) {
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3553 3554
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3555
	}
3556
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3557 3558 3559 3560
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3561
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3562 3563 3564 3565 3566 3567
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3568
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3569
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3570 3571 3572
{
	int rv;

3573
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3574

3575
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3576
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3577

3578 3579 3580
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3581

3582
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3583
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3584 3585 3586 3587

	return rv;
}

3588
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3589
{
3590 3591 3592
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3593 3594 3595
	}
}

3596
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3597
{
3598 3599 3600
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3601 3602 3603 3604 3605 3606 3607 3608 3609
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3610
	struct drbd_device *device = (struct drbd_device *) data;
3611
	drbd_device_post_work(device, MD_SYNC);
P
Philipp Reisner 已提交
3612 3613
}

3614
const char *cmdname(enum drbd_packet cmd)
3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3644
		[P_SUPERSEDED]          = "Superseded",
3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3659
		[P_RETRY_WRITE]		= "RetryWrite",
3660 3661 3662
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3663 3664
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3665 3666 3667 3668 3669

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3670 3671
	};

3672
	/* too big for the array: 0xfffX */
3673 3674 3675 3676
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3677 3678
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3679
	if (cmd >= ARRAY_SIZE(cmdnames))
3680 3681 3682 3683
		return "Unknown";
	return cmdnames[cmd];
}

3684 3685
/**
 * drbd_wait_misc  -  wait for a request to make progress
3686
 * @device:	device associated with the request
3687 3688 3689
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3690
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3691
{
3692
	struct net_conf *nc;
3693 3694 3695
	DEFINE_WAIT(wait);
	long timeout;

3696
	rcu_read_lock();
3697
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3698 3699
	if (!nc) {
		rcu_read_unlock();
3700
		return -ETIMEDOUT;
3701 3702 3703
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3704

3705
	/* Indicate to wake up device->misc_wait on progress.  */
3706
	i->waiting = true;
3707
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3708
	spin_unlock_irq(&device->resource->req_lock);
3709
	timeout = schedule_timeout(timeout);
3710
	finish_wait(&device->misc_wait, &wait);
3711
	spin_lock_irq(&device->resource->req_lock);
3712
	if (!timeout || device->state.conn < C_CONNECTED)
3713 3714 3715 3716
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739
}

#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3740
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3760 3761
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3762 3763 3764 3765 3766 3767
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3768
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3769 3770 3771 3772 3773
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3774
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3775 3776 3777 3778 3779
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3780
		if (__ratelimit(&drbd_ratelimit_state))
3781
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3797 3798 3799 3800
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);