drbd_main.c 109.6 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

#include <linux/module.h>
#include <linux/drbd.h>
#include <asm/uaccess.h>
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
35
#include <linux/mutex.h>
P
Philipp Reisner 已提交
36 37 38 39 40 41 42 43 44 45 46 47
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
48
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
49 50 51 52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>

#include <linux/drbd_limits.h>
#include "drbd_int.h"
55
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
56 57 58 59
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */

#include "drbd_vli.h"

60
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
61
static int drbd_open(struct block_device *bdev, fmode_t mode);
62
static void drbd_release(struct gendisk *gd, fmode_t mode);
63
static int w_md_sync(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
64
static void md_sync_timer_fn(unsigned long data);
65 66
static int w_bitmap_io(struct drbd_work *w, int unused);
static int w_go_diskless(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
67 68 69 70 71 72

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
73
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
74
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
103
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
104 105
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
106 107 108 109 110 111 112 113 114 115 116
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
char usermode_helper[80] = "/sbin/drbdadm";

module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
117
struct idr drbd_devices;
118
struct list_head drbd_resources;
P
Philipp Reisner 已提交
119 120

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
121
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
122 123 124 125
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
126
mempool_t *drbd_md_io_page_pool;
127
struct bio_set *drbd_md_io_bio_set;
P
Philipp Reisner 已提交
128 129 130 131 132 133 134 135 136 137 138 139 140 141

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

142
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
143 144 145 146
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
147

148 149 150
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
151

152 153
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
154

155 156 157 158
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
159 160
}

P
Philipp Reisner 已提交
161 162 163 164
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
165
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
166 167 168
{
	int io_allowed;

169 170
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
171
	if (!io_allowed) {
172 173
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
174 175 176 177 178
	}
	return io_allowed;
}

#endif
179

P
Philipp Reisner 已提交
180
/**
181
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
182
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
183 184 185 186
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
187 188
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
189
 */
190
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
191
		unsigned int set_size)
P
Philipp Reisner 已提交
192 193
{
	struct drbd_request *r;
194 195 196
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
197

198
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
199

200
	/* find oldest not yet barrier-acked write request,
201
	 * count writes in its epoch. */
202
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
203
		const unsigned s = r->rq_state;
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
222
		}
P
Philipp Reisner 已提交
223
	}
224

P
Philipp Reisner 已提交
225
	/* first some paranoia code */
226
	if (req == NULL) {
227
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
228
			 barrier_nr);
P
Philipp Reisner 已提交
229 230
		goto bail;
	}
231
	if (expect_epoch != barrier_nr) {
232
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
233
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
234
		goto bail;
235 236
	}

237
	if (expect_size != set_size) {
238
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
239
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
240 241 242
		goto bail;
	}

243 244 245 246
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
247
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
248 249
		if (req->epoch == expect_epoch)
			break;
250
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
251 252 253
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
254
	}
255
	spin_unlock_irq(&connection->resource->req_lock);
256

P
Philipp Reisner 已提交
257 258 259
	return;

bail:
260
	spin_unlock_irq(&connection->resource->req_lock);
261
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
262 263 264 265
}


/**
266
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
267
 * @device:	DRBD device.
268
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
269
 *
270 271
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
272
 */
273
/* must hold resource->req_lock */
274
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
275
{
276
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
277

278
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
279 280
		_req_mod(req, what);
}
281

282
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
283
{
284
	spin_lock_irq(&connection->resource->req_lock);
285
	_tl_restart(connection, what);
286
	spin_unlock_irq(&connection->resource->req_lock);
287
}
P
Philipp Reisner 已提交
288 289 290

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
291
 * @device:	DRBD device.
P
Philipp Reisner 已提交
292 293 294 295 296
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
297
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
298
{
299
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
300
}
301

302
/**
303 304
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
305
 */
306
void tl_abort_disk_io(struct drbd_device *device)
307
{
308
	struct drbd_connection *connection = first_peer_device(device)->connection;
309
	struct drbd_request *req, *r;
310

311
	spin_lock_irq(&connection->resource->req_lock);
312
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
313 314
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
315
		if (req->device != device)
316 317
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
318
	}
319
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
320 321 322 323 324
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
325
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
326 327 328
	unsigned long flags;
	int retval;

329
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
330
		 thi->name[0],
331
		 resource->name);
332

P
Philipp Reisner 已提交
333 334 335 336 337
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

338
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
339 340 341
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
342
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
343
	 * t_state check and assignment needs to be within the same spinlock,
344 345
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
346 347
	 */

348
	if (thi->t_state == RESTARTING) {
349
		drbd_info(resource, "Restarting %s thread\n", thi->name);
350
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
351 352 353 354 355
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
356
	thi->t_state = NONE;
P
Philipp Reisner 已提交
357
	smp_mb();
L
Lars Ellenberg 已提交
358
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
359 360
	spin_unlock_irqrestore(&thi->t_lock, flags);

361
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
362 363

	/* Release mod reference taken when thread was started */
364

365 366 367
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
368 369 370 371
	module_put(THIS_MODULE);
	return retval;
}

372
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
373
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
374 375 376
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
377
	thi->t_state = NONE;
P
Philipp Reisner 已提交
378
	thi->function = func;
379 380
	thi->resource = resource;
	thi->connection = NULL;
381
	thi->name = name;
P
Philipp Reisner 已提交
382 383 384 385
}

int drbd_thread_start(struct drbd_thread *thi)
{
386
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
387 388 389 390 391 392 393 394
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
395
	case NONE:
396
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
397
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
398 399 400

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
401
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
402
			spin_unlock_irqrestore(&thi->t_lock, flags);
403
			return false;
P
Philipp Reisner 已提交
404 405
		}

406 407 408
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
409

P
Philipp Reisner 已提交
410 411
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
412
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
413 414 415 416
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
417
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
418 419

		if (IS_ERR(nt)) {
420
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
421

422 423 424
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
425
			module_put(THIS_MODULE);
426
			return false;
P
Philipp Reisner 已提交
427 428 429
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
430
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
431 432 433
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
434 435
	case EXITING:
		thi->t_state = RESTARTING;
436
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
437
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
438
		/* fall through */
439 440
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
441 442 443 444 445
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

446
	return true;
P
Philipp Reisner 已提交
447 448 449 450 451 452 453
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

454
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
455 456 457 458

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

459
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

485
int conn_lowest_minor(struct drbd_connection *connection)
486
{
487 488
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
489

490
	rcu_read_lock();
491 492 493
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
494 495
	rcu_read_unlock();

496
	return minor;
497
}
498

P
Philipp Reisner 已提交
499 500 501 502
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
503
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
504 505
 * DRBD's performance. May be overwritten by user's configuration.
 */
506
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
507
{
508
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
509

510 511 512 513
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
514

515 516 517 518
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
519
		}
520 521 522 523 524 525 526 527 528 529 530 531
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
532
	}
533
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
534 535 536 537
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
538
 * @device:	DRBD device.
539
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
540 541 542 543
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
544
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
545
{
546
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
547
	struct task_struct *p = current;
548

P
Philipp Reisner 已提交
549 550 551
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
552
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
553
}
554 555
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
556 557
#endif

558 559 560 561 562 563 564
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
565
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
566
{
567
	if (connection->agreed_pro_version >= 100) {
568 569 570 571 572 573 574 575
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
576
}
P
Philipp Reisner 已提交
577

578
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
579 580 581 582
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
583
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
584
}
P
Philipp Reisner 已提交
585

586
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
587 588
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
589
	h->command = cpu_to_be16(cmd);
590
	h->length = cpu_to_be32(size);
591
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
592
}
P
Philipp Reisner 已提交
593

594 595 596 597 598 599 600 601 602 603
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
604

605
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
606
				   void *buffer, enum drbd_packet cmd, int size)
607
{
608
	if (connection->agreed_pro_version >= 100)
609
		return prepare_header100(buffer, cmd, size, vnr);
610
	else if (connection->agreed_pro_version >= 95 &&
611
		 size > DRBD_MAX_SIZE_H80_PACKET)
612
		return prepare_header95(buffer, cmd, size);
613
	else
614
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
615 616
}

617
static void *__conn_prepare_command(struct drbd_connection *connection,
618
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
619
{
620 621
	if (!sock->socket)
		return NULL;
622
	return sock->sbuf + drbd_header_size(connection);
623
}
P
Philipp Reisner 已提交
624

625
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
626
{
627
	void *p;
P
Philipp Reisner 已提交
628

629
	mutex_lock(&sock->mutex);
630
	p = __conn_prepare_command(connection, sock);
631
	if (!p)
632
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
633

634
	return p;
P
Philipp Reisner 已提交
635 636
}

637
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
638
{
639
	return conn_prepare_command(peer_device->connection, sock);
640
}
P
Philipp Reisner 已提交
641

642
static int __send_command(struct drbd_connection *connection, int vnr,
643 644 645 646 647 648
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
649

650 651 652 653 654 655 656 657 658
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

659
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
660
				      header_size + size);
661
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
662 663
			    msg_flags);
	if (data && !err)
664
		err = drbd_send_all(connection, sock->socket, data, size, 0);
665 666 667
	return err;
}

668
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
669 670 671
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
672
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
673 674
}

675
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
676 677 678 679
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
680

681
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
682 683 684 685
	mutex_unlock(&sock->mutex);
	return err;
}

686
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
687 688 689 690 691
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

692 693
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
694 695 696
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
697

698
int drbd_send_ping(struct drbd_connection *connection)
699
{
700 701
	struct drbd_socket *sock;

702 703
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
704
		return -EIO;
705
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
706
}
P
Philipp Reisner 已提交
707

708
int drbd_send_ping_ack(struct drbd_connection *connection)
709
{
710 711
	struct drbd_socket *sock;

712 713
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
714
		return -EIO;
715
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
716 717
}

718
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
719
{
720
	struct drbd_socket *sock;
721
	struct p_rs_param_95 *p;
722
	int size;
723
	const int apv = peer_device->connection->agreed_pro_version;
724
	enum drbd_packet cmd;
725
	struct net_conf *nc;
P
Philipp Reisner 已提交
726
	struct disk_conf *dc;
727

728 729
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
730 731
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
732

733
	rcu_read_lock();
734
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
735 736 737

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
738
			+ strlen(nc->verify_alg) + 1
739 740
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
741

742
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
743

744 745
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
746

747 748
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
749
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
750 751 752 753
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
754
		put_ldev(peer_device->device);
755
	} else {
756
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
757 758 759 760 761
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
762

763
	if (apv >= 88)
764
		strcpy(p->verify_alg, nc->verify_alg);
765
	if (apv >= 89)
766 767
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
768

769
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
770 771
}

772
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
773
{
774
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
775
	struct p_protocol *p;
776
	struct net_conf *nc;
777
	int size, cf;
P
Philipp Reisner 已提交
778

779 780
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
781 782
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
783

784
	rcu_read_lock();
785
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
786

787
	if (nc->tentative && connection->agreed_pro_version < 92) {
788 789
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
790
		drbd_err(connection, "--dry-run is not supported by peer");
791 792
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
793

794
	size = sizeof(*p);
795
	if (connection->agreed_pro_version >= 87)
796
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
797

798 799 800 801 802
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
803
	cf = 0;
804 805
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
806
	if (nc->tentative)
807
		cf |= CF_DRY_RUN;
808 809
	p->conn_flags    = cpu_to_be32(cf);

810
	if (connection->agreed_pro_version >= 87)
811 812
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
813

814
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
815 816
}

817
int drbd_send_protocol(struct drbd_connection *connection)
818 819 820
{
	int err;

821 822 823
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
824 825

	return err;
P
Philipp Reisner 已提交
826 827
}

828
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
829
{
830
	struct drbd_device *device = peer_device->device;
831 832
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
833 834
	int i;

835
	if (!get_ldev_if_state(device, D_NEGOTIATING))
836
		return 0;
P
Philipp Reisner 已提交
837

838 839
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
840
	if (!p) {
841
		put_ldev(device);
842 843
		return -EIO;
	}
844
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
845
	for (i = UI_CURRENT; i < UI_SIZE; i++)
846 847
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
848

849 850
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
851
	rcu_read_lock();
852
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
853
	rcu_read_unlock();
854 855
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
856
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
857

858
	put_ldev(device);
859
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
860 861
}

862
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
863
{
864
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
865 866
}

867
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
868
{
869
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
870 871
}

872
void drbd_print_uuids(struct drbd_device *device, const char *text)
873
{
874 875
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
876
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
877 878 879 880 881
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
882
		put_ldev(device);
883
	} else {
884
		drbd_info(device, "%s effective data uuid: %016llX\n",
885
				text,
886
				(unsigned long long)device->ed_uuid);
887 888 889
	}
}

890
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
891
{
892
	struct drbd_device *device = peer_device->device;
893 894
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
895 896
	u64 uuid;

897
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
898

899
	uuid = device->ldev->md.uuid[UI_BITMAP];
900 901 902 903
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
904 905 906
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
907

908 909
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
910 911
	if (p) {
		p->uuid = cpu_to_be64(uuid);
912
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
913
	}
P
Philipp Reisner 已提交
914 915
}

916
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
917
{
918
	struct drbd_device *device = peer_device->device;
919 920
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
921
	sector_t d_size, u_size;
922 923
	int q_order_type;
	unsigned int max_bio_size;
P
Philipp Reisner 已提交
924

925
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
926
		D_ASSERT(device, device->ldev->backing_bdev);
927
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
928
		rcu_read_lock();
929
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
930
		rcu_read_unlock();
931 932
		q_order_type = drbd_queue_order_type(device);
		max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9;
933
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
934
		put_ldev(device);
P
Philipp Reisner 已提交
935 936 937 938
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
939
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
P
Philipp Reisner 已提交
940 941
	}

942 943
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
944 945
	if (!p)
		return -EIO;
946

947
	if (peer_device->connection->agreed_pro_version <= 94)
948
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
949
	else if (peer_device->connection->agreed_pro_version < 100)
950
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
951

952 953
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
954
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
955 956 957
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
958
	return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
959 960 961
}

/**
962
 * drbd_send_current_state() - Sends the drbd state to the peer
963
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
964
 */
965
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
966
{
967
	struct drbd_socket *sock;
968
	struct p_state *p;
P
Philipp Reisner 已提交
969

970 971
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
972 973
	if (!p)
		return -EIO;
974 975
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
976 977
}

978 979
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
980
 * @peer_device:      DRBD peer device.
981
 * @state:     the state to send, not necessarily the current state.
982 983 984 985 986 987
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
988
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
989
{
990 991
	struct drbd_socket *sock;
	struct p_state *p;
992

993 994
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
995 996 997
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
998
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
999
}
1000

1001
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1002 1003 1004
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1005

1006 1007
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1008 1009 1010 1011
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1012
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1013
}
1014

1015
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1016
{
1017 1018 1019
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1020

1021 1022 1023
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1024 1025 1026 1027
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1028
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1029 1030
}

1031
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1032
{
1033 1034
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1035

1036 1037
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1038 1039
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1040
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1041
	}
P
Philipp Reisner 已提交
1042 1043
}

1044
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1045
{
1046 1047
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1048
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1049

1050 1051
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1052 1053
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1054
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1055
	}
P
Philipp Reisner 已提交
1056 1057
}

1058
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1059
{
1060 1061 1062
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1063

1064 1065 1066 1067
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1068

1069 1070 1071 1072
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1073 1074
}

1075
static int fill_bitmap_rle_bits(struct drbd_device *device,
1076 1077 1078
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1079 1080 1081 1082 1083 1084 1085
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1086
	int bits, use_rle;
P
Philipp Reisner 已提交
1087 1088

	/* may we use this feature? */
1089
	rcu_read_lock();
1090
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1091
	rcu_read_unlock();
1092
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1093
		return 0;
P
Philipp Reisner 已提交
1094 1095 1096 1097 1098

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1099 1100
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1112 1113
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1114 1115 1116 1117 1118 1119 1120 1121
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1122
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1123 1124 1125 1126
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1127
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1128 1129 1130 1131 1132
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1133
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1134 1135 1136 1137 1138 1139 1140 1141
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1142
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1167
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1168 1169 1170 1171

	return len;
}

1172 1173 1174 1175 1176 1177 1178
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1179
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1180
{
1181 1182
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1183
	struct p_compressed_bm *p = sock->sbuf + header_size;
1184
	int len, err;
P
Philipp Reisner 已提交
1185

1186
	len = fill_bitmap_rle_bits(device, p,
1187
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1188
	if (len < 0)
1189
		return -EIO;
P
Philipp Reisner 已提交
1190 1191

	if (len) {
1192
		dcbp_set_code(p, RLE_VLI_Bits);
1193
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1194 1195
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1196
		c->packets[0]++;
1197
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1198 1199 1200 1201 1202 1203

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1204 1205
		unsigned int data_size;
		unsigned long num_words;
1206
		unsigned long *p = sock->sbuf + header_size;
1207 1208

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1209
		num_words = min_t(size_t, data_size / sizeof(*p),
1210
				  c->bm_words - c->word_offset);
1211
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1212
		if (len)
1213
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1214
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1215 1216 1217 1218
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1219
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1220 1221 1222 1223

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1224
	if (!err) {
1225
		if (len == 0) {
1226
			INFO_bm_xfer_stats(device, "send", c);
1227 1228 1229 1230 1231
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1232 1233 1234
}

/* See the comment at receive_bitmap() */
1235
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1236 1237
{
	struct bm_xfer_ctx c;
1238
	int err;
P
Philipp Reisner 已提交
1239

1240
	if (!expect(device->bitmap))
1241
		return false;
P
Philipp Reisner 已提交
1242

1243 1244
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1245
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1246 1247
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1248 1249 1250
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1251
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1252
			} else {
1253 1254
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1255 1256
			}
		}
1257
		put_ldev(device);
P
Philipp Reisner 已提交
1258 1259 1260
	}

	c = (struct bm_xfer_ctx) {
1261 1262
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1263 1264 1265
	};

	do {
1266
		err = send_bitmap_rle_or_plain(device, &c);
1267
	} while (err > 0);
P
Philipp Reisner 已提交
1268

1269
	return err == 0;
P
Philipp Reisner 已提交
1270 1271
}

1272
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1273
{
1274
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1275
	int err = -1;
P
Philipp Reisner 已提交
1276

1277 1278
	mutex_lock(&sock->mutex);
	if (sock->socket)
1279
		err = !_drbd_send_bitmap(device);
1280
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1281 1282 1283
	return err;
}

1284
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1285
{
1286 1287
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1288

1289
	if (connection->cstate < C_WF_REPORT_PARAMS)
1290
		return;
P
Philipp Reisner 已提交
1291

1292 1293
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1294 1295 1296 1297
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1298
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1299 1300 1301 1302
}

/**
 * _drbd_send_ack() - Sends an ack packet
1303
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1304 1305 1306 1307 1308
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1309
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1310
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1311
{
1312 1313
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1314

1315
	if (peer_device->device->state.conn < C_CONNECTED)
1316
		return -EIO;
P
Philipp Reisner 已提交
1317

1318 1319
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1320
	if (!p)
1321
		return -EIO;
1322 1323 1324
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1325 1326
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1327 1328
}

1329 1330 1331
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1332
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1333
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1334
{
1335 1336 1337
	if (peer_device->connection->peer_integrity_tfm)
		data_size -= crypto_hash_digestsize(peer_device->connection->peer_integrity_tfm);
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1338
		       dp->block_id);
P
Philipp Reisner 已提交
1339 1340
}

1341
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1342
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1343
{
1344
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1345 1346 1347 1348
}

/**
 * drbd_send_ack() - Sends an ack packet
1349
 * @device:	DRBD device
1350 1351
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1352
 */
1353
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1354
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1355
{
1356
	return _drbd_send_ack(peer_device, cmd,
1357 1358 1359
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1360 1361 1362 1363
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1364
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1365 1366
		     sector_t sector, int blksize, u64 block_id)
{
1367
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1368 1369 1370 1371 1372
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1373
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1374 1375
		       sector_t sector, int size, u64 block_id)
{
1376 1377
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1378

1379 1380
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1381 1382 1383 1384 1385
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1386
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1387 1388
}

1389
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1390
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1391
{
1392 1393
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1394

1395
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1396

1397 1398
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1399 1400 1401 1402 1403
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1404
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1405 1406
}

1407
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1408
{
1409 1410
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1411

1412 1413
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1414 1415 1416 1417 1418
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1419
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1420 1421 1422
}

/* called on sndtimeo
1423 1424
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1425
 */
1426
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1427 1428
{
	int drop_it;
1429
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1430

1431 1432 1433 1434
	drop_it =   connection->meta.socket == sock
		|| !connection->asender.task
		|| get_t_state(&connection->asender) != RUNNING
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1435 1436

	if (drop_it)
1437
		return true;
P
Philipp Reisner 已提交
1438

1439
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1440
	if (!drop_it) {
1441
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1442 1443
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1444 1445
	}

1446
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1447 1448
}

1449
static void drbd_update_congested(struct drbd_connection *connection)
1450
{
1451
	struct sock *sk = connection->data.socket->sk;
1452
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1453
		set_bit(NET_CONGESTED, &connection->flags);
1454 1455
}

P
Philipp Reisner 已提交
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1477
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1478
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1479
{
1480 1481 1482 1483
	struct socket *socket;
	void *addr;
	int err;

1484
	socket = peer_device->connection->data.socket;
1485
	addr = kmap(page) + offset;
1486
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1487
	kunmap(page);
1488
	if (!err)
1489
		peer_device->device->send_cnt += size >> 9;
1490
	return err;
P
Philipp Reisner 已提交
1491 1492
}

1493
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1494
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1495
{
1496
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1497 1498
	mm_segment_t oldfs = get_fs();
	int len = size;
1499
	int err = -EIO;
P
Philipp Reisner 已提交
1500 1501 1502 1503 1504 1505 1506 1507

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1508
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1509

L
Lars Ellenberg 已提交
1510
	msg_flags |= MSG_NOSIGNAL;
1511
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1512 1513
	set_fs(KERNEL_DS);
	do {
1514 1515 1516
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1517
		if (sent <= 0) {
1518
			if (sent == -EAGAIN) {
1519
				if (we_should_drop_the_connection(peer_device->connection, socket))
1520 1521 1522
					break;
				continue;
			}
1523
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1524
			     __func__, (int)size, len, sent);
1525 1526
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1527 1528 1529 1530
			break;
		}
		len    -= sent;
		offset += sent;
1531
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
P
Philipp Reisner 已提交
1532
	set_fs(oldfs);
1533
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1534

1535 1536
	if (len == 0) {
		err = 0;
1537
		peer_device->device->send_cnt += size >> 9;
1538 1539
	}
	return err;
P
Philipp Reisner 已提交
1540 1541
}

1542
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1543
{
1544 1545 1546
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1547
	/* hint all but last page with MSG_MORE */
1548
	bio_for_each_segment(bvec, bio, iter) {
1549 1550
		int err;

1551
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1552
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1553
					 bio_iter_last(bvec, iter)
1554
					 ? 0 : MSG_MORE);
1555 1556
		if (err)
			return err;
P
Philipp Reisner 已提交
1557
	}
1558
	return 0;
P
Philipp Reisner 已提交
1559 1560
}

1561
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1562
{
1563 1564 1565
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1566
	/* hint all but last page with MSG_MORE */
1567
	bio_for_each_segment(bvec, bio, iter) {
1568 1569
		int err;

1570
		err = _drbd_send_page(peer_device, bvec.bv_page,
1571
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1572
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1573 1574
		if (err)
			return err;
P
Philipp Reisner 已提交
1575
	}
1576
	return 0;
P
Philipp Reisner 已提交
1577 1578
}

1579
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1580
			    struct drbd_peer_request *peer_req)
1581
{
1582 1583
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1584
	int err;
1585

L
Lars Ellenberg 已提交
1586
	/* hint all but last page with MSG_MORE */
1587 1588
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1589

1590
		err = _drbd_send_page(peer_device, page, 0, l,
1591 1592 1593
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1594 1595
		len -= l;
	}
1596
	return 0;
1597 1598
}

1599
static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw)
1600
{
1601
	if (connection->agreed_pro_version >= 95)
1602 1603 1604 1605 1606
		return  (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bi_rw & REQ_FUA ? DP_FUA : 0) |
			(bi_rw & REQ_FLUSH ? DP_FLUSH : 0) |
			(bi_rw & REQ_DISCARD ? DP_DISCARD : 0);
	else
J
Jens Axboe 已提交
1607
		return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0;
1608 1609
}

P
Philipp Reisner 已提交
1610 1611 1612
/* Used to send write requests
 * R_PRIMARY -> Peer	(P_DATA)
 */
1613
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1614
{
1615
	struct drbd_device *device = peer_device->device;
1616 1617
	struct drbd_socket *sock;
	struct p_data *p;
P
Philipp Reisner 已提交
1618 1619
	unsigned int dp_flags = 0;
	int dgs;
1620
	int err;
P
Philipp Reisner 已提交
1621

1622 1623 1624 1625
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1626

1627 1628 1629 1630
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1631
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
1632
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw);
1633 1634
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1635
		dp_flags |= DP_MAY_SET_IN_SYNC;
1636
	if (peer_device->connection->agreed_pro_version >= 100) {
1637 1638 1639 1640 1641
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
		if (req->rq_state & RQ_EXP_WRITE_ACK)
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1642 1643
	p->dp_flags = cpu_to_be32(dp_flags);
	if (dgs)
1644 1645
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + dgs, NULL, req->i.size);
1646
	if (!err) {
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1658
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || dgs)
1659
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1660
		else
1661
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1662 1663 1664

		/* double check digest, sometimes buffers have been modified in flight. */
		if (dgs > 0 && dgs <= 64) {
B
Bart Van Assche 已提交
1665
			/* 64 byte, 512 bit, is the largest digest size
1666 1667
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1668
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1669
			if (memcmp(p + 1, digest, dgs)) {
1670
				drbd_warn(device,
1671
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1672
					(unsigned long long)req->i.sector, req->i.size);
1673 1674 1675 1676
			}
		} /* else if (dgs > 64) {
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1677
	}
1678
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1679

1680
	return err;
P
Philipp Reisner 已提交
1681 1682 1683 1684 1685 1686
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1687
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1688
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1689
{
1690
	struct drbd_device *device = peer_device->device;
1691 1692
	struct drbd_socket *sock;
	struct p_data *p;
1693
	int err;
P
Philipp Reisner 已提交
1694 1695
	int dgs;

1696 1697
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1698

1699 1700
	dgs = peer_device->connection->integrity_tfm ?
	      crypto_hash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1701

1702 1703 1704 1705 1706
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1707
	p->dp_flags = 0;
1708
	if (dgs)
1709 1710
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + dgs, NULL, peer_req->i.size);
1711
	if (!err)
1712
		err = _drbd_send_zc_ee(peer_device, peer_req);
1713
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1714

1715
	return err;
P
Philipp Reisner 已提交
1716 1717
}

1718
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1719
{
1720 1721
	struct drbd_socket *sock;
	struct p_block_desc *p;
1722

1723 1724
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1725 1726 1727 1728
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1729
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1730 1731
}

P
Philipp Reisner 已提交
1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1748
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1749 1750 1751 1752 1753 1754 1755
	      void *buf, size_t size, unsigned msg_flags)
{
	struct kvec iov;
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1756
		return -EBADR;
P
Philipp Reisner 已提交
1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768

	/* THINK  if (signal_pending) return ... ? */

	iov.iov_base = buf;
	iov.iov_len  = size;

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

1769
	if (sock == connection->data.socket) {
1770
		rcu_read_lock();
1771
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1772
		rcu_read_unlock();
1773
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
	}
	do {
		/* STRANGE
		 * tcp_sendmsg does _not_ use its size parameter at all ?
		 *
		 * -EAGAIN on timeout, -EINTR on signal.
		 */
/* THINK
 * do we need to block DRBD_SIG if sock == &meta.socket ??
 * otherwise wake_asender() might interrupt some send_*Ack !
 */
		rv = kernel_sendmsg(sock, &msg, &iov, 1, size);
		if (rv == -EAGAIN) {
1787
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
		iov.iov_base += rv;
		iov.iov_len  -= rv;
	} while (sent < size);

1803 1804
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1805 1806 1807

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1808
			drbd_err(connection, "%s_sendmsg returned %d\n",
1809
				 sock == connection->meta.socket ? "msock" : "sock",
1810
				 rv);
1811
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1812
		} else
1813
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1814 1815 1816 1817 1818
	}

	return sent;
}

1819 1820 1821 1822 1823
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1824
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1825 1826 1827 1828
		  size_t size, unsigned msg_flags)
{
	int err;

1829
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1830 1831 1832 1833 1834 1835 1836
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1837 1838
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1839
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1840 1841 1842
	unsigned long flags;
	int rv = 0;

1843
	mutex_lock(&drbd_main_mutex);
1844
	spin_lock_irqsave(&device->resource->req_lock, flags);
1845
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1846 1847
	 * and no race with updating open_cnt */

1848
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1849 1850 1851 1852 1853 1854 1855
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1856
		device->open_cnt++;
1857
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1858
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1859 1860 1861 1862

	return rv;
}

1863
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1864
{
1865
	struct drbd_device *device = gd->private_data;
1866
	mutex_lock(&drbd_main_mutex);
1867
	device->open_cnt--;
1868
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1869 1870
}

1871
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1872
{
1873 1874
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1875
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1876 1877 1878 1879 1880 1881 1882 1883
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1884
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1885 1886 1887 1888
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
	atomic_set(&device->md_io_in_use, 0);

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->go_diskless.list);
	INIT_LIST_HEAD(&device->md_sync_work.list);
	INIT_LIST_HEAD(&device->start_resync_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->go_diskless.cb  = w_go_diskless;
	device->md_sync_work.cb = w_md_sync;
	device->bm_io_work.w.cb = w_bitmap_io;
	device->start_resync_work.cb = w_start_resync;

	init_timer(&device->resync_timer);
	init_timer(&device->md_sync_timer);
	init_timer(&device->start_resync_timer);
	init_timer(&device->request_timer);
	device->resync_timer.function = resync_timer_fn;
	device->resync_timer.data = (unsigned long) device;
	device->md_sync_timer.function = md_sync_timer_fn;
	device->md_sync_timer.data = (unsigned long) device;
	device->start_resync_timer.function = start_resync_timer_fn;
	device->start_resync_timer.data = (unsigned long) device;
	device->request_timer.function = request_timer_fn;
	device->request_timer.data = (unsigned long) device;

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
1953
{
1954
	int i;
1955
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
1956
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1957
				first_peer_device(device)->connection->receiver.t_state);
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
1971
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
1972 1973
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
1974
	}
1975
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
1976

1977 1978
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
1979
		/* maybe never allocated. */
1980 1981
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
1982 1983
	}

1984 1985
	drbd_free_bc(device->ldev);
	device->ldev = NULL;
1986

1987
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
1988

1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
	D_ASSERT(device, list_empty(&device->go_diskless.list));
1999

2000
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2015
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2016

2017 2018
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2019 2020
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2034
	drbd_md_io_bio_set   = NULL;
2035
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2049
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2050 2051 2052 2053 2054 2055 2056 2057 2058
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2059
	drbd_md_io_page_pool = NULL;
2060
	drbd_md_io_bio_set   = NULL;
P
Philipp Reisner 已提交
2061 2062 2063 2064 2065 2066 2067 2068

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2069
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2084 2085 2086 2087
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2088 2089 2090 2091
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

P
Philipp Reisner 已提交
2092 2093 2094 2095 2096 2097 2098
	drbd_request_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_request_cache);
	if (drbd_request_mempool == NULL)
		goto Enomem;

	drbd_ee_mempool = mempool_create(number,
		mempool_alloc_slab, mempool_free_slab, drbd_ee_cache);
2099
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

static int drbd_notify_sys(struct notifier_block *this, unsigned long code,
	void *unused)
{
	/* just so we have it.  you never know what interesting things we
	 * might want to do here some day...
	 */

	return NOTIFY_DONE;
}

static struct notifier_block drbd_notifier = {
	.notifier_call = drbd_notify_sys,
};

2135
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2136 2137 2138
{
	int rr;

2139
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2140
	if (rr)
2141
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2142

2143
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2144
	if (rr)
2145
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2146

2147
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2148
	if (rr)
2149
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2150

2151
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2152
	if (rr)
2153
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2154

2155
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2156
	if (rr)
2157
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2158 2159
}

2160
/* caution. no locking. */
2161
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2162
{
2163
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2164 2165
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
P
Philipp Reisner 已提交
2166

2167
	del_timer_sync(&device->request_timer);
2168

P
Philipp Reisner 已提交
2169
	/* paranoia asserts */
2170
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2171 2172 2173 2174 2175
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2176 2177
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2178

2179 2180
	drbd_free_bc(device->ldev);
	device->ldev = NULL;
P
Philipp Reisner 已提交
2181

2182
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2183

2184 2185
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2186

2187 2188
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2189

2190 2191 2192 2193 2194 2195
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
	__free_page(device->md_io_page);
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2196
	kfree(first_peer_device(device));
2197
	kfree(device);
2198

2199 2200 2201
	for_each_connection(connection, resource)
		kref_put(&connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2202 2203
}

2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2226
		struct drbd_device *device = req->device;
2227 2228
		struct bio *bio = req->master_bio;
		unsigned long start_time = req->start_time;
2229 2230
		bool expected;

2231
		expected =
2232 2233 2234 2235 2236 2237
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2238
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2239 2240 2241 2242 2243 2244 2245
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2246
		 */
2247
		kref_put(&req->kref, drbd_req_destroy);
2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2262 2263
		inc_ap_bio(device);
		__drbd_make_request(device, bio, start_time);
2264 2265 2266
	}
}

2267
void drbd_restart_request(struct drbd_request *req)
2268 2269 2270 2271 2272 2273 2274 2275 2276
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2277
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2278

2279
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2280 2281
}

2282 2283 2284 2285 2286
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2287
	idr_destroy(&resource->devices);
2288
	free_cpumask_var(resource->cpu_mask);
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302
	kfree(resource->name);
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
		kref_put(&connection->kref, drbd_destroy_connection);
	}
	kref_put(&resource->kref, drbd_destroy_resource);
}
2303

P
Philipp Reisner 已提交
2304 2305 2306
static void drbd_cleanup(void)
{
	unsigned int i;
2307
	struct drbd_device *device;
2308
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2309 2310 2311

	unregister_reboot_notifier(&drbd_notifier);

2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2323 2324
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2325

2326
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2327

2328
	idr_for_each_entry(&drbd_devices, device, i)
2329
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2330

P
Philipp Reisner 已提交
2331
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2332 2333 2334
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2335
	}
P
Philipp Reisner 已提交
2336

2337
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2338 2339
	unregister_blkdev(DRBD_MAJOR, "drbd");

2340
	idr_destroy(&drbd_devices);
2341

P
Philipp Reisner 已提交
2342 2343 2344 2345
	printk(KERN_INFO "drbd: module cleanup done.\n");
}

/**
2346
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2347
 * @congested_data:	User data
2348
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2349 2350 2351 2352 2353
 *
 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2354
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2355 2356 2357 2358
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2359
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2360 2361 2362 2363 2364 2365
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2366
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2367 2368 2369 2370 2371 2372
		r |= (1 << BDI_async_congested);
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2373
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2374 2375
			r |= (1 << BDI_sync_congested);
		else
2376
			put_ldev(device);
2377 2378 2379 2380 2381
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2382 2383
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
P
Philipp Reisner 已提交
2384
		r = bdi_congested(&q->backing_dev_info, bdi_bits);
2385
		put_ldev(device);
P
Philipp Reisner 已提交
2386 2387 2388 2389
		if (r)
			reason = 'b';
	}

2390 2391
	if (bdi_bits & (1 << BDI_async_congested) &&
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
P
Philipp Reisner 已提交
2392 2393 2394 2395 2396
		r |= (1 << BDI_async_congested);
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2397
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2398 2399 2400
	return r;
}

2401 2402 2403 2404
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2405
	init_waitqueue_head(&wq->q_wait);
2406 2407
}

2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2432
struct drbd_resource *drbd_find_resource(const char *name)
2433
{
2434
	struct drbd_resource *resource;
2435

2436 2437 2438
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2439
	rcu_read_lock();
2440 2441
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2442
			kref_get(&resource->kref);
2443
			goto found;
2444
		}
2445
	}
2446
	resource = NULL;
2447
found:
P
Philipp Reisner 已提交
2448
	rcu_read_unlock();
2449
	return resource;
2450 2451
}

2452
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2453 2454
				     void *peer_addr, int peer_addr_len)
{
2455
	struct drbd_resource *resource;
2456
	struct drbd_connection *connection;
2457 2458

	rcu_read_lock();
2459 2460 2461 2462 2463 2464 2465 2466 2467
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2468 2469
		}
	}
2470
	connection = NULL;
2471 2472
found:
	rcu_read_unlock();
2473
	return connection;
2474 2475
}

2476 2477 2478 2479 2480
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2481 2482 2483
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2484 2485 2486 2487 2488
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2489
	free_page((unsigned long) socket->sbuf);
2490 2491 2492
	free_page((unsigned long) socket->rbuf);
}

2493
void conn_free_crypto(struct drbd_connection *connection)
2494
{
2495
	drbd_free_sock(connection);
2496

2497 2498 2499 2500 2501 2502 2503
	crypto_free_hash(connection->csums_tfm);
	crypto_free_hash(connection->verify_tfm);
	crypto_free_hash(connection->cram_hmac_tfm);
	crypto_free_hash(connection->integrity_tfm);
	crypto_free_hash(connection->peer_integrity_tfm);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2504

2505 2506 2507 2508 2509 2510 2511
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2512 2513
}

2514
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2515
{
2516
	struct drbd_connection *connection;
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;
		/*
		retcode = ERR_NOMEM;
		drbd_msg_put_info("unable to allocate cpumask");
		*/

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2529
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2530
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2531
		if (err) {
2532
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2533 2534 2535 2536
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2537
	resource->res_opts = *res_opts;
2538 2539 2540 2541 2542
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2543 2544 2545 2546
			connection->receiver.reset_cpu_mask = 1;
			connection->asender.reset_cpu_mask = 1;
			connection->worker.reset_cpu_mask = 1;
		}
2547 2548 2549 2550 2551 2552 2553 2554 2555
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2556 2557 2558 2559
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2560
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2561
	if (!resource)
2562
		goto fail;
2563
	resource->name = kstrdup(name, GFP_KERNEL);
2564 2565 2566 2567
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2568
	kref_init(&resource->kref);
2569
	idr_init(&resource->devices);
2570 2571
	INIT_LIST_HEAD(&resource->connections);
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2572 2573
	mutex_init(&resource->conf_update);
	spin_lock_init(&resource->req_lock);
2574
	return resource;
2575 2576 2577 2578 2579 2580 2581

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2582 2583
}

2584
/* caller must be under genl_lock() */
2585
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2586
{
2587
	struct drbd_resource *resource;
2588
	struct drbd_connection *connection;
2589

2590 2591
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2592 2593
		return NULL;

2594
	if (drbd_alloc_socket(&connection->data))
2595
		goto fail;
2596
	if (drbd_alloc_socket(&connection->meta))
2597 2598
		goto fail;

2599 2600
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2601
		goto fail;
2602

2603
	INIT_LIST_HEAD(&connection->transfer_log);
2604

2605 2606 2607 2608
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
	connection->write_ordering = WO_bdev_flush;
2609

2610 2611 2612
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2613

2614 2615 2616 2617
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2618 2619 2620
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2621
	idr_init(&connection->peer_devices);
2622

2623 2624 2625
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2626

2627 2628 2629 2630 2631 2632
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
	drbd_thread_init(resource, &connection->asender, drbd_asender, "asender");
	connection->asender.connection = connection;
2633

2634
	kref_init(&connection->kref);
2635 2636

	connection->resource = resource;
2637

2638 2639 2640 2641 2642
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2643
	return connection;
2644

2645 2646 2647
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2648
fail:
2649 2650 2651 2652
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2653 2654 2655
	return NULL;
}

2656
void drbd_destroy_connection(struct kref *kref)
2657
{
2658
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2659
	struct drbd_resource *resource = connection->resource;
2660

2661
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2662
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2663
	kfree(connection->current_epoch);
2664

2665
	idr_destroy(&connection->peer_devices);
2666

2667 2668 2669 2670 2671
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
	kfree(connection);
2672
	kref_put(&resource->kref, drbd_destroy_resource);
2673 2674
}

2675
static int init_submitter(struct drbd_device *device)
2676 2677 2678
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2679 2680 2681
	device->submit.wq = alloc_workqueue("drbd%u_submit",
			WQ_UNBOUND | WQ_MEM_RECLAIM, 1, device->minor);
	if (!device->submit.wq)
2682 2683
		return -ENOMEM;

2684 2685 2686
	INIT_WORK(&device->submit.worker, do_submit);
	spin_lock_init(&device->submit.lock);
	INIT_LIST_HEAD(&device->submit.writes);
2687 2688 2689
	return 0;
}

2690
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2691
{
2692
	struct drbd_resource *resource = adm_ctx->resource;
2693
	struct drbd_connection *connection;
2694
	struct drbd_device *device;
2695
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2696 2697
	struct gendisk *disk;
	struct request_queue *q;
2698
	int id;
2699
	int vnr = adm_ctx->volume;
2700
	enum drbd_ret_code err = ERR_NOMEM;
2701

2702 2703
	device = minor_to_device(minor);
	if (device)
2704
		return ERR_MINOR_EXISTS;
P
Philipp Reisner 已提交
2705 2706

	/* GFP_KERNEL, we are outside of all write-out paths */
2707 2708
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2709
		return ERR_NOMEM;
2710 2711 2712 2713
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2714 2715
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2716

2717
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2718 2719 2720 2721

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2722 2723
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2724 2725 2726 2727

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2728
	device->vdisk = disk;
P
Philipp Reisner 已提交
2729

2730
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2731 2732 2733 2734 2735 2736

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2737
	disk->private_data = device;
P
Philipp Reisner 已提交
2738

2739
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2740
	/* we have no partitions. we contain only ourselves. */
2741
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2742 2743

	q->backing_dev_info.congested_fn = drbd_congested;
2744
	q->backing_dev_info.congested_data = device;
P
Philipp Reisner 已提交
2745

2746
	blk_queue_make_request(q, drbd_make_request);
2747
	blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
2748 2749 2750
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
P
Philipp Reisner 已提交
2751 2752
	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
	blk_queue_merge_bvec(q, drbd_merge_bvec);
2753
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2754

2755 2756
	device->md_io_page = alloc_page(GFP_KERNEL);
	if (!device->md_io_page)
P
Philipp Reisner 已提交
2757 2758
		goto out_no_io_page;

2759
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2760
		goto out_no_bitmap;
2761 2762
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2763

2764 2765 2766
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
T
Tejun Heo 已提交
2767
			err = ERR_MINOR_EXISTS;
2768
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
T
Tejun Heo 已提交
2769
		}
2770
		goto out_no_minor_idr;
2771
	}
2772 2773 2774 2775 2776 2777
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
		if (id == -ENOSPC) {
			err = ERR_MINOR_EXISTS;
2778
			drbd_msg_put_info(adm_ctx->reply_skb, "requested minor exists already");
2779 2780 2781 2782
		}
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2783

2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
	INIT_LIST_HEAD(&device->peer_devices);
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
			if (id == -ENOSPC) {
				err = ERR_INVALID_REQUEST;
2799
				drbd_msg_put_info(adm_ctx->reply_skb, "requested volume exists already");
2800 2801
			}
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2802
		}
2803
		kref_get(&connection->kref);
2804
	}
T
Tejun Heo 已提交
2805

2806
	if (init_submitter(device)) {
2807
		err = ERR_NOMEM;
2808
		drbd_msg_put_info(adm_ctx->reply_skb, "unable to create submit workqueue");
2809 2810 2811
		goto out_idr_remove_vol;
	}

2812 2813
	add_disk(disk);

2814
	/* inherit the connection state */
2815
	device->state.conn = first_connection(resource)->cstate;
2816 2817 2818 2819
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2820

2821
	return NO_ERROR;
P
Philipp Reisner 已提交
2822

2823
out_idr_remove_vol:
2824
	idr_remove(&connection->peer_devices, vnr);
2825
out_idr_remove_from_resource:
2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
	for_each_connection(connection, resource) {
		peer_device = idr_find(&connection->peer_devices, vnr);
		if (peer_device) {
			idr_remove(&connection->peer_devices, vnr);
			kref_put(&connection->kref, drbd_destroy_connection);
		}
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2837
	idr_remove(&resource->devices, vnr);
2838
out_idr_remove_minor:
2839
	idr_remove(&drbd_devices, minor);
2840
	synchronize_rcu();
2841
out_no_minor_idr:
2842
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2843
out_no_bitmap:
2844
	__free_page(device->md_io_page);
P
Philipp Reisner 已提交
2845 2846 2847 2848 2849
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2850
	kref_put(&resource->kref, drbd_destroy_resource);
2851
	kfree(device);
2852
	return err;
P
Philipp Reisner 已提交
2853 2854
}

2855
void drbd_delete_device(struct drbd_device *device)
2856 2857 2858 2859 2860 2861
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
	int refs = 3;

	for_each_connection(connection, resource) {
2862
		idr_remove(&connection->peer_devices, device->vnr);
2863 2864 2865 2866 2867 2868 2869 2870 2871
		refs++;
	}
	idr_remove(&resource->devices, device->vnr);
	idr_remove(&drbd_devices, device_to_minor(device));
	del_gendisk(device->vdisk);
	synchronize_rcu();
	kref_sub(&device->kref, refs, drbd_destroy_device);
}

P
Philipp Reisner 已提交
2872 2873 2874 2875
int __init drbd_init(void)
{
	int err;

2876
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
P
Philipp Reisner 已提交
2877
		printk(KERN_ERR
2878
		       "drbd: invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2879 2880 2881
#ifdef MODULE
		return -EINVAL;
#else
2882
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
		printk(KERN_ERR
		       "drbd: unable to register block device major %d\n",
		       DRBD_MAJOR);
		return err;
	}

	register_reboot_notifier(&drbd_notifier);

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
2902
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
2903

2904
	rwlock_init(&global_state_lock);
2905
	INIT_LIST_HEAD(&drbd_resources);
2906 2907 2908 2909 2910 2911 2912

	err = drbd_genl_register();
	if (err) {
		printk(KERN_ERR "drbd: unable to register generic netlink family\n");
		goto fail;
	}

P
Philipp Reisner 已提交
2913 2914
	err = drbd_create_mempools();
	if (err)
2915
		goto fail;
P
Philipp Reisner 已提交
2916

2917
	err = -ENOMEM;
2918
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
2919 2920
	if (!drbd_proc)	{
		printk(KERN_ERR "drbd: unable to register proc file\n");
2921
		goto fail;
P
Philipp Reisner 已提交
2922 2923
	}

2924 2925 2926 2927 2928 2929 2930 2931
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
		printk(KERN_ERR "drbd: unable to create retry workqueue\n");
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
2932 2933 2934 2935 2936 2937 2938 2939 2940 2941

	printk(KERN_INFO "drbd: initialized. "
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
	printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
	printk(KERN_INFO "drbd: registered as block device major %d\n",
		DRBD_MAJOR);

	return 0; /* Success! */

2942
fail:
P
Philipp Reisner 已提交
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955
	drbd_cleanup();
	if (err == -ENOMEM)
		printk(KERN_ERR "drbd: ran out of memory\n");
	else
		printk(KERN_ERR "drbd: initialization failure\n");
	return err;
}

void drbd_free_bc(struct drbd_backing_dev *ldev)
{
	if (ldev == NULL)
		return;

2956 2957
	blkdev_put(ldev->backing_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
	blkdev_put(ldev->md_bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
P
Philipp Reisner 已提交
2958

L
Lars Ellenberg 已提交
2959
	kfree(ldev->disk_conf);
P
Philipp Reisner 已提交
2960 2961 2962
	kfree(ldev);
}

2963
void drbd_free_sock(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2964
{
2965 2966 2967 2968 2969 2970
	if (connection->data.socket) {
		mutex_lock(&connection->data.mutex);
		kernel_sock_shutdown(connection->data.socket, SHUT_RDWR);
		sock_release(connection->data.socket);
		connection->data.socket = NULL;
		mutex_unlock(&connection->data.mutex);
P
Philipp Reisner 已提交
2971
	}
2972 2973 2974 2975 2976 2977
	if (connection->meta.socket) {
		mutex_lock(&connection->meta.mutex);
		kernel_sock_shutdown(connection->meta.socket, SHUT_RDWR);
		sock_release(connection->meta.socket);
		connection->meta.socket = NULL;
		mutex_unlock(&connection->meta.mutex);
P
Philipp Reisner 已提交
2978 2979 2980 2981 2982
	}
}

/* meta data management */

2983
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
2984
{
2985
	struct drbd_peer_device *peer_device;
2986
	int vnr;
P
Philipp Reisner 已提交
2987

2988
	rcu_read_lock();
2989 2990 2991
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

2992
		kref_get(&device->kref);
2993
		rcu_read_unlock();
2994
		drbd_md_sync(device);
2995
		kref_put(&device->kref, drbd_destroy_device);
2996 2997 2998
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
2999 3000
}

3001
/* aligned 4kByte */
P
Philipp Reisner 已提交
3002
struct meta_data_on_disk {
3003
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
3004 3005 3006 3007 3008 3009 3010
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3011
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3012
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3013 3014
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3015
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3016

3017 3018 3019 3020 3021
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3022 3023
} __packed;

3024 3025


3026
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3027
{
3028
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3029 3030 3031
	sector_t sector;
	int i;

3032
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3033

3034
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3035
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3036 3037
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3038
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3039

3040 3041 3042
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3043
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3044
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3045

3046 3047
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3048

3049 3050
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3051

3052
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3053
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3054

3055
	if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
P
Philipp Reisner 已提交
3056
		/* this was a try anyways ... */
3057
		drbd_err(device, "meta data update failed!\n");
3058
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3059
	}
3060 3061 3062 3063
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3064
 * @device:	DRBD device.
3065
 */
3066
void drbd_md_sync(struct drbd_device *device)
3067 3068 3069 3070 3071 3072 3073
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3074
	del_timer(&device->md_sync_timer);
3075
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3076
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3077 3078 3079 3080
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3081
	if (!get_ldev_if_state(device, D_FAILED))
3082 3083
		return;

3084
	buffer = drbd_md_get_buffer(device);
3085 3086 3087
	if (!buffer)
		goto out;

3088
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3089

3090
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3091
	 * since we updated it on metadata. */
3092
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3093

3094
	drbd_md_put_buffer(device);
3095
out:
3096
	put_ldev(device);
P
Philipp Reisner 已提交
3097 3098
}

3099
static int check_activity_log_stripe_size(struct drbd_device *device,
3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3139
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3140 3141 3142 3143
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3144
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3211
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3224 3225
/**
 * drbd_md_read() - Reads in the meta data super block
3226
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3227 3228
 * @bdev:	Device from which the meta data should be read in.
 *
3229
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3230
 * something goes wrong.
3231
 *
3232
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3233
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3234
 */
3235
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3236 3237
{
	struct meta_data_on_disk *buffer;
3238
	u32 magic, flags;
P
Philipp Reisner 已提交
3239 3240
	int i, rv = NO_ERROR;

3241
	if (device->state.disk != D_DISKLESS)
3242
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3243

3244
	buffer = drbd_md_get_buffer(device);
3245
	if (!buffer)
3246
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3247

3248 3249
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3250 3251
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
P
Philipp Reisner 已提交
3252

3253
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) {
L
Lucas De Marchi 已提交
3254
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3255
		   called BEFORE disk is attached */
3256
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3257 3258 3259 3260
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3261 3262 3263 3264 3265
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3266
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3267 3268 3269
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3270 3271

	rv = ERR_MD_INVALID;
3272
	if (magic != DRBD_MD_MAGIC_08) {
3273
		if (magic == DRBD_MD_MAGIC_07)
3274
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3275
		else
3276
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3277 3278
		goto err;
	}
3279

3280
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3281
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3282
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3283 3284
		goto err;
	}
3285

3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3298
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3299
		goto err;
3300
	if (check_offsets_and_sizes(device, bdev))
3301 3302
		goto err;

P
Philipp Reisner 已提交
3303
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3304
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3305 3306 3307 3308
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3309
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3310 3311 3312 3313
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3314
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3315

3316
	spin_lock_irq(&device->resource->req_lock);
3317
	if (device->state.conn < C_CONNECTED) {
3318
		unsigned int peer;
3319
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3320
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3321
		device->peer_max_bio_size = peer;
3322
	}
3323
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3324 3325

 err:
3326
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3327 3328 3329 3330 3331 3332

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3333
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3334 3335 3336 3337 3338
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3339
#ifdef DEBUG
3340
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3341
{
3342 3343 3344 3345
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3346 3347 3348
	}
}
#else
3349
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3350
{
3351 3352
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3353
}
3354
#endif
P
Philipp Reisner 已提交
3355

3356
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3357 3358 3359
{
	int i;

3360
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3361
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3362 3363
}

3364
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3365 3366
{
	if (idx == UI_CURRENT) {
3367
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3368 3369 3370 3371
			val |= 1;
		else
			val &= ~((u64)1);

3372
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3373 3374
	}

3375 3376
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3377 3378
}

3379
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3380 3381
{
	unsigned long flags;
3382 3383 3384
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3385
}
P
Philipp Reisner 已提交
3386

3387
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3388
{
3389
	unsigned long flags;
3390 3391 3392 3393
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3394
	}
3395 3396
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3397 3398 3399 3400
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3401
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3402 3403 3404 3405
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3406
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3407 3408
{
	u64 val;
3409 3410 3411 3412
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3413 3414
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3415 3416

	if (bm_uuid)
3417
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3418

3419 3420 3421
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3422

3423
	drbd_print_uuids(device, "new current UUID");
3424
	/* get it to stable storage _now_ */
3425
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3426 3427
}

3428
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3429
{
3430
	unsigned long flags;
3431
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3432 3433
		return;

3434
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3435
	if (val == 0) {
3436 3437 3438
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3439
	} else {
3440
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3441
		if (bm_uuid)
3442
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3443

3444
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3445
	}
3446
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3447

3448
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3449 3450 3451 3452
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3453
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3454 3455 3456
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3457
int drbd_bmio_set_n_write(struct drbd_device *device)
P
Philipp Reisner 已提交
3458 3459 3460
{
	int rv = -EIO;

3461 3462 3463 3464
	if (get_ldev_if_state(device, D_ATTACHING)) {
		drbd_md_set_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
		drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3465

3466
		rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3467 3468

		if (!rv) {
3469 3470
			drbd_md_clear_flag(device, MDF_FULL_SYNC);
			drbd_md_sync(device);
P
Philipp Reisner 已提交
3471 3472
		}

3473
		put_ldev(device);
P
Philipp Reisner 已提交
3474 3475 3476 3477 3478 3479 3480
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3481
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3482 3483 3484
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3485
int drbd_bmio_clear_n_write(struct drbd_device *device)
P
Philipp Reisner 已提交
3486 3487 3488
{
	int rv = -EIO;

3489 3490 3491 3492 3493
	drbd_resume_al(device);
	if (get_ldev_if_state(device, D_ATTACHING)) {
		drbd_bm_clear_all(device);
		rv = drbd_bm_write(device);
		put_ldev(device);
P
Philipp Reisner 已提交
3494 3495 3496 3497 3498
	}

	return rv;
}

3499
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3500
{
3501 3502 3503
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3504
	int rv = -EIO;
P
Philipp Reisner 已提交
3505

3506
	D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0);
P
Philipp Reisner 已提交
3507

3508 3509 3510 3511 3512
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3513
	}
P
Philipp Reisner 已提交
3514

3515 3516
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3517 3518

	if (work->done)
3519
		work->done(device, rv);
P
Philipp Reisner 已提交
3520

3521
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3522
	work->why = NULL;
3523
	work->flags = 0;
P
Philipp Reisner 已提交
3524

3525
	return 0;
P
Philipp Reisner 已提交
3526 3527
}

3528
void drbd_ldev_destroy(struct drbd_device *device)
3529
{
3530 3531 3532 3533
	lc_destroy(device->resync);
	device->resync = NULL;
	lc_destroy(device->act_log);
	device->act_log = NULL;
3534
	__no_warn(local,
3535 3536
		drbd_free_bc(device->ldev);
		device->ldev = NULL;);
3537

3538
	clear_bit(GO_DISKLESS, &device->flags);
3539 3540
}

3541
static int w_go_diskless(struct drbd_work *w, int unused)
3542
{
3543 3544
	struct drbd_device *device =
		container_of(w, struct drbd_device, go_diskless);
3545

3546
	D_ASSERT(device, device->state.disk == D_FAILED);
3547 3548
	/* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
	 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
3549 3550
	 * the protected members anymore, though, so once put_ldev reaches zero
	 * again, it will be safe to free them. */
3551 3552 3553 3554 3555 3556 3557 3558 3559

	/* Try to write changed bitmap pages, read errors may have just
	 * set some bits outside the area covered by the activity log.
	 *
	 * If we have an IO error during the bitmap writeout,
	 * we will want a full sync next time, just in case.
	 * (Do we want a specific meta data flag for this?)
	 *
	 * If that does not make it to stable storage either,
3560 3561 3562 3563 3564
	 * we cannot do anything about that anymore.
	 *
	 * We still need to check if both bitmap and ldev are present, we may
	 * end up here after a failed attach, before ldev was even assigned.
	 */
3565
	if (device->bitmap && device->ldev) {
3566 3567 3568 3569
		/* An interrupted resync or similar is allowed to recounts bits
		 * while we detach.
		 * Any modifications would not be expected anymore, though.
		 */
3570
		if (drbd_bitmap_io_from_worker(device, drbd_bm_write,
3571
					"detach", BM_LOCKED_TEST_ALLOWED)) {
3572 3573 3574
			if (test_bit(WAS_READ_ERROR, &device->flags)) {
				drbd_md_set_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
3575 3576 3577 3578
			}
		}
	}

3579
	drbd_force_state(device, NS(disk, D_DISKLESS));
3580
	return 0;
3581 3582
}

P
Philipp Reisner 已提交
3583 3584
/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3585
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3586 3587 3588 3589 3590 3591 3592 3593 3594
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
 */
3595
void drbd_queue_bitmap_io(struct drbd_device *device,
3596 3597
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3598
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3599
{
3600
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3601

3602 3603 3604
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3605
	if (device->bm_io_work.why)
3606
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3607
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3608

3609 3610 3611 3612
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3613

3614
	spin_lock_irq(&device->resource->req_lock);
3615 3616 3617
	set_bit(BITMAP_IO, &device->flags);
	if (atomic_read(&device->ap_bio_cnt) == 0) {
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3618 3619
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3620
	}
3621
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3622 3623 3624 3625
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3626
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3627 3628 3629 3630 3631 3632
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3633
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3634
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3635 3636 3637
{
	int rv;

3638
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3639

3640
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3641
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3642

3643 3644 3645
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3646

3647
	if ((flags & BM_LOCKED_SET_ALLOWED) == 0)
3648
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3649 3650 3651 3652

	return rv;
}

3653
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3654
{
3655 3656 3657
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3658 3659 3660
	}
}

3661
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3662
{
3663 3664 3665
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3666 3667 3668 3669 3670 3671 3672 3673 3674
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3675
	struct drbd_device *device = (struct drbd_device *) data;
P
Philipp Reisner 已提交
3676

3677
	/* must not double-queue! */
3678
	if (list_empty(&device->md_sync_work.list))
3679 3680
		drbd_queue_work_front(&first_peer_device(device)->connection->sender_work,
				      &device->md_sync_work);
P
Philipp Reisner 已提交
3681 3682
}

3683
static int w_md_sync(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3684
{
3685 3686
	struct drbd_device *device =
		container_of(w, struct drbd_device, md_sync_work);
3687

3688
	drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3689
#ifdef DEBUG
3690
	drbd_warn(device, "last md_mark_dirty: %s:%u\n",
3691
		device->last_md_mark_dirty.func, device->last_md_mark_dirty.line);
3692
#endif
3693
	drbd_md_sync(device);
3694
	return 0;
P
Philipp Reisner 已提交
3695 3696
}

3697
const char *cmdname(enum drbd_packet cmd)
3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3727
		[P_SUPERSEDED]          = "Superseded",
3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3742
		[P_RETRY_WRITE]		= "RetryWrite",
3743 3744 3745
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3746 3747
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3748 3749 3750 3751 3752

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3753 3754
	};

3755
	/* too big for the array: 0xfffX */
3756 3757 3758 3759
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3760 3761
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3762
	if (cmd >= ARRAY_SIZE(cmdnames))
3763 3764 3765 3766
		return "Unknown";
	return cmdnames[cmd];
}

3767 3768
/**
 * drbd_wait_misc  -  wait for a request to make progress
3769
 * @device:	device associated with the request
3770 3771 3772
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3773
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3774
{
3775
	struct net_conf *nc;
3776 3777 3778
	DEFINE_WAIT(wait);
	long timeout;

3779
	rcu_read_lock();
3780
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3781 3782
	if (!nc) {
		rcu_read_unlock();
3783
		return -ETIMEDOUT;
3784 3785 3786
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3787

3788
	/* Indicate to wake up device->misc_wait on progress.  */
3789
	i->waiting = true;
3790
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3791
	spin_unlock_irq(&device->resource->req_lock);
3792
	timeout = schedule_timeout(timeout);
3793
	finish_wait(&device->misc_wait, &wait);
3794
	spin_lock_irq(&device->resource->req_lock);
3795
	if (!timeout || device->state.conn < C_CONNECTED)
3796 3797 3798 3799
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
}

#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3823
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3843 3844
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3845 3846 3847 3848 3849 3850
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3851
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3852 3853 3854 3855 3856
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3857
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3858 3859 3860 3861 3862
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3863
		if (__ratelimit(&drbd_ratelimit_state))
3864
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3880 3881 3882 3883
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);