drbd_main.c 112.3 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

29 30
#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

P
Philipp Reisner 已提交
31
#include <linux/module.h>
32
#include <linux/jiffies.h>
P
Philipp Reisner 已提交
33
#include <linux/drbd.h>
34
#include <linux/uaccess.h>
P
Philipp Reisner 已提交
35 36 37
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
38
#include <linux/mutex.h>
P
Philipp Reisner 已提交
39 40 41 42 43 44 45 46 47 48 49 50
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
51
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>
55
#include <linux/sched/signal.h>
P
Philipp Reisner 已提交
56 57 58

#include <linux/drbd_limits.h>
#include "drbd_int.h"
59
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
60 61
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
#include "drbd_vli.h"
62
#include "drbd_debugfs.h"
P
Philipp Reisner 已提交
63

64
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
65
static int drbd_open(struct block_device *bdev, fmode_t mode);
66
static void drbd_release(struct gendisk *gd, fmode_t mode);
P
Philipp Reisner 已提交
67
static void md_sync_timer_fn(unsigned long data);
68
static int w_bitmap_io(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
69 70 71 72 73 74

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
75
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
76
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* allow_open_on_secondary */
MODULE_PARM_DESC(allow_oos, "DONT USE!");
/* thanks to these macros, if compiled into the kernel (not-module),
 * this becomes the boot parameter drbd.minor_count */
module_param(minor_count, uint, 0444);
module_param(disable_sendpage, bool, 0644);
module_param(allow_oos, bool, 0);
module_param(proc_details, int, 0644);

#ifdef CONFIG_DRBD_FAULT_INJECTION
int enable_faults;
int fault_rate;
static int fault_count;
int fault_devs;
/* bitmap of enabled faults */
module_param(enable_faults, int, 0664);
/* fault rate % value - applies to all enabled faults */
module_param(fault_rate, int, 0664);
/* count of faults inserted */
module_param(fault_count, int, 0664);
/* bitmap of devices to insert faults on */
module_param(fault_devs, int, 0644);
#endif

/* module parameter, defined */
105
unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
106 107
bool disable_sendpage;
bool allow_oos;
P
Philipp Reisner 已提交
108 109 110 111
int proc_details;       /* Detail level in proc drbd*/

/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
112
char drbd_usermode_helper[80] = "/sbin/drbdadm";
P
Philipp Reisner 已提交
113

114
module_param_string(usermode_helper, drbd_usermode_helper, sizeof(drbd_usermode_helper), 0644);
P
Philipp Reisner 已提交
115 116 117 118

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
119
struct idr drbd_devices;
120
struct list_head drbd_resources;
121
struct mutex resources_mutex;
P
Philipp Reisner 已提交
122 123

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
124
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
125 126 127 128
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
129
mempool_t *drbd_md_io_page_pool;
130
struct bio_set *drbd_md_io_bio_set;
131
struct bio_set *drbd_io_bio_set;
P
Philipp Reisner 已提交
132 133 134 135 136 137 138 139 140 141 142 143 144 145

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

146
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
147 148 149 150
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
151

152 153 154
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
155

156 157
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
158

159 160 161 162
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
163 164
}

P
Philipp Reisner 已提交
165 166 167 168
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
169
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
170 171 172
{
	int io_allowed;

173 174
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
175
	if (!io_allowed) {
176 177
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
178 179 180 181 182
	}
	return io_allowed;
}

#endif
183

P
Philipp Reisner 已提交
184
/**
185
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
186
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
187 188 189 190
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
191 192
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
193
 */
194
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
195
		unsigned int set_size)
P
Philipp Reisner 已提交
196 197
{
	struct drbd_request *r;
198 199 200
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
201

202
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
203

204
	/* find oldest not yet barrier-acked write request,
205
	 * count writes in its epoch. */
206
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
207
		const unsigned s = r->rq_state;
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
226
		}
P
Philipp Reisner 已提交
227
	}
228

P
Philipp Reisner 已提交
229
	/* first some paranoia code */
230
	if (req == NULL) {
231
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
232
			 barrier_nr);
P
Philipp Reisner 已提交
233 234
		goto bail;
	}
235
	if (expect_epoch != barrier_nr) {
236
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
237
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
238
		goto bail;
239 240
	}

241
	if (expect_size != set_size) {
242
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
243
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
244 245 246
		goto bail;
	}

247 248 249 250
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
251
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
252 253
		if (req->epoch == expect_epoch)
			break;
254
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
255 256 257
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
258
	}
259
	spin_unlock_irq(&connection->resource->req_lock);
260

P
Philipp Reisner 已提交
261 262 263
	return;

bail:
264
	spin_unlock_irq(&connection->resource->req_lock);
265
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
266 267 268 269
}


/**
270
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
271
 * @connection:	DRBD connection to operate on.
272
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
273
 *
274 275
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
276
 */
277
/* must hold resource->req_lock */
278
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
279
{
280
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
281

282
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
283 284
		_req_mod(req, what);
}
285

286
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
287
{
288
	spin_lock_irq(&connection->resource->req_lock);
289
	_tl_restart(connection, what);
290
	spin_unlock_irq(&connection->resource->req_lock);
291
}
P
Philipp Reisner 已提交
292 293 294

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
295
 * @device:	DRBD device.
P
Philipp Reisner 已提交
296 297 298 299 300
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
301
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
302
{
303
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
304
}
305

306
/**
307 308
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
309
 */
310
void tl_abort_disk_io(struct drbd_device *device)
311
{
312
	struct drbd_connection *connection = first_peer_device(device)->connection;
313
	struct drbd_request *req, *r;
314

315
	spin_lock_irq(&connection->resource->req_lock);
316
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
317 318
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
319
		if (req->device != device)
320 321
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
322
	}
323
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
324 325 326 327 328
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
329
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
330 331 332
	unsigned long flags;
	int retval;

333
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
334
		 thi->name[0],
335
		 resource->name);
336

P
Philipp Reisner 已提交
337 338 339 340 341
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

342
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
343 344 345
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
346
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
347
	 * t_state check and assignment needs to be within the same spinlock,
348 349
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
350 351
	 */

352
	if (thi->t_state == RESTARTING) {
353
		drbd_info(resource, "Restarting %s thread\n", thi->name);
354
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
355 356 357 358 359
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
360
	thi->t_state = NONE;
P
Philipp Reisner 已提交
361
	smp_mb();
L
Lars Ellenberg 已提交
362
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
363 364
	spin_unlock_irqrestore(&thi->t_lock, flags);

365
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
366 367

	/* Release mod reference taken when thread was started */
368

369 370 371
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
372 373 374 375
	module_put(THIS_MODULE);
	return retval;
}

376
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
377
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
378 379 380
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
381
	thi->t_state = NONE;
P
Philipp Reisner 已提交
382
	thi->function = func;
383 384
	thi->resource = resource;
	thi->connection = NULL;
385
	thi->name = name;
P
Philipp Reisner 已提交
386 387 388 389
}

int drbd_thread_start(struct drbd_thread *thi)
{
390
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
391 392 393 394 395 396 397 398
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
399
	case NONE:
400
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
401
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
402 403 404

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
405
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
406
			spin_unlock_irqrestore(&thi->t_lock, flags);
407
			return false;
P
Philipp Reisner 已提交
408 409
		}

410 411 412
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
413

P
Philipp Reisner 已提交
414 415
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
416
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
417 418 419 420
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
421
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
422 423

		if (IS_ERR(nt)) {
424
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
425

426 427 428
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
429
			module_put(THIS_MODULE);
430
			return false;
P
Philipp Reisner 已提交
431 432 433
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
434
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
435 436 437
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
438 439
	case EXITING:
		thi->t_state = RESTARTING;
440
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
441
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
442
		/* fall through */
443 444
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
445 446 447 448 449
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

450
	return true;
P
Philipp Reisner 已提交
451 452 453 454 455 456 457
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

458
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
459 460 461 462

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

463
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

489
int conn_lowest_minor(struct drbd_connection *connection)
490
{
491 492
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
493

494
	rcu_read_lock();
495 496 497
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
498 499
	rcu_read_unlock();

500
	return minor;
501
}
502

P
Philipp Reisner 已提交
503 504 505 506
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
507
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
508 509
 * DRBD's performance. May be overwritten by user's configuration.
 */
510
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
511
{
512
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
513

514 515 516 517
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
518

519 520 521 522
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
523
		}
524 525 526 527 528 529 530 531 532 533 534 535
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
536
	}
537
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
538 539 540 541
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
542
 * @device:	DRBD device.
543
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
544 545 546 547
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
548
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
549
{
550
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
551
	struct task_struct *p = current;
552

P
Philipp Reisner 已提交
553 554 555
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
556
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
557
}
558 559
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
560 561
#endif

562 563 564 565 566 567 568
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
569
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
570
{
571
	if (connection->agreed_pro_version >= 100) {
572 573 574 575 576 577 578 579
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
580
}
P
Philipp Reisner 已提交
581

582
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
583 584 585 586
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
587
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
588
}
P
Philipp Reisner 已提交
589

590
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
591 592
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
593
	h->command = cpu_to_be16(cmd);
594
	h->length = cpu_to_be32(size);
595
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
596
}
P
Philipp Reisner 已提交
597

598 599 600 601 602 603 604 605 606 607
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
608

609
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
610
				   void *buffer, enum drbd_packet cmd, int size)
611
{
612
	if (connection->agreed_pro_version >= 100)
613
		return prepare_header100(buffer, cmd, size, vnr);
614
	else if (connection->agreed_pro_version >= 95 &&
615
		 size > DRBD_MAX_SIZE_H80_PACKET)
616
		return prepare_header95(buffer, cmd, size);
617
	else
618
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
619 620
}

621
static void *__conn_prepare_command(struct drbd_connection *connection,
622
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
623
{
624 625
	if (!sock->socket)
		return NULL;
626
	return sock->sbuf + drbd_header_size(connection);
627
}
P
Philipp Reisner 已提交
628

629
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
630
{
631
	void *p;
P
Philipp Reisner 已提交
632

633
	mutex_lock(&sock->mutex);
634
	p = __conn_prepare_command(connection, sock);
635
	if (!p)
636
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
637

638
	return p;
P
Philipp Reisner 已提交
639 640
}

641
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
642
{
643
	return conn_prepare_command(peer_device->connection, sock);
644
}
P
Philipp Reisner 已提交
645

646
static int __send_command(struct drbd_connection *connection, int vnr,
647 648 649 650 651 652
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
653

654 655 656 657 658 659 660 661 662
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

663
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
664
				      header_size + size);
665
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
666 667
			    msg_flags);
	if (data && !err)
668
		err = drbd_send_all(connection, sock->socket, data, size, 0);
669 670 671 672 673
	/* DRBD protocol "pings" are latency critical.
	 * This is supposed to trigger tcp_push_pending_frames() */
	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
		drbd_tcp_nodelay(sock->socket);

674 675 676
	return err;
}

677
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
678 679 680
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
681
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
682 683
}

684
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
685 686 687 688
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
689

690
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
691 692 693 694
	mutex_unlock(&sock->mutex);
	return err;
}

695
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
696 697 698 699 700
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

701 702
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
703 704 705
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
706

707
int drbd_send_ping(struct drbd_connection *connection)
708
{
709 710
	struct drbd_socket *sock;

711 712
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
713
		return -EIO;
714
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
715
}
P
Philipp Reisner 已提交
716

717
int drbd_send_ping_ack(struct drbd_connection *connection)
718
{
719 720
	struct drbd_socket *sock;

721 722
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
723
		return -EIO;
724
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
725 726
}

727
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
728
{
729
	struct drbd_socket *sock;
730
	struct p_rs_param_95 *p;
731
	int size;
732
	const int apv = peer_device->connection->agreed_pro_version;
733
	enum drbd_packet cmd;
734
	struct net_conf *nc;
P
Philipp Reisner 已提交
735
	struct disk_conf *dc;
736

737 738
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
739 740
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
741

742
	rcu_read_lock();
743
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
744 745 746

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
747
			+ strlen(nc->verify_alg) + 1
748 749
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
750

751
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
752

753 754
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
755

756 757
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
758
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
759 760 761 762
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
763
		put_ldev(peer_device->device);
764
	} else {
765
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
766 767 768 769 770
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
771

772
	if (apv >= 88)
773
		strcpy(p->verify_alg, nc->verify_alg);
774
	if (apv >= 89)
775 776
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
777

778
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
779 780
}

781
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
782
{
783
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
784
	struct p_protocol *p;
785
	struct net_conf *nc;
786
	int size, cf;
P
Philipp Reisner 已提交
787

788 789
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
790 791
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
792

793
	rcu_read_lock();
794
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
795

796
	if (nc->tentative && connection->agreed_pro_version < 92) {
797 798
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
799
		drbd_err(connection, "--dry-run is not supported by peer");
800 801
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
802

803
	size = sizeof(*p);
804
	if (connection->agreed_pro_version >= 87)
805
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
806

807 808 809 810 811
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
812
	cf = 0;
813 814
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
815
	if (nc->tentative)
816
		cf |= CF_DRY_RUN;
817 818
	p->conn_flags    = cpu_to_be32(cf);

819
	if (connection->agreed_pro_version >= 87)
820 821
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
822

823
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
824 825
}

826
int drbd_send_protocol(struct drbd_connection *connection)
827 828 829
{
	int err;

830 831 832
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
833 834

	return err;
P
Philipp Reisner 已提交
835 836
}

837
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
838
{
839
	struct drbd_device *device = peer_device->device;
840 841
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
842 843
	int i;

844
	if (!get_ldev_if_state(device, D_NEGOTIATING))
845
		return 0;
P
Philipp Reisner 已提交
846

847 848
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
849
	if (!p) {
850
		put_ldev(device);
851 852
		return -EIO;
	}
853
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
854
	for (i = UI_CURRENT; i < UI_SIZE; i++)
855 856
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
857

858 859
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
860
	rcu_read_lock();
861
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
862
	rcu_read_unlock();
863 864
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
865
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
866

867
	put_ldev(device);
868
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
869 870
}

871
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
872
{
873
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
874 875
}

876
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
877
{
878
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
879 880
}

881
void drbd_print_uuids(struct drbd_device *device, const char *text)
882
{
883 884
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
885
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
886 887 888 889 890
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
891
		put_ldev(device);
892
	} else {
893
		drbd_info(device, "%s effective data uuid: %016llX\n",
894
				text,
895
				(unsigned long long)device->ed_uuid);
896 897 898
	}
}

899
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
900
{
901
	struct drbd_device *device = peer_device->device;
902 903
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
904 905
	u64 uuid;

906
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
907

908
	uuid = device->ldev->md.uuid[UI_BITMAP];
909 910 911 912
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
913 914 915
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
916

917 918
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
919 920
	if (p) {
		p->uuid = cpu_to_be64(uuid);
921
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
922
	}
P
Philipp Reisner 已提交
923 924
}

925
/* communicated if (agreed_features & DRBD_FF_WSAME) */
926 927 928
static void
assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p,
					struct request_queue *q)
929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
{
	if (q) {
		p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
		p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
		p->qlim->alignment_offset = cpu_to_be32(queue_alignment_offset(q));
		p->qlim->io_min = cpu_to_be32(queue_io_min(q));
		p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
		p->qlim->discard_enabled = blk_queue_discard(q);
		p->qlim->write_same_capable = !!q->limits.max_write_same_sectors;
	} else {
		q = device->rq_queue;
		p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
		p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
		p->qlim->alignment_offset = 0;
		p->qlim->io_min = cpu_to_be32(queue_io_min(q));
		p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
		p->qlim->discard_enabled = 0;
		p->qlim->write_same_capable = 0;
	}
}

950
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
951
{
952
	struct drbd_device *device = peer_device->device;
953 954
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
955
	sector_t d_size, u_size;
956 957
	int q_order_type;
	unsigned int max_bio_size;
958 959 960 961 962 963
	unsigned int packet_size;

	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
964

965 966 967 968 969
	packet_size = sizeof(*p);
	if (peer_device->connection->agreed_features & DRBD_FF_WSAME)
		packet_size += sizeof(p->qlim[0]);

	memset(p, 0, packet_size);
970
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
971
		struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev);
972
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
973
		rcu_read_lock();
974
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
975
		rcu_read_unlock();
976
		q_order_type = drbd_queue_order_type(device);
977
		max_bio_size = queue_max_hw_sectors(q) << 9;
978
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
979
		assign_p_sizes_qlim(device, p, q);
980
		put_ldev(device);
P
Philipp Reisner 已提交
981 982 983 984
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
985
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
986
		assign_p_sizes_qlim(device, p, NULL);
P
Philipp Reisner 已提交
987 988
	}

989
	if (peer_device->connection->agreed_pro_version <= 94)
990
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
991
	else if (peer_device->connection->agreed_pro_version < 100)
992
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
993

994 995
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
996
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
997 998 999
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
1000 1001

	return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0);
P
Philipp Reisner 已提交
1002 1003 1004
}

/**
1005
 * drbd_send_current_state() - Sends the drbd state to the peer
1006
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
1007
 */
1008
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
1009
{
1010
	struct drbd_socket *sock;
1011
	struct p_state *p;
P
Philipp Reisner 已提交
1012

1013 1014
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1015 1016
	if (!p)
		return -EIO;
1017 1018
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1019 1020
}

1021 1022
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
1023
 * @peer_device:      DRBD peer device.
1024
 * @state:     the state to send, not necessarily the current state.
1025 1026 1027 1028 1029 1030
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
1031
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
1032
{
1033 1034
	struct drbd_socket *sock;
	struct p_state *p;
1035

1036 1037
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1038 1039 1040
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
1041
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1042
}
1043

1044
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1045 1046 1047
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1048

1049 1050
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1051 1052 1053 1054
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1055
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1056
}
1057

1058
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1059
{
1060 1061 1062
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1063

1064 1065 1066
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1067 1068 1069 1070
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1071
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1072 1073
}

1074
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1075
{
1076 1077
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1078

1079 1080
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1081 1082
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1083
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1084
	}
P
Philipp Reisner 已提交
1085 1086
}

1087
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1088
{
1089 1090
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1091
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1092

1093 1094
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1095 1096
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1097
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1098
	}
P
Philipp Reisner 已提交
1099 1100
}

1101
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1102
{
1103 1104 1105
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1106

1107 1108 1109 1110
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1111

1112 1113 1114 1115
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1116 1117
}

1118
static int fill_bitmap_rle_bits(struct drbd_device *device,
1119 1120 1121
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1122 1123 1124 1125 1126 1127 1128
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1129
	int bits, use_rle;
P
Philipp Reisner 已提交
1130 1131

	/* may we use this feature? */
1132
	rcu_read_lock();
1133
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1134
	rcu_read_unlock();
1135
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1136
		return 0;
P
Philipp Reisner 已提交
1137 1138 1139 1140 1141

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1142 1143
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1155 1156
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1157 1158 1159 1160 1161 1162 1163 1164
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1165
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1166 1167 1168 1169
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1170
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1171 1172 1173 1174 1175
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1176
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1177 1178 1179 1180 1181 1182 1183 1184
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1185
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1210
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1211 1212 1213 1214

	return len;
}

1215 1216 1217 1218 1219 1220 1221
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1222
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1223
{
1224 1225
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1226
	struct p_compressed_bm *p = sock->sbuf + header_size;
1227
	int len, err;
P
Philipp Reisner 已提交
1228

1229
	len = fill_bitmap_rle_bits(device, p,
1230
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1231
	if (len < 0)
1232
		return -EIO;
P
Philipp Reisner 已提交
1233 1234

	if (len) {
1235
		dcbp_set_code(p, RLE_VLI_Bits);
1236
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1237 1238
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1239
		c->packets[0]++;
1240
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1241 1242 1243 1244 1245 1246

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1247 1248
		unsigned int data_size;
		unsigned long num_words;
1249
		unsigned long *p = sock->sbuf + header_size;
1250 1251

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1252
		num_words = min_t(size_t, data_size / sizeof(*p),
1253
				  c->bm_words - c->word_offset);
1254
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1255
		if (len)
1256
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1257
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1258 1259 1260 1261
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1262
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1263 1264 1265 1266

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1267
	if (!err) {
1268
		if (len == 0) {
1269
			INFO_bm_xfer_stats(device, "send", c);
1270 1271 1272 1273 1274
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1275 1276 1277
}

/* See the comment at receive_bitmap() */
1278
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1279 1280
{
	struct bm_xfer_ctx c;
1281
	int err;
P
Philipp Reisner 已提交
1282

1283
	if (!expect(device->bitmap))
1284
		return false;
P
Philipp Reisner 已提交
1285

1286 1287
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1288
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1289 1290
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1291 1292 1293
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1294
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1295
			} else {
1296 1297
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1298 1299
			}
		}
1300
		put_ldev(device);
P
Philipp Reisner 已提交
1301 1302 1303
	}

	c = (struct bm_xfer_ctx) {
1304 1305
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1306 1307 1308
	};

	do {
1309
		err = send_bitmap_rle_or_plain(device, &c);
1310
	} while (err > 0);
P
Philipp Reisner 已提交
1311

1312
	return err == 0;
P
Philipp Reisner 已提交
1313 1314
}

1315
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1316
{
1317
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1318
	int err = -1;
P
Philipp Reisner 已提交
1319

1320 1321
	mutex_lock(&sock->mutex);
	if (sock->socket)
1322
		err = !_drbd_send_bitmap(device);
1323
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1324 1325 1326
	return err;
}

1327
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1328
{
1329 1330
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1331

1332
	if (connection->cstate < C_WF_REPORT_PARAMS)
1333
		return;
P
Philipp Reisner 已提交
1334

1335 1336
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1337 1338 1339 1340
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1341
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1342 1343 1344 1345
}

/**
 * _drbd_send_ack() - Sends an ack packet
1346
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1347 1348 1349 1350 1351
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1352
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1353
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1354
{
1355 1356
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1357

1358
	if (peer_device->device->state.conn < C_CONNECTED)
1359
		return -EIO;
P
Philipp Reisner 已提交
1360

1361 1362
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1363
	if (!p)
1364
		return -EIO;
1365 1366 1367
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1368 1369
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1370 1371
}

1372 1373 1374
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1375
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1376
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1377
{
1378
	if (peer_device->connection->peer_integrity_tfm)
H
Herbert Xu 已提交
1379
		data_size -= crypto_ahash_digestsize(peer_device->connection->peer_integrity_tfm);
1380
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1381
		       dp->block_id);
P
Philipp Reisner 已提交
1382 1383
}

1384
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1385
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1386
{
1387
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1388 1389 1390 1391
}

/**
 * drbd_send_ack() - Sends an ack packet
1392
 * @device:	DRBD device
1393 1394
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1395
 */
1396
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1397
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1398
{
1399
	return _drbd_send_ack(peer_device, cmd,
1400 1401 1402
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1403 1404 1405 1406
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1407
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1408 1409
		     sector_t sector, int blksize, u64 block_id)
{
1410
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1411 1412 1413 1414 1415
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
			     struct drbd_peer_request *peer_req)
{
	struct drbd_socket *sock;
	struct p_block_desc *p;

	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->blksize = cpu_to_be32(peer_req->i.size);
	p->pad = 0;
	return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0);
}

1432
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1433 1434
		       sector_t sector, int size, u64 block_id)
{
1435 1436
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1437

1438 1439
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1440 1441 1442 1443 1444
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1445
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1446 1447
}

1448
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1449
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1450
{
1451 1452
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1453

1454
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1455

1456 1457
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1458 1459 1460 1461 1462
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1463
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1464 1465
}

1466
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1467
{
1468 1469
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1470

1471 1472
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1473 1474 1475 1476 1477
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1478
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1479 1480 1481
}

/* called on sndtimeo
1482 1483
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1484
 */
1485
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1486 1487
{
	int drop_it;
1488
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1489

1490
	drop_it =   connection->meta.socket == sock
1491 1492
		|| !connection->ack_receiver.task
		|| get_t_state(&connection->ack_receiver) != RUNNING
1493
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1494 1495

	if (drop_it)
1496
		return true;
P
Philipp Reisner 已提交
1497

1498
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1499
	if (!drop_it) {
1500
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1501 1502
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1503 1504
	}

1505
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1506 1507
}

1508
static void drbd_update_congested(struct drbd_connection *connection)
1509
{
1510
	struct sock *sk = connection->data.socket->sk;
1511
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1512
		set_bit(NET_CONGESTED, &connection->flags);
1513 1514
}

P
Philipp Reisner 已提交
1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1536
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1537
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1538
{
1539 1540 1541 1542
	struct socket *socket;
	void *addr;
	int err;

1543
	socket = peer_device->connection->data.socket;
1544
	addr = kmap(page) + offset;
1545
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1546
	kunmap(page);
1547
	if (!err)
1548
		peer_device->device->send_cnt += size >> 9;
1549
	return err;
P
Philipp Reisner 已提交
1550 1551
}

1552
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1553
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1554
{
1555
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1556
	int len = size;
1557
	int err = -EIO;
P
Philipp Reisner 已提交
1558 1559 1560 1561 1562 1563 1564 1565

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1566
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1567

L
Lars Ellenberg 已提交
1568
	msg_flags |= MSG_NOSIGNAL;
1569
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1570
	do {
1571 1572 1573
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1574
		if (sent <= 0) {
1575
			if (sent == -EAGAIN) {
1576
				if (we_should_drop_the_connection(peer_device->connection, socket))
1577 1578 1579
					break;
				continue;
			}
1580
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1581
			     __func__, (int)size, len, sent);
1582 1583
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1584 1585 1586 1587
			break;
		}
		len    -= sent;
		offset += sent;
1588
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
1589
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1590

1591 1592
	if (len == 0) {
		err = 0;
1593
		peer_device->device->send_cnt += size >> 9;
1594 1595
	}
	return err;
P
Philipp Reisner 已提交
1596 1597
}

1598
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1599
{
1600 1601 1602
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1603
	/* hint all but last page with MSG_MORE */
1604
	bio_for_each_segment(bvec, bio, iter) {
1605 1606
		int err;

1607
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1608
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1609
					 bio_iter_last(bvec, iter)
1610
					 ? 0 : MSG_MORE);
1611 1612
		if (err)
			return err;
1613 1614 1615
		/* REQ_OP_WRITE_SAME has only one segment */
		if (bio_op(bio) == REQ_OP_WRITE_SAME)
			break;
P
Philipp Reisner 已提交
1616
	}
1617
	return 0;
P
Philipp Reisner 已提交
1618 1619
}

1620
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1621
{
1622 1623 1624
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1625
	/* hint all but last page with MSG_MORE */
1626
	bio_for_each_segment(bvec, bio, iter) {
1627 1628
		int err;

1629
		err = _drbd_send_page(peer_device, bvec.bv_page,
1630
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1631
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1632 1633
		if (err)
			return err;
1634 1635 1636
		/* REQ_OP_WRITE_SAME has only one segment */
		if (bio_op(bio) == REQ_OP_WRITE_SAME)
			break;
P
Philipp Reisner 已提交
1637
	}
1638
	return 0;
P
Philipp Reisner 已提交
1639 1640
}

1641
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1642
			    struct drbd_peer_request *peer_req)
1643
{
1644 1645
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1646
	int err;
1647

L
Lars Ellenberg 已提交
1648
	/* hint all but last page with MSG_MORE */
1649 1650
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1651

1652
		err = _drbd_send_page(peer_device, page, 0, l,
1653 1654 1655
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1656 1657
		len -= l;
	}
1658
	return 0;
1659 1660
}

M
Mike Christie 已提交
1661 1662
static u32 bio_flags_to_wire(struct drbd_connection *connection,
			     struct bio *bio)
1663
{
1664
	if (connection->agreed_pro_version >= 95)
J
Jens Axboe 已提交
1665 1666 1667
		return  (bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bio->bi_opf & REQ_FUA ? DP_FUA : 0) |
			(bio->bi_opf & REQ_PREFLUSH ? DP_FLUSH : 0) |
1668
			(bio_op(bio) == REQ_OP_WRITE_SAME ? DP_WSAME : 0) |
1669 1670
			(bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0) |
			(bio_op(bio) == REQ_OP_WRITE_ZEROES ? DP_DISCARD : 0);
1671
	else
J
Jens Axboe 已提交
1672
		return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0;
1673 1674
}

1675 1676
/* Used to send write or TRIM aka REQ_DISCARD requests
 * R_PRIMARY -> Peer	(P_DATA, P_TRIM)
P
Philipp Reisner 已提交
1677
 */
1678
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1679
{
1680
	struct drbd_device *device = peer_device->device;
1681 1682
	struct drbd_socket *sock;
	struct p_data *p;
1683 1684
	struct p_wsame *wsame = NULL;
	void *digest_out;
P
Philipp Reisner 已提交
1685
	unsigned int dp_flags = 0;
1686
	int digest_size;
1687
	int err;
P
Philipp Reisner 已提交
1688

1689 1690
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1691
	digest_size = peer_device->connection->integrity_tfm ?
H
Herbert Xu 已提交
1692
		      crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1693

1694 1695 1696 1697
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1698
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
M
Mike Christie 已提交
1699
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio);
1700 1701
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1702
		dp_flags |= DP_MAY_SET_IN_SYNC;
1703
	if (peer_device->connection->agreed_pro_version >= 100) {
1704 1705
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
1706 1707 1708 1709
		/* During resync, request an explicit write ack,
		 * even in protocol != C */
		if (req->rq_state & RQ_EXP_WRITE_ACK
		|| (dp_flags & DP_MAY_SET_IN_SYNC))
1710 1711
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1712
	p->dp_flags = cpu_to_be32(dp_flags);
1713 1714 1715 1716 1717 1718 1719

	if (dp_flags & DP_DISCARD) {
		struct p_trim *t = (struct p_trim*)p;
		t->size = cpu_to_be32(req->i.size);
		err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0);
		goto out;
	}
1720 1721 1722 1723 1724 1725 1726 1727 1728
	if (dp_flags & DP_WSAME) {
		/* this will only work if DRBD_FF_WSAME is set AND the
		 * handshake agreed that all nodes and backend devices are
		 * WRITE_SAME capable and agree on logical_block_size */
		wsame = (struct p_wsame*)p;
		digest_out = wsame + 1;
		wsame->size = cpu_to_be32(req->i.size);
	} else
		digest_out = p + 1;
1729 1730 1731

	/* our digest is still only over the payload.
	 * TRIM does not carry any payload. */
1732
	if (digest_size)
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out);
	if (wsame) {
		err =
		    __send_command(peer_device->connection, device->vnr, sock, P_WSAME,
				   sizeof(*wsame) + digest_size, NULL,
				   bio_iovec(req->master_bio).bv_len);
	} else
		err =
		    __send_command(peer_device->connection, device->vnr, sock, P_DATA,
				   sizeof(*p) + digest_size, NULL, req->i.size);
1743
	if (!err) {
1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1755
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || digest_size)
1756
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1757
		else
1758
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1759 1760

		/* double check digest, sometimes buffers have been modified in flight. */
1761
		if (digest_size > 0 && digest_size <= 64) {
B
Bart Van Assche 已提交
1762
			/* 64 byte, 512 bit, is the largest digest size
1763 1764
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1765
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1766
			if (memcmp(p + 1, digest, digest_size)) {
1767
				drbd_warn(device,
1768
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1769
					(unsigned long long)req->i.sector, req->i.size);
1770
			}
1771
		} /* else if (digest_size > 64) {
1772 1773
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1774
	}
1775
out:
1776
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1777

1778
	return err;
P
Philipp Reisner 已提交
1779 1780 1781 1782 1783 1784
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1785
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1786
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1787
{
1788
	struct drbd_device *device = peer_device->device;
1789 1790
	struct drbd_socket *sock;
	struct p_data *p;
1791
	int err;
1792
	int digest_size;
P
Philipp Reisner 已提交
1793

1794 1795
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1796

1797
	digest_size = peer_device->connection->integrity_tfm ?
H
Herbert Xu 已提交
1798
		      crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1799

1800 1801 1802 1803 1804
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1805
	p->dp_flags = 0;
1806
	if (digest_size)
1807
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
1808
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + digest_size, NULL, peer_req->i.size);
1809
	if (!err)
1810
		err = _drbd_send_zc_ee(peer_device, peer_req);
1811
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1812

1813
	return err;
P
Philipp Reisner 已提交
1814 1815
}

1816
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1817
{
1818 1819
	struct drbd_socket *sock;
	struct p_block_desc *p;
1820

1821 1822
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1823 1824 1825 1826
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1827
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1828 1829
}

P
Philipp Reisner 已提交
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1846
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1847 1848
	      void *buf, size_t size, unsigned msg_flags)
{
A
Al Viro 已提交
1849
	struct kvec iov = {.iov_base = buf, .iov_len = size};
P
Philipp Reisner 已提交
1850 1851 1852 1853
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1854
		return -EBADR;
P
Philipp Reisner 已提交
1855 1856 1857 1858 1859 1860 1861 1862 1863

	/* THINK  if (signal_pending) return ... ? */

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

A
Al Viro 已提交
1864 1865
	iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, size);

1866
	if (sock == connection->data.socket) {
1867
		rcu_read_lock();
1868
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1869
		rcu_read_unlock();
1870
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1871 1872
	}
	do {
A
Al Viro 已提交
1873
		rv = sock_sendmsg(sock, &msg);
P
Philipp Reisner 已提交
1874
		if (rv == -EAGAIN) {
1875
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
	} while (sent < size);

1889 1890
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1891 1892 1893

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1894
			drbd_err(connection, "%s_sendmsg returned %d\n",
1895
				 sock == connection->meta.socket ? "msock" : "sock",
1896
				 rv);
1897
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1898
		} else
1899
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1900 1901 1902 1903 1904
	}

	return sent;
}

1905 1906 1907 1908 1909
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1910
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1911 1912 1913 1914
		  size_t size, unsigned msg_flags)
{
	int err;

1915
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1916 1917 1918 1919 1920 1921 1922
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1923 1924
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1925
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1926 1927 1928
	unsigned long flags;
	int rv = 0;

1929
	mutex_lock(&drbd_main_mutex);
1930
	spin_lock_irqsave(&device->resource->req_lock, flags);
1931
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1932 1933
	 * and no race with updating open_cnt */

1934
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1935 1936 1937 1938 1939 1940 1941
		if (mode & FMODE_WRITE)
			rv = -EROFS;
		else if (!allow_oos)
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1942
		device->open_cnt++;
1943
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1944
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1945 1946 1947 1948

	return rv;
}

1949
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1950
{
1951
	struct drbd_device *device = gd->private_data;
1952
	mutex_lock(&drbd_main_mutex);
1953
	device->open_cnt--;
1954
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1955 1956
}

1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
/* need to hold resource->req_lock */
void drbd_queue_unplug(struct drbd_device *device)
{
	if (device->state.pdsk >= D_INCONSISTENT && device->state.conn >= C_CONNECTED) {
		D_ASSERT(device, device->state.role == R_PRIMARY);
		if (test_and_clear_bit(UNPLUG_REMOTE, &device->flags)) {
			drbd_queue_work_if_unqueued(
				&first_peer_device(device)->connection->sender_work,
				&device->unplug_work);
		}
	}
}

1970
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1971
{
1972 1973
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1974
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1975 1976 1977 1978 1979 1980 1981 1982
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1983
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1984 1985 1986 1987
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1988 1989 1990
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
1991
	atomic_set(&device->ap_actlog_cnt, 0);
1992 1993 1994 1995 1996 1997 1998 1999
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
2000
	atomic_set(&device->md_io.in_use, 0);
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);
2017 2018 2019 2020
	INIT_LIST_HEAD(&device->pending_master_completion[0]);
	INIT_LIST_HEAD(&device->pending_master_completion[1]);
	INIT_LIST_HEAD(&device->pending_completion[0]);
	INIT_LIST_HEAD(&device->pending_completion[1]);
2021 2022 2023 2024 2025

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->bm_io_work.w.cb = w_bitmap_io;

2026 2027 2028 2029 2030 2031 2032 2033
	setup_timer(&device->resync_timer, resync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->md_sync_timer, md_sync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->start_resync_timer, start_resync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->request_timer, request_timer_fn,
			(unsigned long)device);
2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
2047
{
2048
	int i;
2049
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
2050
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
2051
				first_peer_device(device)->connection->receiver.t_state);
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
2065
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
2066 2067
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
2068
	}
2069
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
2070

2071 2072
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
2073
		/* maybe never allocated. */
2074 2075
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2076 2077
	}

2078
	drbd_backing_dev_free(device, device->ldev);
2079
	device->ldev = NULL;
2080

2081
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
2082

2083 2084 2085 2086 2087 2088 2089 2090 2091
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
2092

2093
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2108
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2109

2110 2111
	if (drbd_io_bio_set)
		bioset_free(drbd_io_bio_set);
2112 2113
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2114 2115
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2129
	drbd_io_bio_set      = NULL;
2130
	drbd_md_io_bio_set   = NULL;
2131
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2145
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
P
Philipp Reisner 已提交
2146 2147 2148 2149 2150 2151 2152 2153 2154
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2155
	drbd_md_io_page_pool = NULL;
2156
	drbd_md_io_bio_set   = NULL;
2157
	drbd_io_bio_set      = NULL;
P
Philipp Reisner 已提交
2158 2159 2160 2161 2162 2163 2164 2165

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2166
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2181 2182 2183 2184
	drbd_io_bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_RESCUER);
	if (drbd_io_bio_set == NULL)
		goto Enomem;

2185 2186 2187
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0,
					   BIOSET_NEED_BVECS |
					   BIOSET_NEED_RESCUER);
2188 2189 2190
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2191 2192 2193 2194
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

2195 2196
	drbd_request_mempool = mempool_create_slab_pool(number,
		drbd_request_cache);
P
Philipp Reisner 已提交
2197 2198 2199
	if (drbd_request_mempool == NULL)
		goto Enomem;

2200
	drbd_ee_mempool = mempool_create_slab_pool(number, drbd_ee_cache);
2201
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

2223
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2224 2225 2226
{
	int rr;

2227
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2228
	if (rr)
2229
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2230

2231
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2232
	if (rr)
2233
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2234

2235
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2236
	if (rr)
2237
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2238

2239
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2240
	if (rr)
2241
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2242

2243
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2244
	if (rr)
2245
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2246 2247
}

2248
/* caution. no locking. */
2249
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2250
{
2251
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2252
	struct drbd_resource *resource = device->resource;
2253
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2254

2255
	del_timer_sync(&device->request_timer);
2256

P
Philipp Reisner 已提交
2257
	/* paranoia asserts */
2258
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2259 2260 2261 2262 2263
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2264 2265
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2266

2267
	drbd_backing_dev_free(device, device->ldev);
2268
	device->ldev = NULL;
P
Philipp Reisner 已提交
2269

2270
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2271

2272 2273
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2274

2275 2276
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2277

2278 2279
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
2280
	__free_page(device->md_io.page);
2281 2282 2283
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2284 2285 2286 2287 2288 2289 2290 2291

	/* not for_each_connection(connection, resource):
	 * those may have been cleaned up and disassociated already.
	 */
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
		kfree(peer_device);
	}
2292
	memset(device, 0xfd, sizeof(*device));
2293
	kfree(device);
2294
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2295 2296
}

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2319
		struct drbd_device *device = req->device;
2320
		struct bio *bio = req->master_bio;
2321
		unsigned long start_jif = req->start_jif;
2322 2323
		bool expected;

2324
		expected =
2325 2326 2327 2328 2329 2330
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2331
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2332 2333 2334 2335 2336 2337 2338
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2339
		 */
2340
		kref_put(&req->kref, drbd_req_destroy);
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2355
		inc_ap_bio(device);
2356
		__drbd_make_request(device, bio, start_jif);
2357 2358 2359
	}
}

2360 2361
/* called via drbd_req_put_completion_ref(),
 * holds resource->req_lock */
2362
void drbd_restart_request(struct drbd_request *req)
2363 2364 2365 2366 2367 2368 2369 2370 2371
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2372
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2373

2374
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2375 2376
}

2377 2378 2379 2380 2381
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2382
	idr_destroy(&resource->devices);
2383
	free_cpumask_var(resource->cpu_mask);
2384
	kfree(resource->name);
2385
	memset(resource, 0xf2, sizeof(*resource));
2386 2387 2388 2389 2390 2391 2392 2393 2394
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
2395
		drbd_debugfs_connection_cleanup(connection);
2396 2397
		kref_put(&connection->kref, drbd_destroy_connection);
	}
2398
	drbd_debugfs_resource_cleanup(resource);
2399 2400
	kref_put(&resource->kref, drbd_destroy_resource);
}
2401

P
Philipp Reisner 已提交
2402 2403 2404
static void drbd_cleanup(void)
{
	unsigned int i;
2405
	struct drbd_device *device;
2406
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2407

2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2419 2420
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2421

2422
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2423

2424
	idr_for_each_entry(&drbd_devices, device, i)
2425
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2426

P
Philipp Reisner 已提交
2427
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2428 2429 2430
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2431
	}
P
Philipp Reisner 已提交
2432

2433 2434
	drbd_debugfs_cleanup();

2435
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2436 2437
	unregister_blkdev(DRBD_MAJOR, "drbd");

2438
	idr_destroy(&drbd_devices);
2439

2440
	pr_info("module cleanup done.\n");
P
Philipp Reisner 已提交
2441 2442 2443
}

/**
2444
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2445
 * @congested_data:	User data
2446
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2447
 *
2448
 * Returns 1<<WB_async_congested and/or 1<<WB_sync_congested if we are congested.
P
Philipp Reisner 已提交
2449 2450 2451
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2452
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2453 2454 2455 2456
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2457
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2458 2459 2460 2461 2462 2463
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2464
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2465
		r |= (1 << WB_async_congested);
2466 2467 2468 2469 2470
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2471
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2472
			r |= (1 << WB_sync_congested);
2473
		else
2474
			put_ldev(device);
2475 2476 2477 2478 2479
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2480 2481
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
2482
		r = bdi_congested(q->backing_dev_info, bdi_bits);
2483
		put_ldev(device);
P
Philipp Reisner 已提交
2484 2485 2486 2487
		if (r)
			reason = 'b';
	}

2488
	if (bdi_bits & (1 << WB_async_congested) &&
2489
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
2490
		r |= (1 << WB_async_congested);
P
Philipp Reisner 已提交
2491 2492 2493 2494
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2495
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2496 2497 2498
	return r;
}

2499 2500 2501 2502
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2503
	init_waitqueue_head(&wq->q_wait);
2504 2505
}

2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2530
struct drbd_resource *drbd_find_resource(const char *name)
2531
{
2532
	struct drbd_resource *resource;
2533

2534 2535 2536
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2537
	rcu_read_lock();
2538 2539
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2540
			kref_get(&resource->kref);
2541
			goto found;
2542
		}
2543
	}
2544
	resource = NULL;
2545
found:
P
Philipp Reisner 已提交
2546
	rcu_read_unlock();
2547
	return resource;
2548 2549
}

2550
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2551 2552
				     void *peer_addr, int peer_addr_len)
{
2553
	struct drbd_resource *resource;
2554
	struct drbd_connection *connection;
2555 2556

	rcu_read_lock();
2557 2558 2559 2560 2561 2562 2563 2564 2565
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2566 2567
		}
	}
2568
	connection = NULL;
2569 2570
found:
	rcu_read_unlock();
2571
	return connection;
2572 2573
}

2574 2575 2576 2577 2578
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2579 2580 2581
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2582 2583 2584 2585 2586
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2587
	free_page((unsigned long) socket->sbuf);
2588 2589 2590
	free_page((unsigned long) socket->rbuf);
}

2591
void conn_free_crypto(struct drbd_connection *connection)
2592
{
2593
	drbd_free_sock(connection);
2594

H
Herbert Xu 已提交
2595 2596 2597 2598 2599
	crypto_free_ahash(connection->csums_tfm);
	crypto_free_ahash(connection->verify_tfm);
	crypto_free_shash(connection->cram_hmac_tfm);
	crypto_free_ahash(connection->integrity_tfm);
	crypto_free_ahash(connection->peer_integrity_tfm);
2600 2601
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2602

2603 2604 2605 2606 2607 2608 2609
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2610 2611
}

2612
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2613
{
2614
	struct drbd_connection *connection;
2615 2616 2617 2618 2619 2620 2621 2622
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2623
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2624
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638
		if (err == -EOVERFLOW) {
			/* So what. mask it out. */
			cpumask_var_t tmp_cpu_mask;
			if (zalloc_cpumask_var(&tmp_cpu_mask, GFP_KERNEL)) {
				cpumask_setall(tmp_cpu_mask);
				cpumask_and(new_cpu_mask, new_cpu_mask, tmp_cpu_mask);
				drbd_warn(resource, "Overflow in bitmap_parse(%.12s%s), truncating to %u bits\n",
					res_opts->cpu_mask,
					strlen(res_opts->cpu_mask) > 12 ? "..." : "",
					nr_cpu_ids);
				free_cpumask_var(tmp_cpu_mask);
				err = 0;
			}
		}
2639
		if (err) {
2640
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2641 2642 2643 2644
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2645
	resource->res_opts = *res_opts;
2646 2647 2648 2649 2650
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2651
			connection->receiver.reset_cpu_mask = 1;
2652
			connection->ack_receiver.reset_cpu_mask = 1;
2653 2654
			connection->worker.reset_cpu_mask = 1;
		}
2655 2656 2657 2658 2659 2660 2661 2662 2663
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2664 2665 2666 2667
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2668
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2669
	if (!resource)
2670
		goto fail;
2671
	resource->name = kstrdup(name, GFP_KERNEL);
2672 2673 2674 2675
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2676
	kref_init(&resource->kref);
2677
	idr_init(&resource->devices);
2678
	INIT_LIST_HEAD(&resource->connections);
2679
	resource->write_ordering = WO_BDEV_FLUSH;
2680
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2681
	mutex_init(&resource->conf_update);
2682
	mutex_init(&resource->adm_mutex);
2683
	spin_lock_init(&resource->req_lock);
2684
	drbd_debugfs_resource_add(resource);
2685
	return resource;
2686 2687 2688 2689 2690 2691 2692

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2693 2694
}

2695
/* caller must be under adm_mutex */
2696
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2697
{
2698
	struct drbd_resource *resource;
2699
	struct drbd_connection *connection;
2700

2701 2702
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2703 2704
		return NULL;

2705
	if (drbd_alloc_socket(&connection->data))
2706
		goto fail;
2707
	if (drbd_alloc_socket(&connection->meta))
2708 2709
		goto fail;

2710 2711
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2712
		goto fail;
2713

2714
	INIT_LIST_HEAD(&connection->transfer_log);
2715

2716 2717 2718
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
2719

2720 2721 2722
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2723

2724 2725 2726 2727
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2728 2729 2730
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2731
	idr_init(&connection->peer_devices);
2732

2733 2734 2735
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2736

2737 2738 2739 2740
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
2741 2742
	drbd_thread_init(resource, &connection->ack_receiver, drbd_ack_receiver, "ack_recv");
	connection->ack_receiver.connection = connection;
2743

2744
	kref_init(&connection->kref);
2745 2746

	connection->resource = resource;
2747

2748 2749 2750 2751 2752
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2753
	drbd_debugfs_connection_add(connection);
2754
	return connection;
2755

2756 2757 2758
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2759
fail:
2760 2761 2762 2763
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2764 2765 2766
	return NULL;
}

2767
void drbd_destroy_connection(struct kref *kref)
2768
{
2769
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2770
	struct drbd_resource *resource = connection->resource;
2771

2772
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2773
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2774
	kfree(connection->current_epoch);
2775

2776
	idr_destroy(&connection->peer_devices);
2777

2778 2779 2780 2781
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2782
	memset(connection, 0xfc, sizeof(*connection));
2783
	kfree(connection);
2784
	kref_put(&resource->kref, drbd_destroy_resource);
2785 2786
}

2787
static int init_submitter(struct drbd_device *device)
2788 2789 2790
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2791 2792
	device->submit.wq =
		alloc_ordered_workqueue("drbd%u_submit", WQ_MEM_RECLAIM, device->minor);
2793
	if (!device->submit.wq)
2794 2795
		return -ENOMEM;

2796 2797
	INIT_WORK(&device->submit.worker, do_submit);
	INIT_LIST_HEAD(&device->submit.writes);
2798 2799 2800
	return 0;
}

2801
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2802
{
2803
	struct drbd_resource *resource = adm_ctx->resource;
2804
	struct drbd_connection *connection;
2805
	struct drbd_device *device;
2806
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2807 2808
	struct gendisk *disk;
	struct request_queue *q;
2809
	int id;
2810
	int vnr = adm_ctx->volume;
2811
	enum drbd_ret_code err = ERR_NOMEM;
2812

2813 2814
	device = minor_to_device(minor);
	if (device)
A
Andreas Gruenbacher 已提交
2815
		return ERR_MINOR_OR_VOLUME_EXISTS;
P
Philipp Reisner 已提交
2816 2817

	/* GFP_KERNEL, we are outside of all write-out paths */
2818 2819
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2820
		return ERR_NOMEM;
2821 2822 2823 2824
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2825 2826
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2827

2828
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2829 2830 2831 2832

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2833 2834
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2835 2836 2837 2838

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2839
	device->vdisk = disk;
P
Philipp Reisner 已提交
2840

2841
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2842 2843 2844 2845 2846 2847

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2848
	disk->private_data = device;
P
Philipp Reisner 已提交
2849

2850
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2851
	/* we have no partitions. we contain only ourselves. */
2852
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2853

2854 2855
	q->backing_dev_info->congested_fn = drbd_congested;
	q->backing_dev_info->congested_data = device;
P
Philipp Reisner 已提交
2856

2857
	blk_queue_make_request(q, drbd_make_request);
2858
	blk_queue_write_cache(q, true, true);
2859 2860 2861
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
2862
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2863

2864 2865
	device->md_io.page = alloc_page(GFP_KERNEL);
	if (!device->md_io.page)
P
Philipp Reisner 已提交
2866 2867
		goto out_no_io_page;

2868
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2869
		goto out_no_bitmap;
2870 2871
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2872

2873 2874
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
2875
		if (id == -ENOSPC)
A
Andreas Gruenbacher 已提交
2876
			err = ERR_MINOR_OR_VOLUME_EXISTS;
2877
		goto out_no_minor_idr;
2878
	}
2879 2880 2881 2882
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
2883
		if (id == -ENOSPC)
A
Andreas Gruenbacher 已提交
2884
			err = ERR_MINOR_OR_VOLUME_EXISTS;
2885 2886 2887
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2888

2889
	INIT_LIST_HEAD(&device->peer_devices);
2890
	INIT_LIST_HEAD(&device->pending_bitmap_io);
2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
2903
			if (id == -ENOSPC)
2904 2905
				err = ERR_INVALID_REQUEST;
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2906
		}
2907
		kref_get(&connection->kref);
2908
		INIT_WORK(&peer_device->send_acks_work, drbd_send_acks_wf);
2909
	}
T
Tejun Heo 已提交
2910

2911
	if (init_submitter(device)) {
2912 2913 2914 2915
		err = ERR_NOMEM;
		goto out_idr_remove_vol;
	}

2916 2917
	add_disk(disk);

2918
	/* inherit the connection state */
2919
	device->state.conn = first_connection(resource)->cstate;
2920 2921 2922 2923
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2924 2925 2926 2927
	/* move to create_peer_device() */
	for_each_peer_device(peer_device, device)
		drbd_debugfs_peer_device_add(peer_device);
	drbd_debugfs_device_add(device);
2928
	return NO_ERROR;
P
Philipp Reisner 已提交
2929

2930
out_idr_remove_vol:
2931
	idr_remove(&connection->peer_devices, vnr);
2932
out_idr_remove_from_resource:
2933
	for_each_connection(connection, resource) {
2934 2935
		peer_device = idr_remove(&connection->peer_devices, vnr);
		if (peer_device)
2936 2937 2938 2939 2940 2941
			kref_put(&connection->kref, drbd_destroy_connection);
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2942
	idr_remove(&resource->devices, vnr);
2943
out_idr_remove_minor:
2944
	idr_remove(&drbd_devices, minor);
2945
	synchronize_rcu();
2946
out_no_minor_idr:
2947
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2948
out_no_bitmap:
2949
	__free_page(device->md_io.page);
P
Philipp Reisner 已提交
2950 2951 2952 2953 2954
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2955
	kref_put(&resource->kref, drbd_destroy_resource);
2956
	kfree(device);
2957
	return err;
P
Philipp Reisner 已提交
2958 2959
}

2960
void drbd_delete_device(struct drbd_device *device)
2961 2962 2963
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
2964
	struct drbd_peer_device *peer_device;
2965

2966 2967 2968 2969
	/* move to free_peer_device() */
	for_each_peer_device(peer_device, device)
		drbd_debugfs_peer_device_cleanup(peer_device);
	drbd_debugfs_device_cleanup(device);
2970
	for_each_connection(connection, resource) {
2971
		idr_remove(&connection->peer_devices, device->vnr);
2972
		kref_put(&device->kref, drbd_destroy_device);
2973 2974
	}
	idr_remove(&resource->devices, device->vnr);
2975
	kref_put(&device->kref, drbd_destroy_device);
2976
	idr_remove(&drbd_devices, device_to_minor(device));
2977
	kref_put(&device->kref, drbd_destroy_device);
2978 2979
	del_gendisk(device->vdisk);
	synchronize_rcu();
2980
	kref_put(&device->kref, drbd_destroy_device);
2981 2982
}

2983
static int __init drbd_init(void)
P
Philipp Reisner 已提交
2984 2985 2986
{
	int err;

2987
	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
2988
		pr_err("invalid minor_count (%d)\n", minor_count);
P
Philipp Reisner 已提交
2989 2990 2991
#ifdef MODULE
		return -EINVAL;
#else
2992
		minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2993 2994 2995 2996 2997
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
2998
		pr_err("unable to register block device major %d\n",
P
Philipp Reisner 已提交
2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
		       DRBD_MAJOR);
		return err;
	}

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
3009
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
3010

3011
	mutex_init(&resources_mutex);
3012
	INIT_LIST_HEAD(&drbd_resources);
3013 3014 3015

	err = drbd_genl_register();
	if (err) {
3016
		pr_err("unable to register generic netlink family\n");
3017 3018 3019
		goto fail;
	}

P
Philipp Reisner 已提交
3020 3021
	err = drbd_create_mempools();
	if (err)
3022
		goto fail;
P
Philipp Reisner 已提交
3023

3024
	err = -ENOMEM;
3025
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
3026
	if (!drbd_proc)	{
3027
		pr_err("unable to register proc file\n");
3028
		goto fail;
P
Philipp Reisner 已提交
3029 3030
	}

3031 3032
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
3033
		pr_err("unable to create retry workqueue\n");
3034 3035 3036 3037 3038
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
3039

3040 3041 3042
	if (drbd_debugfs_init())
		pr_notice("failed to initialize debugfs -- will not be available\n");

3043
	pr_info("initialized. "
P
Philipp Reisner 已提交
3044 3045
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
3046 3047
	pr_info("%s\n", drbd_buildtag());
	pr_info("registered as block device major %d\n", DRBD_MAJOR);
P
Philipp Reisner 已提交
3048 3049
	return 0; /* Success! */

3050
fail:
P
Philipp Reisner 已提交
3051 3052
	drbd_cleanup();
	if (err == -ENOMEM)
3053
		pr_err("ran out of memory\n");
P
Philipp Reisner 已提交
3054
	else
3055
		pr_err("initialization failure\n");
P
Philipp Reisner 已提交
3056 3057 3058
	return err;
}

3059
static void drbd_free_one_sock(struct drbd_socket *ds)
P
Philipp Reisner 已提交
3060
{
3061 3062 3063 3064 3065 3066 3067 3068 3069 3070
	struct socket *s;
	mutex_lock(&ds->mutex);
	s = ds->socket;
	ds->socket = NULL;
	mutex_unlock(&ds->mutex);
	if (s) {
		/* so debugfs does not need to mutex_lock() */
		synchronize_rcu();
		kernel_sock_shutdown(s, SHUT_RDWR);
		sock_release(s);
P
Philipp Reisner 已提交
3071 3072 3073
	}
}

3074 3075 3076 3077 3078 3079 3080 3081
void drbd_free_sock(struct drbd_connection *connection)
{
	if (connection->data.socket)
		drbd_free_one_sock(&connection->data);
	if (connection->meta.socket)
		drbd_free_one_sock(&connection->meta);
}

P
Philipp Reisner 已提交
3082 3083
/* meta data management */

3084
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
3085
{
3086
	struct drbd_peer_device *peer_device;
3087
	int vnr;
P
Philipp Reisner 已提交
3088

3089
	rcu_read_lock();
3090 3091 3092
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

3093
		kref_get(&device->kref);
3094
		rcu_read_unlock();
3095
		drbd_md_sync(device);
3096
		kref_put(&device->kref, drbd_destroy_device);
3097 3098 3099
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
3100 3101
}

3102
/* aligned 4kByte */
P
Philipp Reisner 已提交
3103
struct meta_data_on_disk {
3104
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
3105 3106 3107 3108 3109 3110 3111
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3112
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3113
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3114 3115
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3116
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3117

3118 3119 3120 3121 3122
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3123 3124
} __packed;

3125 3126


3127
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3128
{
3129
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3130 3131 3132
	sector_t sector;
	int i;

3133
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3134

3135
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3136
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3137 3138
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3139
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3140

3141 3142 3143
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3144
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3145
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3146

3147 3148
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3149

3150 3151
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3152

3153
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3154
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3155

M
Mike Christie 已提交
3156
	if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
P
Philipp Reisner 已提交
3157
		/* this was a try anyways ... */
3158
		drbd_err(device, "meta data update failed!\n");
3159
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3160
	}
3161 3162 3163 3164
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3165
 * @device:	DRBD device.
3166
 */
3167
void drbd_md_sync(struct drbd_device *device)
3168 3169 3170 3171 3172 3173 3174
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3175
	del_timer(&device->md_sync_timer);
3176
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3177
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3178 3179 3180 3181
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3182
	if (!get_ldev_if_state(device, D_FAILED))
3183 3184
		return;

3185
	buffer = drbd_md_get_buffer(device, __func__);
3186 3187 3188
	if (!buffer)
		goto out;

3189
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3190

3191
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3192
	 * since we updated it on metadata. */
3193
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3194

3195
	drbd_md_put_buffer(device);
3196
out:
3197
	put_ldev(device);
P
Philipp Reisner 已提交
3198 3199
}

3200
static int check_activity_log_stripe_size(struct drbd_device *device,
3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3240
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3241 3242 3243 3244
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3245
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3312
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3325 3326
/**
 * drbd_md_read() - Reads in the meta data super block
3327
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3328 3329
 * @bdev:	Device from which the meta data should be read in.
 *
3330
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3331
 * something goes wrong.
3332
 *
3333
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3334
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3335
 */
3336
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3337 3338
{
	struct meta_data_on_disk *buffer;
3339
	u32 magic, flags;
P
Philipp Reisner 已提交
3340 3341
	int i, rv = NO_ERROR;

3342
	if (device->state.disk != D_DISKLESS)
3343
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3344

3345
	buffer = drbd_md_get_buffer(device, __func__);
3346
	if (!buffer)
3347
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3348

3349 3350
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3351 3352
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
3353 3354 3355 3356
	/* Even for (flexible or indexed) external meta data,
	 * initially restrict us to the 4k superblock for now.
	 * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */
	bdev->md.md_size_sect = 8;
P
Philipp Reisner 已提交
3357

M
Mike Christie 已提交
3358 3359
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset,
				 REQ_OP_READ)) {
L
Lucas De Marchi 已提交
3360
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3361
		   called BEFORE disk is attached */
3362
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3363 3364 3365 3366
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3367 3368 3369 3370 3371
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3372
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3373 3374 3375
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3376 3377

	rv = ERR_MD_INVALID;
3378
	if (magic != DRBD_MD_MAGIC_08) {
3379
		if (magic == DRBD_MD_MAGIC_07)
3380
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3381
		else
3382
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3383 3384
		goto err;
	}
3385

3386
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3387
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3388
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3389 3390
		goto err;
	}
3391

3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3404
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3405
		goto err;
3406
	if (check_offsets_and_sizes(device, bdev))
3407 3408
		goto err;

P
Philipp Reisner 已提交
3409
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3410
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3411 3412 3413 3414
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3415
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3416 3417 3418 3419
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3420
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3421

3422
	spin_lock_irq(&device->resource->req_lock);
3423
	if (device->state.conn < C_CONNECTED) {
3424
		unsigned int peer;
3425
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3426
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3427
		device->peer_max_bio_size = peer;
3428
	}
3429
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3430 3431

 err:
3432
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3433 3434 3435 3436 3437 3438

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3439
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3440 3441 3442 3443 3444
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3445
#ifdef DEBUG
3446
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3447
{
3448 3449 3450 3451
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3452 3453 3454
	}
}
#else
3455
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3456
{
3457 3458
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3459
}
3460
#endif
P
Philipp Reisner 已提交
3461

3462
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3463 3464 3465
{
	int i;

3466
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3467
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3468 3469
}

3470
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3471 3472
{
	if (idx == UI_CURRENT) {
3473
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3474 3475 3476 3477
			val |= 1;
		else
			val &= ~((u64)1);

3478
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3479 3480
	}

3481 3482
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3483 3484
}

3485
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3486 3487
{
	unsigned long flags;
3488 3489 3490
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3491
}
P
Philipp Reisner 已提交
3492

3493
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3494
{
3495
	unsigned long flags;
3496 3497 3498 3499
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3500
	}
3501 3502
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3503 3504 3505 3506
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3507
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3508 3509 3510 3511
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3512
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3513 3514
{
	u64 val;
3515 3516 3517 3518
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3519 3520
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3521 3522

	if (bm_uuid)
3523
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3524

3525 3526 3527
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3528

3529
	drbd_print_uuids(device, "new current UUID");
3530
	/* get it to stable storage _now_ */
3531
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3532 3533
}

3534
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3535
{
3536
	unsigned long flags;
3537
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3538 3539
		return;

3540
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3541
	if (val == 0) {
3542 3543 3544
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3545
	} else {
3546
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3547
		if (bm_uuid)
3548
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3549

3550
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3551
	}
3552
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3553

3554
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3555 3556 3557 3558
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3559
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3560 3561 3562
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3563
int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3564 3565 3566
{
	int rv = -EIO;

3567 3568 3569
	drbd_md_set_flag(device, MDF_FULL_SYNC);
	drbd_md_sync(device);
	drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3570

3571
	rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3572

3573 3574 3575
	if (!rv) {
		drbd_md_clear_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
P
Philipp Reisner 已提交
3576 3577 3578 3579 3580 3581 3582
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3583
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3584 3585 3586
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3587
int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3588
{
3589
	drbd_resume_al(device);
3590 3591
	drbd_bm_clear_all(device);
	return drbd_bm_write(device);
P
Philipp Reisner 已提交
3592 3593
}

3594
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3595
{
3596 3597 3598
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3599
	int rv = -EIO;
P
Philipp Reisner 已提交
3600

3601 3602 3603 3604 3605 3606
	if (work->flags != BM_LOCKED_CHANGE_ALLOWED) {
		int cnt = atomic_read(&device->ap_bio_cnt);
		if (cnt)
			drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n",
					cnt, work->why);
	}
P
Philipp Reisner 已提交
3607

3608 3609 3610 3611 3612
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3613
	}
P
Philipp Reisner 已提交
3614

3615 3616
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3617 3618

	if (work->done)
3619
		work->done(device, rv);
P
Philipp Reisner 已提交
3620

3621
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3622
	work->why = NULL;
3623
	work->flags = 0;
P
Philipp Reisner 已提交
3624

3625
	return 0;
P
Philipp Reisner 已提交
3626 3627 3628 3629
}

/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3630
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3631 3632 3633 3634 3635 3636 3637 3638
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
3639 3640 3641
 *
 * Its worker function encloses the call of io_fn() by get_ldev() and
 * put_ldev().
P
Philipp Reisner 已提交
3642
 */
3643
void drbd_queue_bitmap_io(struct drbd_device *device,
3644 3645
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3646
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3647
{
3648
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3649

3650 3651 3652
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3653
	if (device->bm_io_work.why)
3654
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3655
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3656

3657 3658 3659 3660
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3661

3662
	spin_lock_irq(&device->resource->req_lock);
3663
	set_bit(BITMAP_IO, &device->flags);
3664 3665 3666
	/* don't wait for pending application IO if the caller indicates that
	 * application IO does not conflict anyways. */
	if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
3667
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3668 3669
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3670
	}
3671
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3672 3673 3674 3675
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3676
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3677 3678 3679 3680 3681 3682
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3683
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3684
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3685
{
3686 3687
	/* Only suspend io, if some operation is supposed to be locked out */
	const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST);
P
Philipp Reisner 已提交
3688 3689
	int rv;

3690
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3691

3692
	if (do_suspend_io)
3693
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3694

3695 3696 3697
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3698

3699
	if (do_suspend_io)
3700
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3701 3702 3703 3704

	return rv;
}

3705
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3706
{
3707 3708 3709
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3710 3711 3712
	}
}

3713
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3714
{
3715 3716 3717
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3718 3719 3720 3721 3722 3723 3724 3725 3726
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3727
	struct drbd_device *device = (struct drbd_device *) data;
3728
	drbd_device_post_work(device, MD_SYNC);
P
Philipp Reisner 已提交
3729 3730
}

3731
const char *cmdname(enum drbd_packet cmd)
3732 3733 3734 3735 3736 3737
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
3738 3739
		[P_WSAME]	        = "WriteSame",
		[P_TRIM]	        = "Trim",
3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3763
		[P_SUPERSEDED]          = "Superseded",
3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3778
		[P_RETRY_WRITE]		= "RetryWrite",
3779 3780 3781
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3782 3783
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3784 3785
		[P_RS_THIN_REQ]         = "rs_thin_req",
		[P_RS_DEALLOCATED]      = "rs_deallocated",
3786 3787 3788 3789 3790

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3791 3792
	};

3793
	/* too big for the array: 0xfffX */
3794 3795 3796 3797
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3798 3799
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3800
	if (cmd >= ARRAY_SIZE(cmdnames))
3801 3802 3803 3804
		return "Unknown";
	return cmdnames[cmd];
}

3805 3806
/**
 * drbd_wait_misc  -  wait for a request to make progress
3807
 * @device:	device associated with the request
3808 3809 3810
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3811
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3812
{
3813
	struct net_conf *nc;
3814 3815 3816
	DEFINE_WAIT(wait);
	long timeout;

3817
	rcu_read_lock();
3818
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3819 3820
	if (!nc) {
		rcu_read_unlock();
3821
		return -ETIMEDOUT;
3822 3823 3824
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3825

3826
	/* Indicate to wake up device->misc_wait on progress.  */
3827
	i->waiting = true;
3828
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3829
	spin_unlock_irq(&device->resource->req_lock);
3830
	timeout = schedule_timeout(timeout);
3831
	finish_wait(&device->misc_wait, &wait);
3832
	spin_lock_irq(&device->resource->req_lock);
3833
	if (!timeout || device->state.conn < C_CONNECTED)
3834 3835 3836 3837
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3838 3839
}

3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860
void lock_all_resources(void)
{
	struct drbd_resource *resource;
	int __maybe_unused i = 0;

	mutex_lock(&resources_mutex);
	local_irq_disable();
	for_each_resource(resource, &drbd_resources)
		spin_lock_nested(&resource->req_lock, i++);
}

void unlock_all_resources(void)
{
	struct drbd_resource *resource;

	for_each_resource(resource, &drbd_resources)
		spin_unlock(&resource->req_lock);
	local_irq_enable();
	mutex_unlock(&resources_mutex);
}

P
Philipp Reisner 已提交
3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881
#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3882
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3902 3903
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3904 3905 3906 3907 3908 3909
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3910
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3911 3912 3913 3914 3915
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
		(fault_devs == 0 ||
3916
			((1 << device_to_minor(device)) & fault_devs) != 0) &&
P
Philipp Reisner 已提交
3917 3918 3919 3920 3921
		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));

	if (ret) {
		fault_count++;

3922
		if (__ratelimit(&drbd_ratelimit_state))
3923
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3939 3940 3941 3942
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);