drbd_main.c 112.7 KB
Newer Older
P
Philipp Reisner 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
   drbd.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.

 */

29 30
#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

P
Philipp Reisner 已提交
31
#include <linux/module.h>
32
#include <linux/jiffies.h>
P
Philipp Reisner 已提交
33
#include <linux/drbd.h>
34
#include <linux/uaccess.h>
P
Philipp Reisner 已提交
35 36 37
#include <asm/types.h>
#include <net/sock.h>
#include <linux/ctype.h>
38
#include <linux/mutex.h>
P
Philipp Reisner 已提交
39 40 41 42 43 44 45 46 47 48 49 50
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/proc_fs.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/slab.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
51
#include <linux/workqueue.h>
P
Philipp Reisner 已提交
52 53 54
#define __KERNEL_SYSCALLS__
#include <linux/unistd.h>
#include <linux/vmalloc.h>
55
#include <linux/sched/signal.h>
P
Philipp Reisner 已提交
56 57 58

#include <linux/drbd_limits.h>
#include "drbd_int.h"
59
#include "drbd_protocol.h"
P
Philipp Reisner 已提交
60 61
#include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
#include "drbd_vli.h"
62
#include "drbd_debugfs.h"
P
Philipp Reisner 已提交
63

64
static DEFINE_MUTEX(drbd_main_mutex);
P
Philipp Reisner 已提交
65
static int drbd_open(struct block_device *bdev, fmode_t mode);
66
static void drbd_release(struct gendisk *gd, fmode_t mode);
P
Philipp Reisner 已提交
67
static void md_sync_timer_fn(unsigned long data);
68
static int w_bitmap_io(struct drbd_work *w, int unused);
P
Philipp Reisner 已提交
69 70 71 72 73 74

MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
	      "Lars Ellenberg <lars@linbit.com>");
MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
MODULE_VERSION(REL_VERSION);
MODULE_LICENSE("GPL");
75
MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
76
		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
P
Philipp Reisner 已提交
77 78 79 80
MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);

#include <linux/moduleparam.h>
/* thanks to these macros, if compiled into the kernel (not-module),
81
 * these become boot parameters (e.g., drbd.minor_count) */
P
Philipp Reisner 已提交
82 83

#ifdef CONFIG_DRBD_FAULT_INJECTION
84 85 86 87
int drbd_enable_faults;
int drbd_fault_rate;
static int drbd_fault_count;
static int drbd_fault_devs;
P
Philipp Reisner 已提交
88
/* bitmap of enabled faults */
89
module_param_named(enable_faults, drbd_enable_faults, int, 0664);
P
Philipp Reisner 已提交
90
/* fault rate % value - applies to all enabled faults */
91
module_param_named(fault_rate, drbd_fault_rate, int, 0664);
P
Philipp Reisner 已提交
92
/* count of faults inserted */
93
module_param_named(fault_count, drbd_fault_count, int, 0664);
P
Philipp Reisner 已提交
94
/* bitmap of devices to insert faults on */
95
module_param_named(fault_devs, drbd_fault_devs, int, 0644);
P
Philipp Reisner 已提交
96 97
#endif

98 99 100 101 102 103 104 105 106 107 108 109
/* module parameters we can keep static */
static bool drbd_allow_oos; /* allow_open_on_secondary */
static bool drbd_disable_sendpage;
MODULE_PARM_DESC(allow_oos, "DONT USE!");
module_param_named(allow_oos, drbd_allow_oos, bool, 0);
module_param_named(disable_sendpage, drbd_disable_sendpage, bool, 0644);

/* module parameters we share */
int drbd_proc_details; /* Detail level in proc drbd*/
module_param_named(proc_details, drbd_proc_details, int, 0644);
/* module parameters shared with defaults */
unsigned int drbd_minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
110 111
/* Module parameter for setting the user mode helper program
 * to run. Default is /sbin/drbdadm */
112
char drbd_usermode_helper[80] = "/sbin/drbdadm";
113
module_param_named(minor_count, drbd_minor_count, uint, 0444);
114
module_param_string(usermode_helper, drbd_usermode_helper, sizeof(drbd_usermode_helper), 0644);
P
Philipp Reisner 已提交
115 116 117 118

/* in 2.6.x, our device mapping and config info contains our virtual gendisks
 * as member "struct gendisk *vdisk;"
 */
119
struct idr drbd_devices;
120
struct list_head drbd_resources;
121
struct mutex resources_mutex;
P
Philipp Reisner 已提交
122 123

struct kmem_cache *drbd_request_cache;
A
Andreas Gruenbacher 已提交
124
struct kmem_cache *drbd_ee_cache;	/* peer requests */
P
Philipp Reisner 已提交
125 126 127 128
struct kmem_cache *drbd_bm_ext_cache;	/* bitmap extents */
struct kmem_cache *drbd_al_ext_cache;	/* activity log extents */
mempool_t *drbd_request_mempool;
mempool_t *drbd_ee_mempool;
129
mempool_t *drbd_md_io_page_pool;
130
struct bio_set *drbd_md_io_bio_set;
131
struct bio_set *drbd_io_bio_set;
P
Philipp Reisner 已提交
132 133 134 135 136 137 138 139 140 141 142 143 144 145

/* I do not use a standard mempool, because:
   1) I want to hand out the pre-allocated objects first.
   2) I want to be able to interrupt sleeping allocation with a signal.
   Note: This is a single linked list, the next pointer is the private
	 member of struct page.
 */
struct page *drbd_pp_pool;
spinlock_t   drbd_pp_lock;
int          drbd_pp_vacant;
wait_queue_head_t drbd_pp_wait;

DEFINE_RATELIMIT_STATE(drbd_ratelimit_state, 5 * HZ, 5);

146
static const struct block_device_operations drbd_ops = {
P
Philipp Reisner 已提交
147 148 149 150
	.owner =   THIS_MODULE,
	.open =    drbd_open,
	.release = drbd_release,
};
151

152 153 154
struct bio *bio_alloc_drbd(gfp_t gfp_mask)
{
	struct bio *bio;
155

156 157
	if (!drbd_md_io_bio_set)
		return bio_alloc(gfp_mask, 1);
158

159 160 161 162
	bio = bio_alloc_bioset(gfp_mask, 1, drbd_md_io_bio_set);
	if (!bio)
		return NULL;
	return bio;
163 164
}

P
Philipp Reisner 已提交
165 166 167 168
#ifdef __CHECKER__
/* When checking with sparse, and this is an inline function, sparse will
   give tons of false positives. When this is a real functions sparse works.
 */
169
int _get_ldev_if_state(struct drbd_device *device, enum drbd_disk_state mins)
P
Philipp Reisner 已提交
170 171 172
{
	int io_allowed;

173 174
	atomic_inc(&device->local_cnt);
	io_allowed = (device->state.disk >= mins);
P
Philipp Reisner 已提交
175
	if (!io_allowed) {
176 177
		if (atomic_dec_and_test(&device->local_cnt))
			wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
178 179 180 181 182
	}
	return io_allowed;
}

#endif
183

P
Philipp Reisner 已提交
184
/**
185
 * tl_release() - mark as BARRIER_ACKED all requests in the corresponding transfer log epoch
186
 * @connection:	DRBD connection.
P
Philipp Reisner 已提交
187 188 189 190
 * @barrier_nr:	Expected identifier of the DRBD write barrier packet.
 * @set_size:	Expected number of requests before that barrier.
 *
 * In case the passed barrier_nr or set_size does not match the oldest
191 192
 * epoch of not yet barrier-acked requests, this function will cause a
 * termination of the connection.
P
Philipp Reisner 已提交
193
 */
194
void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
195
		unsigned int set_size)
P
Philipp Reisner 已提交
196 197
{
	struct drbd_request *r;
198 199 200
	struct drbd_request *req = NULL;
	int expect_epoch = 0;
	int expect_size = 0;
P
Philipp Reisner 已提交
201

202
	spin_lock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
203

204
	/* find oldest not yet barrier-acked write request,
205
	 * count writes in its epoch. */
206
	list_for_each_entry(r, &connection->transfer_log, tl_requests) {
207
		const unsigned s = r->rq_state;
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
		if (!req) {
			if (!(s & RQ_WRITE))
				continue;
			if (!(s & RQ_NET_MASK))
				continue;
			if (s & RQ_NET_DONE)
				continue;
			req = r;
			expect_epoch = req->epoch;
			expect_size ++;
		} else {
			if (r->epoch != expect_epoch)
				break;
			if (!(s & RQ_WRITE))
				continue;
			/* if (s & RQ_DONE): not expected */
			/* if (!(s & RQ_NET_MASK)): not expected */
			expect_size++;
226
		}
P
Philipp Reisner 已提交
227
	}
228

P
Philipp Reisner 已提交
229
	/* first some paranoia code */
230
	if (req == NULL) {
231
		drbd_err(connection, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
232
			 barrier_nr);
P
Philipp Reisner 已提交
233 234
		goto bail;
	}
235
	if (expect_epoch != barrier_nr) {
236
		drbd_err(connection, "BAD! BarrierAck #%u received, expected #%u!\n",
237
			 barrier_nr, expect_epoch);
P
Philipp Reisner 已提交
238
		goto bail;
239 240
	}

241
	if (expect_size != set_size) {
242
		drbd_err(connection, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
243
			 barrier_nr, set_size, expect_size);
P
Philipp Reisner 已提交
244 245 246
		goto bail;
	}

247 248 249 250
	/* Clean up list of requests processed during current epoch. */
	/* this extra list walk restart is paranoia,
	 * to catch requests being barrier-acked "unexpectedly".
	 * It usually should find the same req again, or some READ preceding it. */
251
	list_for_each_entry(req, &connection->transfer_log, tl_requests)
252 253
		if (req->epoch == expect_epoch)
			break;
254
	list_for_each_entry_safe_from(req, r, &connection->transfer_log, tl_requests) {
255 256 257
		if (req->epoch != expect_epoch)
			break;
		_req_mod(req, BARRIER_ACKED);
258
	}
259
	spin_unlock_irq(&connection->resource->req_lock);
260

P
Philipp Reisner 已提交
261 262 263
	return;

bail:
264
	spin_unlock_irq(&connection->resource->req_lock);
265
	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
P
Philipp Reisner 已提交
266 267 268 269
}


/**
270
 * _tl_restart() - Walks the transfer log, and applies an action to all requests
271
 * @connection:	DRBD connection to operate on.
272
 * @what:       The action/event to perform with all request objects
P
Philipp Reisner 已提交
273
 *
274 275
 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
 * RESTART_FROZEN_DISK_IO.
P
Philipp Reisner 已提交
276
 */
277
/* must hold resource->req_lock */
278
void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
P
Philipp Reisner 已提交
279
{
280
	struct drbd_request *req, *r;
P
Philipp Reisner 已提交
281

282
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests)
283 284
		_req_mod(req, what);
}
285

286
void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
287
{
288
	spin_lock_irq(&connection->resource->req_lock);
289
	_tl_restart(connection, what);
290
	spin_unlock_irq(&connection->resource->req_lock);
291
}
P
Philipp Reisner 已提交
292 293 294

/**
 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
295
 * @device:	DRBD device.
P
Philipp Reisner 已提交
296 297 298 299 300
 *
 * This is called after the connection to the peer was lost. The storage covered
 * by the requests on the transfer gets marked as our of sync. Called from the
 * receiver thread and the worker thread.
 */
301
void tl_clear(struct drbd_connection *connection)
P
Philipp Reisner 已提交
302
{
303
	tl_restart(connection, CONNECTION_LOST_WHILE_PENDING);
P
Philipp Reisner 已提交
304
}
305

306
/**
307 308
 * tl_abort_disk_io() - Abort disk I/O for all requests for a certain device in the TL
 * @device:	DRBD device.
309
 */
310
void tl_abort_disk_io(struct drbd_device *device)
311
{
312
	struct drbd_connection *connection = first_peer_device(device)->connection;
313
	struct drbd_request *req, *r;
314

315
	spin_lock_irq(&connection->resource->req_lock);
316
	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
317 318
		if (!(req->rq_state & RQ_LOCAL_PENDING))
			continue;
319
		if (req->device != device)
320 321
			continue;
		_req_mod(req, ABORT_DISK_IO);
P
Philipp Reisner 已提交
322
	}
323
	spin_unlock_irq(&connection->resource->req_lock);
P
Philipp Reisner 已提交
324 325 326 327 328
}

static int drbd_thread_setup(void *arg)
{
	struct drbd_thread *thi = (struct drbd_thread *) arg;
329
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
330 331 332
	unsigned long flags;
	int retval;

333
	snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
334
		 thi->name[0],
335
		 resource->name);
336

P
Philipp Reisner 已提交
337 338 339 340 341
restart:
	retval = thi->function(thi);

	spin_lock_irqsave(&thi->t_lock, flags);

342
	/* if the receiver has been "EXITING", the last thing it did
P
Philipp Reisner 已提交
343 344 345
	 * was set the conn state to "StandAlone",
	 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
	 * and receiver thread will be "started".
346
	 * drbd_thread_start needs to set "RESTARTING" in that case.
P
Philipp Reisner 已提交
347
	 * t_state check and assignment needs to be within the same spinlock,
348 349
	 * so either thread_start sees EXITING, and can remap to RESTARTING,
	 * or thread_start see NONE, and can proceed as normal.
P
Philipp Reisner 已提交
350 351
	 */

352
	if (thi->t_state == RESTARTING) {
353
		drbd_info(resource, "Restarting %s thread\n", thi->name);
354
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
355 356 357 358 359
		spin_unlock_irqrestore(&thi->t_lock, flags);
		goto restart;
	}

	thi->task = NULL;
360
	thi->t_state = NONE;
P
Philipp Reisner 已提交
361
	smp_mb();
L
Lars Ellenberg 已提交
362
	complete_all(&thi->stop);
P
Philipp Reisner 已提交
363 364
	spin_unlock_irqrestore(&thi->t_lock, flags);

365
	drbd_info(resource, "Terminating %s\n", current->comm);
P
Philipp Reisner 已提交
366 367

	/* Release mod reference taken when thread was started */
368

369 370 371
	if (thi->connection)
		kref_put(&thi->connection->kref, drbd_destroy_connection);
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
372 373 374 375
	module_put(THIS_MODULE);
	return retval;
}

376
static void drbd_thread_init(struct drbd_resource *resource, struct drbd_thread *thi,
377
			     int (*func) (struct drbd_thread *), const char *name)
P
Philipp Reisner 已提交
378 379 380
{
	spin_lock_init(&thi->t_lock);
	thi->task    = NULL;
381
	thi->t_state = NONE;
P
Philipp Reisner 已提交
382
	thi->function = func;
383 384
	thi->resource = resource;
	thi->connection = NULL;
385
	thi->name = name;
P
Philipp Reisner 已提交
386 387 388 389
}

int drbd_thread_start(struct drbd_thread *thi)
{
390
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
391 392 393 394 395 396 397 398
	struct task_struct *nt;
	unsigned long flags;

	/* is used from state engine doing drbd_thread_stop_nowait,
	 * while holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

	switch (thi->t_state) {
399
	case NONE:
400
		drbd_info(resource, "Starting %s thread (from %s [%d])\n",
401
			 thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
402 403 404

		/* Get ref on module for thread - this is released when thread exits */
		if (!try_module_get(THIS_MODULE)) {
405
			drbd_err(resource, "Failed to get module reference in drbd_thread_start\n");
P
Philipp Reisner 已提交
406
			spin_unlock_irqrestore(&thi->t_lock, flags);
407
			return false;
P
Philipp Reisner 已提交
408 409
		}

410 411 412
		kref_get(&resource->kref);
		if (thi->connection)
			kref_get(&thi->connection->kref);
413

P
Philipp Reisner 已提交
414 415
		init_completion(&thi->stop);
		thi->reset_cpu_mask = 1;
416
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
417 418 419 420
		spin_unlock_irqrestore(&thi->t_lock, flags);
		flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

		nt = kthread_create(drbd_thread_setup, (void *) thi,
421
				    "drbd_%c_%s", thi->name[0], thi->resource->name);
P
Philipp Reisner 已提交
422 423

		if (IS_ERR(nt)) {
424
			drbd_err(resource, "Couldn't start thread\n");
P
Philipp Reisner 已提交
425

426 427 428
			if (thi->connection)
				kref_put(&thi->connection->kref, drbd_destroy_connection);
			kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
429
			module_put(THIS_MODULE);
430
			return false;
P
Philipp Reisner 已提交
431 432 433
		}
		spin_lock_irqsave(&thi->t_lock, flags);
		thi->task = nt;
434
		thi->t_state = RUNNING;
P
Philipp Reisner 已提交
435 436 437
		spin_unlock_irqrestore(&thi->t_lock, flags);
		wake_up_process(nt);
		break;
438 439
	case EXITING:
		thi->t_state = RESTARTING;
440
		drbd_info(resource, "Restarting %s thread (from %s [%d])\n",
441
				thi->name, current->comm, current->pid);
P
Philipp Reisner 已提交
442
		/* fall through */
443 444
	case RUNNING:
	case RESTARTING:
P
Philipp Reisner 已提交
445 446 447 448 449
	default:
		spin_unlock_irqrestore(&thi->t_lock, flags);
		break;
	}

450
	return true;
P
Philipp Reisner 已提交
451 452 453 454 455 456 457
}


void _drbd_thread_stop(struct drbd_thread *thi, int restart, int wait)
{
	unsigned long flags;

458
	enum drbd_thread_state ns = restart ? RESTARTING : EXITING;
P
Philipp Reisner 已提交
459 460 461 462

	/* may be called from state engine, holding the req lock irqsave */
	spin_lock_irqsave(&thi->t_lock, flags);

463
	if (thi->t_state == NONE) {
P
Philipp Reisner 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
		spin_unlock_irqrestore(&thi->t_lock, flags);
		if (restart)
			drbd_thread_start(thi);
		return;
	}

	if (thi->t_state != ns) {
		if (thi->task == NULL) {
			spin_unlock_irqrestore(&thi->t_lock, flags);
			return;
		}

		thi->t_state = ns;
		smp_mb();
		init_completion(&thi->stop);
		if (thi->task != current)
			force_sig(DRBD_SIGKILL, thi->task);
	}

	spin_unlock_irqrestore(&thi->t_lock, flags);

	if (wait)
		wait_for_completion(&thi->stop);
}

489
int conn_lowest_minor(struct drbd_connection *connection)
490
{
491 492
	struct drbd_peer_device *peer_device;
	int vnr = 0, minor = -1;
493

494
	rcu_read_lock();
495 496 497
	peer_device = idr_get_next(&connection->peer_devices, &vnr);
	if (peer_device)
		minor = device_to_minor(peer_device->device);
498 499
	rcu_read_unlock();

500
	return minor;
501
}
502

P
Philipp Reisner 已提交
503 504 505 506
#ifdef CONFIG_SMP
/**
 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
 *
507
 * Forces all threads of a resource onto the same CPU. This is beneficial for
P
Philipp Reisner 已提交
508 509
 * DRBD's performance. May be overwritten by user's configuration.
 */
510
static void drbd_calc_cpu_mask(cpumask_var_t *cpu_mask)
P
Philipp Reisner 已提交
511
{
512
	unsigned int *resources_per_cpu, min_index = ~0;
P
Philipp Reisner 已提交
513

514 515 516 517
	resources_per_cpu = kzalloc(nr_cpu_ids * sizeof(*resources_per_cpu), GFP_KERNEL);
	if (resources_per_cpu) {
		struct drbd_resource *resource;
		unsigned int cpu, min = ~0;
P
Philipp Reisner 已提交
518

519 520 521 522
		rcu_read_lock();
		for_each_resource_rcu(resource, &drbd_resources) {
			for_each_cpu(cpu, resource->cpu_mask)
				resources_per_cpu[cpu]++;
P
Philipp Reisner 已提交
523
		}
524 525 526 527 528 529 530 531 532 533 534 535
		rcu_read_unlock();
		for_each_online_cpu(cpu) {
			if (resources_per_cpu[cpu] < min) {
				min = resources_per_cpu[cpu];
				min_index = cpu;
			}
		}
		kfree(resources_per_cpu);
	}
	if (min_index == ~0) {
		cpumask_setall(*cpu_mask);
		return;
P
Philipp Reisner 已提交
536
	}
537
	cpumask_set_cpu(min_index, *cpu_mask);
P
Philipp Reisner 已提交
538 539 540 541
}

/**
 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
542
 * @device:	DRBD device.
543
 * @thi:	drbd_thread object
P
Philipp Reisner 已提交
544 545 546 547
 *
 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
 * prematurely.
 */
548
void drbd_thread_current_set_cpu(struct drbd_thread *thi)
P
Philipp Reisner 已提交
549
{
550
	struct drbd_resource *resource = thi->resource;
P
Philipp Reisner 已提交
551
	struct task_struct *p = current;
552

P
Philipp Reisner 已提交
553 554 555
	if (!thi->reset_cpu_mask)
		return;
	thi->reset_cpu_mask = 0;
556
	set_cpus_allowed_ptr(p, resource->cpu_mask);
P
Philipp Reisner 已提交
557
}
558 559
#else
#define drbd_calc_cpu_mask(A) ({})
P
Philipp Reisner 已提交
560 561
#endif

562 563 564 565 566 567 568
/**
 * drbd_header_size  -  size of a packet header
 *
 * The header size is a multiple of 8, so any payload following the header is
 * word aligned on 64-bit architectures.  (The bitmap send and receive code
 * relies on this.)
 */
569
unsigned int drbd_header_size(struct drbd_connection *connection)
P
Philipp Reisner 已提交
570
{
571
	if (connection->agreed_pro_version >= 100) {
572 573 574 575 576 577 578 579
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100), 8));
		return sizeof(struct p_header100);
	} else {
		BUILD_BUG_ON(sizeof(struct p_header80) !=
			     sizeof(struct p_header95));
		BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80), 8));
		return sizeof(struct p_header80);
	}
580
}
P
Philipp Reisner 已提交
581

582
static unsigned int prepare_header80(struct p_header80 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
583 584 585 586
{
	h->magic   = cpu_to_be32(DRBD_MAGIC);
	h->command = cpu_to_be16(cmd);
	h->length  = cpu_to_be16(size);
587
	return sizeof(struct p_header80);
P
Philipp Reisner 已提交
588
}
P
Philipp Reisner 已提交
589

590
static unsigned int prepare_header95(struct p_header95 *h, enum drbd_packet cmd, int size)
P
Philipp Reisner 已提交
591 592
{
	h->magic   = cpu_to_be16(DRBD_MAGIC_BIG);
P
Philipp Reisner 已提交
593
	h->command = cpu_to_be16(cmd);
594
	h->length = cpu_to_be32(size);
595
	return sizeof(struct p_header95);
P
Philipp Reisner 已提交
596
}
P
Philipp Reisner 已提交
597

598 599 600 601 602 603 604 605 606 607
static unsigned int prepare_header100(struct p_header100 *h, enum drbd_packet cmd,
				      int size, int vnr)
{
	h->magic = cpu_to_be32(DRBD_MAGIC_100);
	h->volume = cpu_to_be16(vnr);
	h->command = cpu_to_be16(cmd);
	h->length = cpu_to_be32(size);
	h->pad = 0;
	return sizeof(struct p_header100);
}
P
Philipp Reisner 已提交
608

609
static unsigned int prepare_header(struct drbd_connection *connection, int vnr,
610
				   void *buffer, enum drbd_packet cmd, int size)
611
{
612
	if (connection->agreed_pro_version >= 100)
613
		return prepare_header100(buffer, cmd, size, vnr);
614
	else if (connection->agreed_pro_version >= 95 &&
615
		 size > DRBD_MAX_SIZE_H80_PACKET)
616
		return prepare_header95(buffer, cmd, size);
617
	else
618
		return prepare_header80(buffer, cmd, size);
P
Philipp Reisner 已提交
619 620
}

621
static void *__conn_prepare_command(struct drbd_connection *connection,
622
				    struct drbd_socket *sock)
P
Philipp Reisner 已提交
623
{
624 625
	if (!sock->socket)
		return NULL;
626
	return sock->sbuf + drbd_header_size(connection);
627
}
P
Philipp Reisner 已提交
628

629
void *conn_prepare_command(struct drbd_connection *connection, struct drbd_socket *sock)
630
{
631
	void *p;
P
Philipp Reisner 已提交
632

633
	mutex_lock(&sock->mutex);
634
	p = __conn_prepare_command(connection, sock);
635
	if (!p)
636
		mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
637

638
	return p;
P
Philipp Reisner 已提交
639 640
}

641
void *drbd_prepare_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock)
P
Philipp Reisner 已提交
642
{
643
	return conn_prepare_command(peer_device->connection, sock);
644
}
P
Philipp Reisner 已提交
645

646
static int __send_command(struct drbd_connection *connection, int vnr,
647 648 649 650 651 652
			  struct drbd_socket *sock, enum drbd_packet cmd,
			  unsigned int header_size, void *data,
			  unsigned int size)
{
	int msg_flags;
	int err;
P
Philipp Reisner 已提交
653

654 655 656 657 658 659 660 661 662
	/*
	 * Called with @data == NULL and the size of the data blocks in @size
	 * for commands that send data blocks.  For those commands, omit the
	 * MSG_MORE flag: this will increase the likelihood that data blocks
	 * which are page aligned on the sender will end up page aligned on the
	 * receiver.
	 */
	msg_flags = data ? MSG_MORE : 0;

663
	header_size += prepare_header(connection, vnr, sock->sbuf, cmd,
664
				      header_size + size);
665
	err = drbd_send_all(connection, sock->socket, sock->sbuf, header_size,
666 667
			    msg_flags);
	if (data && !err)
668
		err = drbd_send_all(connection, sock->socket, data, size, 0);
669 670 671 672 673
	/* DRBD protocol "pings" are latency critical.
	 * This is supposed to trigger tcp_push_pending_frames() */
	if (!err && (cmd == P_PING || cmd == P_PING_ACK))
		drbd_tcp_nodelay(sock->socket);

674 675 676
	return err;
}

677
static int __conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
678 679 680
			       enum drbd_packet cmd, unsigned int header_size,
			       void *data, unsigned int size)
{
681
	return __send_command(connection, 0, sock, cmd, header_size, data, size);
682 683
}

684
int conn_send_command(struct drbd_connection *connection, struct drbd_socket *sock,
685 686 687 688
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;
P
Philipp Reisner 已提交
689

690
	err = __conn_send_command(connection, sock, cmd, header_size, data, size);
691 692 693 694
	mutex_unlock(&sock->mutex);
	return err;
}

695
int drbd_send_command(struct drbd_peer_device *peer_device, struct drbd_socket *sock,
696 697 698 699 700
		      enum drbd_packet cmd, unsigned int header_size,
		      void *data, unsigned int size)
{
	int err;

701 702
	err = __send_command(peer_device->connection, peer_device->device->vnr,
			     sock, cmd, header_size, data, size);
703 704 705
	mutex_unlock(&sock->mutex);
	return err;
}
P
Philipp Reisner 已提交
706

707
int drbd_send_ping(struct drbd_connection *connection)
708
{
709 710
	struct drbd_socket *sock;

711 712
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
713
		return -EIO;
714
	return conn_send_command(connection, sock, P_PING, 0, NULL, 0);
715
}
P
Philipp Reisner 已提交
716

717
int drbd_send_ping_ack(struct drbd_connection *connection)
718
{
719 720
	struct drbd_socket *sock;

721 722
	sock = &connection->meta;
	if (!conn_prepare_command(connection, sock))
723
		return -EIO;
724
	return conn_send_command(connection, sock, P_PING_ACK, 0, NULL, 0);
P
Philipp Reisner 已提交
725 726
}

727
int drbd_send_sync_param(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
728
{
729
	struct drbd_socket *sock;
730
	struct p_rs_param_95 *p;
731
	int size;
732
	const int apv = peer_device->connection->agreed_pro_version;
733
	enum drbd_packet cmd;
734
	struct net_conf *nc;
P
Philipp Reisner 已提交
735
	struct disk_conf *dc;
736

737 738
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
739 740
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
741

742
	rcu_read_lock();
743
	nc = rcu_dereference(peer_device->connection->net_conf);
P
Philipp Reisner 已提交
744 745 746

	size = apv <= 87 ? sizeof(struct p_rs_param)
		: apv == 88 ? sizeof(struct p_rs_param)
747
			+ strlen(nc->verify_alg) + 1
748 749
		: apv <= 94 ? sizeof(struct p_rs_param_89)
		: /* apv >= 95 */ sizeof(struct p_rs_param_95);
P
Philipp Reisner 已提交
750

751
	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
P
Philipp Reisner 已提交
752

753 754
	/* initialize verify_alg and csums_alg */
	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
P
Philipp Reisner 已提交
755

756 757
	if (get_ldev(peer_device->device)) {
		dc = rcu_dereference(peer_device->device->ldev->disk_conf);
758
		p->resync_rate = cpu_to_be32(dc->resync_rate);
P
Philipp Reisner 已提交
759 760 761 762
		p->c_plan_ahead = cpu_to_be32(dc->c_plan_ahead);
		p->c_delay_target = cpu_to_be32(dc->c_delay_target);
		p->c_fill_target = cpu_to_be32(dc->c_fill_target);
		p->c_max_rate = cpu_to_be32(dc->c_max_rate);
763
		put_ldev(peer_device->device);
764
	} else {
765
		p->resync_rate = cpu_to_be32(DRBD_RESYNC_RATE_DEF);
766 767 768 769 770
		p->c_plan_ahead = cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF);
		p->c_delay_target = cpu_to_be32(DRBD_C_DELAY_TARGET_DEF);
		p->c_fill_target = cpu_to_be32(DRBD_C_FILL_TARGET_DEF);
		p->c_max_rate = cpu_to_be32(DRBD_C_MAX_RATE_DEF);
	}
P
Philipp Reisner 已提交
771

772
	if (apv >= 88)
773
		strcpy(p->verify_alg, nc->verify_alg);
774
	if (apv >= 89)
775 776
		strcpy(p->csums_alg, nc->csums_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
777

778
	return drbd_send_command(peer_device, sock, cmd, size, NULL, 0);
P
Philipp Reisner 已提交
779 780
}

781
int __drbd_send_protocol(struct drbd_connection *connection, enum drbd_packet cmd)
P
Philipp Reisner 已提交
782
{
783
	struct drbd_socket *sock;
P
Philipp Reisner 已提交
784
	struct p_protocol *p;
785
	struct net_conf *nc;
786
	int size, cf;
P
Philipp Reisner 已提交
787

788 789
	sock = &connection->data;
	p = __conn_prepare_command(connection, sock);
790 791
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
792

793
	rcu_read_lock();
794
	nc = rcu_dereference(connection->net_conf);
P
Philipp Reisner 已提交
795

796
	if (nc->tentative && connection->agreed_pro_version < 92) {
797 798
		rcu_read_unlock();
		mutex_unlock(&sock->mutex);
799
		drbd_err(connection, "--dry-run is not supported by peer");
800 801
		return -EOPNOTSUPP;
	}
P
Philipp Reisner 已提交
802

803
	size = sizeof(*p);
804
	if (connection->agreed_pro_version >= 87)
805
		size += strlen(nc->integrity_alg) + 1;
P
Philipp Reisner 已提交
806

807 808 809 810 811
	p->protocol      = cpu_to_be32(nc->wire_protocol);
	p->after_sb_0p   = cpu_to_be32(nc->after_sb_0p);
	p->after_sb_1p   = cpu_to_be32(nc->after_sb_1p);
	p->after_sb_2p   = cpu_to_be32(nc->after_sb_2p);
	p->two_primaries = cpu_to_be32(nc->two_primaries);
812
	cf = 0;
813 814
	if (nc->discard_my_data)
		cf |= CF_DISCARD_MY_DATA;
815
	if (nc->tentative)
816
		cf |= CF_DRY_RUN;
817 818
	p->conn_flags    = cpu_to_be32(cf);

819
	if (connection->agreed_pro_version >= 87)
820 821
		strcpy(p->integrity_alg, nc->integrity_alg);
	rcu_read_unlock();
P
Philipp Reisner 已提交
822

823
	return __conn_send_command(connection, sock, cmd, size, NULL, 0);
824 825
}

826
int drbd_send_protocol(struct drbd_connection *connection)
827 828 829
{
	int err;

830 831 832
	mutex_lock(&connection->data.mutex);
	err = __drbd_send_protocol(connection, P_PROTOCOL);
	mutex_unlock(&connection->data.mutex);
833 834

	return err;
P
Philipp Reisner 已提交
835 836
}

837
static int _drbd_send_uuids(struct drbd_peer_device *peer_device, u64 uuid_flags)
P
Philipp Reisner 已提交
838
{
839
	struct drbd_device *device = peer_device->device;
840 841
	struct drbd_socket *sock;
	struct p_uuids *p;
P
Philipp Reisner 已提交
842 843
	int i;

844
	if (!get_ldev_if_state(device, D_NEGOTIATING))
845
		return 0;
P
Philipp Reisner 已提交
846

847 848
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
849
	if (!p) {
850
		put_ldev(device);
851 852
		return -EIO;
	}
853
	spin_lock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
854
	for (i = UI_CURRENT; i < UI_SIZE; i++)
855 856
		p->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
857

858 859
	device->comm_bm_set = drbd_bm_total_weight(device);
	p->uuid[UI_SIZE] = cpu_to_be64(device->comm_bm_set);
860
	rcu_read_lock();
861
	uuid_flags |= rcu_dereference(peer_device->connection->net_conf)->discard_my_data ? 1 : 0;
862
	rcu_read_unlock();
863 864
	uuid_flags |= test_bit(CRASHED_PRIMARY, &device->flags) ? 2 : 0;
	uuid_flags |= device->new_state_tmp.disk == D_INCONSISTENT ? 4 : 0;
865
	p->uuid[UI_FLAGS] = cpu_to_be64(uuid_flags);
P
Philipp Reisner 已提交
866

867
	put_ldev(device);
868
	return drbd_send_command(peer_device, sock, P_UUIDS, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
869 870
}

871
int drbd_send_uuids(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
872
{
873
	return _drbd_send_uuids(peer_device, 0);
P
Philipp Reisner 已提交
874 875
}

876
int drbd_send_uuids_skip_initial_sync(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
877
{
878
	return _drbd_send_uuids(peer_device, 8);
P
Philipp Reisner 已提交
879 880
}

881
void drbd_print_uuids(struct drbd_device *device, const char *text)
882
{
883 884
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
		u64 *uuid = device->ldev->md.uuid;
885
		drbd_info(device, "%s %016llX:%016llX:%016llX:%016llX\n",
886 887 888 889 890
		     text,
		     (unsigned long long)uuid[UI_CURRENT],
		     (unsigned long long)uuid[UI_BITMAP],
		     (unsigned long long)uuid[UI_HISTORY_START],
		     (unsigned long long)uuid[UI_HISTORY_END]);
891
		put_ldev(device);
892
	} else {
893
		drbd_info(device, "%s effective data uuid: %016llX\n",
894
				text,
895
				(unsigned long long)device->ed_uuid);
896 897 898
	}
}

899
void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
900
{
901
	struct drbd_device *device = peer_device->device;
902 903
	struct drbd_socket *sock;
	struct p_rs_uuid *p;
904 905
	u64 uuid;

906
	D_ASSERT(device, device->state.disk == D_UP_TO_DATE);
P
Philipp Reisner 已提交
907

908
	uuid = device->ldev->md.uuid[UI_BITMAP];
909 910 911 912
	if (uuid && uuid != UUID_JUST_CREATED)
		uuid = uuid + UUID_NEW_BM_OFFSET;
	else
		get_random_bytes(&uuid, sizeof(u64));
913 914 915
	drbd_uuid_set(device, UI_BITMAP, uuid);
	drbd_print_uuids(device, "updated sync UUID");
	drbd_md_sync(device);
P
Philipp Reisner 已提交
916

917 918
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
919 920
	if (p) {
		p->uuid = cpu_to_be64(uuid);
921
		drbd_send_command(peer_device, sock, P_SYNC_UUID, sizeof(*p), NULL, 0);
922
	}
P
Philipp Reisner 已提交
923 924
}

925
/* communicated if (agreed_features & DRBD_FF_WSAME) */
926 927 928
static void
assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p,
					struct request_queue *q)
929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
{
	if (q) {
		p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
		p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
		p->qlim->alignment_offset = cpu_to_be32(queue_alignment_offset(q));
		p->qlim->io_min = cpu_to_be32(queue_io_min(q));
		p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
		p->qlim->discard_enabled = blk_queue_discard(q);
		p->qlim->write_same_capable = !!q->limits.max_write_same_sectors;
	} else {
		q = device->rq_queue;
		p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
		p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q));
		p->qlim->alignment_offset = 0;
		p->qlim->io_min = cpu_to_be32(queue_io_min(q));
		p->qlim->io_opt = cpu_to_be32(queue_io_opt(q));
		p->qlim->discard_enabled = 0;
		p->qlim->write_same_capable = 0;
	}
}

950
int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags)
P
Philipp Reisner 已提交
951
{
952
	struct drbd_device *device = peer_device->device;
953 954
	struct drbd_socket *sock;
	struct p_sizes *p;
P
Philipp Reisner 已提交
955
	sector_t d_size, u_size;
956 957
	int q_order_type;
	unsigned int max_bio_size;
958 959 960 961 962 963
	unsigned int packet_size;

	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	if (!p)
		return -EIO;
P
Philipp Reisner 已提交
964

965 966 967 968 969
	packet_size = sizeof(*p);
	if (peer_device->connection->agreed_features & DRBD_FF_WSAME)
		packet_size += sizeof(p->qlim[0]);

	memset(p, 0, packet_size);
970
	if (get_ldev_if_state(device, D_NEGOTIATING)) {
971
		struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev);
972
		d_size = drbd_get_max_capacity(device->ldev);
P
Philipp Reisner 已提交
973
		rcu_read_lock();
974
		u_size = rcu_dereference(device->ldev->disk_conf)->disk_size;
P
Philipp Reisner 已提交
975
		rcu_read_unlock();
976
		q_order_type = drbd_queue_order_type(device);
977
		max_bio_size = queue_max_hw_sectors(q) << 9;
978
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE);
979
		assign_p_sizes_qlim(device, p, q);
980
		put_ldev(device);
P
Philipp Reisner 已提交
981 982 983 984
	} else {
		d_size = 0;
		u_size = 0;
		q_order_type = QUEUE_ORDERED_NONE;
985
		max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */
986
		assign_p_sizes_qlim(device, p, NULL);
P
Philipp Reisner 已提交
987 988
	}

989
	if (peer_device->connection->agreed_pro_version <= 94)
990
		max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET);
991
	else if (peer_device->connection->agreed_pro_version < 100)
992
		max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE_P95);
P
Philipp Reisner 已提交
993

994 995
	p->d_size = cpu_to_be64(d_size);
	p->u_size = cpu_to_be64(u_size);
996
	p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
997 998 999
	p->max_bio_size = cpu_to_be32(max_bio_size);
	p->queue_order_type = cpu_to_be16(q_order_type);
	p->dds_flags = cpu_to_be16(flags);
1000 1001

	return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0);
P
Philipp Reisner 已提交
1002 1003 1004
}

/**
1005
 * drbd_send_current_state() - Sends the drbd state to the peer
1006
 * @peer_device:	DRBD peer device.
P
Philipp Reisner 已提交
1007
 */
1008
int drbd_send_current_state(struct drbd_peer_device *peer_device)
P
Philipp Reisner 已提交
1009
{
1010
	struct drbd_socket *sock;
1011
	struct p_state *p;
P
Philipp Reisner 已提交
1012

1013 1014
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1015 1016
	if (!p)
		return -EIO;
1017 1018
	p->state = cpu_to_be32(peer_device->device->state.i); /* Within the send mutex */
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1019 1020
}

1021 1022
/**
 * drbd_send_state() - After a state change, sends the new state to the peer
1023
 * @peer_device:      DRBD peer device.
1024
 * @state:     the state to send, not necessarily the current state.
1025 1026 1027 1028 1029 1030
 *
 * Each state change queues an "after_state_ch" work, which will eventually
 * send the resulting new state to the peer. If more state changes happen
 * between queuing and processing of the after_state_ch work, we still
 * want to send each intermediary state in the order it occurred.
 */
1031
int drbd_send_state(struct drbd_peer_device *peer_device, union drbd_state state)
1032
{
1033 1034
	struct drbd_socket *sock;
	struct p_state *p;
1035

1036 1037
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1038 1039 1040
	if (!p)
		return -EIO;
	p->state = cpu_to_be32(state.i); /* Within the send mutex */
1041
	return drbd_send_command(peer_device, sock, P_STATE, sizeof(*p), NULL, 0);
1042
}
1043

1044
int drbd_send_state_req(struct drbd_peer_device *peer_device, union drbd_state mask, union drbd_state val)
1045 1046 1047
{
	struct drbd_socket *sock;
	struct p_req_state *p;
1048

1049 1050
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1051 1052 1053 1054
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1055
	return drbd_send_command(peer_device, sock, P_STATE_CHG_REQ, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1056
}
1057

1058
int conn_send_state_req(struct drbd_connection *connection, union drbd_state mask, union drbd_state val)
P
Philipp Reisner 已提交
1059
{
1060 1061 1062
	enum drbd_packet cmd;
	struct drbd_socket *sock;
	struct p_req_state *p;
1063

1064 1065 1066
	cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
	sock = &connection->data;
	p = conn_prepare_command(connection, sock);
1067 1068 1069 1070
	if (!p)
		return -EIO;
	p->mask = cpu_to_be32(mask.i);
	p->val = cpu_to_be32(val.i);
1071
	return conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1072 1073
}

1074
void drbd_send_sr_reply(struct drbd_peer_device *peer_device, enum drbd_state_rv retcode)
P
Philipp Reisner 已提交
1075
{
1076 1077
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
P
Philipp Reisner 已提交
1078

1079 1080
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1081 1082
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1083
		drbd_send_command(peer_device, sock, P_STATE_CHG_REPLY, sizeof(*p), NULL, 0);
1084
	}
P
Philipp Reisner 已提交
1085 1086
}

1087
void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode)
1088
{
1089 1090
	struct drbd_socket *sock;
	struct p_req_state_reply *p;
1091
	enum drbd_packet cmd = connection->agreed_pro_version < 100 ? P_STATE_CHG_REPLY : P_CONN_ST_CHG_REPLY;
P
Philipp Reisner 已提交
1092

1093 1094
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1095 1096
	if (p) {
		p->retcode = cpu_to_be32(retcode);
1097
		conn_send_command(connection, sock, cmd, sizeof(*p), NULL, 0);
1098
	}
P
Philipp Reisner 已提交
1099 1100
}

1101
static void dcbp_set_code(struct p_compressed_bm *p, enum drbd_bitmap_code code)
P
Philipp Reisner 已提交
1102
{
1103 1104 1105
	BUG_ON(code & ~0xf);
	p->encoding = (p->encoding & ~0xf) | code;
}
P
Philipp Reisner 已提交
1106

1107 1108 1109 1110
static void dcbp_set_start(struct p_compressed_bm *p, int set)
{
	p->encoding = (p->encoding & ~0x80) | (set ? 0x80 : 0);
}
P
Philipp Reisner 已提交
1111

1112 1113 1114 1115
static void dcbp_set_pad_bits(struct p_compressed_bm *p, int n)
{
	BUG_ON(n & ~0x7);
	p->encoding = (p->encoding & (~0x7 << 4)) | (n << 4);
P
Philipp Reisner 已提交
1116 1117
}

1118
static int fill_bitmap_rle_bits(struct drbd_device *device,
1119 1120 1121
			 struct p_compressed_bm *p,
			 unsigned int size,
			 struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1122 1123 1124 1125 1126 1127 1128
{
	struct bitstream bs;
	unsigned long plain_bits;
	unsigned long tmp;
	unsigned long rl;
	unsigned len;
	unsigned toggle;
1129
	int bits, use_rle;
P
Philipp Reisner 已提交
1130 1131

	/* may we use this feature? */
1132
	rcu_read_lock();
1133
	use_rle = rcu_dereference(first_peer_device(device)->connection->net_conf)->use_rle;
1134
	rcu_read_unlock();
1135
	if (!use_rle || first_peer_device(device)->connection->agreed_pro_version < 90)
1136
		return 0;
P
Philipp Reisner 已提交
1137 1138 1139 1140 1141

	if (c->bit_offset >= c->bm_bits)
		return 0; /* nothing to do. */

	/* use at most thus many bytes */
1142 1143
	bitstream_init(&bs, p->code, size, 0);
	memset(p->code, 0, size);
P
Philipp Reisner 已提交
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
	/* plain bits covered in this code string */
	plain_bits = 0;

	/* p->encoding & 0x80 stores whether the first run length is set.
	 * bit offset is implicit.
	 * start with toggle == 2 to be able to tell the first iteration */
	toggle = 2;

	/* see how much plain bits we can stuff into one packet
	 * using RLE and VLI. */
	do {
1155 1156
		tmp = (toggle == 0) ? _drbd_bm_find_next_zero(device, c->bit_offset)
				    : _drbd_bm_find_next(device, c->bit_offset);
P
Philipp Reisner 已提交
1157 1158 1159 1160 1161 1162 1163 1164
		if (tmp == -1UL)
			tmp = c->bm_bits;
		rl = tmp - c->bit_offset;

		if (toggle == 2) { /* first iteration */
			if (rl == 0) {
				/* the first checked bit was set,
				 * store start value, */
1165
				dcbp_set_start(p, 1);
P
Philipp Reisner 已提交
1166 1167 1168 1169
				/* but skip encoding of zero run length */
				toggle = !toggle;
				continue;
			}
1170
			dcbp_set_start(p, 0);
P
Philipp Reisner 已提交
1171 1172 1173 1174 1175
		}

		/* paranoia: catch zero runlength.
		 * can only happen if bitmap is modified while we scan it. */
		if (rl == 0) {
1176
			drbd_err(device, "unexpected zero runlength while encoding bitmap "
P
Philipp Reisner 已提交
1177 1178 1179 1180 1181 1182 1183 1184
			    "t:%u bo:%lu\n", toggle, c->bit_offset);
			return -1;
		}

		bits = vli_encode_bits(&bs, rl);
		if (bits == -ENOBUFS) /* buffer full */
			break;
		if (bits <= 0) {
1185
			drbd_err(device, "error while encoding bitmap: %d\n", bits);
P
Philipp Reisner 已提交
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
			return 0;
		}

		toggle = !toggle;
		plain_bits += rl;
		c->bit_offset = tmp;
	} while (c->bit_offset < c->bm_bits);

	len = bs.cur.b - p->code + !!bs.cur.bit;

	if (plain_bits < (len << 3)) {
		/* incompressible with this method.
		 * we need to rewind both word and bit position. */
		c->bit_offset -= plain_bits;
		bm_xfer_ctx_bit_to_word_offset(c);
		c->bit_offset = c->word_offset * BITS_PER_LONG;
		return 0;
	}

	/* RLE + VLI was able to compress it just fine.
	 * update c->word_offset. */
	bm_xfer_ctx_bit_to_word_offset(c);

	/* store pad_bits */
1210
	dcbp_set_pad_bits(p, (8 - bs.cur.bit) & 0x7);
P
Philipp Reisner 已提交
1211 1212 1213 1214

	return len;
}

1215 1216 1217 1218 1219 1220 1221
/**
 * send_bitmap_rle_or_plain
 *
 * Return 0 when done, 1 when another iteration is needed, and a negative error
 * code upon failure.
 */
static int
1222
send_bitmap_rle_or_plain(struct drbd_device *device, struct bm_xfer_ctx *c)
P
Philipp Reisner 已提交
1223
{
1224 1225
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
	unsigned int header_size = drbd_header_size(first_peer_device(device)->connection);
1226
	struct p_compressed_bm *p = sock->sbuf + header_size;
1227
	int len, err;
P
Philipp Reisner 已提交
1228

1229
	len = fill_bitmap_rle_bits(device, p,
1230
			DRBD_SOCKET_BUFFER_SIZE - header_size - sizeof(*p), c);
P
Philipp Reisner 已提交
1231
	if (len < 0)
1232
		return -EIO;
P
Philipp Reisner 已提交
1233 1234

	if (len) {
1235
		dcbp_set_code(p, RLE_VLI_Bits);
1236
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock,
1237 1238
				     P_COMPRESSED_BITMAP, sizeof(*p) + len,
				     NULL, 0);
P
Philipp Reisner 已提交
1239
		c->packets[0]++;
1240
		c->bytes[0] += header_size + sizeof(*p) + len;
P
Philipp Reisner 已提交
1241 1242 1243 1244 1245 1246

		if (c->bit_offset >= c->bm_bits)
			len = 0; /* DONE */
	} else {
		/* was not compressible.
		 * send a buffer full of plain text bits instead. */
1247 1248
		unsigned int data_size;
		unsigned long num_words;
1249
		unsigned long *p = sock->sbuf + header_size;
1250 1251

		data_size = DRBD_SOCKET_BUFFER_SIZE - header_size;
1252
		num_words = min_t(size_t, data_size / sizeof(*p),
1253
				  c->bm_words - c->word_offset);
1254
		len = num_words * sizeof(*p);
P
Philipp Reisner 已提交
1255
		if (len)
1256
			drbd_bm_get_lel(device, c->word_offset, num_words, p);
1257
		err = __send_command(first_peer_device(device)->connection, device->vnr, sock, P_BITMAP, len, NULL, 0);
P
Philipp Reisner 已提交
1258 1259 1260 1261
		c->word_offset += num_words;
		c->bit_offset = c->word_offset * BITS_PER_LONG;

		c->packets[1]++;
1262
		c->bytes[1] += header_size + len;
P
Philipp Reisner 已提交
1263 1264 1265 1266

		if (c->bit_offset > c->bm_bits)
			c->bit_offset = c->bm_bits;
	}
1267
	if (!err) {
1268
		if (len == 0) {
1269
			INFO_bm_xfer_stats(device, "send", c);
1270 1271 1272 1273 1274
			return 0;
		} else
			return 1;
	}
	return -EIO;
P
Philipp Reisner 已提交
1275 1276 1277
}

/* See the comment at receive_bitmap() */
1278
static int _drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1279 1280
{
	struct bm_xfer_ctx c;
1281
	int err;
P
Philipp Reisner 已提交
1282

1283
	if (!expect(device->bitmap))
1284
		return false;
P
Philipp Reisner 已提交
1285

1286 1287
	if (get_ldev(device)) {
		if (drbd_md_test_flag(device->ldev, MDF_FULL_SYNC)) {
1288
			drbd_info(device, "Writing the whole bitmap, MDF_FullSync was set.\n");
1289 1290
			drbd_bm_set_all(device);
			if (drbd_bm_write(device)) {
P
Philipp Reisner 已提交
1291 1292 1293
				/* write_bm did fail! Leave full sync flag set in Meta P_DATA
				 * but otherwise process as per normal - need to tell other
				 * side that a full resync is required! */
1294
				drbd_err(device, "Failed to write bitmap to disk!\n");
P
Philipp Reisner 已提交
1295
			} else {
1296 1297
				drbd_md_clear_flag(device, MDF_FULL_SYNC);
				drbd_md_sync(device);
P
Philipp Reisner 已提交
1298 1299
			}
		}
1300
		put_ldev(device);
P
Philipp Reisner 已提交
1301 1302 1303
	}

	c = (struct bm_xfer_ctx) {
1304 1305
		.bm_bits = drbd_bm_bits(device),
		.bm_words = drbd_bm_words(device),
P
Philipp Reisner 已提交
1306 1307 1308
	};

	do {
1309
		err = send_bitmap_rle_or_plain(device, &c);
1310
	} while (err > 0);
P
Philipp Reisner 已提交
1311

1312
	return err == 0;
P
Philipp Reisner 已提交
1313 1314
}

1315
int drbd_send_bitmap(struct drbd_device *device)
P
Philipp Reisner 已提交
1316
{
1317
	struct drbd_socket *sock = &first_peer_device(device)->connection->data;
1318
	int err = -1;
P
Philipp Reisner 已提交
1319

1320 1321
	mutex_lock(&sock->mutex);
	if (sock->socket)
1322
		err = !_drbd_send_bitmap(device);
1323
	mutex_unlock(&sock->mutex);
P
Philipp Reisner 已提交
1324 1325 1326
	return err;
}

1327
void drbd_send_b_ack(struct drbd_connection *connection, u32 barrier_nr, u32 set_size)
P
Philipp Reisner 已提交
1328
{
1329 1330
	struct drbd_socket *sock;
	struct p_barrier_ack *p;
P
Philipp Reisner 已提交
1331

1332
	if (connection->cstate < C_WF_REPORT_PARAMS)
1333
		return;
P
Philipp Reisner 已提交
1334

1335 1336
	sock = &connection->meta;
	p = conn_prepare_command(connection, sock);
1337 1338 1339 1340
	if (!p)
		return;
	p->barrier = barrier_nr;
	p->set_size = cpu_to_be32(set_size);
1341
	conn_send_command(connection, sock, P_BARRIER_ACK, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1342 1343 1344 1345
}

/**
 * _drbd_send_ack() - Sends an ack packet
1346
 * @device:	DRBD device.
P
Philipp Reisner 已提交
1347 1348 1349 1350 1351
 * @cmd:	Packet command code.
 * @sector:	sector, needs to be in big endian byte order
 * @blksize:	size in byte, needs to be in big endian byte order
 * @block_id:	Id, big endian byte order
 */
1352
static int _drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1353
			  u64 sector, u32 blksize, u64 block_id)
P
Philipp Reisner 已提交
1354
{
1355 1356
	struct drbd_socket *sock;
	struct p_block_ack *p;
P
Philipp Reisner 已提交
1357

1358
	if (peer_device->device->state.conn < C_CONNECTED)
1359
		return -EIO;
P
Philipp Reisner 已提交
1360

1361 1362
	sock = &peer_device->connection->meta;
	p = drbd_prepare_command(peer_device, sock);
1363
	if (!p)
1364
		return -EIO;
1365 1366 1367
	p->sector = sector;
	p->block_id = block_id;
	p->blksize = blksize;
1368 1369
	p->seq_num = cpu_to_be32(atomic_inc_return(&peer_device->device->packet_seq));
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1370 1371
}

1372 1373 1374
/* dp->sector and dp->block_id already/still in network byte order,
 * data_size is payload size according to dp->head,
 * and may need to be corrected for digest size. */
1375
void drbd_send_ack_dp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1376
		      struct p_data *dp, int data_size)
P
Philipp Reisner 已提交
1377
{
1378
	if (peer_device->connection->peer_integrity_tfm)
H
Herbert Xu 已提交
1379
		data_size -= crypto_ahash_digestsize(peer_device->connection->peer_integrity_tfm);
1380
	_drbd_send_ack(peer_device, cmd, dp->sector, cpu_to_be32(data_size),
1381
		       dp->block_id);
P
Philipp Reisner 已提交
1382 1383
}

1384
void drbd_send_ack_rp(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1385
		      struct p_block_req *rp)
P
Philipp Reisner 已提交
1386
{
1387
	_drbd_send_ack(peer_device, cmd, rp->sector, rp->blksize, rp->block_id);
P
Philipp Reisner 已提交
1388 1389 1390 1391
}

/**
 * drbd_send_ack() - Sends an ack packet
1392
 * @device:	DRBD device
1393 1394
 * @cmd:	packet command code
 * @peer_req:	peer request
P
Philipp Reisner 已提交
1395
 */
1396
int drbd_send_ack(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1397
		  struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1398
{
1399
	return _drbd_send_ack(peer_device, cmd,
1400 1401 1402
			      cpu_to_be64(peer_req->i.sector),
			      cpu_to_be32(peer_req->i.size),
			      peer_req->block_id);
P
Philipp Reisner 已提交
1403 1404 1405 1406
}

/* This function misuses the block_id field to signal if the blocks
 * are is sync or not. */
1407
int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
P
Philipp Reisner 已提交
1408 1409
		     sector_t sector, int blksize, u64 block_id)
{
1410
	return _drbd_send_ack(peer_device, cmd,
P
Philipp Reisner 已提交
1411 1412 1413 1414 1415
			      cpu_to_be64(sector),
			      cpu_to_be32(blksize),
			      cpu_to_be64(block_id));
}

1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device,
			     struct drbd_peer_request *peer_req)
{
	struct drbd_socket *sock;
	struct p_block_desc *p;

	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->blksize = cpu_to_be32(peer_req->i.size);
	p->pad = 0;
	return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0);
}

1432
int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd,
P
Philipp Reisner 已提交
1433 1434
		       sector_t sector, int size, u64 block_id)
{
1435 1436
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1437

1438 1439
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1440 1441 1442 1443 1444
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = block_id;
	p->blksize = cpu_to_be32(size);
1445
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1446 1447
}

1448
int drbd_send_drequest_csum(struct drbd_peer_device *peer_device, sector_t sector, int size,
1449
			    void *digest, int digest_size, enum drbd_packet cmd)
P
Philipp Reisner 已提交
1450
{
1451 1452
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1453

1454
	/* FIXME: Put the digest into the preallocated socket buffer.  */
P
Philipp Reisner 已提交
1455

1456 1457
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1458 1459 1460 1461 1462
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1463
	return drbd_send_command(peer_device, sock, cmd, sizeof(*p), digest, digest_size);
P
Philipp Reisner 已提交
1464 1465
}

1466
int drbd_send_ov_request(struct drbd_peer_device *peer_device, sector_t sector, int size)
P
Philipp Reisner 已提交
1467
{
1468 1469
	struct drbd_socket *sock;
	struct p_block_req *p;
P
Philipp Reisner 已提交
1470

1471 1472
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1473 1474 1475 1476 1477
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(sector);
	p->block_id = ID_SYNCER /* unused */;
	p->blksize = cpu_to_be32(size);
1478
	return drbd_send_command(peer_device, sock, P_OV_REQUEST, sizeof(*p), NULL, 0);
P
Philipp Reisner 已提交
1479 1480 1481
}

/* called on sndtimeo
1482 1483
 * returns false if we should retry,
 * true if we think connection is dead
P
Philipp Reisner 已提交
1484
 */
1485
static int we_should_drop_the_connection(struct drbd_connection *connection, struct socket *sock)
P
Philipp Reisner 已提交
1486 1487
{
	int drop_it;
1488
	/* long elapsed = (long)(jiffies - device->last_received); */
P
Philipp Reisner 已提交
1489

1490
	drop_it =   connection->meta.socket == sock
1491 1492
		|| !connection->ack_receiver.task
		|| get_t_state(&connection->ack_receiver) != RUNNING
1493
		|| connection->cstate < C_WF_REPORT_PARAMS;
P
Philipp Reisner 已提交
1494 1495

	if (drop_it)
1496
		return true;
P
Philipp Reisner 已提交
1497

1498
	drop_it = !--connection->ko_count;
P
Philipp Reisner 已提交
1499
	if (!drop_it) {
1500
		drbd_err(connection, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1501 1502
			 current->comm, current->pid, connection->ko_count);
		request_ping(connection);
P
Philipp Reisner 已提交
1503 1504
	}

1505
	return drop_it; /* && (device->state == R_PRIMARY) */;
P
Philipp Reisner 已提交
1506 1507
}

1508
static void drbd_update_congested(struct drbd_connection *connection)
1509
{
1510
	struct sock *sk = connection->data.socket->sk;
1511
	if (sk->sk_wmem_queued > sk->sk_sndbuf * 4 / 5)
1512
		set_bit(NET_CONGESTED, &connection->flags);
1513 1514
}

P
Philipp Reisner 已提交
1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
/* The idea of sendpage seems to be to put some kind of reference
 * to the page into the skb, and to hand it over to the NIC. In
 * this process get_page() gets called.
 *
 * As soon as the page was really sent over the network put_page()
 * gets called by some part of the network layer. [ NIC driver? ]
 *
 * [ get_page() / put_page() increment/decrement the count. If count
 *   reaches 0 the page will be freed. ]
 *
 * This works nicely with pages from FSs.
 * But this means that in protocol A we might signal IO completion too early!
 *
 * In order not to corrupt data during a resync we must make sure
 * that we do not reuse our own buffer pages (EEs) to early, therefore
 * we have the net_ee list.
 *
 * XFS seems to have problems, still, it submits pages with page_count == 0!
 * As a workaround, we disable sendpage on pages
 * with page_count == 0 or PageSlab.
 */
1536
static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page *page,
1537
			      int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1538
{
1539 1540 1541 1542
	struct socket *socket;
	void *addr;
	int err;

1543
	socket = peer_device->connection->data.socket;
1544
	addr = kmap(page) + offset;
1545
	err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags);
P
Philipp Reisner 已提交
1546
	kunmap(page);
1547
	if (!err)
1548
		peer_device->device->send_cnt += size >> 9;
1549
	return err;
P
Philipp Reisner 已提交
1550 1551
}

1552
static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *page,
L
Lars Ellenberg 已提交
1553
		    int offset, size_t size, unsigned msg_flags)
P
Philipp Reisner 已提交
1554
{
1555
	struct socket *socket = peer_device->connection->data.socket;
P
Philipp Reisner 已提交
1556
	int len = size;
1557
	int err = -EIO;
P
Philipp Reisner 已提交
1558 1559 1560 1561 1562 1563 1564

	/* e.g. XFS meta- & log-data is in slab pages, which have a
	 * page_count of 0 and/or have PageSlab() set.
	 * we cannot use send_page for those, as that does get_page();
	 * put_page(); and would cause either a VM_BUG directly, or
	 * __page_cache_release a page that would actually still be referenced
	 * by someone, leading to some obscure delayed Oops somewhere else. */
1565
	if (drbd_disable_sendpage || (page_count(page) < 1) || PageSlab(page))
1566
		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
P
Philipp Reisner 已提交
1567

L
Lars Ellenberg 已提交
1568
	msg_flags |= MSG_NOSIGNAL;
1569
	drbd_update_congested(peer_device->connection);
P
Philipp Reisner 已提交
1570
	do {
1571 1572 1573
		int sent;

		sent = socket->ops->sendpage(socket, page, offset, len, msg_flags);
P
Philipp Reisner 已提交
1574
		if (sent <= 0) {
1575
			if (sent == -EAGAIN) {
1576
				if (we_should_drop_the_connection(peer_device->connection, socket))
1577 1578 1579
					break;
				continue;
			}
1580
			drbd_warn(peer_device->device, "%s: size=%d len=%d sent=%d\n",
P
Philipp Reisner 已提交
1581
			     __func__, (int)size, len, sent);
1582 1583
			if (sent < 0)
				err = sent;
P
Philipp Reisner 已提交
1584 1585 1586 1587
			break;
		}
		len    -= sent;
		offset += sent;
1588
	} while (len > 0 /* THINK && device->cstate >= C_CONNECTED*/);
1589
	clear_bit(NET_CONGESTED, &peer_device->connection->flags);
P
Philipp Reisner 已提交
1590

1591 1592
	if (len == 0) {
		err = 0;
1593
		peer_device->device->send_cnt += size >> 9;
1594 1595
	}
	return err;
P
Philipp Reisner 已提交
1596 1597
}

1598
static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1599
{
1600 1601 1602
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1603
	/* hint all but last page with MSG_MORE */
1604
	bio_for_each_segment(bvec, bio, iter) {
1605 1606
		int err;

1607
		err = _drbd_no_send_page(peer_device, bvec.bv_page,
1608
					 bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1609
					 bio_iter_last(bvec, iter)
1610
					 ? 0 : MSG_MORE);
1611 1612
		if (err)
			return err;
1613 1614 1615
		/* REQ_OP_WRITE_SAME has only one segment */
		if (bio_op(bio) == REQ_OP_WRITE_SAME)
			break;
P
Philipp Reisner 已提交
1616
	}
1617
	return 0;
P
Philipp Reisner 已提交
1618 1619
}

1620
static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *bio)
P
Philipp Reisner 已提交
1621
{
1622 1623 1624
	struct bio_vec bvec;
	struct bvec_iter iter;

L
Lars Ellenberg 已提交
1625
	/* hint all but last page with MSG_MORE */
1626
	bio_for_each_segment(bvec, bio, iter) {
1627 1628
		int err;

1629
		err = _drbd_send_page(peer_device, bvec.bv_page,
1630
				      bvec.bv_offset, bvec.bv_len,
K
Kent Overstreet 已提交
1631
				      bio_iter_last(bvec, iter) ? 0 : MSG_MORE);
1632 1633
		if (err)
			return err;
1634 1635 1636
		/* REQ_OP_WRITE_SAME has only one segment */
		if (bio_op(bio) == REQ_OP_WRITE_SAME)
			break;
P
Philipp Reisner 已提交
1637
	}
1638
	return 0;
P
Philipp Reisner 已提交
1639 1640
}

1641
static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device,
1642
			    struct drbd_peer_request *peer_req)
1643
{
1644 1645
	struct page *page = peer_req->pages;
	unsigned len = peer_req->i.size;
1646
	int err;
1647

L
Lars Ellenberg 已提交
1648
	/* hint all but last page with MSG_MORE */
1649 1650
	page_chain_for_each(page) {
		unsigned l = min_t(unsigned, len, PAGE_SIZE);
1651

1652
		err = _drbd_send_page(peer_device, page, 0, l,
1653 1654 1655
				      page_chain_next(page) ? MSG_MORE : 0);
		if (err)
			return err;
1656 1657
		len -= l;
	}
1658
	return 0;
1659 1660
}

M
Mike Christie 已提交
1661 1662
static u32 bio_flags_to_wire(struct drbd_connection *connection,
			     struct bio *bio)
1663
{
1664
	if (connection->agreed_pro_version >= 95)
J
Jens Axboe 已提交
1665 1666 1667
		return  (bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0) |
			(bio->bi_opf & REQ_FUA ? DP_FUA : 0) |
			(bio->bi_opf & REQ_PREFLUSH ? DP_FLUSH : 0) |
1668
			(bio_op(bio) == REQ_OP_WRITE_SAME ? DP_WSAME : 0) |
1669 1670
			(bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0) |
			(bio_op(bio) == REQ_OP_WRITE_ZEROES ? DP_DISCARD : 0);
1671
	else
J
Jens Axboe 已提交
1672
		return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0;
1673 1674
}

1675 1676
/* Used to send write or TRIM aka REQ_DISCARD requests
 * R_PRIMARY -> Peer	(P_DATA, P_TRIM)
P
Philipp Reisner 已提交
1677
 */
1678
int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request *req)
P
Philipp Reisner 已提交
1679
{
1680
	struct drbd_device *device = peer_device->device;
1681 1682
	struct drbd_socket *sock;
	struct p_data *p;
1683 1684
	struct p_wsame *wsame = NULL;
	void *digest_out;
P
Philipp Reisner 已提交
1685
	unsigned int dp_flags = 0;
1686
	int digest_size;
1687
	int err;
P
Philipp Reisner 已提交
1688

1689 1690
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1691
	digest_size = peer_device->connection->integrity_tfm ?
H
Herbert Xu 已提交
1692
		      crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1693

1694 1695 1696 1697
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->block_id = (unsigned long)req;
1698
	p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq));
M
Mike Christie 已提交
1699
	dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio);
1700 1701
	if (device->state.conn >= C_SYNC_SOURCE &&
	    device->state.conn <= C_PAUSED_SYNC_T)
P
Philipp Reisner 已提交
1702
		dp_flags |= DP_MAY_SET_IN_SYNC;
1703
	if (peer_device->connection->agreed_pro_version >= 100) {
1704 1705
		if (req->rq_state & RQ_EXP_RECEIVE_ACK)
			dp_flags |= DP_SEND_RECEIVE_ACK;
1706 1707 1708 1709
		/* During resync, request an explicit write ack,
		 * even in protocol != C */
		if (req->rq_state & RQ_EXP_WRITE_ACK
		|| (dp_flags & DP_MAY_SET_IN_SYNC))
1710 1711
			dp_flags |= DP_SEND_WRITE_ACK;
	}
1712
	p->dp_flags = cpu_to_be32(dp_flags);
1713 1714 1715 1716 1717 1718 1719

	if (dp_flags & DP_DISCARD) {
		struct p_trim *t = (struct p_trim*)p;
		t->size = cpu_to_be32(req->i.size);
		err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0);
		goto out;
	}
1720 1721 1722 1723 1724 1725 1726 1727 1728
	if (dp_flags & DP_WSAME) {
		/* this will only work if DRBD_FF_WSAME is set AND the
		 * handshake agreed that all nodes and backend devices are
		 * WRITE_SAME capable and agree on logical_block_size */
		wsame = (struct p_wsame*)p;
		digest_out = wsame + 1;
		wsame->size = cpu_to_be32(req->i.size);
	} else
		digest_out = p + 1;
1729 1730 1731

	/* our digest is still only over the payload.
	 * TRIM does not carry any payload. */
1732
	if (digest_size)
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742
		drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out);
	if (wsame) {
		err =
		    __send_command(peer_device->connection, device->vnr, sock, P_WSAME,
				   sizeof(*wsame) + digest_size, NULL,
				   bio_iovec(req->master_bio).bv_len);
	} else
		err =
		    __send_command(peer_device->connection, device->vnr, sock, P_DATA,
				   sizeof(*p) + digest_size, NULL, req->i.size);
1743
	if (!err) {
1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
		/* For protocol A, we have to memcpy the payload into
		 * socket buffers, as we may complete right away
		 * as soon as we handed it over to tcp, at which point the data
		 * pages may become invalid.
		 *
		 * For data-integrity enabled, we copy it as well, so we can be
		 * sure that even if the bio pages may still be modified, it
		 * won't change the data on the wire, thus if the digest checks
		 * out ok after sending on this side, but does not fit on the
		 * receiving side, we sure have detected corruption elsewhere.
		 */
1755
		if (!(req->rq_state & (RQ_EXP_RECEIVE_ACK | RQ_EXP_WRITE_ACK)) || digest_size)
1756
			err = _drbd_send_bio(peer_device, req->master_bio);
P
Philipp Reisner 已提交
1757
		else
1758
			err = _drbd_send_zc_bio(peer_device, req->master_bio);
1759 1760

		/* double check digest, sometimes buffers have been modified in flight. */
1761
		if (digest_size > 0 && digest_size <= 64) {
B
Bart Van Assche 已提交
1762
			/* 64 byte, 512 bit, is the largest digest size
1763 1764
			 * currently supported in kernel crypto. */
			unsigned char digest[64];
1765
			drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest);
1766
			if (memcmp(p + 1, digest, digest_size)) {
1767
				drbd_warn(device,
1768
					"Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1769
					(unsigned long long)req->i.sector, req->i.size);
1770
			}
1771
		} /* else if (digest_size > 64) {
1772 1773
		     ... Be noisy about digest too large ...
		} */
P
Philipp Reisner 已提交
1774
	}
1775
out:
1776
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
P
Philipp Reisner 已提交
1777

1778
	return err;
P
Philipp Reisner 已提交
1779 1780 1781 1782 1783 1784
}

/* answer packet, used to send data back for read requests:
 *  Peer       -> (diskless) R_PRIMARY   (P_DATA_REPLY)
 *  C_SYNC_SOURCE -> C_SYNC_TARGET         (P_RS_DATA_REPLY)
 */
1785
int drbd_send_block(struct drbd_peer_device *peer_device, enum drbd_packet cmd,
1786
		    struct drbd_peer_request *peer_req)
P
Philipp Reisner 已提交
1787
{
1788
	struct drbd_device *device = peer_device->device;
1789 1790
	struct drbd_socket *sock;
	struct p_data *p;
1791
	int err;
1792
	int digest_size;
P
Philipp Reisner 已提交
1793

1794 1795
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
P
Philipp Reisner 已提交
1796

1797
	digest_size = peer_device->connection->integrity_tfm ?
H
Herbert Xu 已提交
1798
		      crypto_ahash_digestsize(peer_device->connection->integrity_tfm) : 0;
P
Philipp Reisner 已提交
1799

1800 1801 1802 1803 1804
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(peer_req->i.sector);
	p->block_id = peer_req->block_id;
	p->seq_num = 0;  /* unused */
1805
	p->dp_flags = 0;
1806
	if (digest_size)
1807
		drbd_csum_ee(peer_device->connection->integrity_tfm, peer_req, p + 1);
1808
	err = __send_command(peer_device->connection, device->vnr, sock, cmd, sizeof(*p) + digest_size, NULL, peer_req->i.size);
1809
	if (!err)
1810
		err = _drbd_send_zc_ee(peer_device, peer_req);
1811
	mutex_unlock(&sock->mutex);  /* locked by drbd_prepare_command() */
1812

1813
	return err;
P
Philipp Reisner 已提交
1814 1815
}

1816
int drbd_send_out_of_sync(struct drbd_peer_device *peer_device, struct drbd_request *req)
1817
{
1818 1819
	struct drbd_socket *sock;
	struct p_block_desc *p;
1820

1821 1822
	sock = &peer_device->connection->data;
	p = drbd_prepare_command(peer_device, sock);
1823 1824 1825 1826
	if (!p)
		return -EIO;
	p->sector = cpu_to_be64(req->i.sector);
	p->blksize = cpu_to_be32(req->i.size);
1827
	return drbd_send_command(peer_device, sock, P_OUT_OF_SYNC, sizeof(*p), NULL, 0);
1828 1829
}

P
Philipp Reisner 已提交
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
/*
  drbd_send distinguishes two cases:

  Packets sent via the data socket "sock"
  and packets sent via the meta data socket "msock"

		    sock                      msock
  -----------------+-------------------------+------------------------------
  timeout           conf.timeout / 2          conf.timeout / 2
  timeout action    send a ping via msock     Abort communication
					      and close all sockets
*/

/*
 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
 */
1846
int drbd_send(struct drbd_connection *connection, struct socket *sock,
P
Philipp Reisner 已提交
1847 1848
	      void *buf, size_t size, unsigned msg_flags)
{
A
Al Viro 已提交
1849
	struct kvec iov = {.iov_base = buf, .iov_len = size};
P
Philipp Reisner 已提交
1850 1851 1852 1853
	struct msghdr msg;
	int rv, sent = 0;

	if (!sock)
1854
		return -EBADR;
P
Philipp Reisner 已提交
1855 1856 1857 1858 1859 1860 1861 1862 1863

	/* THINK  if (signal_pending) return ... ? */

	msg.msg_name       = NULL;
	msg.msg_namelen    = 0;
	msg.msg_control    = NULL;
	msg.msg_controllen = 0;
	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;

A
Al Viro 已提交
1864 1865
	iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, size);

1866
	if (sock == connection->data.socket) {
1867
		rcu_read_lock();
1868
		connection->ko_count = rcu_dereference(connection->net_conf)->ko_count;
1869
		rcu_read_unlock();
1870
		drbd_update_congested(connection);
P
Philipp Reisner 已提交
1871 1872
	}
	do {
A
Al Viro 已提交
1873
		rv = sock_sendmsg(sock, &msg);
P
Philipp Reisner 已提交
1874
		if (rv == -EAGAIN) {
1875
			if (we_should_drop_the_connection(connection, sock))
P
Philipp Reisner 已提交
1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
				break;
			else
				continue;
		}
		if (rv == -EINTR) {
			flush_signals(current);
			rv = 0;
		}
		if (rv < 0)
			break;
		sent += rv;
	} while (sent < size);

1889 1890
	if (sock == connection->data.socket)
		clear_bit(NET_CONGESTED, &connection->flags);
P
Philipp Reisner 已提交
1891 1892 1893

	if (rv <= 0) {
		if (rv != -EAGAIN) {
1894
			drbd_err(connection, "%s_sendmsg returned %d\n",
1895
				 sock == connection->meta.socket ? "msock" : "sock",
1896
				 rv);
1897
			conn_request_state(connection, NS(conn, C_BROKEN_PIPE), CS_HARD);
P
Philipp Reisner 已提交
1898
		} else
1899
			conn_request_state(connection, NS(conn, C_TIMEOUT), CS_HARD);
P
Philipp Reisner 已提交
1900 1901 1902 1903 1904
	}

	return sent;
}

1905 1906 1907 1908 1909
/**
 * drbd_send_all  -  Send an entire buffer
 *
 * Returns 0 upon success and a negative error value otherwise.
 */
1910
int drbd_send_all(struct drbd_connection *connection, struct socket *sock, void *buffer,
1911 1912 1913 1914
		  size_t size, unsigned msg_flags)
{
	int err;

1915
	err = drbd_send(connection, sock, buffer, size, msg_flags);
1916 1917 1918 1919 1920 1921 1922
	if (err < 0)
		return err;
	if (err != size)
		return -EIO;
	return 0;
}

P
Philipp Reisner 已提交
1923 1924
static int drbd_open(struct block_device *bdev, fmode_t mode)
{
1925
	struct drbd_device *device = bdev->bd_disk->private_data;
P
Philipp Reisner 已提交
1926 1927 1928
	unsigned long flags;
	int rv = 0;

1929
	mutex_lock(&drbd_main_mutex);
1930
	spin_lock_irqsave(&device->resource->req_lock, flags);
1931
	/* to have a stable device->state.role
P
Philipp Reisner 已提交
1932 1933
	 * and no race with updating open_cnt */

1934
	if (device->state.role != R_PRIMARY) {
P
Philipp Reisner 已提交
1935 1936
		if (mode & FMODE_WRITE)
			rv = -EROFS;
1937
		else if (!drbd_allow_oos)
P
Philipp Reisner 已提交
1938 1939 1940 1941
			rv = -EMEDIUMTYPE;
	}

	if (!rv)
1942
		device->open_cnt++;
1943
	spin_unlock_irqrestore(&device->resource->req_lock, flags);
1944
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1945 1946 1947 1948

	return rv;
}

1949
static void drbd_release(struct gendisk *gd, fmode_t mode)
P
Philipp Reisner 已提交
1950
{
1951
	struct drbd_device *device = gd->private_data;
1952
	mutex_lock(&drbd_main_mutex);
1953
	device->open_cnt--;
1954
	mutex_unlock(&drbd_main_mutex);
P
Philipp Reisner 已提交
1955 1956
}

1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
/* need to hold resource->req_lock */
void drbd_queue_unplug(struct drbd_device *device)
{
	if (device->state.pdsk >= D_INCONSISTENT && device->state.conn >= C_CONNECTED) {
		D_ASSERT(device, device->state.role == R_PRIMARY);
		if (test_and_clear_bit(UNPLUG_REMOTE, &device->flags)) {
			drbd_queue_work_if_unqueued(
				&first_peer_device(device)->connection->sender_work,
				&device->unplug_work);
		}
	}
}

1970
static void drbd_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1971
{
1972 1973
	/* Beware! The actual layout differs
	 * between big endian and little endian */
1974
	device->state = (union drbd_dev_state) {
P
Philipp Reisner 已提交
1975 1976 1977 1978 1979 1980 1981 1982
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = C_STANDALONE,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
}

1983
void drbd_init_set_defaults(struct drbd_device *device)
P
Philipp Reisner 已提交
1984 1985 1986 1987
{
	/* the memset(,0,) did most of this.
	 * note: only assignments, no allocation in here */

1988 1989 1990
	drbd_set_defaults(device);

	atomic_set(&device->ap_bio_cnt, 0);
1991
	atomic_set(&device->ap_actlog_cnt, 0);
1992 1993 1994 1995 1996 1997 1998 1999
	atomic_set(&device->ap_pending_cnt, 0);
	atomic_set(&device->rs_pending_cnt, 0);
	atomic_set(&device->unacked_cnt, 0);
	atomic_set(&device->local_cnt, 0);
	atomic_set(&device->pp_in_use_by_net, 0);
	atomic_set(&device->rs_sect_in, 0);
	atomic_set(&device->rs_sect_ev, 0);
	atomic_set(&device->ap_in_flight, 0);
2000
	atomic_set(&device->md_io.in_use, 0);
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016

	mutex_init(&device->own_state_mutex);
	device->state_mutex = &device->own_state_mutex;

	spin_lock_init(&device->al_lock);
	spin_lock_init(&device->peer_seq_lock);

	INIT_LIST_HEAD(&device->active_ee);
	INIT_LIST_HEAD(&device->sync_ee);
	INIT_LIST_HEAD(&device->done_ee);
	INIT_LIST_HEAD(&device->read_ee);
	INIT_LIST_HEAD(&device->net_ee);
	INIT_LIST_HEAD(&device->resync_reads);
	INIT_LIST_HEAD(&device->resync_work.list);
	INIT_LIST_HEAD(&device->unplug_work.list);
	INIT_LIST_HEAD(&device->bm_io_work.w.list);
2017 2018 2019 2020
	INIT_LIST_HEAD(&device->pending_master_completion[0]);
	INIT_LIST_HEAD(&device->pending_master_completion[1]);
	INIT_LIST_HEAD(&device->pending_completion[0]);
	INIT_LIST_HEAD(&device->pending_completion[1]);
2021 2022 2023 2024 2025

	device->resync_work.cb  = w_resync_timer;
	device->unplug_work.cb  = w_send_write_hint;
	device->bm_io_work.w.cb = w_bitmap_io;

2026 2027 2028 2029 2030 2031 2032 2033
	setup_timer(&device->resync_timer, resync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->md_sync_timer, md_sync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->start_resync_timer, start_resync_timer_fn,
			(unsigned long)device);
	setup_timer(&device->request_timer, request_timer_fn,
			(unsigned long)device);
2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046

	init_waitqueue_head(&device->misc_wait);
	init_waitqueue_head(&device->state_wait);
	init_waitqueue_head(&device->ee_wait);
	init_waitqueue_head(&device->al_wait);
	init_waitqueue_head(&device->seq_wait);

	device->resync_wenr = LC_FREE;
	device->peer_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
	device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}

void drbd_device_cleanup(struct drbd_device *device)
P
Philipp Reisner 已提交
2047
{
2048
	int i;
2049
	if (first_peer_device(device)->connection->receiver.t_state != NONE)
2050
		drbd_err(device, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
2051
				first_peer_device(device)->connection->receiver.t_state);
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064

	device->al_writ_cnt  =
	device->bm_writ_cnt  =
	device->read_cnt     =
	device->recv_cnt     =
	device->send_cnt     =
	device->writ_cnt     =
	device->p_size       =
	device->rs_start     =
	device->rs_total     =
	device->rs_failed    = 0;
	device->rs_last_events = 0;
	device->rs_last_sect_ev = 0;
2065
	for (i = 0; i < DRBD_SYNC_MARKS; i++) {
2066 2067
		device->rs_mark_left[i] = 0;
		device->rs_mark_time[i] = 0;
2068
	}
2069
	D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
P
Philipp Reisner 已提交
2070

2071 2072
	drbd_set_my_capacity(device, 0);
	if (device->bitmap) {
P
Philipp Reisner 已提交
2073
		/* maybe never allocated. */
2074 2075
		drbd_bm_resize(device, 0, 1);
		drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2076 2077
	}

2078
	drbd_backing_dev_free(device, device->ldev);
2079
	device->ldev = NULL;
2080

2081
	clear_bit(AL_SUSPENDED, &device->flags);
P
Philipp Reisner 已提交
2082

2083 2084 2085 2086 2087 2088 2089 2090 2091
	D_ASSERT(device, list_empty(&device->active_ee));
	D_ASSERT(device, list_empty(&device->sync_ee));
	D_ASSERT(device, list_empty(&device->done_ee));
	D_ASSERT(device, list_empty(&device->read_ee));
	D_ASSERT(device, list_empty(&device->net_ee));
	D_ASSERT(device, list_empty(&device->resync_reads));
	D_ASSERT(device, list_empty(&first_peer_device(device)->connection->sender_work.q));
	D_ASSERT(device, list_empty(&device->resync_work.list));
	D_ASSERT(device, list_empty(&device->unplug_work.list));
2092

2093
	drbd_set_defaults(device);
P
Philipp Reisner 已提交
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
}


static void drbd_destroy_mempools(void)
{
	struct page *page;

	while (drbd_pp_pool) {
		page = drbd_pp_pool;
		drbd_pp_pool = (struct page *)page_private(page);
		__free_page(page);
		drbd_pp_vacant--;
	}

2108
	/* D_ASSERT(device, atomic_read(&drbd_pp_vacant)==0); */
P
Philipp Reisner 已提交
2109

2110 2111
	if (drbd_io_bio_set)
		bioset_free(drbd_io_bio_set);
2112 2113
	if (drbd_md_io_bio_set)
		bioset_free(drbd_md_io_bio_set);
2114 2115
	if (drbd_md_io_page_pool)
		mempool_destroy(drbd_md_io_page_pool);
P
Philipp Reisner 已提交
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
	if (drbd_ee_mempool)
		mempool_destroy(drbd_ee_mempool);
	if (drbd_request_mempool)
		mempool_destroy(drbd_request_mempool);
	if (drbd_ee_cache)
		kmem_cache_destroy(drbd_ee_cache);
	if (drbd_request_cache)
		kmem_cache_destroy(drbd_request_cache);
	if (drbd_bm_ext_cache)
		kmem_cache_destroy(drbd_bm_ext_cache);
	if (drbd_al_ext_cache)
		kmem_cache_destroy(drbd_al_ext_cache);

2129
	drbd_io_bio_set      = NULL;
2130
	drbd_md_io_bio_set   = NULL;
2131
	drbd_md_io_page_pool = NULL;
P
Philipp Reisner 已提交
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
	drbd_ee_mempool      = NULL;
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;

	return;
}

static int drbd_create_mempools(void)
{
	struct page *page;
2145
	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * drbd_minor_count;
P
Philipp Reisner 已提交
2146 2147 2148 2149 2150 2151 2152 2153 2154
	int i;

	/* prepare our caches and mempools */
	drbd_request_mempool = NULL;
	drbd_ee_cache        = NULL;
	drbd_request_cache   = NULL;
	drbd_bm_ext_cache    = NULL;
	drbd_al_ext_cache    = NULL;
	drbd_pp_pool         = NULL;
2155
	drbd_md_io_page_pool = NULL;
2156
	drbd_md_io_bio_set   = NULL;
2157
	drbd_io_bio_set      = NULL;
P
Philipp Reisner 已提交
2158 2159 2160 2161 2162 2163 2164 2165

	/* caches */
	drbd_request_cache = kmem_cache_create(
		"drbd_req", sizeof(struct drbd_request), 0, 0, NULL);
	if (drbd_request_cache == NULL)
		goto Enomem;

	drbd_ee_cache = kmem_cache_create(
2166
		"drbd_ee", sizeof(struct drbd_peer_request), 0, 0, NULL);
P
Philipp Reisner 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
	if (drbd_ee_cache == NULL)
		goto Enomem;

	drbd_bm_ext_cache = kmem_cache_create(
		"drbd_bm", sizeof(struct bm_extent), 0, 0, NULL);
	if (drbd_bm_ext_cache == NULL)
		goto Enomem;

	drbd_al_ext_cache = kmem_cache_create(
		"drbd_al", sizeof(struct lc_element), 0, 0, NULL);
	if (drbd_al_ext_cache == NULL)
		goto Enomem;

	/* mempools */
2181
	drbd_io_bio_set = bioset_create(BIO_POOL_SIZE, 0, 0);
2182 2183 2184
	if (drbd_io_bio_set == NULL)
		goto Enomem;

2185
	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0,
2186
					   BIOSET_NEED_BVECS);
2187 2188 2189
	if (drbd_md_io_bio_set == NULL)
		goto Enomem;

2190 2191 2192 2193
	drbd_md_io_page_pool = mempool_create_page_pool(DRBD_MIN_POOL_PAGES, 0);
	if (drbd_md_io_page_pool == NULL)
		goto Enomem;

2194 2195
	drbd_request_mempool = mempool_create_slab_pool(number,
		drbd_request_cache);
P
Philipp Reisner 已提交
2196 2197 2198
	if (drbd_request_mempool == NULL)
		goto Enomem;

2199
	drbd_ee_mempool = mempool_create_slab_pool(number, drbd_ee_cache);
2200
	if (drbd_ee_mempool == NULL)
P
Philipp Reisner 已提交
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
		goto Enomem;

	/* drbd's page pool */
	spin_lock_init(&drbd_pp_lock);

	for (i = 0; i < number; i++) {
		page = alloc_page(GFP_HIGHUSER);
		if (!page)
			goto Enomem;
		set_page_private(page, (unsigned long)drbd_pp_pool);
		drbd_pp_pool = page;
	}
	drbd_pp_vacant = number;

	return 0;

Enomem:
	drbd_destroy_mempools(); /* in case we allocated some */
	return -ENOMEM;
}

2222
static void drbd_release_all_peer_reqs(struct drbd_device *device)
P
Philipp Reisner 已提交
2223 2224 2225
{
	int rr;

2226
	rr = drbd_free_peer_reqs(device, &device->active_ee);
P
Philipp Reisner 已提交
2227
	if (rr)
2228
		drbd_err(device, "%d EEs in active list found!\n", rr);
P
Philipp Reisner 已提交
2229

2230
	rr = drbd_free_peer_reqs(device, &device->sync_ee);
P
Philipp Reisner 已提交
2231
	if (rr)
2232
		drbd_err(device, "%d EEs in sync list found!\n", rr);
P
Philipp Reisner 已提交
2233

2234
	rr = drbd_free_peer_reqs(device, &device->read_ee);
P
Philipp Reisner 已提交
2235
	if (rr)
2236
		drbd_err(device, "%d EEs in read list found!\n", rr);
P
Philipp Reisner 已提交
2237

2238
	rr = drbd_free_peer_reqs(device, &device->done_ee);
P
Philipp Reisner 已提交
2239
	if (rr)
2240
		drbd_err(device, "%d EEs in done list found!\n", rr);
P
Philipp Reisner 已提交
2241

2242
	rr = drbd_free_peer_reqs(device, &device->net_ee);
P
Philipp Reisner 已提交
2243
	if (rr)
2244
		drbd_err(device, "%d EEs in net list found!\n", rr);
P
Philipp Reisner 已提交
2245 2246
}

2247
/* caution. no locking. */
2248
void drbd_destroy_device(struct kref *kref)
P
Philipp Reisner 已提交
2249
{
2250
	struct drbd_device *device = container_of(kref, struct drbd_device, kref);
2251
	struct drbd_resource *resource = device->resource;
2252
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2253

2254
	del_timer_sync(&device->request_timer);
2255

P
Philipp Reisner 已提交
2256
	/* paranoia asserts */
2257
	D_ASSERT(device, device->open_cnt == 0);
P
Philipp Reisner 已提交
2258 2259 2260 2261 2262
	/* end paranoia asserts */

	/* cleanup stuff that may have been allocated during
	 * device (re-)configuration or state changes */

2263 2264
	if (device->this_bdev)
		bdput(device->this_bdev);
P
Philipp Reisner 已提交
2265

2266
	drbd_backing_dev_free(device, device->ldev);
2267
	device->ldev = NULL;
P
Philipp Reisner 已提交
2268

2269
	drbd_release_all_peer_reqs(device);
P
Philipp Reisner 已提交
2270

2271 2272
	lc_destroy(device->act_log);
	lc_destroy(device->resync);
P
Philipp Reisner 已提交
2273

2274 2275
	kfree(device->p_uuid);
	/* device->p_uuid = NULL; */
P
Philipp Reisner 已提交
2276

2277 2278
	if (device->bitmap) /* should no longer be there. */
		drbd_bm_cleanup(device);
2279
	__free_page(device->md_io.page);
2280 2281 2282
	put_disk(device->vdisk);
	blk_cleanup_queue(device->rq_queue);
	kfree(device->rs_plan_s);
2283 2284 2285 2286 2287 2288 2289 2290

	/* not for_each_connection(connection, resource):
	 * those may have been cleaned up and disassociated already.
	 */
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		kref_put(&peer_device->connection->kref, drbd_destroy_connection);
		kfree(peer_device);
	}
2291
	memset(device, 0xfd, sizeof(*device));
2292
	kfree(device);
2293
	kref_put(&resource->kref, drbd_destroy_resource);
P
Philipp Reisner 已提交
2294 2295
}

2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317
/* One global retry thread, if we need to push back some bio and have it
 * reinserted through our make request function.
 */
static struct retry_worker {
	struct workqueue_struct *wq;
	struct work_struct worker;

	spinlock_t lock;
	struct list_head writes;
} retry;

static void do_retry(struct work_struct *ws)
{
	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
	LIST_HEAD(writes);
	struct drbd_request *req, *tmp;

	spin_lock_irq(&retry->lock);
	list_splice_init(&retry->writes, &writes);
	spin_unlock_irq(&retry->lock);

	list_for_each_entry_safe(req, tmp, &writes, tl_requests) {
2318
		struct drbd_device *device = req->device;
2319
		struct bio *bio = req->master_bio;
2320
		unsigned long start_jif = req->start_jif;
2321 2322
		bool expected;

2323
		expected =
2324 2325 2326 2327 2328 2329
			expect(atomic_read(&req->completion_ref) == 0) &&
			expect(req->rq_state & RQ_POSTPONED) &&
			expect((req->rq_state & RQ_LOCAL_PENDING) == 0 ||
				(req->rq_state & RQ_LOCAL_ABORTED) != 0);

		if (!expected)
2330
			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
2331 2332 2333 2334 2335 2336 2337
				req, atomic_read(&req->completion_ref),
				req->rq_state);

		/* We still need to put one kref associated with the
		 * "completion_ref" going zero in the code path that queued it
		 * here.  The request object may still be referenced by a
		 * frozen local req->private_bio, in case we force-detached.
2338
		 */
2339
		kref_put(&req->kref, drbd_req_destroy);
2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353

		/* A single suspended or otherwise blocking device may stall
		 * all others as well.  Fortunately, this code path is to
		 * recover from a situation that "should not happen":
		 * concurrent writes in multi-primary setup.
		 * In a "normal" lifecycle, this workqueue is supposed to be
		 * destroyed without ever doing anything.
		 * If it turns out to be an issue anyways, we can do per
		 * resource (replication group) or per device (minor) retry
		 * workqueues instead.
		 */

		/* We are not just doing generic_make_request(),
		 * as we want to keep the start_time information. */
2354
		inc_ap_bio(device);
2355
		__drbd_make_request(device, bio, start_jif);
2356 2357 2358
	}
}

2359 2360
/* called via drbd_req_put_completion_ref(),
 * holds resource->req_lock */
2361
void drbd_restart_request(struct drbd_request *req)
2362 2363 2364 2365 2366 2367 2368 2369 2370
{
	unsigned long flags;
	spin_lock_irqsave(&retry.lock, flags);
	list_move_tail(&req->tl_requests, &retry.writes);
	spin_unlock_irqrestore(&retry.lock, flags);

	/* Drop the extra reference that would otherwise
	 * have been dropped by complete_master_bio.
	 * do_retry() needs to grab a new one. */
2371
	dec_ap_bio(req->device);
P
Philipp Reisner 已提交
2372

2373
	queue_work(retry.wq, &retry.worker);
P
Philipp Reisner 已提交
2374 2375
}

2376 2377 2378 2379 2380
void drbd_destroy_resource(struct kref *kref)
{
	struct drbd_resource *resource =
		container_of(kref, struct drbd_resource, kref);

2381
	idr_destroy(&resource->devices);
2382
	free_cpumask_var(resource->cpu_mask);
2383
	kfree(resource->name);
2384
	memset(resource, 0xf2, sizeof(*resource));
2385 2386 2387 2388 2389 2390 2391 2392 2393
	kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
	struct drbd_connection *connection, *tmp;

	for_each_connection_safe(connection, tmp, resource) {
		list_del(&connection->connections);
2394
		drbd_debugfs_connection_cleanup(connection);
2395 2396
		kref_put(&connection->kref, drbd_destroy_connection);
	}
2397
	drbd_debugfs_resource_cleanup(resource);
2398 2399
	kref_put(&resource->kref, drbd_destroy_resource);
}
2400

P
Philipp Reisner 已提交
2401 2402 2403
static void drbd_cleanup(void)
{
	unsigned int i;
2404
	struct drbd_device *device;
2405
	struct drbd_resource *resource, *tmp;
P
Philipp Reisner 已提交
2406

2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
	/* first remove proc,
	 * drbdsetup uses it's presence to detect
	 * whether DRBD is loaded.
	 * If we would get stuck in proc removal,
	 * but have netlink already deregistered,
	 * some drbdsetup commands may wait forever
	 * for an answer.
	 */
	if (drbd_proc)
		remove_proc_entry("drbd", NULL);

2418 2419
	if (retry.wq)
		destroy_workqueue(retry.wq);
P
Philipp Reisner 已提交
2420

2421
	drbd_genl_unregister();
P
Philipp Reisner 已提交
2422

2423
	idr_for_each_entry(&drbd_devices, device, i)
2424
		drbd_delete_device(device);
P
Philipp Reisner 已提交
2425

P
Philipp Reisner 已提交
2426
	/* not _rcu since, no other updater anymore. Genl already unregistered */
2427 2428 2429
	for_each_resource_safe(resource, tmp, &drbd_resources) {
		list_del(&resource->resources);
		drbd_free_resource(resource);
2430
	}
P
Philipp Reisner 已提交
2431

2432 2433
	drbd_debugfs_cleanup();

2434
	drbd_destroy_mempools();
P
Philipp Reisner 已提交
2435 2436
	unregister_blkdev(DRBD_MAJOR, "drbd");

2437
	idr_destroy(&drbd_devices);
2438

2439
	pr_info("module cleanup done.\n");
P
Philipp Reisner 已提交
2440 2441 2442
}

/**
2443
 * drbd_congested() - Callback for the flusher thread
P
Philipp Reisner 已提交
2444
 * @congested_data:	User data
2445
 * @bdi_bits:		Bits the BDI flusher thread is currently interested in
P
Philipp Reisner 已提交
2446
 *
2447
 * Returns 1<<WB_async_congested and/or 1<<WB_sync_congested if we are congested.
P
Philipp Reisner 已提交
2448 2449 2450
 */
static int drbd_congested(void *congested_data, int bdi_bits)
{
2451
	struct drbd_device *device = congested_data;
P
Philipp Reisner 已提交
2452 2453 2454 2455
	struct request_queue *q;
	char reason = '-';
	int r = 0;

2456
	if (!may_inc_ap_bio(device)) {
P
Philipp Reisner 已提交
2457 2458 2459 2460 2461 2462
		/* DRBD has frozen IO */
		r = bdi_bits;
		reason = 'd';
		goto out;
	}

2463
	if (test_bit(CALLBACK_PENDING, &first_peer_device(device)->connection->flags)) {
2464
		r |= (1 << WB_async_congested);
2465 2466 2467 2468 2469
		/* Without good local data, we would need to read from remote,
		 * and that would need the worker thread as well, which is
		 * currently blocked waiting for that usermode helper to
		 * finish.
		 */
2470
		if (!get_ldev_if_state(device, D_UP_TO_DATE))
2471
			r |= (1 << WB_sync_congested);
2472
		else
2473
			put_ldev(device);
2474 2475 2476 2477 2478
		r &= bdi_bits;
		reason = 'c';
		goto out;
	}

2479 2480
	if (get_ldev(device)) {
		q = bdev_get_queue(device->ldev->backing_bdev);
2481
		r = bdi_congested(q->backing_dev_info, bdi_bits);
2482
		put_ldev(device);
P
Philipp Reisner 已提交
2483 2484 2485 2486
		if (r)
			reason = 'b';
	}

2487
	if (bdi_bits & (1 << WB_async_congested) &&
2488
	    test_bit(NET_CONGESTED, &first_peer_device(device)->connection->flags)) {
2489
		r |= (1 << WB_async_congested);
P
Philipp Reisner 已提交
2490 2491 2492 2493
		reason = reason == 'b' ? 'a' : 'n';
	}

out:
2494
	device->congestion_reason = reason;
P
Philipp Reisner 已提交
2495 2496 2497
	return r;
}

2498 2499 2500 2501
static void drbd_init_workqueue(struct drbd_work_queue* wq)
{
	spin_lock_init(&wq->q_lock);
	INIT_LIST_HEAD(&wq->q);
2502
	init_waitqueue_head(&wq->q_wait);
2503 2504
}

2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
struct completion_work {
	struct drbd_work w;
	struct completion done;
};

static int w_complete(struct drbd_work *w, int cancel)
{
	struct completion_work *completion_work =
		container_of(w, struct completion_work, w);

	complete(&completion_work->done);
	return 0;
}

void drbd_flush_workqueue(struct drbd_work_queue *work_queue)
{
	struct completion_work completion_work;

	completion_work.w.cb = w_complete;
	init_completion(&completion_work.done);
	drbd_queue_work(work_queue, &completion_work.w);
	wait_for_completion(&completion_work.done);
}

2529
struct drbd_resource *drbd_find_resource(const char *name)
2530
{
2531
	struct drbd_resource *resource;
2532

2533 2534 2535
	if (!name || !name[0])
		return NULL;

P
Philipp Reisner 已提交
2536
	rcu_read_lock();
2537 2538
	for_each_resource_rcu(resource, &drbd_resources) {
		if (!strcmp(resource->name, name)) {
2539
			kref_get(&resource->kref);
2540
			goto found;
2541
		}
2542
	}
2543
	resource = NULL;
2544
found:
P
Philipp Reisner 已提交
2545
	rcu_read_unlock();
2546
	return resource;
2547 2548
}

2549
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
2550 2551
				     void *peer_addr, int peer_addr_len)
{
2552
	struct drbd_resource *resource;
2553
	struct drbd_connection *connection;
2554 2555

	rcu_read_lock();
2556 2557 2558 2559 2560 2561 2562 2563 2564
	for_each_resource_rcu(resource, &drbd_resources) {
		for_each_connection_rcu(connection, resource) {
			if (connection->my_addr_len == my_addr_len &&
			    connection->peer_addr_len == peer_addr_len &&
			    !memcmp(&connection->my_addr, my_addr, my_addr_len) &&
			    !memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
				kref_get(&connection->kref);
				goto found;
			}
2565 2566
		}
	}
2567
	connection = NULL;
2568 2569
found:
	rcu_read_unlock();
2570
	return connection;
2571 2572
}

2573 2574 2575 2576 2577
static int drbd_alloc_socket(struct drbd_socket *socket)
{
	socket->rbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->rbuf)
		return -ENOMEM;
2578 2579 2580
	socket->sbuf = (void *) __get_free_page(GFP_KERNEL);
	if (!socket->sbuf)
		return -ENOMEM;
2581 2582 2583 2584 2585
	return 0;
}

static void drbd_free_socket(struct drbd_socket *socket)
{
2586
	free_page((unsigned long) socket->sbuf);
2587 2588 2589
	free_page((unsigned long) socket->rbuf);
}

2590
void conn_free_crypto(struct drbd_connection *connection)
2591
{
2592
	drbd_free_sock(connection);
2593

H
Herbert Xu 已提交
2594 2595 2596 2597 2598
	crypto_free_ahash(connection->csums_tfm);
	crypto_free_ahash(connection->verify_tfm);
	crypto_free_shash(connection->cram_hmac_tfm);
	crypto_free_ahash(connection->integrity_tfm);
	crypto_free_ahash(connection->peer_integrity_tfm);
2599 2600
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2601

2602 2603 2604 2605 2606 2607 2608
	connection->csums_tfm = NULL;
	connection->verify_tfm = NULL;
	connection->cram_hmac_tfm = NULL;
	connection->integrity_tfm = NULL;
	connection->peer_integrity_tfm = NULL;
	connection->int_dig_in = NULL;
	connection->int_dig_vv = NULL;
2609 2610
}

2611
int set_resource_options(struct drbd_resource *resource, struct res_opts *res_opts)
2612
{
2613
	struct drbd_connection *connection;
2614 2615 2616 2617 2618 2619 2620 2621
	cpumask_var_t new_cpu_mask;
	int err;

	if (!zalloc_cpumask_var(&new_cpu_mask, GFP_KERNEL))
		return -ENOMEM;

	/* silently ignore cpu mask on UP kernel */
	if (nr_cpu_ids > 1 && res_opts->cpu_mask[0] != 0) {
2622
		err = bitmap_parse(res_opts->cpu_mask, DRBD_CPU_MASK_SIZE,
2623
				   cpumask_bits(new_cpu_mask), nr_cpu_ids);
2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
		if (err == -EOVERFLOW) {
			/* So what. mask it out. */
			cpumask_var_t tmp_cpu_mask;
			if (zalloc_cpumask_var(&tmp_cpu_mask, GFP_KERNEL)) {
				cpumask_setall(tmp_cpu_mask);
				cpumask_and(new_cpu_mask, new_cpu_mask, tmp_cpu_mask);
				drbd_warn(resource, "Overflow in bitmap_parse(%.12s%s), truncating to %u bits\n",
					res_opts->cpu_mask,
					strlen(res_opts->cpu_mask) > 12 ? "..." : "",
					nr_cpu_ids);
				free_cpumask_var(tmp_cpu_mask);
				err = 0;
			}
		}
2638
		if (err) {
2639
			drbd_warn(resource, "bitmap_parse() failed with %d\n", err);
2640 2641 2642 2643
			/* retcode = ERR_CPU_MASK_PARSE; */
			goto fail;
		}
	}
2644
	resource->res_opts = *res_opts;
2645 2646 2647 2648 2649
	if (cpumask_empty(new_cpu_mask))
		drbd_calc_cpu_mask(&new_cpu_mask);
	if (!cpumask_equal(resource->cpu_mask, new_cpu_mask)) {
		cpumask_copy(resource->cpu_mask, new_cpu_mask);
		for_each_connection_rcu(connection, resource) {
2650
			connection->receiver.reset_cpu_mask = 1;
2651
			connection->ack_receiver.reset_cpu_mask = 1;
2652 2653
			connection->worker.reset_cpu_mask = 1;
		}
2654 2655 2656 2657 2658 2659 2660 2661 2662
	}
	err = 0;

fail:
	free_cpumask_var(new_cpu_mask);
	return err;

}

2663 2664 2665 2666
struct drbd_resource *drbd_create_resource(const char *name)
{
	struct drbd_resource *resource;

2667
	resource = kzalloc(sizeof(struct drbd_resource), GFP_KERNEL);
2668
	if (!resource)
2669
		goto fail;
2670
	resource->name = kstrdup(name, GFP_KERNEL);
2671 2672 2673 2674
	if (!resource->name)
		goto fail_free_resource;
	if (!zalloc_cpumask_var(&resource->cpu_mask, GFP_KERNEL))
		goto fail_free_name;
2675
	kref_init(&resource->kref);
2676
	idr_init(&resource->devices);
2677
	INIT_LIST_HEAD(&resource->connections);
2678
	resource->write_ordering = WO_BDEV_FLUSH;
2679
	list_add_tail_rcu(&resource->resources, &drbd_resources);
2680
	mutex_init(&resource->conf_update);
2681
	mutex_init(&resource->adm_mutex);
2682
	spin_lock_init(&resource->req_lock);
2683
	drbd_debugfs_resource_add(resource);
2684
	return resource;
2685 2686 2687 2688 2689 2690 2691

fail_free_name:
	kfree(resource->name);
fail_free_resource:
	kfree(resource);
fail:
	return NULL;
2692 2693
}

2694
/* caller must be under adm_mutex */
2695
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
2696
{
2697
	struct drbd_resource *resource;
2698
	struct drbd_connection *connection;
2699

2700 2701
	connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
	if (!connection)
2702 2703
		return NULL;

2704
	if (drbd_alloc_socket(&connection->data))
2705
		goto fail;
2706
	if (drbd_alloc_socket(&connection->meta))
2707 2708
		goto fail;

2709 2710
	connection->current_epoch = kzalloc(sizeof(struct drbd_epoch), GFP_KERNEL);
	if (!connection->current_epoch)
2711
		goto fail;
2712

2713
	INIT_LIST_HEAD(&connection->transfer_log);
2714

2715 2716 2717
	INIT_LIST_HEAD(&connection->current_epoch->list);
	connection->epochs = 1;
	spin_lock_init(&connection->epoch_lock);
2718

2719 2720 2721
	connection->send.seen_any_write_yet = false;
	connection->send.current_epoch_nr = 0;
	connection->send.current_epoch_writes = 0;
2722

2723 2724 2725 2726
	resource = drbd_create_resource(name);
	if (!resource)
		goto fail;

2727 2728 2729
	connection->cstate = C_STANDALONE;
	mutex_init(&connection->cstate_mutex);
	init_waitqueue_head(&connection->ping_wait);
2730
	idr_init(&connection->peer_devices);
2731

2732 2733 2734
	drbd_init_workqueue(&connection->sender_work);
	mutex_init(&connection->data.mutex);
	mutex_init(&connection->meta.mutex);
2735

2736 2737 2738 2739
	drbd_thread_init(resource, &connection->receiver, drbd_receiver, "receiver");
	connection->receiver.connection = connection;
	drbd_thread_init(resource, &connection->worker, drbd_worker, "worker");
	connection->worker.connection = connection;
2740 2741
	drbd_thread_init(resource, &connection->ack_receiver, drbd_ack_receiver, "ack_recv");
	connection->ack_receiver.connection = connection;
2742

2743
	kref_init(&connection->kref);
2744 2745

	connection->resource = resource;
2746

2747 2748 2749 2750 2751
	if (set_resource_options(resource, res_opts))
		goto fail_resource;

	kref_get(&resource->kref);
	list_add_tail_rcu(&connection->connections, &resource->connections);
2752
	drbd_debugfs_connection_add(connection);
2753
	return connection;
2754

2755 2756 2757
fail_resource:
	list_del(&resource->resources);
	drbd_free_resource(resource);
2758
fail:
2759 2760 2761 2762
	kfree(connection->current_epoch);
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection);
2763 2764 2765
	return NULL;
}

2766
void drbd_destroy_connection(struct kref *kref)
2767
{
2768
	struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
2769
	struct drbd_resource *resource = connection->resource;
2770

2771
	if (atomic_read(&connection->current_epoch->epoch_size) !=  0)
2772
		drbd_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
2773
	kfree(connection->current_epoch);
2774

2775
	idr_destroy(&connection->peer_devices);
2776

2777 2778 2779 2780
	drbd_free_socket(&connection->meta);
	drbd_free_socket(&connection->data);
	kfree(connection->int_dig_in);
	kfree(connection->int_dig_vv);
2781
	memset(connection, 0xfc, sizeof(*connection));
2782
	kfree(connection);
2783
	kref_put(&resource->kref, drbd_destroy_resource);
2784 2785
}

2786
static int init_submitter(struct drbd_device *device)
2787 2788 2789
{
	/* opencoded create_singlethread_workqueue(),
	 * to be able to say "drbd%d", ..., minor */
2790 2791
	device->submit.wq =
		alloc_ordered_workqueue("drbd%u_submit", WQ_MEM_RECLAIM, device->minor);
2792
	if (!device->submit.wq)
2793 2794
		return -ENOMEM;

2795 2796
	INIT_WORK(&device->submit.worker, do_submit);
	INIT_LIST_HEAD(&device->submit.writes);
2797 2798 2799
	return 0;
}

2800
enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsigned int minor)
P
Philipp Reisner 已提交
2801
{
2802
	struct drbd_resource *resource = adm_ctx->resource;
2803
	struct drbd_connection *connection;
2804
	struct drbd_device *device;
2805
	struct drbd_peer_device *peer_device, *tmp_peer_device;
P
Philipp Reisner 已提交
2806 2807
	struct gendisk *disk;
	struct request_queue *q;
2808
	int id;
2809
	int vnr = adm_ctx->volume;
2810
	enum drbd_ret_code err = ERR_NOMEM;
2811

2812 2813
	device = minor_to_device(minor);
	if (device)
A
Andreas Gruenbacher 已提交
2814
		return ERR_MINOR_OR_VOLUME_EXISTS;
P
Philipp Reisner 已提交
2815 2816

	/* GFP_KERNEL, we are outside of all write-out paths */
2817 2818
	device = kzalloc(sizeof(struct drbd_device), GFP_KERNEL);
	if (!device)
2819
		return ERR_NOMEM;
2820 2821 2822 2823
	kref_init(&device->kref);

	kref_get(&resource->kref);
	device->resource = resource;
2824 2825
	device->minor = minor;
	device->vnr = vnr;
P
Philipp Reisner 已提交
2826

2827
	drbd_init_set_defaults(device);
P
Philipp Reisner 已提交
2828 2829 2830 2831

	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		goto out_no_q;
2832 2833
	device->rq_queue = q;
	q->queuedata   = device;
P
Philipp Reisner 已提交
2834 2835 2836 2837

	disk = alloc_disk(1);
	if (!disk)
		goto out_no_disk;
2838
	device->vdisk = disk;
P
Philipp Reisner 已提交
2839

2840
	set_disk_ro(disk, true);
P
Philipp Reisner 已提交
2841 2842 2843 2844 2845 2846

	disk->queue = q;
	disk->major = DRBD_MAJOR;
	disk->first_minor = minor;
	disk->fops = &drbd_ops;
	sprintf(disk->disk_name, "drbd%d", minor);
2847
	disk->private_data = device;
P
Philipp Reisner 已提交
2848

2849
	device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
P
Philipp Reisner 已提交
2850
	/* we have no partitions. we contain only ourselves. */
2851
	device->this_bdev->bd_contains = device->this_bdev;
P
Philipp Reisner 已提交
2852

2853 2854
	q->backing_dev_info->congested_fn = drbd_congested;
	q->backing_dev_info->congested_data = device;
P
Philipp Reisner 已提交
2855

2856
	blk_queue_make_request(q, drbd_make_request);
2857
	blk_queue_write_cache(q, true, true);
2858 2859 2860
	/* Setting the max_hw_sectors to an odd value of 8kibyte here
	   This triggers a max_bio_size message upon first attach or connect */
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
2861
	q->queue_lock = &resource->req_lock;
P
Philipp Reisner 已提交
2862

2863 2864
	device->md_io.page = alloc_page(GFP_KERNEL);
	if (!device->md_io.page)
P
Philipp Reisner 已提交
2865 2866
		goto out_no_io_page;

2867
	if (drbd_bm_init(device))
P
Philipp Reisner 已提交
2868
		goto out_no_bitmap;
2869 2870
	device->read_requests = RB_ROOT;
	device->write_requests = RB_ROOT;
P
Philipp Reisner 已提交
2871

2872 2873
	id = idr_alloc(&drbd_devices, device, minor, minor + 1, GFP_KERNEL);
	if (id < 0) {
2874
		if (id == -ENOSPC)
A
Andreas Gruenbacher 已提交
2875
			err = ERR_MINOR_OR_VOLUME_EXISTS;
2876
		goto out_no_minor_idr;
2877
	}
2878 2879 2880 2881
	kref_get(&device->kref);

	id = idr_alloc(&resource->devices, device, vnr, vnr + 1, GFP_KERNEL);
	if (id < 0) {
2882
		if (id == -ENOSPC)
A
Andreas Gruenbacher 已提交
2883
			err = ERR_MINOR_OR_VOLUME_EXISTS;
2884 2885 2886
		goto out_idr_remove_minor;
	}
	kref_get(&device->kref);
2887

2888
	INIT_LIST_HEAD(&device->peer_devices);
2889
	INIT_LIST_HEAD(&device->pending_bitmap_io);
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
	for_each_connection(connection, resource) {
		peer_device = kzalloc(sizeof(struct drbd_peer_device), GFP_KERNEL);
		if (!peer_device)
			goto out_idr_remove_from_resource;
		peer_device->connection = connection;
		peer_device->device = device;

		list_add(&peer_device->peer_devices, &device->peer_devices);
		kref_get(&device->kref);

		id = idr_alloc(&connection->peer_devices, peer_device, vnr, vnr + 1, GFP_KERNEL);
		if (id < 0) {
2902
			if (id == -ENOSPC)
2903 2904
				err = ERR_INVALID_REQUEST;
			goto out_idr_remove_from_resource;
T
Tejun Heo 已提交
2905
		}
2906
		kref_get(&connection->kref);
2907
		INIT_WORK(&peer_device->send_acks_work, drbd_send_acks_wf);
2908
	}
T
Tejun Heo 已提交
2909

2910
	if (init_submitter(device)) {
2911 2912 2913 2914
		err = ERR_NOMEM;
		goto out_idr_remove_vol;
	}

2915 2916
	add_disk(disk);

2917
	/* inherit the connection state */
2918
	device->state.conn = first_connection(resource)->cstate;
2919 2920 2921 2922
	if (device->state.conn == C_WF_REPORT_PARAMS) {
		for_each_peer_device(peer_device, device)
			drbd_connected(peer_device);
	}
2923 2924 2925 2926
	/* move to create_peer_device() */
	for_each_peer_device(peer_device, device)
		drbd_debugfs_peer_device_add(peer_device);
	drbd_debugfs_device_add(device);
2927
	return NO_ERROR;
P
Philipp Reisner 已提交
2928

2929
out_idr_remove_vol:
2930
	idr_remove(&connection->peer_devices, vnr);
2931
out_idr_remove_from_resource:
2932
	for_each_connection(connection, resource) {
2933 2934
		peer_device = idr_remove(&connection->peer_devices, vnr);
		if (peer_device)
2935 2936 2937 2938 2939 2940
			kref_put(&connection->kref, drbd_destroy_connection);
	}
	for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
		list_del(&peer_device->peer_devices);
		kfree(peer_device);
	}
2941
	idr_remove(&resource->devices, vnr);
2942
out_idr_remove_minor:
2943
	idr_remove(&drbd_devices, minor);
2944
	synchronize_rcu();
2945
out_no_minor_idr:
2946
	drbd_bm_cleanup(device);
P
Philipp Reisner 已提交
2947
out_no_bitmap:
2948
	__free_page(device->md_io.page);
P
Philipp Reisner 已提交
2949 2950 2951 2952 2953
out_no_io_page:
	put_disk(disk);
out_no_disk:
	blk_cleanup_queue(q);
out_no_q:
2954
	kref_put(&resource->kref, drbd_destroy_resource);
2955
	kfree(device);
2956
	return err;
P
Philipp Reisner 已提交
2957 2958
}

2959
void drbd_delete_device(struct drbd_device *device)
2960 2961 2962
{
	struct drbd_resource *resource = device->resource;
	struct drbd_connection *connection;
2963
	struct drbd_peer_device *peer_device;
2964

2965 2966 2967 2968
	/* move to free_peer_device() */
	for_each_peer_device(peer_device, device)
		drbd_debugfs_peer_device_cleanup(peer_device);
	drbd_debugfs_device_cleanup(device);
2969
	for_each_connection(connection, resource) {
2970
		idr_remove(&connection->peer_devices, device->vnr);
2971
		kref_put(&device->kref, drbd_destroy_device);
2972 2973
	}
	idr_remove(&resource->devices, device->vnr);
2974
	kref_put(&device->kref, drbd_destroy_device);
2975
	idr_remove(&drbd_devices, device_to_minor(device));
2976
	kref_put(&device->kref, drbd_destroy_device);
2977 2978
	del_gendisk(device->vdisk);
	synchronize_rcu();
2979
	kref_put(&device->kref, drbd_destroy_device);
2980 2981
}

2982
static int __init drbd_init(void)
P
Philipp Reisner 已提交
2983 2984 2985
{
	int err;

2986 2987
	if (drbd_minor_count < DRBD_MINOR_COUNT_MIN || drbd_minor_count > DRBD_MINOR_COUNT_MAX) {
		pr_err("invalid minor_count (%d)\n", drbd_minor_count);
P
Philipp Reisner 已提交
2988 2989 2990
#ifdef MODULE
		return -EINVAL;
#else
2991
		drbd_minor_count = DRBD_MINOR_COUNT_DEF;
P
Philipp Reisner 已提交
2992 2993 2994 2995 2996
#endif
	}

	err = register_blkdev(DRBD_MAJOR, "drbd");
	if (err) {
2997
		pr_err("unable to register block device major %d\n",
P
Philipp Reisner 已提交
2998 2999 3000 3001 3002 3003 3004 3005 3006 3007
		       DRBD_MAJOR);
		return err;
	}

	/*
	 * allocate all necessary structs
	 */
	init_waitqueue_head(&drbd_pp_wait);

	drbd_proc = NULL; /* play safe for drbd_cleanup */
3008
	idr_init(&drbd_devices);
P
Philipp Reisner 已提交
3009

3010
	mutex_init(&resources_mutex);
3011
	INIT_LIST_HEAD(&drbd_resources);
3012 3013 3014

	err = drbd_genl_register();
	if (err) {
3015
		pr_err("unable to register generic netlink family\n");
3016 3017 3018
		goto fail;
	}

P
Philipp Reisner 已提交
3019 3020
	err = drbd_create_mempools();
	if (err)
3021
		goto fail;
P
Philipp Reisner 已提交
3022

3023
	err = -ENOMEM;
3024
	drbd_proc = proc_create_data("drbd", S_IFREG | S_IRUGO , NULL, &drbd_proc_fops, NULL);
P
Philipp Reisner 已提交
3025
	if (!drbd_proc)	{
3026
		pr_err("unable to register proc file\n");
3027
		goto fail;
P
Philipp Reisner 已提交
3028 3029
	}

3030 3031
	retry.wq = create_singlethread_workqueue("drbd-reissue");
	if (!retry.wq) {
3032
		pr_err("unable to create retry workqueue\n");
3033 3034 3035 3036 3037
		goto fail;
	}
	INIT_WORK(&retry.worker, do_retry);
	spin_lock_init(&retry.lock);
	INIT_LIST_HEAD(&retry.writes);
P
Philipp Reisner 已提交
3038

3039 3040 3041
	if (drbd_debugfs_init())
		pr_notice("failed to initialize debugfs -- will not be available\n");

3042
	pr_info("initialized. "
P
Philipp Reisner 已提交
3043 3044
	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
	       API_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
3045 3046
	pr_info("%s\n", drbd_buildtag());
	pr_info("registered as block device major %d\n", DRBD_MAJOR);
P
Philipp Reisner 已提交
3047 3048
	return 0; /* Success! */

3049
fail:
P
Philipp Reisner 已提交
3050 3051
	drbd_cleanup();
	if (err == -ENOMEM)
3052
		pr_err("ran out of memory\n");
P
Philipp Reisner 已提交
3053
	else
3054
		pr_err("initialization failure\n");
P
Philipp Reisner 已提交
3055 3056 3057
	return err;
}

3058
static void drbd_free_one_sock(struct drbd_socket *ds)
P
Philipp Reisner 已提交
3059
{
3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
	struct socket *s;
	mutex_lock(&ds->mutex);
	s = ds->socket;
	ds->socket = NULL;
	mutex_unlock(&ds->mutex);
	if (s) {
		/* so debugfs does not need to mutex_lock() */
		synchronize_rcu();
		kernel_sock_shutdown(s, SHUT_RDWR);
		sock_release(s);
P
Philipp Reisner 已提交
3070 3071 3072
	}
}

3073 3074 3075 3076 3077 3078 3079 3080
void drbd_free_sock(struct drbd_connection *connection)
{
	if (connection->data.socket)
		drbd_free_one_sock(&connection->data);
	if (connection->meta.socket)
		drbd_free_one_sock(&connection->meta);
}

P
Philipp Reisner 已提交
3081 3082
/* meta data management */

3083
void conn_md_sync(struct drbd_connection *connection)
P
Philipp Reisner 已提交
3084
{
3085
	struct drbd_peer_device *peer_device;
3086
	int vnr;
P
Philipp Reisner 已提交
3087

3088
	rcu_read_lock();
3089 3090 3091
	idr_for_each_entry(&connection->peer_devices, peer_device, vnr) {
		struct drbd_device *device = peer_device->device;

3092
		kref_get(&device->kref);
3093
		rcu_read_unlock();
3094
		drbd_md_sync(device);
3095
		kref_put(&device->kref, drbd_destroy_device);
3096 3097 3098
		rcu_read_lock();
	}
	rcu_read_unlock();
P
Philipp Reisner 已提交
3099 3100
}

3101
/* aligned 4kByte */
P
Philipp Reisner 已提交
3102
struct meta_data_on_disk {
3103
	u64 la_size_sect;      /* last agreed size. */
P
Philipp Reisner 已提交
3104 3105 3106 3107 3108 3109 3110
	u64 uuid[UI_SIZE];   /* UUIDs. */
	u64 device_uuid;
	u64 reserved_u64_1;
	u32 flags;             /* MDF */
	u32 magic;
	u32 md_size_sect;
	u32 al_offset;         /* offset to this block */
3111
	u32 al_nr_extents;     /* important for restoring the AL (userspace) */
3112
	      /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
P
Philipp Reisner 已提交
3113 3114
	u32 bm_offset;         /* offset to the bitmap, from here */
	u32 bm_bytes_per_bit;  /* BM_BLOCK_SIZE */
3115
	u32 la_peer_max_bio_size;   /* last peer max_bio_size */
P
Philipp Reisner 已提交
3116

3117 3118 3119 3120 3121
	/* see al_tr_number_to_on_disk_sector() */
	u32 al_stripes;
	u32 al_stripe_size_4k;

	u8 reserved_u8[4096 - (7*8 + 10*4)];
P
Philipp Reisner 已提交
3122 3123
} __packed;

3124 3125


3126
void drbd_md_write(struct drbd_device *device, void *b)
P
Philipp Reisner 已提交
3127
{
3128
	struct meta_data_on_disk *buffer = b;
P
Philipp Reisner 已提交
3129 3130 3131
	sector_t sector;
	int i;

3132
	memset(buffer, 0, sizeof(*buffer));
P
Philipp Reisner 已提交
3133

3134
	buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
P
Philipp Reisner 已提交
3135
	for (i = UI_CURRENT; i < UI_SIZE; i++)
3136 3137
		buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
	buffer->flags = cpu_to_be32(device->ldev->md.flags);
3138
	buffer->magic = cpu_to_be32(DRBD_MD_MAGIC_84_UNCLEAN);
P
Philipp Reisner 已提交
3139

3140 3141 3142
	buffer->md_size_sect  = cpu_to_be32(device->ldev->md.md_size_sect);
	buffer->al_offset     = cpu_to_be32(device->ldev->md.al_offset);
	buffer->al_nr_extents = cpu_to_be32(device->act_log->nr_elements);
P
Philipp Reisner 已提交
3143
	buffer->bm_bytes_per_bit = cpu_to_be32(BM_BLOCK_SIZE);
3144
	buffer->device_uuid = cpu_to_be64(device->ldev->md.device_uuid);
P
Philipp Reisner 已提交
3145

3146 3147
	buffer->bm_offset = cpu_to_be32(device->ldev->md.bm_offset);
	buffer->la_peer_max_bio_size = cpu_to_be32(device->peer_max_bio_size);
P
Philipp Reisner 已提交
3148

3149 3150
	buffer->al_stripes = cpu_to_be32(device->ldev->md.al_stripes);
	buffer->al_stripe_size_4k = cpu_to_be32(device->ldev->md.al_stripe_size_4k);
3151

3152
	D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset);
3153
	sector = device->ldev->md.md_offset;
P
Philipp Reisner 已提交
3154

M
Mike Christie 已提交
3155
	if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) {
P
Philipp Reisner 已提交
3156
		/* this was a try anyways ... */
3157
		drbd_err(device, "meta data update failed!\n");
3158
		drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
P
Philipp Reisner 已提交
3159
	}
3160 3161 3162 3163
}

/**
 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
3164
 * @device:	DRBD device.
3165
 */
3166
void drbd_md_sync(struct drbd_device *device)
3167 3168 3169 3170 3171 3172 3173
{
	struct meta_data_on_disk *buffer;

	/* Don't accidentally change the DRBD meta data layout. */
	BUILD_BUG_ON(UI_SIZE != 4);
	BUILD_BUG_ON(sizeof(struct meta_data_on_disk) != 4096);

3174
	del_timer(&device->md_sync_timer);
3175
	/* timer may be rearmed by drbd_md_mark_dirty() now. */
3176
	if (!test_and_clear_bit(MD_DIRTY, &device->flags))
3177 3178 3179 3180
		return;

	/* We use here D_FAILED and not D_ATTACHING because we try to write
	 * metadata even if we detach due to a disk failure! */
3181
	if (!get_ldev_if_state(device, D_FAILED))
3182 3183
		return;

3184
	buffer = drbd_md_get_buffer(device, __func__);
3185 3186 3187
	if (!buffer)
		goto out;

3188
	drbd_md_write(device, buffer);
P
Philipp Reisner 已提交
3189

3190
	/* Update device->ldev->md.la_size_sect,
P
Philipp Reisner 已提交
3191
	 * since we updated it on metadata. */
3192
	device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
P
Philipp Reisner 已提交
3193

3194
	drbd_md_put_buffer(device);
3195
out:
3196
	put_ldev(device);
P
Philipp Reisner 已提交
3197 3198
}

3199
static int check_activity_log_stripe_size(struct drbd_device *device,
3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238
		struct meta_data_on_disk *on_disk,
		struct drbd_md *in_core)
{
	u32 al_stripes = be32_to_cpu(on_disk->al_stripes);
	u32 al_stripe_size_4k = be32_to_cpu(on_disk->al_stripe_size_4k);
	u64 al_size_4k;

	/* both not set: default to old fixed size activity log */
	if (al_stripes == 0 && al_stripe_size_4k == 0) {
		al_stripes = 1;
		al_stripe_size_4k = MD_32kB_SECT/8;
	}

	/* some paranoia plausibility checks */

	/* we need both values to be set */
	if (al_stripes == 0 || al_stripe_size_4k == 0)
		goto err;

	al_size_4k = (u64)al_stripes * al_stripe_size_4k;

	/* Upper limit of activity log area, to avoid potential overflow
	 * problems in al_tr_number_to_on_disk_sector(). As right now, more
	 * than 72 * 4k blocks total only increases the amount of history,
	 * limiting this arbitrarily to 16 GB is not a real limitation ;-)  */
	if (al_size_4k > (16 * 1024 * 1024/4))
		goto err;

	/* Lower limit: we need at least 8 transaction slots (32kB)
	 * to not break existing setups */
	if (al_size_4k < MD_32kB_SECT/8)
		goto err;

	in_core->al_stripe_size_4k = al_stripe_size_4k;
	in_core->al_stripes = al_stripes;
	in_core->al_size_4k = al_size_4k;

	return 0;
err:
3239
	drbd_err(device, "invalid activity log striping: al_stripes=%u, al_stripe_size_4k=%u\n",
3240 3241 3242 3243
			al_stripes, al_stripe_size_4k);
	return -EINVAL;
}

3244
static int check_offsets_and_sizes(struct drbd_device *device, struct drbd_backing_dev *bdev)
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310
{
	sector_t capacity = drbd_get_capacity(bdev->md_bdev);
	struct drbd_md *in_core = &bdev->md;
	s32 on_disk_al_sect;
	s32 on_disk_bm_sect;

	/* The on-disk size of the activity log, calculated from offsets, and
	 * the size of the activity log calculated from the stripe settings,
	 * should match.
	 * Though we could relax this a bit: it is ok, if the striped activity log
	 * fits in the available on-disk activity log size.
	 * Right now, that would break how resize is implemented.
	 * TODO: make drbd_determine_dev_size() (and the drbdmeta tool) aware
	 * of possible unused padding space in the on disk layout. */
	if (in_core->al_offset < 0) {
		if (in_core->bm_offset > in_core->al_offset)
			goto err;
		on_disk_al_sect = -in_core->al_offset;
		on_disk_bm_sect = in_core->al_offset - in_core->bm_offset;
	} else {
		if (in_core->al_offset != MD_4kB_SECT)
			goto err;
		if (in_core->bm_offset < in_core->al_offset + in_core->al_size_4k * MD_4kB_SECT)
			goto err;

		on_disk_al_sect = in_core->bm_offset - MD_4kB_SECT;
		on_disk_bm_sect = in_core->md_size_sect - in_core->bm_offset;
	}

	/* old fixed size meta data is exactly that: fixed. */
	if (in_core->meta_dev_idx >= 0) {
		if (in_core->md_size_sect != MD_128MB_SECT
		||  in_core->al_offset != MD_4kB_SECT
		||  in_core->bm_offset != MD_4kB_SECT + MD_32kB_SECT
		||  in_core->al_stripes != 1
		||  in_core->al_stripe_size_4k != MD_32kB_SECT/8)
			goto err;
	}

	if (capacity < in_core->md_size_sect)
		goto err;
	if (capacity - in_core->md_size_sect < drbd_md_first_sector(bdev))
		goto err;

	/* should be aligned, and at least 32k */
	if ((on_disk_al_sect & 7) || (on_disk_al_sect < MD_32kB_SECT))
		goto err;

	/* should fit (for now: exactly) into the available on-disk space;
	 * overflow prevention is in check_activity_log_stripe_size() above. */
	if (on_disk_al_sect != in_core->al_size_4k * MD_4kB_SECT)
		goto err;

	/* again, should be aligned */
	if (in_core->bm_offset & 7)
		goto err;

	/* FIXME check for device grow with flex external meta data? */

	/* can the available bitmap space cover the last agreed device size? */
	if (on_disk_bm_sect < (in_core->la_size_sect+7)/MD_4kB_SECT/8/512)
		goto err;

	return 0;

err:
3311
	drbd_err(device, "meta data offsets don't make sense: idx=%d "
3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323
			"al_s=%u, al_sz4k=%u, al_offset=%d, bm_offset=%d, "
			"md_size_sect=%u, la_size=%llu, md_capacity=%llu\n",
			in_core->meta_dev_idx,
			in_core->al_stripes, in_core->al_stripe_size_4k,
			in_core->al_offset, in_core->bm_offset, in_core->md_size_sect,
			(unsigned long long)in_core->la_size_sect,
			(unsigned long long)capacity);

	return -EINVAL;
}


P
Philipp Reisner 已提交
3324 3325
/**
 * drbd_md_read() - Reads in the meta data super block
3326
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3327 3328
 * @bdev:	Device from which the meta data should be read in.
 *
3329
 * Return NO_ERROR on success, and an enum drbd_ret_code in case
3330
 * something goes wrong.
3331
 *
3332
 * Called exactly once during drbd_adm_attach(), while still being D_DISKLESS,
3333
 * even before @bdev is assigned to @device->ldev.
P
Philipp Reisner 已提交
3334
 */
3335
int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)
P
Philipp Reisner 已提交
3336 3337
{
	struct meta_data_on_disk *buffer;
3338
	u32 magic, flags;
P
Philipp Reisner 已提交
3339 3340
	int i, rv = NO_ERROR;

3341
	if (device->state.disk != D_DISKLESS)
3342
		return ERR_DISK_CONFIGURED;
P
Philipp Reisner 已提交
3343

3344
	buffer = drbd_md_get_buffer(device, __func__);
3345
	if (!buffer)
3346
		return ERR_NOMEM;
P
Philipp Reisner 已提交
3347

3348 3349
	/* First, figure out where our meta data superblock is located,
	 * and read it. */
3350 3351
	bdev->md.meta_dev_idx = bdev->disk_conf->meta_dev_idx;
	bdev->md.md_offset = drbd_md_ss(bdev);
3352 3353 3354 3355
	/* Even for (flexible or indexed) external meta data,
	 * initially restrict us to the 4k superblock for now.
	 * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */
	bdev->md.md_size_sect = 8;
P
Philipp Reisner 已提交
3356

M
Mike Christie 已提交
3357 3358
	if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset,
				 REQ_OP_READ)) {
L
Lucas De Marchi 已提交
3359
		/* NOTE: can't do normal error processing here as this is
P
Philipp Reisner 已提交
3360
		   called BEFORE disk is attached */
3361
		drbd_err(device, "Error while reading metadata.\n");
P
Philipp Reisner 已提交
3362 3363 3364 3365
		rv = ERR_IO_MD_DISK;
		goto err;
	}

3366 3367 3368 3369 3370
	magic = be32_to_cpu(buffer->magic);
	flags = be32_to_cpu(buffer->flags);
	if (magic == DRBD_MD_MAGIC_84_UNCLEAN ||
	    (magic == DRBD_MD_MAGIC_08 && !(flags & MDF_AL_CLEAN))) {
			/* btw: that's Activity Log clean, not "all" clean. */
3371
		drbd_err(device, "Found unclean meta data. Did you \"drbdadm apply-al\"?\n");
3372 3373 3374
		rv = ERR_MD_UNCLEAN;
		goto err;
	}
3375 3376

	rv = ERR_MD_INVALID;
3377
	if (magic != DRBD_MD_MAGIC_08) {
3378
		if (magic == DRBD_MD_MAGIC_07)
3379
			drbd_err(device, "Found old (0.7) meta data magic. Did you \"drbdadm create-md\"?\n");
3380
		else
3381
			drbd_err(device, "Meta data magic not found. Did you \"drbdadm create-md\"?\n");
P
Philipp Reisner 已提交
3382 3383
		goto err;
	}
3384

3385
	if (be32_to_cpu(buffer->bm_bytes_per_bit) != BM_BLOCK_SIZE) {
3386
		drbd_err(device, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
3387
		    be32_to_cpu(buffer->bm_bytes_per_bit), BM_BLOCK_SIZE);
P
Philipp Reisner 已提交
3388 3389
		goto err;
	}
3390

3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402

	/* convert to in_core endian */
	bdev->md.la_size_sect = be64_to_cpu(buffer->la_size_sect);
	for (i = UI_CURRENT; i < UI_SIZE; i++)
		bdev->md.uuid[i] = be64_to_cpu(buffer->uuid[i]);
	bdev->md.flags = be32_to_cpu(buffer->flags);
	bdev->md.device_uuid = be64_to_cpu(buffer->device_uuid);

	bdev->md.md_size_sect = be32_to_cpu(buffer->md_size_sect);
	bdev->md.al_offset = be32_to_cpu(buffer->al_offset);
	bdev->md.bm_offset = be32_to_cpu(buffer->bm_offset);

3403
	if (check_activity_log_stripe_size(device, buffer, &bdev->md))
P
Philipp Reisner 已提交
3404
		goto err;
3405
	if (check_offsets_and_sizes(device, bdev))
3406 3407
		goto err;

P
Philipp Reisner 已提交
3408
	if (be32_to_cpu(buffer->bm_offset) != bdev->md.bm_offset) {
3409
		drbd_err(device, "unexpected bm_offset: %d (expected %d)\n",
P
Philipp Reisner 已提交
3410 3411 3412 3413
		    be32_to_cpu(buffer->bm_offset), bdev->md.bm_offset);
		goto err;
	}
	if (be32_to_cpu(buffer->md_size_sect) != bdev->md.md_size_sect) {
3414
		drbd_err(device, "unexpected md_size: %u (expected %u)\n",
P
Philipp Reisner 已提交
3415 3416 3417 3418
		    be32_to_cpu(buffer->md_size_sect), bdev->md.md_size_sect);
		goto err;
	}

3419
	rv = NO_ERROR;
P
Philipp Reisner 已提交
3420

3421
	spin_lock_irq(&device->resource->req_lock);
3422
	if (device->state.conn < C_CONNECTED) {
3423
		unsigned int peer;
3424
		peer = be32_to_cpu(buffer->la_peer_max_bio_size);
3425
		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
3426
		device->peer_max_bio_size = peer;
3427
	}
3428
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3429 3430

 err:
3431
	drbd_md_put_buffer(device);
P
Philipp Reisner 已提交
3432 3433 3434 3435 3436 3437

	return rv;
}

/**
 * drbd_md_mark_dirty() - Mark meta data super block as dirty
3438
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3439 3440 3441 3442 3443
 *
 * Call this function if you change anything that should be written to
 * the meta-data super block. This function sets MD_DIRTY, and starts a
 * timer that ensures that within five seconds you have to call drbd_md_sync().
 */
3444
#ifdef DEBUG
3445
void drbd_md_mark_dirty_(struct drbd_device *device, unsigned int line, const char *func)
3446
{
3447 3448 3449 3450
	if (!test_and_set_bit(MD_DIRTY, &device->flags)) {
		mod_timer(&device->md_sync_timer, jiffies + HZ);
		device->last_md_mark_dirty.line = line;
		device->last_md_mark_dirty.func = func;
3451 3452 3453
	}
}
#else
3454
void drbd_md_mark_dirty(struct drbd_device *device)
P
Philipp Reisner 已提交
3455
{
3456 3457
	if (!test_and_set_bit(MD_DIRTY, &device->flags))
		mod_timer(&device->md_sync_timer, jiffies + 5*HZ);
P
Philipp Reisner 已提交
3458
}
3459
#endif
P
Philipp Reisner 已提交
3460

3461
void drbd_uuid_move_history(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3462 3463 3464
{
	int i;

3465
	for (i = UI_HISTORY_START; i < UI_HISTORY_END; i++)
3466
		device->ldev->md.uuid[i+1] = device->ldev->md.uuid[i];
P
Philipp Reisner 已提交
3467 3468
}

3469
void __drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3470 3471
{
	if (idx == UI_CURRENT) {
3472
		if (device->state.role == R_PRIMARY)
P
Philipp Reisner 已提交
3473 3474 3475 3476
			val |= 1;
		else
			val &= ~((u64)1);

3477
		drbd_set_ed_uuid(device, val);
P
Philipp Reisner 已提交
3478 3479
	}

3480 3481
	device->ldev->md.uuid[idx] = val;
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3482 3483
}

3484
void _drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
3485 3486
{
	unsigned long flags;
3487 3488 3489
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3490
}
P
Philipp Reisner 已提交
3491

3492
void drbd_uuid_set(struct drbd_device *device, int idx, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3493
{
3494
	unsigned long flags;
3495 3496 3497 3498
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
	if (device->ldev->md.uuid[idx]) {
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[idx];
P
Philipp Reisner 已提交
3499
	}
3500 3501
	__drbd_uuid_set(device, idx, val);
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3502 3503 3504 3505
}

/**
 * drbd_uuid_new_current() - Creates a new current UUID
3506
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3507 3508 3509 3510
 *
 * Creates a new current UUID, and rotates the old current UUID into
 * the bitmap slot. Causes an incremental resync upon next connect.
 */
3511
void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3512 3513
{
	u64 val;
3514 3515 3516 3517
	unsigned long long bm_uuid;

	get_random_bytes(&val, sizeof(u64));

3518 3519
	spin_lock_irq(&device->ldev->md.uuid_lock);
	bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3520 3521

	if (bm_uuid)
3522
		drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3523

3524 3525 3526
	device->ldev->md.uuid[UI_BITMAP] = device->ldev->md.uuid[UI_CURRENT];
	__drbd_uuid_set(device, UI_CURRENT, val);
	spin_unlock_irq(&device->ldev->md.uuid_lock);
P
Philipp Reisner 已提交
3527

3528
	drbd_print_uuids(device, "new current UUID");
3529
	/* get it to stable storage _now_ */
3530
	drbd_md_sync(device);
P
Philipp Reisner 已提交
3531 3532
}

3533
void drbd_uuid_set_bm(struct drbd_device *device, u64 val) __must_hold(local)
P
Philipp Reisner 已提交
3534
{
3535
	unsigned long flags;
3536
	if (device->ldev->md.uuid[UI_BITMAP] == 0 && val == 0)
P
Philipp Reisner 已提交
3537 3538
		return;

3539
	spin_lock_irqsave(&device->ldev->md.uuid_lock, flags);
P
Philipp Reisner 已提交
3540
	if (val == 0) {
3541 3542 3543
		drbd_uuid_move_history(device);
		device->ldev->md.uuid[UI_HISTORY_START] = device->ldev->md.uuid[UI_BITMAP];
		device->ldev->md.uuid[UI_BITMAP] = 0;
P
Philipp Reisner 已提交
3544
	} else {
3545
		unsigned long long bm_uuid = device->ldev->md.uuid[UI_BITMAP];
3546
		if (bm_uuid)
3547
			drbd_warn(device, "bm UUID was already set: %llX\n", bm_uuid);
P
Philipp Reisner 已提交
3548

3549
		device->ldev->md.uuid[UI_BITMAP] = val & ~((u64)1);
P
Philipp Reisner 已提交
3550
	}
3551
	spin_unlock_irqrestore(&device->ldev->md.uuid_lock, flags);
3552

3553
	drbd_md_mark_dirty(device);
P
Philipp Reisner 已提交
3554 3555 3556 3557
}

/**
 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3558
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3559 3560 3561
 *
 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3562
int drbd_bmio_set_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3563 3564 3565
{
	int rv = -EIO;

3566 3567 3568
	drbd_md_set_flag(device, MDF_FULL_SYNC);
	drbd_md_sync(device);
	drbd_bm_set_all(device);
P
Philipp Reisner 已提交
3569

3570
	rv = drbd_bm_write(device);
P
Philipp Reisner 已提交
3571

3572 3573 3574
	if (!rv) {
		drbd_md_clear_flag(device, MDF_FULL_SYNC);
		drbd_md_sync(device);
P
Philipp Reisner 已提交
3575 3576 3577 3578 3579 3580 3581
	}

	return rv;
}

/**
 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3582
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3583 3584 3585
 *
 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
 */
3586
int drbd_bmio_clear_n_write(struct drbd_device *device) __must_hold(local)
P
Philipp Reisner 已提交
3587
{
3588
	drbd_resume_al(device);
3589 3590
	drbd_bm_clear_all(device);
	return drbd_bm_write(device);
P
Philipp Reisner 已提交
3591 3592
}

3593
static int w_bitmap_io(struct drbd_work *w, int unused)
P
Philipp Reisner 已提交
3594
{
3595 3596 3597
	struct drbd_device *device =
		container_of(w, struct drbd_device, bm_io_work.w);
	struct bm_io_work *work = &device->bm_io_work;
3598
	int rv = -EIO;
P
Philipp Reisner 已提交
3599

3600 3601 3602 3603 3604 3605
	if (work->flags != BM_LOCKED_CHANGE_ALLOWED) {
		int cnt = atomic_read(&device->ap_bio_cnt);
		if (cnt)
			drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n",
					cnt, work->why);
	}
P
Philipp Reisner 已提交
3606

3607 3608 3609 3610 3611
	if (get_ldev(device)) {
		drbd_bm_lock(device, work->why, work->flags);
		rv = work->io_fn(device);
		drbd_bm_unlock(device);
		put_ldev(device);
3612
	}
P
Philipp Reisner 已提交
3613

3614 3615
	clear_bit_unlock(BITMAP_IO, &device->flags);
	wake_up(&device->misc_wait);
P
Philipp Reisner 已提交
3616 3617

	if (work->done)
3618
		work->done(device, rv);
P
Philipp Reisner 已提交
3619

3620
	clear_bit(BITMAP_IO_QUEUED, &device->flags);
P
Philipp Reisner 已提交
3621
	work->why = NULL;
3622
	work->flags = 0;
P
Philipp Reisner 已提交
3623

3624
	return 0;
P
Philipp Reisner 已提交
3625 3626 3627 3628
}

/**
 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3629
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3630 3631 3632 3633 3634 3635 3636 3637
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @done:	callback to be called after the bitmap IO was performed
 * @why:	Descriptive text of the reason for doing the IO
 *
 * While IO on the bitmap happens we freeze application IO thus we ensure
 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
 * called from worker context. It MUST NOT be used while a previous such
 * work is still pending!
3638 3639 3640
 *
 * Its worker function encloses the call of io_fn() by get_ldev() and
 * put_ldev().
P
Philipp Reisner 已提交
3641
 */
3642
void drbd_queue_bitmap_io(struct drbd_device *device,
3643 3644
			  int (*io_fn)(struct drbd_device *),
			  void (*done)(struct drbd_device *, int),
3645
			  char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3646
{
3647
	D_ASSERT(device, current == first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3648

3649 3650 3651
	D_ASSERT(device, !test_bit(BITMAP_IO_QUEUED, &device->flags));
	D_ASSERT(device, !test_bit(BITMAP_IO, &device->flags));
	D_ASSERT(device, list_empty(&device->bm_io_work.w.list));
3652
	if (device->bm_io_work.why)
3653
		drbd_err(device, "FIXME going to queue '%s' but '%s' still pending?\n",
3654
			why, device->bm_io_work.why);
P
Philipp Reisner 已提交
3655

3656 3657 3658 3659
	device->bm_io_work.io_fn = io_fn;
	device->bm_io_work.done = done;
	device->bm_io_work.why = why;
	device->bm_io_work.flags = flags;
P
Philipp Reisner 已提交
3660

3661
	spin_lock_irq(&device->resource->req_lock);
3662
	set_bit(BITMAP_IO, &device->flags);
3663 3664 3665
	/* don't wait for pending application IO if the caller indicates that
	 * application IO does not conflict anyways. */
	if (flags == BM_LOCKED_CHANGE_ALLOWED || atomic_read(&device->ap_bio_cnt) == 0) {
3666
		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
3667 3668
			drbd_queue_work(&first_peer_device(device)->connection->sender_work,
					&device->bm_io_work.w);
P
Philipp Reisner 已提交
3669
	}
3670
	spin_unlock_irq(&device->resource->req_lock);
P
Philipp Reisner 已提交
3671 3672 3673 3674
}

/**
 * drbd_bitmap_io() -  Does an IO operation on the whole bitmap
3675
 * @device:	DRBD device.
P
Philipp Reisner 已提交
3676 3677 3678 3679 3680 3681
 * @io_fn:	IO callback to be called when bitmap IO is possible
 * @why:	Descriptive text of the reason for doing the IO
 *
 * freezes application IO while that the actual IO operations runs. This
 * functions MAY NOT be called from worker context.
 */
3682
int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *),
3683
		char *why, enum bm_flag flags)
P
Philipp Reisner 已提交
3684
{
3685 3686
	/* Only suspend io, if some operation is supposed to be locked out */
	const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST);
P
Philipp Reisner 已提交
3687 3688
	int rv;

3689
	D_ASSERT(device, current != first_peer_device(device)->connection->worker.task);
P
Philipp Reisner 已提交
3690

3691
	if (do_suspend_io)
3692
		drbd_suspend_io(device);
P
Philipp Reisner 已提交
3693

3694 3695 3696
	drbd_bm_lock(device, why, flags);
	rv = io_fn(device);
	drbd_bm_unlock(device);
P
Philipp Reisner 已提交
3697

3698
	if (do_suspend_io)
3699
		drbd_resume_io(device);
P
Philipp Reisner 已提交
3700 3701 3702 3703

	return rv;
}

3704
void drbd_md_set_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3705
{
3706 3707 3708
	if ((device->ldev->md.flags & flag) != flag) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags |= flag;
P
Philipp Reisner 已提交
3709 3710 3711
	}
}

3712
void drbd_md_clear_flag(struct drbd_device *device, int flag) __must_hold(local)
P
Philipp Reisner 已提交
3713
{
3714 3715 3716
	if ((device->ldev->md.flags & flag) != 0) {
		drbd_md_mark_dirty(device);
		device->ldev->md.flags &= ~flag;
P
Philipp Reisner 已提交
3717 3718 3719 3720 3721 3722 3723 3724 3725
	}
}
int drbd_md_test_flag(struct drbd_backing_dev *bdev, int flag)
{
	return (bdev->md.flags & flag) != 0;
}

static void md_sync_timer_fn(unsigned long data)
{
3726
	struct drbd_device *device = (struct drbd_device *) data;
3727
	drbd_device_post_work(device, MD_SYNC);
P
Philipp Reisner 已提交
3728 3729
}

3730
const char *cmdname(enum drbd_packet cmd)
3731 3732 3733 3734 3735 3736
{
	/* THINK may need to become several global tables
	 * when we want to support more than
	 * one PRO_VERSION */
	static const char *cmdnames[] = {
		[P_DATA]	        = "Data",
3737 3738
		[P_WSAME]	        = "WriteSame",
		[P_TRIM]	        = "Trim",
3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761
		[P_DATA_REPLY]	        = "DataReply",
		[P_RS_DATA_REPLY]	= "RSDataReply",
		[P_BARRIER]	        = "Barrier",
		[P_BITMAP]	        = "ReportBitMap",
		[P_BECOME_SYNC_TARGET]  = "BecomeSyncTarget",
		[P_BECOME_SYNC_SOURCE]  = "BecomeSyncSource",
		[P_UNPLUG_REMOTE]	= "UnplugRemote",
		[P_DATA_REQUEST]	= "DataRequest",
		[P_RS_DATA_REQUEST]     = "RSDataRequest",
		[P_SYNC_PARAM]	        = "SyncParam",
		[P_SYNC_PARAM89]	= "SyncParam89",
		[P_PROTOCOL]            = "ReportProtocol",
		[P_UUIDS]	        = "ReportUUIDs",
		[P_SIZES]	        = "ReportSizes",
		[P_STATE]	        = "ReportState",
		[P_SYNC_UUID]           = "ReportSyncUUID",
		[P_AUTH_CHALLENGE]      = "AuthChallenge",
		[P_AUTH_RESPONSE]	= "AuthResponse",
		[P_PING]		= "Ping",
		[P_PING_ACK]	        = "PingAck",
		[P_RECV_ACK]	        = "RecvAck",
		[P_WRITE_ACK]	        = "WriteAck",
		[P_RS_WRITE_ACK]	= "RSWriteAck",
3762
		[P_SUPERSEDED]          = "Superseded",
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776
		[P_NEG_ACK]	        = "NegAck",
		[P_NEG_DREPLY]	        = "NegDReply",
		[P_NEG_RS_DREPLY]	= "NegRSDReply",
		[P_BARRIER_ACK]	        = "BarrierAck",
		[P_STATE_CHG_REQ]       = "StateChgRequest",
		[P_STATE_CHG_REPLY]     = "StateChgReply",
		[P_OV_REQUEST]          = "OVRequest",
		[P_OV_REPLY]            = "OVReply",
		[P_OV_RESULT]           = "OVResult",
		[P_CSUM_RS_REQUEST]     = "CsumRSRequest",
		[P_RS_IS_IN_SYNC]	= "CsumRSIsInSync",
		[P_COMPRESSED_BITMAP]   = "CBitmap",
		[P_DELAY_PROBE]         = "DelayProbe",
		[P_OUT_OF_SYNC]		= "OutOfSync",
3777
		[P_RETRY_WRITE]		= "RetryWrite",
3778 3779 3780
		[P_RS_CANCEL]		= "RSCancel",
		[P_CONN_ST_CHG_REQ]	= "conn_st_chg_req",
		[P_CONN_ST_CHG_REPLY]	= "conn_st_chg_reply",
3781 3782
		[P_RETRY_WRITE]		= "retry_write",
		[P_PROTOCOL_UPDATE]	= "protocol_update",
3783 3784
		[P_RS_THIN_REQ]         = "rs_thin_req",
		[P_RS_DEALLOCATED]      = "rs_deallocated",
3785 3786 3787 3788 3789

		/* enum drbd_packet, but not commands - obsoleted flags:
		 *	P_MAY_IGNORE
		 *	P_MAX_OPT_CMD
		 */
3790 3791
	};

3792
	/* too big for the array: 0xfffX */
3793 3794 3795 3796
	if (cmd == P_INITIAL_META)
		return "InitialMeta";
	if (cmd == P_INITIAL_DATA)
		return "InitialData";
3797 3798
	if (cmd == P_CONNECTION_FEATURES)
		return "ConnectionFeatures";
A
Andreas Gruenbacher 已提交
3799
	if (cmd >= ARRAY_SIZE(cmdnames))
3800 3801 3802 3803
		return "Unknown";
	return cmdnames[cmd];
}

3804 3805
/**
 * drbd_wait_misc  -  wait for a request to make progress
3806
 * @device:	device associated with the request
3807 3808 3809
 * @i:		the struct drbd_interval embedded in struct drbd_request or
 *		struct drbd_peer_request
 */
3810
int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
3811
{
3812
	struct net_conf *nc;
3813 3814 3815
	DEFINE_WAIT(wait);
	long timeout;

3816
	rcu_read_lock();
3817
	nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
3818 3819
	if (!nc) {
		rcu_read_unlock();
3820
		return -ETIMEDOUT;
3821 3822 3823
	}
	timeout = nc->ko_count ? nc->timeout * HZ / 10 * nc->ko_count : MAX_SCHEDULE_TIMEOUT;
	rcu_read_unlock();
3824

3825
	/* Indicate to wake up device->misc_wait on progress.  */
3826
	i->waiting = true;
3827
	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
3828
	spin_unlock_irq(&device->resource->req_lock);
3829
	timeout = schedule_timeout(timeout);
3830
	finish_wait(&device->misc_wait, &wait);
3831
	spin_lock_irq(&device->resource->req_lock);
3832
	if (!timeout || device->state.conn < C_CONNECTED)
3833 3834 3835 3836
		return -ETIMEDOUT;
	if (signal_pending(current))
		return -ERESTARTSYS;
	return 0;
P
Philipp Reisner 已提交
3837 3838
}

3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859
void lock_all_resources(void)
{
	struct drbd_resource *resource;
	int __maybe_unused i = 0;

	mutex_lock(&resources_mutex);
	local_irq_disable();
	for_each_resource(resource, &drbd_resources)
		spin_lock_nested(&resource->req_lock, i++);
}

void unlock_all_resources(void)
{
	struct drbd_resource *resource;

	for_each_resource(resource, &drbd_resources)
		spin_unlock(&resource->req_lock);
	local_irq_enable();
	mutex_unlock(&resources_mutex);
}

P
Philipp Reisner 已提交
3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880
#ifdef CONFIG_DRBD_FAULT_INJECTION
/* Fault insertion support including random number generator shamelessly
 * stolen from kernel/rcutorture.c */
struct fault_random_state {
	unsigned long state;
	unsigned long count;
};

#define FAULT_RANDOM_MULT 39916801  /* prime */
#define FAULT_RANDOM_ADD	479001701 /* prime */
#define FAULT_RANDOM_REFRESH 10000

/*
 * Crude but fast random-number generator.  Uses a linear congruential
 * generator, with occasional help from get_random_bytes().
 */
static unsigned long
_drbd_fault_random(struct fault_random_state *rsp)
{
	long refresh;

3881
	if (!rsp->count--) {
P
Philipp Reisner 已提交
3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900
		get_random_bytes(&refresh, sizeof(refresh));
		rsp->state += refresh;
		rsp->count = FAULT_RANDOM_REFRESH;
	}
	rsp->state = rsp->state * FAULT_RANDOM_MULT + FAULT_RANDOM_ADD;
	return swahw32(rsp->state);
}

static char *
_drbd_fault_str(unsigned int type) {
	static char *_faults[] = {
		[DRBD_FAULT_MD_WR] = "Meta-data write",
		[DRBD_FAULT_MD_RD] = "Meta-data read",
		[DRBD_FAULT_RS_WR] = "Resync write",
		[DRBD_FAULT_RS_RD] = "Resync read",
		[DRBD_FAULT_DT_WR] = "Data write",
		[DRBD_FAULT_DT_RD] = "Data read",
		[DRBD_FAULT_DT_RA] = "Data read ahead",
		[DRBD_FAULT_BM_ALLOC] = "BM allocation",
3901 3902
		[DRBD_FAULT_AL_EE] = "EE allocation",
		[DRBD_FAULT_RECEIVE] = "receive data corruption",
P
Philipp Reisner 已提交
3903 3904 3905 3906 3907 3908
	};

	return (type < DRBD_FAULT_MAX) ? _faults[type] : "**Unknown**";
}

unsigned int
3909
_drbd_insert_fault(struct drbd_device *device, unsigned int type)
P
Philipp Reisner 已提交
3910 3911 3912 3913
{
	static struct fault_random_state rrs = {0, 0};

	unsigned int ret = (
3914 3915 3916
		(drbd_fault_devs == 0 ||
			((1 << device_to_minor(device)) & drbd_fault_devs) != 0) &&
		(((_drbd_fault_random(&rrs) % 100) + 1) <= drbd_fault_rate));
P
Philipp Reisner 已提交
3917 3918

	if (ret) {
3919
		drbd_fault_count++;
P
Philipp Reisner 已提交
3920

3921
		if (__ratelimit(&drbd_ratelimit_state))
3922
			drbd_warn(device, "***Simulating %s failure\n",
P
Philipp Reisner 已提交
3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937
				_drbd_fault_str(type));
	}

	return ret;
}
#endif

const char *drbd_buildtag(void)
{
	/* DRBD built from external sources has here a reference to the
	   git hash of the source code. */

	static char buildtag[38] = "\0uilt-in";

	if (buildtag[0] == 0) {
C
Cong Wang 已提交
3938 3939 3940 3941
#ifdef MODULE
		sprintf(buildtag, "srcversion: %-24s", THIS_MODULE->srcversion);
#else
		buildtag[0] = 'b';
P
Philipp Reisner 已提交
3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954
#endif
	}

	return buildtag;
}

module_init(drbd_init)
module_exit(drbd_cleanup)

EXPORT_SYMBOL(drbd_conn_str);
EXPORT_SYMBOL(drbd_role_str);
EXPORT_SYMBOL(drbd_disk_str);
EXPORT_SYMBOL(drbd_set_st_err_str);