messenger.c 85.4 KB
Newer Older
1
#include <linux/ceph/ceph_debug.h>
S
Sage Weil 已提交
2 3 4 5 6 7 8

#include <linux/crc32c.h>
#include <linux/ctype.h>
#include <linux/highmem.h>
#include <linux/inet.h>
#include <linux/kthread.h>
#include <linux/net.h>
9
#include <linux/nsproxy.h>
10
#include <linux/slab.h>
S
Sage Weil 已提交
11 12
#include <linux/socket.h>
#include <linux/string.h>
13
#ifdef	CONFIG_BLOCK
14
#include <linux/bio.h>
15
#endif	/* CONFIG_BLOCK */
N
Noah Watkins 已提交
16
#include <linux/dns_resolver.h>
S
Sage Weil 已提交
17 18
#include <net/tcp.h>

I
Ilya Dryomov 已提交
19
#include <linux/ceph/ceph_features.h>
20 21 22 23
#include <linux/ceph/libceph.h>
#include <linux/ceph/messenger.h>
#include <linux/ceph/decode.h>
#include <linux/ceph/pagelist.h>
24
#include <linux/export.h>
S
Sage Weil 已提交
25 26 27 28 29 30 31 32 33 34

/*
 * Ceph uses the messenger to exchange ceph_msg messages with other
 * hosts in the system.  The messenger provides ordered and reliable
 * delivery.  We tolerate TCP disconnects by reconnecting (with
 * exponential backoff) in the case of a fault (disconnection, bad
 * crc, protocol error).  Acks allow sent messages to be discarded by
 * the sender.
 */

A
Alex Elder 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
/*
 * We track the state of the socket on a given connection using
 * values defined below.  The transition to a new socket state is
 * handled by a function which verifies we aren't coming from an
 * unexpected state.
 *
 *      --------
 *      | NEW* |  transient initial state
 *      --------
 *          | con_sock_state_init()
 *          v
 *      ----------
 *      | CLOSED |  initialized, but no socket (and no
 *      ----------  TCP connection)
 *       ^      \
 *       |       \ con_sock_state_connecting()
 *       |        ----------------------
 *       |                              \
 *       + con_sock_state_closed()       \
54 55 56 57 58 59 60 61 62 63 64
 *       |+---------------------------    \
 *       | \                          \    \
 *       |  -----------                \    \
 *       |  | CLOSING |  socket event;  \    \
 *       |  -----------  await close     \    \
 *       |       ^                        \   |
 *       |       |                         \  |
 *       |       + con_sock_state_closing() \ |
 *       |      / \                         | |
 *       |     /   ---------------          | |
 *       |    /                   \         v v
A
Alex Elder 已提交
65 66 67 68 69 70 71 72 73 74 75
 *       |   /                    --------------
 *       |  /    -----------------| CONNECTING |  socket created, TCP
 *       |  |   /                 --------------  connect initiated
 *       |  |   | con_sock_state_connected()
 *       |  |   v
 *      -------------
 *      | CONNECTED |  TCP connection established
 *      -------------
 *
 * State values for ceph_connection->sock_state; NEW is assumed to be 0.
 */
76 77 78 79 80 81 82

#define CON_SOCK_STATE_NEW		0	/* -> CLOSED */
#define CON_SOCK_STATE_CLOSED		1	/* -> CONNECTING */
#define CON_SOCK_STATE_CONNECTING	2	/* -> CONNECTED or -> CLOSING */
#define CON_SOCK_STATE_CONNECTED	3	/* -> CLOSING or -> CLOSED */
#define CON_SOCK_STATE_CLOSING		4	/* -> CLOSED */

83 84 85 86 87 88 89 90 91 92
/*
 * connection states
 */
#define CON_STATE_CLOSED        1  /* -> PREOPEN */
#define CON_STATE_PREOPEN       2  /* -> CONNECTING, CLOSED */
#define CON_STATE_CONNECTING    3  /* -> NEGOTIATING, CLOSED */
#define CON_STATE_NEGOTIATING   4  /* -> OPEN, CLOSED */
#define CON_STATE_OPEN          5  /* -> STANDBY, CLOSED */
#define CON_STATE_STANDBY       6  /* -> PREOPEN, CLOSED */

S
Sage Weil 已提交
93 94 95 96 97 98 99 100 101
/*
 * ceph_connection flag bits
 */
#define CON_FLAG_LOSSYTX           0  /* we can close channel or drop
				       * messages on errors */
#define CON_FLAG_KEEPALIVE_PENDING 1  /* we need to send a keepalive */
#define CON_FLAG_WRITE_PENDING	   2  /* we have data ready to send */
#define CON_FLAG_SOCK_CLOSED	   3  /* socket state changed to closed */
#define CON_FLAG_BACKOFF           4  /* need to retry queuing delayed work */
102

103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
static bool con_flag_valid(unsigned long con_flag)
{
	switch (con_flag) {
	case CON_FLAG_LOSSYTX:
	case CON_FLAG_KEEPALIVE_PENDING:
	case CON_FLAG_WRITE_PENDING:
	case CON_FLAG_SOCK_CLOSED:
	case CON_FLAG_BACKOFF:
		return true;
	default:
		return false;
	}
}

static void con_flag_clear(struct ceph_connection *con, unsigned long con_flag)
{
	BUG_ON(!con_flag_valid(con_flag));

	clear_bit(con_flag, &con->flags);
}

static void con_flag_set(struct ceph_connection *con, unsigned long con_flag)
{
	BUG_ON(!con_flag_valid(con_flag));

	set_bit(con_flag, &con->flags);
}

static bool con_flag_test(struct ceph_connection *con, unsigned long con_flag)
{
	BUG_ON(!con_flag_valid(con_flag));

	return test_bit(con_flag, &con->flags);
}

static bool con_flag_test_and_clear(struct ceph_connection *con,
					unsigned long con_flag)
{
	BUG_ON(!con_flag_valid(con_flag));

	return test_and_clear_bit(con_flag, &con->flags);
}

static bool con_flag_test_and_set(struct ceph_connection *con,
					unsigned long con_flag)
{
	BUG_ON(!con_flag_valid(con_flag));

	return test_and_set_bit(con_flag, &con->flags);
}

154 155 156
/* Slab caches for frequently-allocated structures */

static struct kmem_cache	*ceph_msg_cache;
157
static struct kmem_cache	*ceph_msg_data_cache;
158

S
Sage Weil 已提交
159 160 161 162
/* static tag bytes (protocol control messages) */
static char tag_msg = CEPH_MSGR_TAG_MSG;
static char tag_ack = CEPH_MSGR_TAG_ACK;
static char tag_keepalive = CEPH_MSGR_TAG_KEEPALIVE;
163
static char tag_keepalive2 = CEPH_MSGR_TAG_KEEPALIVE2;
S
Sage Weil 已提交
164

165 166 167 168
#ifdef CONFIG_LOCKDEP
static struct lock_class_key socket_class;
#endif

A
Alex Elder 已提交
169 170 171 172 173
/*
 * When skipping (ignoring) a block of input we read it into a "skip
 * buffer," which is this many bytes in size.
 */
#define SKIP_BUF_SIZE	1024
S
Sage Weil 已提交
174 175

static void queue_con(struct ceph_connection *con);
176
static void cancel_con(struct ceph_connection *con);
177
static void ceph_con_workfn(struct work_struct *);
178
static void con_fault(struct ceph_connection *con);
S
Sage Weil 已提交
179 180

/*
181 182
 * Nicely render a sockaddr as a string.  An array of formatted
 * strings is used, to approximate reentrancy.
S
Sage Weil 已提交
183
 */
184 185 186 187 188 189 190
#define ADDR_STR_COUNT_LOG	5	/* log2(# address strings in array) */
#define ADDR_STR_COUNT		(1 << ADDR_STR_COUNT_LOG)
#define ADDR_STR_COUNT_MASK	(ADDR_STR_COUNT - 1)
#define MAX_ADDR_STR_LEN	64	/* 54 is enough */

static char addr_str[ADDR_STR_COUNT][MAX_ADDR_STR_LEN];
static atomic_t addr_str_seq = ATOMIC_INIT(0);
S
Sage Weil 已提交
191

192 193
static struct page *zero_page;		/* used in certain error cases */

194
const char *ceph_pr_addr(const struct sockaddr_storage *ss)
S
Sage Weil 已提交
195 196 197
{
	int i;
	char *s;
A
Alex Elder 已提交
198 199
	struct sockaddr_in *in4 = (struct sockaddr_in *) ss;
	struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) ss;
S
Sage Weil 已提交
200

201
	i = atomic_inc_return(&addr_str_seq) & ADDR_STR_COUNT_MASK;
S
Sage Weil 已提交
202 203 204 205
	s = addr_str[i];

	switch (ss->ss_family) {
	case AF_INET:
A
Alex Elder 已提交
206 207
		snprintf(s, MAX_ADDR_STR_LEN, "%pI4:%hu", &in4->sin_addr,
			 ntohs(in4->sin_port));
S
Sage Weil 已提交
208 209 210
		break;

	case AF_INET6:
A
Alex Elder 已提交
211 212
		snprintf(s, MAX_ADDR_STR_LEN, "[%pI6c]:%hu", &in6->sin6_addr,
			 ntohs(in6->sin6_port));
S
Sage Weil 已提交
213 214 215
		break;

	default:
A
Alex Elder 已提交
216 217
		snprintf(s, MAX_ADDR_STR_LEN, "(unknown sockaddr family %hu)",
			 ss->ss_family);
S
Sage Weil 已提交
218 219 220 221
	}

	return s;
}
222
EXPORT_SYMBOL(ceph_pr_addr);
S
Sage Weil 已提交
223

224 225 226 227 228 229
static void encode_my_addr(struct ceph_messenger *msgr)
{
	memcpy(&msgr->my_enc_addr, &msgr->inst.addr, sizeof(msgr->my_enc_addr));
	ceph_encode_addr(&msgr->my_enc_addr);
}

S
Sage Weil 已提交
230 231 232
/*
 * work queue for all reading and writing to/from the socket.
 */
A
Alex Elder 已提交
233
static struct workqueue_struct *ceph_msgr_wq;
S
Sage Weil 已提交
234

235 236 237 238 239 240
static int ceph_msgr_slab_init(void)
{
	BUG_ON(ceph_msg_cache);
	ceph_msg_cache = kmem_cache_create("ceph_msg",
					sizeof (struct ceph_msg),
					__alignof__(struct ceph_msg), 0, NULL);
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256

	if (!ceph_msg_cache)
		return -ENOMEM;

	BUG_ON(ceph_msg_data_cache);
	ceph_msg_data_cache = kmem_cache_create("ceph_msg_data",
					sizeof (struct ceph_msg_data),
					__alignof__(struct ceph_msg_data),
					0, NULL);
	if (ceph_msg_data_cache)
		return 0;

	kmem_cache_destroy(ceph_msg_cache);
	ceph_msg_cache = NULL;

	return -ENOMEM;
257 258 259 260
}

static void ceph_msgr_slab_exit(void)
{
261 262 263 264
	BUG_ON(!ceph_msg_data_cache);
	kmem_cache_destroy(ceph_msg_data_cache);
	ceph_msg_data_cache = NULL;

265 266 267 268 269
	BUG_ON(!ceph_msg_cache);
	kmem_cache_destroy(ceph_msg_cache);
	ceph_msg_cache = NULL;
}

A
Alex Elder 已提交
270
static void _ceph_msgr_exit(void)
271
{
A
Alex Elder 已提交
272
	if (ceph_msgr_wq) {
273
		destroy_workqueue(ceph_msgr_wq);
A
Alex Elder 已提交
274 275
		ceph_msgr_wq = NULL;
	}
276 277 278 279

	BUG_ON(zero_page == NULL);
	page_cache_release(zero_page);
	zero_page = NULL;
280 281

	ceph_msgr_slab_exit();
282 283
}

284
int ceph_msgr_init(void)
S
Sage Weil 已提交
285
{
286 287 288
	if (ceph_msgr_slab_init())
		return -ENOMEM;

289 290 291 292
	BUG_ON(zero_page != NULL);
	zero_page = ZERO_PAGE(0);
	page_cache_get(zero_page);

293 294 295 296 297
	/*
	 * The number of active work items is limited by the number of
	 * connections, so leave @max_active at default.
	 */
	ceph_msgr_wq = alloc_workqueue("ceph-msgr", WQ_MEM_RECLAIM, 0);
298 299
	if (ceph_msgr_wq)
		return 0;
300

301 302
	pr_err("msgr_init failed to create workqueue\n");
	_ceph_msgr_exit();
303

304
	return -ENOMEM;
S
Sage Weil 已提交
305
}
306
EXPORT_SYMBOL(ceph_msgr_init);
S
Sage Weil 已提交
307 308 309

void ceph_msgr_exit(void)
{
310 311
	BUG_ON(ceph_msgr_wq == NULL);

312
	_ceph_msgr_exit();
S
Sage Weil 已提交
313
}
314
EXPORT_SYMBOL(ceph_msgr_exit);
S
Sage Weil 已提交
315

Y
Yehuda Sadeh 已提交
316
void ceph_msgr_flush(void)
317 318 319
{
	flush_workqueue(ceph_msgr_wq);
}
320
EXPORT_SYMBOL(ceph_msgr_flush);
321

322 323 324 325 326 327 328 329 330
/* Connection socket state transition functions */

static void con_sock_state_init(struct ceph_connection *con)
{
	int old_state;

	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
	if (WARN_ON(old_state != CON_SOCK_STATE_NEW))
		printk("%s: unexpected old state %d\n", __func__, old_state);
331 332
	dout("%s con %p sock %d -> %d\n", __func__, con, old_state,
	     CON_SOCK_STATE_CLOSED);
333 334 335 336 337 338 339 340 341
}

static void con_sock_state_connecting(struct ceph_connection *con)
{
	int old_state;

	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTING);
	if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED))
		printk("%s: unexpected old state %d\n", __func__, old_state);
342 343
	dout("%s con %p sock %d -> %d\n", __func__, con, old_state,
	     CON_SOCK_STATE_CONNECTING);
344 345 346 347 348 349 350 351 352
}

static void con_sock_state_connected(struct ceph_connection *con)
{
	int old_state;

	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED);
	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING))
		printk("%s: unexpected old state %d\n", __func__, old_state);
353 354
	dout("%s con %p sock %d -> %d\n", __func__, con, old_state,
	     CON_SOCK_STATE_CONNECTED);
355 356 357 358 359 360 361 362 363 364 365
}

static void con_sock_state_closing(struct ceph_connection *con)
{
	int old_state;

	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING);
	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING &&
			old_state != CON_SOCK_STATE_CONNECTED &&
			old_state != CON_SOCK_STATE_CLOSING))
		printk("%s: unexpected old state %d\n", __func__, old_state);
366 367
	dout("%s con %p sock %d -> %d\n", __func__, con, old_state,
	     CON_SOCK_STATE_CLOSING);
368 369 370 371 372 373 374 375
}

static void con_sock_state_closed(struct ceph_connection *con)
{
	int old_state;

	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED &&
376
		    old_state != CON_SOCK_STATE_CLOSING &&
377 378
		    old_state != CON_SOCK_STATE_CONNECTING &&
		    old_state != CON_SOCK_STATE_CLOSED))
379
		printk("%s: unexpected old state %d\n", __func__, old_state);
380 381
	dout("%s con %p sock %d -> %d\n", __func__, con, old_state,
	     CON_SOCK_STATE_CLOSED);
382
}
383

S
Sage Weil 已提交
384 385 386 387 388
/*
 * socket callback functions
 */

/* data available on socket, or listen socket received a connect */
389
static void ceph_sock_data_ready(struct sock *sk)
S
Sage Weil 已提交
390
{
A
Alex Elder 已提交
391
	struct ceph_connection *con = sk->sk_user_data;
392 393 394
	if (atomic_read(&con->msgr->stopping)) {
		return;
	}
A
Alex Elder 已提交
395

S
Sage Weil 已提交
396
	if (sk->sk_state != TCP_CLOSE_WAIT) {
A
Alex Elder 已提交
397
		dout("%s on %p state = %lu, queueing work\n", __func__,
S
Sage Weil 已提交
398 399 400 401 402 403
		     con, con->state);
		queue_con(con);
	}
}

/* socket has buffer space for writing */
A
Alex Elder 已提交
404
static void ceph_sock_write_space(struct sock *sk)
S
Sage Weil 已提交
405
{
A
Alex Elder 已提交
406
	struct ceph_connection *con = sk->sk_user_data;
S
Sage Weil 已提交
407

408 409
	/* only queue to workqueue if there is data we want to write,
	 * and there is sufficient space in the socket buffer to accept
A
Alex Elder 已提交
410
	 * more data.  clear SOCK_NOSPACE so that ceph_sock_write_space()
411 412 413 414
	 * doesn't get called again until try_write() fills the socket
	 * buffer. See net/ipv4/tcp_input.c:tcp_check_space()
	 * and net/core/stream.c:sk_stream_write_space().
	 */
415
	if (con_flag_test(con, CON_FLAG_WRITE_PENDING)) {
416
		if (sk_stream_is_writeable(sk)) {
A
Alex Elder 已提交
417
			dout("%s %p queueing write work\n", __func__, con);
418 419 420
			clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
			queue_con(con);
		}
S
Sage Weil 已提交
421
	} else {
A
Alex Elder 已提交
422
		dout("%s %p nothing to write\n", __func__, con);
S
Sage Weil 已提交
423 424 425 426
	}
}

/* socket's state has changed */
A
Alex Elder 已提交
427
static void ceph_sock_state_change(struct sock *sk)
S
Sage Weil 已提交
428
{
A
Alex Elder 已提交
429
	struct ceph_connection *con = sk->sk_user_data;
S
Sage Weil 已提交
430

A
Alex Elder 已提交
431
	dout("%s %p state = %lu sk_state = %u\n", __func__,
S
Sage Weil 已提交
432 433 434 435
	     con, con->state, sk->sk_state);

	switch (sk->sk_state) {
	case TCP_CLOSE:
A
Alex Elder 已提交
436
		dout("%s TCP_CLOSE\n", __func__);
S
Sage Weil 已提交
437
	case TCP_CLOSE_WAIT:
A
Alex Elder 已提交
438
		dout("%s TCP_CLOSE_WAIT\n", __func__);
439
		con_sock_state_closing(con);
440
		con_flag_set(con, CON_FLAG_SOCK_CLOSED);
441
		queue_con(con);
S
Sage Weil 已提交
442 443
		break;
	case TCP_ESTABLISHED:
A
Alex Elder 已提交
444
		dout("%s TCP_ESTABLISHED\n", __func__);
445
		con_sock_state_connected(con);
S
Sage Weil 已提交
446 447
		queue_con(con);
		break;
A
Alex Elder 已提交
448 449
	default:	/* Everything else is uninteresting */
		break;
S
Sage Weil 已提交
450 451 452 453 454 455 456 457 458 459
	}
}

/*
 * set up socket callbacks
 */
static void set_sock_callbacks(struct socket *sock,
			       struct ceph_connection *con)
{
	struct sock *sk = sock->sk;
A
Alex Elder 已提交
460
	sk->sk_user_data = con;
A
Alex Elder 已提交
461 462 463
	sk->sk_data_ready = ceph_sock_data_ready;
	sk->sk_write_space = ceph_sock_write_space;
	sk->sk_state_change = ceph_sock_state_change;
S
Sage Weil 已提交
464 465 466 467 468 469 470 471 472 473
}


/*
 * socket helpers
 */

/*
 * initiate connection to a remote socket.
 */
474
static int ceph_tcp_connect(struct ceph_connection *con)
S
Sage Weil 已提交
475
{
S
Sage Weil 已提交
476
	struct sockaddr_storage *paddr = &con->peer_addr.in_addr;
S
Sage Weil 已提交
477 478 479 480
	struct socket *sock;
	int ret;

	BUG_ON(con->sock);
481
	ret = sock_create_kern(read_pnet(&con->msgr->net), paddr->ss_family,
482
			       SOCK_STREAM, IPPROTO_TCP, &sock);
S
Sage Weil 已提交
483
	if (ret)
484
		return ret;
485
	sock->sk->sk_allocation = GFP_NOFS;
S
Sage Weil 已提交
486

487 488 489 490
#ifdef CONFIG_LOCKDEP
	lockdep_set_class(&sock->sk->sk_lock, &socket_class);
#endif

S
Sage Weil 已提交
491 492
	set_sock_callbacks(sock, con);

493
	dout("connect %s\n", ceph_pr_addr(&con->peer_addr.in_addr));
S
Sage Weil 已提交
494

495
	con_sock_state_connecting(con);
S
Sage Weil 已提交
496 497
	ret = sock->ops->connect(sock, (struct sockaddr *)paddr, sizeof(*paddr),
				 O_NONBLOCK);
S
Sage Weil 已提交
498 499
	if (ret == -EINPROGRESS) {
		dout("connect %s EINPROGRESS sk_state = %u\n",
500
		     ceph_pr_addr(&con->peer_addr.in_addr),
S
Sage Weil 已提交
501
		     sock->sk->sk_state);
502
	} else if (ret < 0) {
S
Sage Weil 已提交
503
		pr_err("connect %s error %d\n",
504
		       ceph_pr_addr(&con->peer_addr.in_addr), ret);
S
Sage Weil 已提交
505
		sock_release(sock);
506
		return ret;
507
	}
508

509
	if (ceph_test_opt(from_msgr(con->msgr), TCP_NODELAY)) {
C
Chaitanya Huilgol 已提交
510 511 512 513 514 515 516 517 518
		int optval = 1;

		ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY,
					(char *)&optval, sizeof(optval));
		if (ret)
			pr_err("kernel_setsockopt(TCP_NODELAY) failed: %d",
			       ret);
	}

519
	con->sock = sock;
520
	return 0;
S
Sage Weil 已提交
521 522 523 524 525 526
}

static int ceph_tcp_recvmsg(struct socket *sock, void *buf, size_t len)
{
	struct kvec iov = {buf, len};
	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL };
527
	int r;
S
Sage Weil 已提交
528

529 530 531 532
	r = kernel_recvmsg(sock, &msg, &iov, 1, len, msg.msg_flags);
	if (r == -EAGAIN)
		r = 0;
	return r;
S
Sage Weil 已提交
533 534
}

535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static int ceph_tcp_recvpage(struct socket *sock, struct page *page,
		     int page_offset, size_t length)
{
	void *kaddr;
	int ret;

	BUG_ON(page_offset + length > PAGE_SIZE);

	kaddr = kmap(page);
	BUG_ON(!kaddr);
	ret = ceph_tcp_recvmsg(sock, kaddr + page_offset, length);
	kunmap(page);

	return ret;
}

S
Sage Weil 已提交
551 552 553 554 555 556 557 558
/*
 * write something.  @more is true if caller will be sending more data
 * shortly.
 */
static int ceph_tcp_sendmsg(struct socket *sock, struct kvec *iov,
		     size_t kvlen, size_t len, int more)
{
	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL };
559
	int r;
S
Sage Weil 已提交
560 561 562 563 564 565

	if (more)
		msg.msg_flags |= MSG_MORE;
	else
		msg.msg_flags |= MSG_EOR;  /* superfluous, but what the hell */

566 567 568 569
	r = kernel_sendmsg(sock, &msg, iov, kvlen, len);
	if (r == -EAGAIN)
		r = 0;
	return r;
S
Sage Weil 已提交
570 571
}

572
static int __ceph_tcp_sendpage(struct socket *sock, struct page *page,
573
		     int offset, size_t size, bool more)
574 575 576 577 578 579 580 581 582 583 584
{
	int flags = MSG_DONTWAIT | MSG_NOSIGNAL | (more ? MSG_MORE : MSG_EOR);
	int ret;

	ret = kernel_sendpage(sock, page, offset, size, flags);
	if (ret == -EAGAIN)
		ret = 0;

	return ret;
}

585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
static int ceph_tcp_sendpage(struct socket *sock, struct page *page,
		     int offset, size_t size, bool more)
{
	int ret;
	struct kvec iov;

	/* sendpage cannot properly handle pages with page_count == 0,
	 * we need to fallback to sendmsg if that's the case */
	if (page_count(page) >= 1)
		return __ceph_tcp_sendpage(sock, page, offset, size, more);

	iov.iov_base = kmap(page) + offset;
	iov.iov_len = size;
	ret = ceph_tcp_sendmsg(sock, &iov, 1, size, more);
	kunmap(page);

	return ret;
}
S
Sage Weil 已提交
603 604 605 606 607 608

/*
 * Shutdown/close the socket for the given connection.
 */
static int con_close_socket(struct ceph_connection *con)
{
609
	int rc = 0;
S
Sage Weil 已提交
610 611

	dout("con_close_socket on %p sock %p\n", con, con->sock);
612 613 614 615 616
	if (con->sock) {
		rc = con->sock->ops->shutdown(con->sock, SHUT_RDWR);
		sock_release(con->sock);
		con->sock = NULL;
	}
617 618

	/*
S
Sage Weil 已提交
619
	 * Forcibly clear the SOCK_CLOSED flag.  It gets set
620 621 622 623
	 * independent of the connection mutex, and we could have
	 * received a socket close event before we had the chance to
	 * shut the socket down.
	 */
624
	con_flag_clear(con, CON_FLAG_SOCK_CLOSED);
625

626
	con_sock_state_closed(con);
S
Sage Weil 已提交
627 628 629 630 631 632 633 634 635 636
	return rc;
}

/*
 * Reset a connection.  Discard all incoming and outgoing messages
 * and clear *_seq state.
 */
static void ceph_msg_remove(struct ceph_msg *msg)
{
	list_del_init(&msg->list_head);
637

S
Sage Weil 已提交
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
	ceph_msg_put(msg);
}
static void ceph_msg_remove_list(struct list_head *head)
{
	while (!list_empty(head)) {
		struct ceph_msg *msg = list_first_entry(head, struct ceph_msg,
							list_head);
		ceph_msg_remove(msg);
	}
}

static void reset_connection(struct ceph_connection *con)
{
	/* reset connection, out_queue, msg_ and connect_seq */
	/* discard existing out_queue and msg_seq */
653
	dout("reset_connection %p\n", con);
S
Sage Weil 已提交
654 655 656
	ceph_msg_remove_list(&con->out_queue);
	ceph_msg_remove_list(&con->out_sent);

657
	if (con->in_msg) {
658
		BUG_ON(con->in_msg->con != con);
659 660 661 662
		ceph_msg_put(con->in_msg);
		con->in_msg = NULL;
	}

S
Sage Weil 已提交
663 664
	con->connect_seq = 0;
	con->out_seq = 0;
665
	if (con->out_msg) {
666
		BUG_ON(con->out_msg->con != con);
667 668 669
		ceph_msg_put(con->out_msg);
		con->out_msg = NULL;
	}
S
Sage Weil 已提交
670
	con->in_seq = 0;
671
	con->in_seq_acked = 0;
I
Ilya Dryomov 已提交
672 673

	con->out_skip = 0;
S
Sage Weil 已提交
674 675 676 677 678 679 680
}

/*
 * mark a peer down.  drop any open connections.
 */
void ceph_con_close(struct ceph_connection *con)
{
681
	mutex_lock(&con->mutex);
682 683
	dout("con_close %p peer %s\n", con,
	     ceph_pr_addr(&con->peer_addr.in_addr));
684
	con->state = CON_STATE_CLOSED;
685

686 687 688 689
	con_flag_clear(con, CON_FLAG_LOSSYTX);	/* so we retry next connect */
	con_flag_clear(con, CON_FLAG_KEEPALIVE_PENDING);
	con_flag_clear(con, CON_FLAG_WRITE_PENDING);
	con_flag_clear(con, CON_FLAG_BACKOFF);
690

S
Sage Weil 已提交
691
	reset_connection(con);
S
Sage Weil 已提交
692
	con->peer_global_seq = 0;
693
	cancel_con(con);
694
	con_close_socket(con);
695
	mutex_unlock(&con->mutex);
S
Sage Weil 已提交
696
}
697
EXPORT_SYMBOL(ceph_con_close);
S
Sage Weil 已提交
698 699 700 701

/*
 * Reopen a closed connection, with a new peer address.
 */
702 703 704
void ceph_con_open(struct ceph_connection *con,
		   __u8 entity_type, __u64 entity_num,
		   struct ceph_entity_addr *addr)
S
Sage Weil 已提交
705
{
706
	mutex_lock(&con->mutex);
707
	dout("con_open %p %s\n", con, ceph_pr_addr(&addr->in_addr));
708

709
	WARN_ON(con->state != CON_STATE_CLOSED);
710
	con->state = CON_STATE_PREOPEN;
711

712 713 714
	con->peer_name.type = (__u8) entity_type;
	con->peer_name.num = cpu_to_le64(entity_num);

S
Sage Weil 已提交
715
	memcpy(&con->peer_addr, addr, sizeof(*addr));
716
	con->delay = 0;      /* reset backoff memory */
717
	mutex_unlock(&con->mutex);
S
Sage Weil 已提交
718 719
	queue_con(con);
}
720
EXPORT_SYMBOL(ceph_con_open);
S
Sage Weil 已提交
721

722 723 724 725 726 727 728 729
/*
 * return true if this connection ever successfully opened
 */
bool ceph_con_opened(struct ceph_connection *con)
{
	return con->connect_seq > 0;
}

S
Sage Weil 已提交
730 731 732
/*
 * initialize a new connection.
 */
733 734
void ceph_con_init(struct ceph_connection *con, void *private,
	const struct ceph_connection_operations *ops,
735
	struct ceph_messenger *msgr)
S
Sage Weil 已提交
736 737 738
{
	dout("con_init %p\n", con);
	memset(con, 0, sizeof(*con));
739 740
	con->private = private;
	con->ops = ops;
S
Sage Weil 已提交
741
	con->msgr = msgr;
742 743 744

	con_sock_state_init(con);

745
	mutex_init(&con->mutex);
S
Sage Weil 已提交
746 747
	INIT_LIST_HEAD(&con->out_queue);
	INIT_LIST_HEAD(&con->out_sent);
748
	INIT_DELAYED_WORK(&con->work, ceph_con_workfn);
749

750
	con->state = CON_STATE_CLOSED;
S
Sage Weil 已提交
751
}
752
EXPORT_SYMBOL(ceph_con_init);
S
Sage Weil 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770


/*
 * We maintain a global counter to order connection attempts.  Get
 * a unique seq greater than @gt.
 */
static u32 get_global_seq(struct ceph_messenger *msgr, u32 gt)
{
	u32 ret;

	spin_lock(&msgr->global_seq_lock);
	if (msgr->global_seq < gt)
		msgr->global_seq = gt;
	ret = ++msgr->global_seq;
	spin_unlock(&msgr->global_seq_lock);
	return ret;
}

771
static void con_out_kvec_reset(struct ceph_connection *con)
772
{
I
Ilya Dryomov 已提交
773 774
	BUG_ON(con->out_skip);

775 776 777 778 779
	con->out_kvec_left = 0;
	con->out_kvec_bytes = 0;
	con->out_kvec_cur = &con->out_kvec[0];
}

780
static void con_out_kvec_add(struct ceph_connection *con,
781 782
				size_t size, void *data)
{
I
Ilya Dryomov 已提交
783
	int index = con->out_kvec_left;
784

I
Ilya Dryomov 已提交
785
	BUG_ON(con->out_skip);
786 787 788 789 790 791 792
	BUG_ON(index >= ARRAY_SIZE(con->out_kvec));

	con->out_kvec[index].iov_len = size;
	con->out_kvec[index].iov_base = data;
	con->out_kvec_left++;
	con->out_kvec_bytes += size;
}
S
Sage Weil 已提交
793

I
Ilya Dryomov 已提交
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
/*
 * Chop off a kvec from the end.  Return residual number of bytes for
 * that kvec, i.e. how many bytes would have been written if the kvec
 * hadn't been nuked.
 */
static int con_out_kvec_skip(struct ceph_connection *con)
{
	int off = con->out_kvec_cur - con->out_kvec;
	int skip = 0;

	if (con->out_kvec_bytes > 0) {
		skip = con->out_kvec[off + con->out_kvec_left - 1].iov_len;
		BUG_ON(con->out_kvec_bytes < skip);
		BUG_ON(!con->out_kvec_left);
		con->out_kvec_bytes -= skip;
		con->out_kvec_left--;
	}

	return skip;
}

815
#ifdef CONFIG_BLOCK
816 817 818 819 820 821

/*
 * For a bio data item, a piece is whatever remains of the next
 * entry in the current bio iovec, or the first entry in the next
 * bio in the list.
 */
A
Alex Elder 已提交
822
static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,
823
					size_t length)
824
{
A
Alex Elder 已提交
825
	struct ceph_msg_data *data = cursor->data;
826 827 828 829 830 831 832
	struct bio *bio;

	BUG_ON(data->type != CEPH_MSG_DATA_BIO);

	bio = data->bio;
	BUG_ON(!bio);

833
	cursor->resid = min(length, data->bio_length);
834
	cursor->bio = bio;
835 836 837
	cursor->bvec_iter = bio->bi_iter;
	cursor->last_piece =
		cursor->resid <= bio_iter_len(bio, cursor->bvec_iter);
838 839
}

A
Alex Elder 已提交
840
static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,
841 842 843
						size_t *page_offset,
						size_t *length)
{
A
Alex Elder 已提交
844
	struct ceph_msg_data *data = cursor->data;
845
	struct bio *bio;
846
	struct bio_vec bio_vec;
847 848 849 850 851 852

	BUG_ON(data->type != CEPH_MSG_DATA_BIO);

	bio = cursor->bio;
	BUG_ON(!bio);

853
	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);
854

855
	*page_offset = (size_t) bio_vec.bv_offset;
856
	BUG_ON(*page_offset >= PAGE_SIZE);
857 858 859
	if (cursor->last_piece) /* pagelist offset is always 0 */
		*length = cursor->resid;
	else
860
		*length = (size_t) bio_vec.bv_len;
861
	BUG_ON(*length > cursor->resid);
862
	BUG_ON(*page_offset + *length > PAGE_SIZE);
863

864
	return bio_vec.bv_page;
865 866
}

A
Alex Elder 已提交
867 868
static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,
					size_t bytes)
869 870
{
	struct bio *bio;
871
	struct bio_vec bio_vec;
872

A
Alex Elder 已提交
873
	BUG_ON(cursor->data->type != CEPH_MSG_DATA_BIO);
874 875 876 877

	bio = cursor->bio;
	BUG_ON(!bio);

878
	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);
879 880 881

	/* Advance the cursor offset */

882 883
	BUG_ON(cursor->resid < bytes);
	cursor->resid -= bytes;
884 885 886 887

	bio_advance_iter(bio, &cursor->bvec_iter, bytes);

	if (bytes < bio_vec.bv_len)
888 889 890 891
		return false;	/* more bytes to process in this segment */

	/* Move on to the next segment, and possibly the next bio */

892
	if (!cursor->bvec_iter.bi_size) {
893
		bio = bio->bi_next;
894 895 896 897 898 899
		cursor->bio = bio;
		if (bio)
			cursor->bvec_iter = bio->bi_iter;
		else
			memset(&cursor->bvec_iter, 0,
			       sizeof(cursor->bvec_iter));
900 901
	}

902 903 904 905
	if (!cursor->last_piece) {
		BUG_ON(!cursor->resid);
		BUG_ON(!bio);
		/* A short read is OK, so use <= rather than == */
906
		if (cursor->resid <= bio_iter_len(bio, cursor->bvec_iter))
907
			cursor->last_piece = true;
908
	}
909 910 911

	return true;
}
912
#endif /* CONFIG_BLOCK */
913

914 915 916 917
/*
 * For a page array, a piece comes from the first page in the array
 * that has not already been fully consumed.
 */
A
Alex Elder 已提交
918
static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,
919
					size_t length)
920
{
A
Alex Elder 已提交
921
	struct ceph_msg_data *data = cursor->data;
922 923 924 925 926 927 928
	int page_count;

	BUG_ON(data->type != CEPH_MSG_DATA_PAGES);

	BUG_ON(!data->pages);
	BUG_ON(!data->length);

929
	cursor->resid = min(length, data->length);
930 931 932
	page_count = calc_pages_for(data->alignment, (u64)data->length);
	cursor->page_offset = data->alignment & ~PAGE_MASK;
	cursor->page_index = 0;
933 934 935
	BUG_ON(page_count > (int)USHRT_MAX);
	cursor->page_count = (unsigned short)page_count;
	BUG_ON(length > SIZE_MAX - cursor->page_offset);
936
	cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;
937 938
}

A
Alex Elder 已提交
939 940 941
static struct page *
ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,
					size_t *page_offset, size_t *length)
942
{
A
Alex Elder 已提交
943
	struct ceph_msg_data *data = cursor->data;
944 945 946 947 948 949 950

	BUG_ON(data->type != CEPH_MSG_DATA_PAGES);

	BUG_ON(cursor->page_index >= cursor->page_count);
	BUG_ON(cursor->page_offset >= PAGE_SIZE);

	*page_offset = cursor->page_offset;
951
	if (cursor->last_piece)
952
		*length = cursor->resid;
953
	else
954 955 956 957 958
		*length = PAGE_SIZE - *page_offset;

	return data->pages[cursor->page_index];
}

A
Alex Elder 已提交
959
static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,
960 961
						size_t bytes)
{
A
Alex Elder 已提交
962
	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);
963 964 965 966 967 968

	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);

	/* Advance the cursor page offset */

	cursor->resid -= bytes;
969 970
	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;
	if (!bytes || cursor->page_offset)
971 972
		return false;	/* more bytes to process in the current page */

973 974 975
	if (!cursor->resid)
		return false;   /* no more data */

976
	/* Move on to the next page; offset is already at 0 */
977 978 979

	BUG_ON(cursor->page_index >= cursor->page_count);
	cursor->page_index++;
980
	cursor->last_piece = cursor->resid <= PAGE_SIZE;
981 982 983 984

	return true;
}

985
/*
986 987
 * For a pagelist, a piece is whatever remains to be consumed in the
 * first page in the list, or the front of the next page.
988
 */
A
Alex Elder 已提交
989 990
static void
ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,
991
					size_t length)
992
{
A
Alex Elder 已提交
993
	struct ceph_msg_data *data = cursor->data;
994 995 996
	struct ceph_pagelist *pagelist;
	struct page *page;

997
	BUG_ON(data->type != CEPH_MSG_DATA_PAGELIST);
998 999 1000

	pagelist = data->pagelist;
	BUG_ON(!pagelist);
1001 1002

	if (!length)
1003 1004 1005 1006 1007
		return;		/* pagelist can be assigned but empty */

	BUG_ON(list_empty(&pagelist->head));
	page = list_first_entry(&pagelist->head, struct page, lru);

1008
	cursor->resid = min(length, pagelist->length);
1009 1010
	cursor->page = page;
	cursor->offset = 0;
A
Alex Elder 已提交
1011
	cursor->last_piece = cursor->resid <= PAGE_SIZE;
1012 1013
}

A
Alex Elder 已提交
1014 1015 1016
static struct page *
ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,
				size_t *page_offset, size_t *length)
1017
{
A
Alex Elder 已提交
1018
	struct ceph_msg_data *data = cursor->data;
1019 1020 1021 1022 1023 1024 1025 1026
	struct ceph_pagelist *pagelist;

	BUG_ON(data->type != CEPH_MSG_DATA_PAGELIST);

	pagelist = data->pagelist;
	BUG_ON(!pagelist);

	BUG_ON(!cursor->page);
1027
	BUG_ON(cursor->offset + cursor->resid != pagelist->length);
1028

1029
	/* offset of first page in pagelist is always 0 */
1030
	*page_offset = cursor->offset & ~PAGE_MASK;
1031
	if (cursor->last_piece)
1032 1033 1034
		*length = cursor->resid;
	else
		*length = PAGE_SIZE - *page_offset;
1035

A
Alex Elder 已提交
1036
	return cursor->page;
1037 1038
}

A
Alex Elder 已提交
1039
static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,
1040
						size_t bytes)
1041
{
A
Alex Elder 已提交
1042
	struct ceph_msg_data *data = cursor->data;
1043 1044 1045 1046 1047 1048
	struct ceph_pagelist *pagelist;

	BUG_ON(data->type != CEPH_MSG_DATA_PAGELIST);

	pagelist = data->pagelist;
	BUG_ON(!pagelist);
1049 1050

	BUG_ON(cursor->offset + cursor->resid != pagelist->length);
1051 1052 1053 1054
	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);

	/* Advance the cursor offset */

1055
	cursor->resid -= bytes;
1056
	cursor->offset += bytes;
1057
	/* offset of first page in pagelist is always 0 */
1058 1059 1060
	if (!bytes || cursor->offset & ~PAGE_MASK)
		return false;	/* more bytes to process in the current page */

1061 1062 1063
	if (!cursor->resid)
		return false;   /* no more data */

1064 1065 1066
	/* Move on to the next page */

	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));
1067
	cursor->page = list_next_entry(cursor->page, lru);
1068
	cursor->last_piece = cursor->resid <= PAGE_SIZE;
1069 1070 1071 1072

	return true;
}

1073 1074 1075 1076 1077 1078 1079 1080
/*
 * Message data is handled (sent or received) in pieces, where each
 * piece resides on a single page.  The network layer might not
 * consume an entire piece at once.  A data item's cursor keeps
 * track of which piece is next to process and how much remains to
 * be processed in that piece.  It also tracks whether the current
 * piece is the last one in the data item.
 */
1081
static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)
1082
{
1083
	size_t length = cursor->total_resid;
A
Alex Elder 已提交
1084 1085

	switch (cursor->data->type) {
1086
	case CEPH_MSG_DATA_PAGELIST:
A
Alex Elder 已提交
1087
		ceph_msg_data_pagelist_cursor_init(cursor, length);
1088
		break;
1089
	case CEPH_MSG_DATA_PAGES:
A
Alex Elder 已提交
1090
		ceph_msg_data_pages_cursor_init(cursor, length);
1091
		break;
1092 1093
#ifdef CONFIG_BLOCK
	case CEPH_MSG_DATA_BIO:
A
Alex Elder 已提交
1094
		ceph_msg_data_bio_cursor_init(cursor, length);
1095
		break;
1096
#endif /* CONFIG_BLOCK */
1097
	case CEPH_MSG_DATA_NONE:
1098 1099 1100 1101
	default:
		/* BUG(); */
		break;
	}
A
Alex Elder 已提交
1102
	cursor->need_crc = true;
1103 1104
}

1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
static void ceph_msg_data_cursor_init(struct ceph_msg *msg, size_t length)
{
	struct ceph_msg_data_cursor *cursor = &msg->cursor;
	struct ceph_msg_data *data;

	BUG_ON(!length);
	BUG_ON(length > msg->data_length);
	BUG_ON(list_empty(&msg->data));

	cursor->data_head = &msg->data;
	cursor->total_resid = length;
	data = list_first_entry(&msg->data, struct ceph_msg_data, links);
	cursor->data = data;

	__ceph_msg_data_cursor_init(cursor);
}

1122 1123 1124 1125 1126
/*
 * Return the page containing the next piece to process for a given
 * data item, and supply the page offset and length of that piece.
 * Indicate whether this is the last piece in this data item.
 */
A
Alex Elder 已提交
1127 1128
static struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,
					size_t *page_offset, size_t *length,
1129 1130 1131 1132
					bool *last_piece)
{
	struct page *page;

A
Alex Elder 已提交
1133
	switch (cursor->data->type) {
1134
	case CEPH_MSG_DATA_PAGELIST:
A
Alex Elder 已提交
1135
		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);
1136
		break;
1137
	case CEPH_MSG_DATA_PAGES:
A
Alex Elder 已提交
1138
		page = ceph_msg_data_pages_next(cursor, page_offset, length);
1139
		break;
1140 1141
#ifdef CONFIG_BLOCK
	case CEPH_MSG_DATA_BIO:
A
Alex Elder 已提交
1142
		page = ceph_msg_data_bio_next(cursor, page_offset, length);
1143
		break;
1144
#endif /* CONFIG_BLOCK */
1145
	case CEPH_MSG_DATA_NONE:
1146 1147 1148 1149 1150 1151 1152 1153
	default:
		page = NULL;
		break;
	}
	BUG_ON(!page);
	BUG_ON(*page_offset + *length > PAGE_SIZE);
	BUG_ON(!*length);
	if (last_piece)
A
Alex Elder 已提交
1154
		*last_piece = cursor->last_piece;
1155 1156 1157 1158 1159 1160 1161 1162

	return page;
}

/*
 * Returns true if the result moves the cursor on to the next piece
 * of the data item.
 */
A
Alex Elder 已提交
1163 1164
static bool ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,
				size_t bytes)
1165 1166 1167
{
	bool new_piece;

1168
	BUG_ON(bytes > cursor->resid);
A
Alex Elder 已提交
1169
	switch (cursor->data->type) {
1170
	case CEPH_MSG_DATA_PAGELIST:
A
Alex Elder 已提交
1171
		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);
1172
		break;
1173
	case CEPH_MSG_DATA_PAGES:
A
Alex Elder 已提交
1174
		new_piece = ceph_msg_data_pages_advance(cursor, bytes);
1175
		break;
1176 1177
#ifdef CONFIG_BLOCK
	case CEPH_MSG_DATA_BIO:
A
Alex Elder 已提交
1178
		new_piece = ceph_msg_data_bio_advance(cursor, bytes);
1179
		break;
1180
#endif /* CONFIG_BLOCK */
1181
	case CEPH_MSG_DATA_NONE:
1182 1183 1184 1185
	default:
		BUG();
		break;
	}
1186
	cursor->total_resid -= bytes;
1187

1188 1189 1190
	if (!cursor->resid && cursor->total_resid) {
		WARN_ON(!cursor->last_piece);
		BUG_ON(list_is_last(&cursor->data->links, cursor->data_head));
1191
		cursor->data = list_next_entry(cursor->data, links);
1192
		__ceph_msg_data_cursor_init(cursor);
A
Alex Elder 已提交
1193
		new_piece = true;
1194
	}
A
Alex Elder 已提交
1195
	cursor->need_crc = new_piece;
1196

1197 1198 1199
	return new_piece;
}

1200
static void prepare_message_data(struct ceph_msg *msg, u32 data_len)
1201 1202
{
	BUG_ON(!msg);
1203
	BUG_ON(!data_len);
1204

1205
	/* Initialize data cursor */
1206

A
Alex Elder 已提交
1207
	ceph_msg_data_cursor_init(msg, (size_t)data_len);
1208 1209
}

S
Sage Weil 已提交
1210 1211 1212 1213
/*
 * Prepare footer for currently outgoing message, and finish things
 * off.  Assumes out_kvec* are already valid.. we just add on to the end.
 */
1214
static void prepare_write_message_footer(struct ceph_connection *con)
S
Sage Weil 已提交
1215 1216
{
	struct ceph_msg *m = con->out_msg;
1217
	int v = con->out_kvec_left;
S
Sage Weil 已提交
1218

1219 1220
	m->footer.flags |= CEPH_MSG_FOOTER_COMPLETE;

S
Sage Weil 已提交
1221 1222
	dout("prepare_write_message_footer %p\n", con);
	con->out_kvec[v].iov_base = &m->footer;
Y
Yan, Zheng 已提交
1223 1224
	if (con->peer_features & CEPH_FEATURE_MSG_AUTH) {
		if (con->ops->sign_message)
1225
			con->ops->sign_message(m);
Y
Yan, Zheng 已提交
1226 1227 1228 1229 1230 1231 1232 1233 1234
		else
			m->footer.sig = 0;
		con->out_kvec[v].iov_len = sizeof(m->footer);
		con->out_kvec_bytes += sizeof(m->footer);
	} else {
		m->old_footer.flags = m->footer.flags;
		con->out_kvec[v].iov_len = sizeof(m->old_footer);
		con->out_kvec_bytes += sizeof(m->old_footer);
	}
S
Sage Weil 已提交
1235 1236
	con->out_kvec_left++;
	con->out_more = m->more_to_follow;
1237
	con->out_msg_done = true;
S
Sage Weil 已提交
1238 1239 1240 1241 1242 1243 1244 1245
}

/*
 * Prepare headers for the next outgoing message.
 */
static void prepare_write_message(struct ceph_connection *con)
{
	struct ceph_msg *m;
1246
	u32 crc;
S
Sage Weil 已提交
1247

1248
	con_out_kvec_reset(con);
1249
	con->out_msg_done = false;
S
Sage Weil 已提交
1250 1251 1252 1253 1254

	/* Sneak an ack in there first?  If we can get it into the same
	 * TCP packet that's a good thing. */
	if (con->in_seq > con->in_seq_acked) {
		con->in_seq_acked = con->in_seq;
1255
		con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
S
Sage Weil 已提交
1256
		con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
1257
		con_out_kvec_add(con, sizeof (con->out_temp_ack),
1258
			&con->out_temp_ack);
S
Sage Weil 已提交
1259 1260
	}

1261
	BUG_ON(list_empty(&con->out_queue));
1262
	m = list_first_entry(&con->out_queue, struct ceph_msg, list_head);
1263
	con->out_msg = m;
1264
	BUG_ON(m->con != con);
1265 1266 1267 1268

	/* put message on sent list */
	ceph_msg_get(m);
	list_move_tail(&m->list_head, &con->out_sent);
S
Sage Weil 已提交
1269

1270 1271 1272 1273 1274 1275 1276 1277
	/*
	 * only assign outgoing seq # if we haven't sent this message
	 * yet.  if it is requeued, resend with it's original seq.
	 */
	if (m->needs_out_seq) {
		m->hdr.seq = cpu_to_le64(++con->out_seq);
		m->needs_out_seq = false;
	}
1278
	WARN_ON(m->data_length != le32_to_cpu(m->hdr.data_len));
S
Sage Weil 已提交
1279

1280
	dout("prepare_write_message %p seq %lld type %d len %d+%d+%zd\n",
S
Sage Weil 已提交
1281 1282
	     m, con->out_seq, le16_to_cpu(m->hdr.type),
	     le32_to_cpu(m->hdr.front_len), le32_to_cpu(m->hdr.middle_len),
1283
	     m->data_length);
S
Sage Weil 已提交
1284 1285 1286
	BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);

	/* tag + hdr + front + middle */
1287
	con_out_kvec_add(con, sizeof (tag_msg), &tag_msg);
I
Ilya Dryomov 已提交
1288
	con_out_kvec_add(con, sizeof(con->out_hdr), &con->out_hdr);
1289
	con_out_kvec_add(con, m->front.iov_len, m->front.iov_base);
1290

S
Sage Weil 已提交
1291
	if (m->middle)
1292
		con_out_kvec_add(con, m->middle->vec.iov_len,
1293
			m->middle->vec.iov_base);
S
Sage Weil 已提交
1294

I
Ilya Dryomov 已提交
1295
	/* fill in hdr crc and finalize hdr */
1296 1297
	crc = crc32c(0, &m->hdr, offsetof(struct ceph_msg_header, crc));
	con->out_msg->hdr.crc = cpu_to_le32(crc);
I
Ilya Dryomov 已提交
1298
	memcpy(&con->out_hdr, &con->out_msg->hdr, sizeof(con->out_hdr));
1299

I
Ilya Dryomov 已提交
1300
	/* fill in front and middle crc, footer */
1301 1302 1303 1304 1305 1306 1307
	crc = crc32c(0, m->front.iov_base, m->front.iov_len);
	con->out_msg->footer.front_crc = cpu_to_le32(crc);
	if (m->middle) {
		crc = crc32c(0, m->middle->vec.iov_base,
				m->middle->vec.iov_len);
		con->out_msg->footer.middle_crc = cpu_to_le32(crc);
	} else
S
Sage Weil 已提交
1308
		con->out_msg->footer.middle_crc = 0;
1309
	dout("%s front_crc %u middle_crc %u\n", __func__,
S
Sage Weil 已提交
1310 1311
	     le32_to_cpu(con->out_msg->footer.front_crc),
	     le32_to_cpu(con->out_msg->footer.middle_crc));
I
Ilya Dryomov 已提交
1312
	con->out_msg->footer.flags = 0;
S
Sage Weil 已提交
1313 1314

	/* is there a data payload? */
1315
	con->out_msg->footer.data_crc = 0;
1316 1317
	if (m->data_length) {
		prepare_message_data(con->out_msg, m->data_length);
1318 1319
		con->out_more = 1;  /* data + footer will follow */
	} else {
S
Sage Weil 已提交
1320
		/* no, queue up footer too and be done */
1321
		prepare_write_message_footer(con);
1322
	}
S
Sage Weil 已提交
1323

1324
	con_flag_set(con, CON_FLAG_WRITE_PENDING);
S
Sage Weil 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
}

/*
 * Prepare an ack.
 */
static void prepare_write_ack(struct ceph_connection *con)
{
	dout("prepare_write_ack %p %llu -> %llu\n", con,
	     con->in_seq_acked, con->in_seq);
	con->in_seq_acked = con->in_seq;

1336
	con_out_kvec_reset(con);
1337

1338
	con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
1339

S
Sage Weil 已提交
1340
	con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
1341
	con_out_kvec_add(con, sizeof (con->out_temp_ack),
1342 1343
				&con->out_temp_ack);

S
Sage Weil 已提交
1344
	con->out_more = 1;  /* more will follow.. eventually.. */
1345
	con_flag_set(con, CON_FLAG_WRITE_PENDING);
S
Sage Weil 已提交
1346 1347
}

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
/*
 * Prepare to share the seq during handshake
 */
static void prepare_write_seq(struct ceph_connection *con)
{
	dout("prepare_write_seq %p %llu -> %llu\n", con,
	     con->in_seq_acked, con->in_seq);
	con->in_seq_acked = con->in_seq;

	con_out_kvec_reset(con);

	con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
	con_out_kvec_add(con, sizeof (con->out_temp_ack),
			 &con->out_temp_ack);

	con_flag_set(con, CON_FLAG_WRITE_PENDING);
}

S
Sage Weil 已提交
1366 1367 1368 1369 1370 1371
/*
 * Prepare to write keepalive byte.
 */
static void prepare_write_keepalive(struct ceph_connection *con)
{
	dout("prepare_write_keepalive %p\n", con);
1372
	con_out_kvec_reset(con);
1373
	if (con->peer_features & CEPH_FEATURE_MSGR_KEEPALIVE2) {
1374 1375
		struct timespec now = CURRENT_TIME;

1376
		con_out_kvec_add(con, sizeof(tag_keepalive2), &tag_keepalive2);
1377 1378 1379
		ceph_encode_timespec(&con->out_temp_keepalive2, &now);
		con_out_kvec_add(con, sizeof(con->out_temp_keepalive2),
				 &con->out_temp_keepalive2);
1380 1381 1382
	} else {
		con_out_kvec_add(con, sizeof(tag_keepalive), &tag_keepalive);
	}
1383
	con_flag_set(con, CON_FLAG_WRITE_PENDING);
S
Sage Weil 已提交
1384 1385 1386 1387 1388 1389
}

/*
 * Connection negotiation.
 */

1390 1391
static struct ceph_auth_handshake *get_connect_authorizer(struct ceph_connection *con,
						int *auth_proto)
1392
{
1393
	struct ceph_auth_handshake *auth;
1394 1395 1396 1397

	if (!con->ops->get_authorizer) {
		con->out_connect.authorizer_protocol = CEPH_AUTH_UNKNOWN;
		con->out_connect.authorizer_len = 0;
1398
		return NULL;
1399 1400 1401
	}

	/* Can't hold the mutex while getting authorizer */
1402
	mutex_unlock(&con->mutex);
1403
	auth = con->ops->get_authorizer(con, auth_proto, con->auth_retry);
1404
	mutex_lock(&con->mutex);
1405

1406
	if (IS_ERR(auth))
1407
		return auth;
1408
	if (con->state != CON_STATE_NEGOTIATING)
1409
		return ERR_PTR(-EAGAIN);
1410

1411 1412
	con->auth_reply_buf = auth->authorizer_reply_buf;
	con->auth_reply_buf_len = auth->authorizer_reply_buf_len;
1413
	return auth;
1414 1415
}

S
Sage Weil 已提交
1416 1417 1418
/*
 * We connected to a peer and are saying hello.
 */
1419
static void prepare_write_banner(struct ceph_connection *con)
S
Sage Weil 已提交
1420
{
1421 1422
	con_out_kvec_add(con, strlen(CEPH_BANNER), CEPH_BANNER);
	con_out_kvec_add(con, sizeof (con->msgr->my_enc_addr),
1423
					&con->msgr->my_enc_addr);
1424 1425

	con->out_more = 0;
1426
	con_flag_set(con, CON_FLAG_WRITE_PENDING);
1427 1428
}

1429
static int prepare_write_connect(struct ceph_connection *con)
1430
{
1431
	unsigned int global_seq = get_global_seq(con->msgr, 0);
S
Sage Weil 已提交
1432
	int proto;
1433
	int auth_proto;
1434
	struct ceph_auth_handshake *auth;
S
Sage Weil 已提交
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451

	switch (con->peer_name.type) {
	case CEPH_ENTITY_TYPE_MON:
		proto = CEPH_MONC_PROTOCOL;
		break;
	case CEPH_ENTITY_TYPE_OSD:
		proto = CEPH_OSDC_PROTOCOL;
		break;
	case CEPH_ENTITY_TYPE_MDS:
		proto = CEPH_MDSC_PROTOCOL;
		break;
	default:
		BUG();
	}

	dout("prepare_write_connect %p cseq=%d gseq=%d proto=%d\n", con,
	     con->connect_seq, global_seq, proto);
1452

1453 1454
	con->out_connect.features =
	    cpu_to_le64(from_msgr(con->msgr)->supported_features);
S
Sage Weil 已提交
1455 1456 1457 1458 1459 1460
	con->out_connect.host_type = cpu_to_le32(CEPH_ENTITY_TYPE_CLIENT);
	con->out_connect.connect_seq = cpu_to_le32(con->connect_seq);
	con->out_connect.global_seq = cpu_to_le32(global_seq);
	con->out_connect.protocol_version = cpu_to_le32(proto);
	con->out_connect.flags = 0;

1461 1462
	auth_proto = CEPH_AUTH_UNKNOWN;
	auth = get_connect_authorizer(con, &auth_proto);
1463 1464
	if (IS_ERR(auth))
		return PTR_ERR(auth);
1465

1466
	con->out_connect.authorizer_protocol = cpu_to_le32(auth_proto);
1467 1468 1469
	con->out_connect.authorizer_len = auth ?
		cpu_to_le32(auth->authorizer_buf_len) : 0;

1470
	con_out_kvec_add(con, sizeof (con->out_connect),
1471 1472
					&con->out_connect);
	if (auth && auth->authorizer_buf_len)
1473
		con_out_kvec_add(con, auth->authorizer_buf_len,
1474
					auth->authorizer_buf);
1475

S
Sage Weil 已提交
1476
	con->out_more = 0;
1477
	con_flag_set(con, CON_FLAG_WRITE_PENDING);
1478

1479
	return 0;
S
Sage Weil 已提交
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
}

/*
 * write as much of pending kvecs to the socket as we can.
 *  1 -> done
 *  0 -> socket full, but more to do
 * <0 -> error
 */
static int write_partial_kvec(struct ceph_connection *con)
{
	int ret;

	dout("write_partial_kvec %p %d left\n", con, con->out_kvec_bytes);
	while (con->out_kvec_bytes > 0) {
		ret = ceph_tcp_sendmsg(con->sock, con->out_kvec_cur,
				       con->out_kvec_left, con->out_kvec_bytes,
				       con->out_more);
		if (ret <= 0)
			goto out;
		con->out_kvec_bytes -= ret;
		if (con->out_kvec_bytes == 0)
			break;            /* done */
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513

		/* account for full iov entries consumed */
		while (ret >= con->out_kvec_cur->iov_len) {
			BUG_ON(!con->out_kvec_left);
			ret -= con->out_kvec_cur->iov_len;
			con->out_kvec_cur++;
			con->out_kvec_left--;
		}
		/* and for a partially-consumed entry */
		if (ret) {
			con->out_kvec_cur->iov_len -= ret;
			con->out_kvec_cur->iov_base += ret;
S
Sage Weil 已提交
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
		}
	}
	con->out_kvec_left = 0;
	ret = 1;
out:
	dout("write_partial_kvec %p %d left in %d kvecs ret = %d\n", con,
	     con->out_kvec_bytes, con->out_kvec_left, ret);
	return ret;  /* done! */
}

1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
static u32 ceph_crc32c_page(u32 crc, struct page *page,
				unsigned int page_offset,
				unsigned int length)
{
	char *kaddr;

	kaddr = kmap(page);
	BUG_ON(kaddr == NULL);
	crc = crc32c(crc, kaddr + page_offset, length);
	kunmap(page);

	return crc;
}
S
Sage Weil 已提交
1537 1538 1539 1540 1541 1542 1543
/*
 * Write as much message data payload as we can.  If we finish, queue
 * up the footer.
 *  1 -> done, footer is now queued in out_kvec[].
 *  0 -> socket full, but more to do
 * <0 -> error
 */
1544
static int write_partial_message_data(struct ceph_connection *con)
S
Sage Weil 已提交
1545 1546
{
	struct ceph_msg *msg = con->out_msg;
A
Alex Elder 已提交
1547
	struct ceph_msg_data_cursor *cursor = &msg->cursor;
1548
	bool do_datacrc = !ceph_test_opt(from_msgr(con->msgr), NOCRC);
A
Alex Elder 已提交
1549
	u32 crc;
S
Sage Weil 已提交
1550

A
Alex Elder 已提交
1551
	dout("%s %p msg %p\n", __func__, con, msg);
S
Sage Weil 已提交
1552

1553
	if (list_empty(&msg->data))
1554 1555
		return -EINVAL;

1556 1557 1558 1559 1560 1561 1562 1563
	/*
	 * Iterate through each page that contains data to be
	 * written, and send as much as possible for each.
	 *
	 * If we are calculating the data crc (the default), we will
	 * need to map the page.  If we have no pages, they have
	 * been revoked, so use the zero page.
	 */
A
Alex Elder 已提交
1564
	crc = do_datacrc ? le32_to_cpu(msg->footer.data_crc) : 0;
1565
	while (cursor->resid) {
1566
		struct page *page;
1567 1568
		size_t page_offset;
		size_t length;
1569
		bool last_piece;
1570
		bool need_crc;
A
Alex Elder 已提交
1571
		int ret;
1572

1573 1574
		page = ceph_msg_data_next(cursor, &page_offset, &length,
					  &last_piece);
1575
		ret = ceph_tcp_sendpage(con->sock, page, page_offset,
1576
					length, !last_piece);
A
Alex Elder 已提交
1577 1578 1579
		if (ret <= 0) {
			if (do_datacrc)
				msg->footer.data_crc = cpu_to_le32(crc);
S
Sage Weil 已提交
1580

A
Alex Elder 已提交
1581 1582
			return ret;
		}
1583 1584
		if (do_datacrc && cursor->need_crc)
			crc = ceph_crc32c_page(crc, page, page_offset, length);
1585
		need_crc = ceph_msg_data_advance(cursor, (size_t)ret);
S
Sage Weil 已提交
1586 1587
	}

1588
	dout("%s %p msg %p done\n", __func__, con, msg);
S
Sage Weil 已提交
1589 1590

	/* prepare and queue up footer, too */
A
Alex Elder 已提交
1591 1592 1593
	if (do_datacrc)
		msg->footer.data_crc = cpu_to_le32(crc);
	else
1594
		msg->footer.flags |= CEPH_MSG_FOOTER_NOCRC;
1595
	con_out_kvec_reset(con);
1596
	prepare_write_message_footer(con);
A
Alex Elder 已提交
1597 1598

	return 1;	/* must return > 0 to indicate success */
S
Sage Weil 已提交
1599 1600 1601 1602 1603 1604 1605 1606 1607
}

/*
 * write some zeros
 */
static int write_partial_skip(struct ceph_connection *con)
{
	int ret;

I
Ilya Dryomov 已提交
1608
	dout("%s %p %d left\n", __func__, con, con->out_skip);
S
Sage Weil 已提交
1609
	while (con->out_skip > 0) {
1610
		size_t size = min(con->out_skip, (int) PAGE_CACHE_SIZE);
S
Sage Weil 已提交
1611

1612
		ret = ceph_tcp_sendpage(con->sock, zero_page, 0, size, true);
S
Sage Weil 已提交
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
		if (ret <= 0)
			goto out;
		con->out_skip -= ret;
	}
	ret = 1;
out:
	return ret;
}

/*
 * Prepare to read connection handshake, or an ack.
 */
1625 1626 1627 1628 1629 1630
static void prepare_read_banner(struct ceph_connection *con)
{
	dout("prepare_read_banner %p\n", con);
	con->in_base_pos = 0;
}

S
Sage Weil 已提交
1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
static void prepare_read_connect(struct ceph_connection *con)
{
	dout("prepare_read_connect %p\n", con);
	con->in_base_pos = 0;
}

static void prepare_read_ack(struct ceph_connection *con)
{
	dout("prepare_read_ack %p\n", con);
	con->in_base_pos = 0;
}

1643 1644 1645 1646 1647 1648 1649
static void prepare_read_seq(struct ceph_connection *con)
{
	dout("prepare_read_seq %p\n", con);
	con->in_base_pos = 0;
	con->in_tag = CEPH_MSGR_TAG_SEQ;
}

S
Sage Weil 已提交
1650 1651 1652 1653 1654 1655 1656
static void prepare_read_tag(struct ceph_connection *con)
{
	dout("prepare_read_tag %p\n", con);
	con->in_base_pos = 0;
	con->in_tag = CEPH_MSGR_TAG_READY;
}

1657 1658 1659 1660 1661 1662
static void prepare_read_keepalive_ack(struct ceph_connection *con)
{
	dout("prepare_read_keepalive_ack %p\n", con);
	con->in_base_pos = 0;
}

S
Sage Weil 已提交
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
/*
 * Prepare to read a message.
 */
static int prepare_read_message(struct ceph_connection *con)
{
	dout("prepare_read_message %p\n", con);
	BUG_ON(con->in_msg != NULL);
	con->in_base_pos = 0;
	con->in_front_crc = con->in_middle_crc = con->in_data_crc = 0;
	return 0;
}


static int read_partial(struct ceph_connection *con,
1677
			int end, int size, void *object)
S
Sage Weil 已提交
1678
{
1679 1680
	while (con->in_base_pos < end) {
		int left = end - con->in_base_pos;
S
Sage Weil 已提交
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
		int have = size - left;
		int ret = ceph_tcp_recvmsg(con->sock, object + have, left);
		if (ret <= 0)
			return ret;
		con->in_base_pos += ret;
	}
	return 1;
}


/*
 * Read all or part of the connect-side handshake on a new connection
 */
1694
static int read_partial_banner(struct ceph_connection *con)
S
Sage Weil 已提交
1695
{
1696 1697 1698
	int size;
	int end;
	int ret;
S
Sage Weil 已提交
1699

1700
	dout("read_partial_banner %p at %d\n", con, con->in_base_pos);
S
Sage Weil 已提交
1701 1702

	/* peer's banner */
1703 1704 1705
	size = strlen(CEPH_BANNER);
	end = size;
	ret = read_partial(con, end, size, con->in_banner);
S
Sage Weil 已提交
1706 1707
	if (ret <= 0)
		goto out;
1708 1709 1710 1711

	size = sizeof (con->actual_peer_addr);
	end += size;
	ret = read_partial(con, end, size, &con->actual_peer_addr);
S
Sage Weil 已提交
1712 1713
	if (ret <= 0)
		goto out;
1714 1715 1716 1717

	size = sizeof (con->peer_addr_for_me);
	end += size;
	ret = read_partial(con, end, size, &con->peer_addr_for_me);
S
Sage Weil 已提交
1718 1719
	if (ret <= 0)
		goto out;
1720

1721 1722 1723 1724 1725 1726
out:
	return ret;
}

static int read_partial_connect(struct ceph_connection *con)
{
1727 1728 1729
	int size;
	int end;
	int ret;
1730 1731 1732

	dout("read_partial_connect %p at %d\n", con, con->in_base_pos);

1733 1734 1735
	size = sizeof (con->in_reply);
	end = size;
	ret = read_partial(con, end, size, &con->in_reply);
S
Sage Weil 已提交
1736 1737
	if (ret <= 0)
		goto out;
1738 1739 1740 1741

	size = le32_to_cpu(con->in_reply.authorizer_len);
	end += size;
	ret = read_partial(con, end, size, con->auth_reply_buf);
1742 1743
	if (ret <= 0)
		goto out;
S
Sage Weil 已提交
1744

1745 1746 1747
	dout("read_partial_connect %p tag %d, con_seq = %u, g_seq = %u\n",
	     con, (int)con->in_reply.tag,
	     le32_to_cpu(con->in_reply.connect_seq),
S
Sage Weil 已提交
1748 1749 1750
	     le32_to_cpu(con->in_reply.global_seq));
out:
	return ret;
1751

S
Sage Weil 已提交
1752 1753 1754 1755 1756 1757 1758 1759
}

/*
 * Verify the hello banner looks okay.
 */
static int verify_hello(struct ceph_connection *con)
{
	if (memcmp(con->in_banner, CEPH_BANNER, strlen(CEPH_BANNER))) {
1760
		pr_err("connect to %s got bad banner\n",
1761
		       ceph_pr_addr(&con->peer_addr.in_addr));
S
Sage Weil 已提交
1762 1763 1764 1765 1766 1767 1768 1769
		con->error_msg = "protocol error, bad banner";
		return -1;
	}
	return 0;
}

static bool addr_is_blank(struct sockaddr_storage *ss)
{
1770 1771 1772
	struct in_addr *addr = &((struct sockaddr_in *)ss)->sin_addr;
	struct in6_addr *addr6 = &((struct sockaddr_in6 *)ss)->sin6_addr;

S
Sage Weil 已提交
1773 1774
	switch (ss->ss_family) {
	case AF_INET:
1775
		return addr->s_addr == htonl(INADDR_ANY);
S
Sage Weil 已提交
1776
	case AF_INET6:
1777 1778 1779
		return ipv6_addr_any(addr6);
	default:
		return true;
S
Sage Weil 已提交
1780 1781 1782 1783 1784 1785 1786
	}
}

static int addr_port(struct sockaddr_storage *ss)
{
	switch (ss->ss_family) {
	case AF_INET:
S
Sage Weil 已提交
1787
		return ntohs(((struct sockaddr_in *)ss)->sin_port);
S
Sage Weil 已提交
1788
	case AF_INET6:
S
Sage Weil 已提交
1789
		return ntohs(((struct sockaddr_in6 *)ss)->sin6_port);
S
Sage Weil 已提交
1790 1791 1792 1793 1794 1795 1796 1797 1798
	}
	return 0;
}

static void addr_set_port(struct sockaddr_storage *ss, int p)
{
	switch (ss->ss_family) {
	case AF_INET:
		((struct sockaddr_in *)ss)->sin_port = htons(p);
1799
		break;
S
Sage Weil 已提交
1800 1801
	case AF_INET6:
		((struct sockaddr_in6 *)ss)->sin6_port = htons(p);
1802
		break;
S
Sage Weil 已提交
1803 1804 1805
	}
}

N
Noah Watkins 已提交
1806 1807 1808 1809 1810 1811
/*
 * Unlike other *_pton function semantics, zero indicates success.
 */
static int ceph_pton(const char *str, size_t len, struct sockaddr_storage *ss,
		char delim, const char **ipend)
{
A
Alex Elder 已提交
1812 1813
	struct sockaddr_in *in4 = (struct sockaddr_in *) ss;
	struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) ss;
N
Noah Watkins 已提交
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900

	memset(ss, 0, sizeof(*ss));

	if (in4_pton(str, len, (u8 *)&in4->sin_addr.s_addr, delim, ipend)) {
		ss->ss_family = AF_INET;
		return 0;
	}

	if (in6_pton(str, len, (u8 *)&in6->sin6_addr.s6_addr, delim, ipend)) {
		ss->ss_family = AF_INET6;
		return 0;
	}

	return -EINVAL;
}

/*
 * Extract hostname string and resolve using kernel DNS facility.
 */
#ifdef CONFIG_CEPH_LIB_USE_DNS_RESOLVER
static int ceph_dns_resolve_name(const char *name, size_t namelen,
		struct sockaddr_storage *ss, char delim, const char **ipend)
{
	const char *end, *delim_p;
	char *colon_p, *ip_addr = NULL;
	int ip_len, ret;

	/*
	 * The end of the hostname occurs immediately preceding the delimiter or
	 * the port marker (':') where the delimiter takes precedence.
	 */
	delim_p = memchr(name, delim, namelen);
	colon_p = memchr(name, ':', namelen);

	if (delim_p && colon_p)
		end = delim_p < colon_p ? delim_p : colon_p;
	else if (!delim_p && colon_p)
		end = colon_p;
	else {
		end = delim_p;
		if (!end) /* case: hostname:/ */
			end = name + namelen;
	}

	if (end <= name)
		return -EINVAL;

	/* do dns_resolve upcall */
	ip_len = dns_query(NULL, name, end - name, NULL, &ip_addr, NULL);
	if (ip_len > 0)
		ret = ceph_pton(ip_addr, ip_len, ss, -1, NULL);
	else
		ret = -ESRCH;

	kfree(ip_addr);

	*ipend = end;

	pr_info("resolve '%.*s' (ret=%d): %s\n", (int)(end - name), name,
			ret, ret ? "failed" : ceph_pr_addr(ss));

	return ret;
}
#else
static inline int ceph_dns_resolve_name(const char *name, size_t namelen,
		struct sockaddr_storage *ss, char delim, const char **ipend)
{
	return -EINVAL;
}
#endif

/*
 * Parse a server name (IP or hostname). If a valid IP address is not found
 * then try to extract a hostname to resolve using userspace DNS upcall.
 */
static int ceph_parse_server_name(const char *name, size_t namelen,
			struct sockaddr_storage *ss, char delim, const char **ipend)
{
	int ret;

	ret = ceph_pton(name, namelen, ss, delim, ipend);
	if (ret)
		ret = ceph_dns_resolve_name(name, namelen, ss, delim, ipend);

	return ret;
}

S
Sage Weil 已提交
1901 1902 1903 1904 1905 1906 1907 1908
/*
 * Parse an ip[:port] list into an addr array.  Use the default
 * monitor port if a port isn't specified.
 */
int ceph_parse_ips(const char *c, const char *end,
		   struct ceph_entity_addr *addr,
		   int max_count, int *count)
{
N
Noah Watkins 已提交
1909
	int i, ret = -EINVAL;
S
Sage Weil 已提交
1910 1911 1912 1913 1914 1915 1916
	const char *p = c;

	dout("parse_ips on '%.*s'\n", (int)(end-c), c);
	for (i = 0; i < max_count; i++) {
		const char *ipend;
		struct sockaddr_storage *ss = &addr[i].in_addr;
		int port;
S
Sage Weil 已提交
1917 1918 1919 1920 1921 1922
		char delim = ',';

		if (*p == '[') {
			delim = ']';
			p++;
		}
S
Sage Weil 已提交
1923

N
Noah Watkins 已提交
1924 1925
		ret = ceph_parse_server_name(p, end - p, ss, delim, &ipend);
		if (ret)
S
Sage Weil 已提交
1926
			goto bad;
N
Noah Watkins 已提交
1927 1928
		ret = -EINVAL;

S
Sage Weil 已提交
1929 1930
		p = ipend;

S
Sage Weil 已提交
1931 1932 1933 1934 1935 1936 1937 1938
		if (delim == ']') {
			if (*p != ']') {
				dout("missing matching ']'\n");
				goto bad;
			}
			p++;
		}

S
Sage Weil 已提交
1939 1940 1941 1942 1943 1944 1945 1946
		/* port? */
		if (p < end && *p == ':') {
			port = 0;
			p++;
			while (p < end && *p >= '0' && *p <= '9') {
				port = (port * 10) + (*p - '0');
				p++;
			}
1947 1948 1949
			if (port == 0)
				port = CEPH_MON_PORT;
			else if (port > 65535)
S
Sage Weil 已提交
1950 1951 1952 1953 1954 1955 1956
				goto bad;
		} else {
			port = CEPH_MON_PORT;
		}

		addr_set_port(ss, port);

1957
		dout("parse_ips got %s\n", ceph_pr_addr(ss));
S
Sage Weil 已提交
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973

		if (p == end)
			break;
		if (*p != ',')
			goto bad;
		p++;
	}

	if (p != end)
		goto bad;

	if (count)
		*count = i + 1;
	return 0;

bad:
S
Sage Weil 已提交
1974
	pr_err("parse_ips bad ip '%.*s'\n", (int)(end - c), c);
N
Noah Watkins 已提交
1975
	return ret;
S
Sage Weil 已提交
1976
}
1977
EXPORT_SYMBOL(ceph_parse_ips);
S
Sage Weil 已提交
1978

1979
static int process_banner(struct ceph_connection *con)
S
Sage Weil 已提交
1980
{
1981
	dout("process_banner on %p\n", con);
S
Sage Weil 已提交
1982 1983 1984 1985

	if (verify_hello(con) < 0)
		return -1;

1986 1987 1988
	ceph_decode_addr(&con->actual_peer_addr);
	ceph_decode_addr(&con->peer_addr_for_me);

S
Sage Weil 已提交
1989 1990 1991 1992 1993
	/*
	 * Make sure the other end is who we wanted.  note that the other
	 * end may not yet know their ip address, so if it's 0.0.0.0, give
	 * them the benefit of the doubt.
	 */
S
Sage Weil 已提交
1994 1995
	if (memcmp(&con->peer_addr, &con->actual_peer_addr,
		   sizeof(con->peer_addr)) != 0 &&
S
Sage Weil 已提交
1996 1997
	    !(addr_is_blank(&con->actual_peer_addr.in_addr) &&
	      con->actual_peer_addr.nonce == con->peer_addr.nonce)) {
1998 1999 2000 2001 2002
		pr_warn("wrong peer, want %s/%d, got %s/%d\n",
			ceph_pr_addr(&con->peer_addr.in_addr),
			(int)le32_to_cpu(con->peer_addr.nonce),
			ceph_pr_addr(&con->actual_peer_addr.in_addr),
			(int)le32_to_cpu(con->actual_peer_addr.nonce));
2003
		con->error_msg = "wrong peer at address";
S
Sage Weil 已提交
2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
		return -1;
	}

	/*
	 * did we learn our address?
	 */
	if (addr_is_blank(&con->msgr->inst.addr.in_addr)) {
		int port = addr_port(&con->msgr->inst.addr.in_addr);

		memcpy(&con->msgr->inst.addr.in_addr,
		       &con->peer_addr_for_me.in_addr,
		       sizeof(con->peer_addr_for_me.in_addr));
		addr_set_port(&con->msgr->inst.addr.in_addr, port);
2017
		encode_my_addr(con->msgr);
2018
		dout("process_banner learned my addr is %s\n",
2019
		     ceph_pr_addr(&con->msgr->inst.addr.in_addr));
S
Sage Weil 已提交
2020 2021
	}

2022 2023 2024 2025 2026
	return 0;
}

static int process_connect(struct ceph_connection *con)
{
2027 2028
	u64 sup_feat = from_msgr(con->msgr)->supported_features;
	u64 req_feat = from_msgr(con->msgr)->required_features;
I
Ilya Dryomov 已提交
2029 2030
	u64 server_feat = ceph_sanitize_features(
				le64_to_cpu(con->in_reply.features));
2031
	int ret;
2032

2033 2034
	dout("process_connect on %p tag %d\n", con, (int)con->in_tag);

S
Sage Weil 已提交
2035
	switch (con->in_reply.tag) {
2036 2037 2038 2039
	case CEPH_MSGR_TAG_FEATURES:
		pr_err("%s%lld %s feature set mismatch,"
		       " my %llx < server's %llx, missing %llx\n",
		       ENTITY_NAME(con->peer_name),
2040
		       ceph_pr_addr(&con->peer_addr.in_addr),
2041 2042
		       sup_feat, server_feat, server_feat & ~sup_feat);
		con->error_msg = "missing required protocol features";
2043
		reset_connection(con);
2044 2045
		return -1;

S
Sage Weil 已提交
2046 2047 2048 2049
	case CEPH_MSGR_TAG_BADPROTOVER:
		pr_err("%s%lld %s protocol version mismatch,"
		       " my %d != server's %d\n",
		       ENTITY_NAME(con->peer_name),
2050
		       ceph_pr_addr(&con->peer_addr.in_addr),
S
Sage Weil 已提交
2051 2052 2053
		       le32_to_cpu(con->out_connect.protocol_version),
		       le32_to_cpu(con->in_reply.protocol_version));
		con->error_msg = "protocol version mismatch";
2054
		reset_connection(con);
S
Sage Weil 已提交
2055 2056
		return -1;

2057 2058 2059 2060 2061 2062 2063 2064
	case CEPH_MSGR_TAG_BADAUTHORIZER:
		con->auth_retry++;
		dout("process_connect %p got BADAUTHORIZER attempt %d\n", con,
		     con->auth_retry);
		if (con->auth_retry == 2) {
			con->error_msg = "connect authorization failure";
			return -1;
		}
2065
		con_out_kvec_reset(con);
2066
		ret = prepare_write_connect(con);
2067 2068
		if (ret < 0)
			return ret;
S
Sage Weil 已提交
2069
		prepare_read_connect(con);
2070
		break;
S
Sage Weil 已提交
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080

	case CEPH_MSGR_TAG_RESETSESSION:
		/*
		 * If we connected with a large connect_seq but the peer
		 * has no record of a session with us (no connection, or
		 * connect_seq == 0), they will send RESETSESION to indicate
		 * that they must have reset their session, and may have
		 * dropped messages.
		 */
		dout("process_connect got RESET peer seq %u\n",
S
Sage Weil 已提交
2081
		     le32_to_cpu(con->in_reply.connect_seq));
S
Sage Weil 已提交
2082 2083
		pr_err("%s%lld %s connection reset\n",
		       ENTITY_NAME(con->peer_name),
2084
		       ceph_pr_addr(&con->peer_addr.in_addr));
S
Sage Weil 已提交
2085
		reset_connection(con);
2086
		con_out_kvec_reset(con);
2087 2088 2089
		ret = prepare_write_connect(con);
		if (ret < 0)
			return ret;
S
Sage Weil 已提交
2090 2091 2092
		prepare_read_connect(con);

		/* Tell ceph about it. */
2093
		mutex_unlock(&con->mutex);
S
Sage Weil 已提交
2094 2095 2096
		pr_info("reset on %s%lld\n", ENTITY_NAME(con->peer_name));
		if (con->ops->peer_reset)
			con->ops->peer_reset(con);
2097
		mutex_lock(&con->mutex);
2098
		if (con->state != CON_STATE_NEGOTIATING)
2099
			return -EAGAIN;
S
Sage Weil 已提交
2100 2101 2102 2103 2104 2105 2106
		break;

	case CEPH_MSGR_TAG_RETRY_SESSION:
		/*
		 * If we sent a smaller connect_seq than the peer has, try
		 * again with a larger value.
		 */
S
Sage Weil 已提交
2107
		dout("process_connect got RETRY_SESSION my seq %u, peer %u\n",
S
Sage Weil 已提交
2108
		     le32_to_cpu(con->out_connect.connect_seq),
S
Sage Weil 已提交
2109 2110
		     le32_to_cpu(con->in_reply.connect_seq));
		con->connect_seq = le32_to_cpu(con->in_reply.connect_seq);
2111
		con_out_kvec_reset(con);
2112 2113 2114
		ret = prepare_write_connect(con);
		if (ret < 0)
			return ret;
S
Sage Weil 已提交
2115 2116 2117 2118 2119 2120 2121 2122
		prepare_read_connect(con);
		break;

	case CEPH_MSGR_TAG_RETRY_GLOBAL:
		/*
		 * If we sent a smaller global_seq than the peer has, try
		 * again with a larger value.
		 */
2123
		dout("process_connect got RETRY_GLOBAL my %u peer_gseq %u\n",
S
Sage Weil 已提交
2124
		     con->peer_global_seq,
S
Sage Weil 已提交
2125
		     le32_to_cpu(con->in_reply.global_seq));
S
Sage Weil 已提交
2126
		get_global_seq(con->msgr,
S
Sage Weil 已提交
2127
			       le32_to_cpu(con->in_reply.global_seq));
2128
		con_out_kvec_reset(con);
2129 2130 2131
		ret = prepare_write_connect(con);
		if (ret < 0)
			return ret;
S
Sage Weil 已提交
2132 2133 2134
		prepare_read_connect(con);
		break;

2135
	case CEPH_MSGR_TAG_SEQ:
S
Sage Weil 已提交
2136
	case CEPH_MSGR_TAG_READY:
2137 2138 2139 2140
		if (req_feat & ~server_feat) {
			pr_err("%s%lld %s protocol feature mismatch,"
			       " my required %llx > server's %llx, need %llx\n",
			       ENTITY_NAME(con->peer_name),
2141
			       ceph_pr_addr(&con->peer_addr.in_addr),
2142 2143
			       req_feat, server_feat, req_feat & ~server_feat);
			con->error_msg = "missing required protocol features";
2144
			reset_connection(con);
2145 2146
			return -1;
		}
2147

2148
		WARN_ON(con->state != CON_STATE_NEGOTIATING);
2149
		con->state = CON_STATE_OPEN;
2150
		con->auth_retry = 0;    /* we authenticated; clear flag */
S
Sage Weil 已提交
2151 2152
		con->peer_global_seq = le32_to_cpu(con->in_reply.global_seq);
		con->connect_seq++;
2153
		con->peer_features = server_feat;
S
Sage Weil 已提交
2154 2155 2156 2157 2158 2159
		dout("process_connect got READY gseq %d cseq %d (%d)\n",
		     con->peer_global_seq,
		     le32_to_cpu(con->in_reply.connect_seq),
		     con->connect_seq);
		WARN_ON(con->connect_seq !=
			le32_to_cpu(con->in_reply.connect_seq));
S
Sage Weil 已提交
2160 2161

		if (con->in_reply.flags & CEPH_MSG_CONNECT_LOSSY)
2162
			con_flag_set(con, CON_FLAG_LOSSYTX);
S
Sage Weil 已提交
2163

2164
		con->delay = 0;      /* reset backoff memory */
S
Sage Weil 已提交
2165

2166 2167 2168 2169 2170 2171
		if (con->in_reply.tag == CEPH_MSGR_TAG_SEQ) {
			prepare_write_seq(con);
			prepare_read_seq(con);
		} else {
			prepare_read_tag(con);
		}
S
Sage Weil 已提交
2172 2173 2174 2175 2176 2177 2178 2179 2180
		break;

	case CEPH_MSGR_TAG_WAIT:
		/*
		 * If there is a connection race (we are opening
		 * connections to each other), one of us may just have
		 * to WAIT.  This shouldn't happen if we are the
		 * client.
		 */
S
Sage Weil 已提交
2181 2182
		con->error_msg = "protocol error, got WAIT as client";
		return -1;
S
Sage Weil 已提交
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196

	default:
		con->error_msg = "protocol error, garbage tag during connect";
		return -1;
	}
	return 0;
}


/*
 * read (part of) an ack
 */
static int read_partial_ack(struct ceph_connection *con)
{
2197 2198
	int size = sizeof (con->in_temp_ack);
	int end = size;
S
Sage Weil 已提交
2199

2200
	return read_partial(con, end, size, &con->in_temp_ack);
S
Sage Weil 已提交
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219
}

/*
 * We can finally discard anything that's been acked.
 */
static void process_ack(struct ceph_connection *con)
{
	struct ceph_msg *m;
	u64 ack = le64_to_cpu(con->in_temp_ack);
	u64 seq;

	while (!list_empty(&con->out_sent)) {
		m = list_first_entry(&con->out_sent, struct ceph_msg,
				     list_head);
		seq = le64_to_cpu(m->hdr.seq);
		if (seq > ack)
			break;
		dout("got ack for seq %llu type %d at %p\n", seq,
		     le16_to_cpu(m->hdr.type), m);
2220
		m->ack_stamp = jiffies;
S
Sage Weil 已提交
2221 2222 2223 2224 2225 2226
		ceph_msg_remove(m);
	}
	prepare_read_tag(con);
}


2227
static int read_partial_message_section(struct ceph_connection *con,
S
Sage Weil 已提交
2228 2229
					struct kvec *section,
					unsigned int sec_len, u32 *crc)
2230
{
2231
	int ret, left;
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243

	BUG_ON(!section);

	while (section->iov_len < sec_len) {
		BUG_ON(section->iov_base == NULL);
		left = sec_len - section->iov_len;
		ret = ceph_tcp_recvmsg(con->sock, (char *)section->iov_base +
				       section->iov_len, left);
		if (ret <= 0)
			return ret;
		section->iov_len += ret;
	}
2244 2245
	if (section->iov_len == sec_len)
		*crc = crc32c(0, section->iov_base, section->iov_len);
S
Sage Weil 已提交
2246

2247 2248
	return 1;
}
S
Sage Weil 已提交
2249

2250 2251 2252
static int read_partial_msg_data(struct ceph_connection *con)
{
	struct ceph_msg *msg = con->in_msg;
A
Alex Elder 已提交
2253
	struct ceph_msg_data_cursor *cursor = &msg->cursor;
2254
	bool do_datacrc = !ceph_test_opt(from_msgr(con->msgr), NOCRC);
A
Alex Elder 已提交
2255 2256 2257
	struct page *page;
	size_t page_offset;
	size_t length;
A
Alex Elder 已提交
2258
	u32 crc = 0;
2259 2260 2261
	int ret;

	BUG_ON(!msg);
2262
	if (list_empty(&msg->data))
2263
		return -EIO;
2264

A
Alex Elder 已提交
2265 2266
	if (do_datacrc)
		crc = con->in_data_crc;
2267
	while (cursor->resid) {
2268
		page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);
A
Alex Elder 已提交
2269
		ret = ceph_tcp_recvpage(con->sock, page, page_offset, length);
A
Alex Elder 已提交
2270 2271 2272 2273
		if (ret <= 0) {
			if (do_datacrc)
				con->in_data_crc = crc;

A
Alex Elder 已提交
2274
			return ret;
A
Alex Elder 已提交
2275
		}
A
Alex Elder 已提交
2276 2277

		if (do_datacrc)
A
Alex Elder 已提交
2278
			crc = ceph_crc32c_page(crc, page, page_offset, ret);
2279
		(void) ceph_msg_data_advance(cursor, (size_t)ret);
2280
	}
A
Alex Elder 已提交
2281 2282
	if (do_datacrc)
		con->in_data_crc = crc;
2283 2284 2285 2286

	return 1;	/* must return > 0 to indicate success */
}

S
Sage Weil 已提交
2287 2288 2289
/*
 * read (part of) a message.
 */
A
Alex Elder 已提交
2290 2291
static int ceph_con_in_msg_alloc(struct ceph_connection *con, int *skip);

S
Sage Weil 已提交
2292 2293 2294
static int read_partial_message(struct ceph_connection *con)
{
	struct ceph_msg *m = con->in_msg;
2295 2296
	int size;
	int end;
S
Sage Weil 已提交
2297
	int ret;
2298
	unsigned int front_len, middle_len, data_len;
2299
	bool do_datacrc = !ceph_test_opt(from_msgr(con->msgr), NOCRC);
Y
Yan, Zheng 已提交
2300
	bool need_sign = (con->peer_features & CEPH_FEATURE_MSG_AUTH);
2301
	u64 seq;
2302
	u32 crc;
S
Sage Weil 已提交
2303 2304 2305 2306

	dout("read_partial_message con %p msg %p\n", con, m);

	/* header */
2307 2308 2309
	size = sizeof (con->in_hdr);
	end = size;
	ret = read_partial(con, end, size, &con->in_hdr);
2310 2311
	if (ret <= 0)
		return ret;
2312 2313 2314

	crc = crc32c(0, &con->in_hdr, offsetof(struct ceph_msg_header, crc));
	if (cpu_to_le32(crc) != con->in_hdr.crc) {
2315
		pr_err("read_partial_message bad hdr crc %u != expected %u\n",
2316 2317 2318 2319
		       crc, con->in_hdr.crc);
		return -EBADMSG;
	}

S
Sage Weil 已提交
2320 2321 2322 2323
	front_len = le32_to_cpu(con->in_hdr.front_len);
	if (front_len > CEPH_MSG_MAX_FRONT_LEN)
		return -EIO;
	middle_len = le32_to_cpu(con->in_hdr.middle_len);
2324
	if (middle_len > CEPH_MSG_MAX_MIDDLE_LEN)
S
Sage Weil 已提交
2325 2326 2327 2328 2329
		return -EIO;
	data_len = le32_to_cpu(con->in_hdr.data_len);
	if (data_len > CEPH_MSG_MAX_DATA_LEN)
		return -EIO;

2330 2331 2332
	/* verify seq# */
	seq = le64_to_cpu(con->in_hdr.seq);
	if ((s64)seq - (s64)con->in_seq < 1) {
S
Sage Weil 已提交
2333
		pr_info("skipping %s%lld %s seq %lld expected %lld\n",
2334
			ENTITY_NAME(con->peer_name),
2335
			ceph_pr_addr(&con->peer_addr.in_addr),
2336 2337 2338 2339 2340 2341 2342 2343 2344
			seq, con->in_seq + 1);
		con->in_base_pos = -front_len - middle_len - data_len -
			sizeof(m->footer);
		con->in_tag = CEPH_MSGR_TAG_READY;
		return 0;
	} else if ((s64)seq - (s64)con->in_seq > 1) {
		pr_err("read_partial_message bad seq %lld expected %lld\n",
		       seq, con->in_seq + 1);
		con->error_msg = "bad message sequence # for incoming message";
2345
		return -EBADE;
2346 2347
	}

S
Sage Weil 已提交
2348 2349
	/* allocate message? */
	if (!con->in_msg) {
2350 2351
		int skip = 0;

S
Sage Weil 已提交
2352
		dout("got hdr type %d front %d data %d\n", con->in_hdr.type,
2353
		     front_len, data_len);
2354 2355 2356
		ret = ceph_con_in_msg_alloc(con, &skip);
		if (ret < 0)
			return ret;
2357 2358

		BUG_ON(!con->in_msg ^ skip);
2359
		if (skip) {
S
Sage Weil 已提交
2360
			/* skip this message */
2361
			dout("alloc_msg said skip message\n");
S
Sage Weil 已提交
2362 2363 2364
			con->in_base_pos = -front_len - middle_len - data_len -
				sizeof(m->footer);
			con->in_tag = CEPH_MSGR_TAG_READY;
2365
			con->in_seq++;
S
Sage Weil 已提交
2366 2367
			return 0;
		}
2368

2369
		BUG_ON(!con->in_msg);
2370
		BUG_ON(con->in_msg->con != con);
S
Sage Weil 已提交
2371 2372
		m = con->in_msg;
		m->front.iov_len = 0;    /* haven't read it yet */
2373 2374
		if (m->middle)
			m->middle->vec.iov_len = 0;
2375

2376
		/* prepare for data payload, if any */
2377

2378
		if (data_len)
2379
			prepare_message_data(con->in_msg, data_len);
S
Sage Weil 已提交
2380 2381 2382
	}

	/* front */
2383 2384 2385 2386
	ret = read_partial_message_section(con, &m->front, front_len,
					   &con->in_front_crc);
	if (ret <= 0)
		return ret;
S
Sage Weil 已提交
2387 2388

	/* middle */
2389
	if (m->middle) {
S
Sage Weil 已提交
2390 2391
		ret = read_partial_message_section(con, &m->middle->vec,
						   middle_len,
2392
						   &con->in_middle_crc);
S
Sage Weil 已提交
2393 2394 2395 2396 2397
		if (ret <= 0)
			return ret;
	}

	/* (page) data */
2398 2399 2400 2401
	if (data_len) {
		ret = read_partial_msg_data(con);
		if (ret <= 0)
			return ret;
S
Sage Weil 已提交
2402 2403 2404
	}

	/* footer */
Y
Yan, Zheng 已提交
2405 2406 2407 2408 2409
	if (need_sign)
		size = sizeof(m->footer);
	else
		size = sizeof(m->old_footer);

2410 2411
	end += size;
	ret = read_partial(con, end, size, &m->footer);
2412 2413 2414
	if (ret <= 0)
		return ret;

Y
Yan, Zheng 已提交
2415 2416 2417 2418 2419
	if (!need_sign) {
		m->footer.flags = m->old_footer.flags;
		m->footer.sig = 0;
	}

S
Sage Weil 已提交
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
	dout("read_partial_message got msg %p %d (%u) + %d (%u) + %d (%u)\n",
	     m, front_len, m->footer.front_crc, middle_len,
	     m->footer.middle_crc, data_len, m->footer.data_crc);

	/* crc ok? */
	if (con->in_front_crc != le32_to_cpu(m->footer.front_crc)) {
		pr_err("read_partial_message %p front crc %u != exp. %u\n",
		       m, con->in_front_crc, m->footer.front_crc);
		return -EBADMSG;
	}
	if (con->in_middle_crc != le32_to_cpu(m->footer.middle_crc)) {
		pr_err("read_partial_message %p middle crc %u != exp %u\n",
		       m, con->in_middle_crc, m->footer.middle_crc);
		return -EBADMSG;
	}
2435
	if (do_datacrc &&
S
Sage Weil 已提交
2436 2437 2438 2439 2440 2441 2442
	    (m->footer.flags & CEPH_MSG_FOOTER_NOCRC) == 0 &&
	    con->in_data_crc != le32_to_cpu(m->footer.data_crc)) {
		pr_err("read_partial_message %p data crc %u != exp. %u\n", m,
		       con->in_data_crc, le32_to_cpu(m->footer.data_crc));
		return -EBADMSG;
	}

Y
Yan, Zheng 已提交
2443
	if (need_sign && con->ops->check_message_signature &&
2444
	    con->ops->check_message_signature(m)) {
Y
Yan, Zheng 已提交
2445 2446 2447 2448
		pr_err("read_partial_message %p signature check failed\n", m);
		return -EBADMSG;
	}

S
Sage Weil 已提交
2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
	return 1; /* done! */
}

/*
 * Process message.  This happens in the worker thread.  The callback should
 * be careful not to do anything that waits on other incoming messages or it
 * may deadlock.
 */
static void process_message(struct ceph_connection *con)
{
2459
	struct ceph_msg *msg = con->in_msg;
S
Sage Weil 已提交
2460

2461
	BUG_ON(con->in_msg->con != con);
S
Sage Weil 已提交
2462 2463 2464 2465
	con->in_msg = NULL;

	/* if first message, set peer_name */
	if (con->peer_name.type == 0)
2466
		con->peer_name = msg->hdr.src;
S
Sage Weil 已提交
2467 2468

	con->in_seq++;
2469
	mutex_unlock(&con->mutex);
S
Sage Weil 已提交
2470 2471 2472

	dout("===== %p %llu from %s%lld %d=%s len %d+%d (%u %u %u) =====\n",
	     msg, le64_to_cpu(msg->hdr.seq),
2473
	     ENTITY_NAME(msg->hdr.src),
S
Sage Weil 已提交
2474 2475 2476 2477 2478 2479
	     le16_to_cpu(msg->hdr.type),
	     ceph_msg_type_name(le16_to_cpu(msg->hdr.type)),
	     le32_to_cpu(msg->hdr.front_len),
	     le32_to_cpu(msg->hdr.data_len),
	     con->in_front_crc, con->in_middle_crc, con->in_data_crc);
	con->ops->dispatch(con, msg);
2480 2481

	mutex_lock(&con->mutex);
S
Sage Weil 已提交
2482 2483
}

2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
static int read_keepalive_ack(struct ceph_connection *con)
{
	struct ceph_timespec ceph_ts;
	size_t size = sizeof(ceph_ts);
	int ret = read_partial(con, size, size, &ceph_ts);
	if (ret <= 0)
		return ret;
	ceph_decode_timespec(&con->last_keepalive_ack, &ceph_ts);
	prepare_read_tag(con);
	return 1;
}
S
Sage Weil 已提交
2495 2496 2497 2498 2499 2500 2501 2502 2503

/*
 * Write something to the socket.  Called in a worker thread when the
 * socket appears to be writeable and we have something ready to send.
 */
static int try_write(struct ceph_connection *con)
{
	int ret = 1;

2504
	dout("try_write start %p state %lu\n", con, con->state);
S
Sage Weil 已提交
2505 2506 2507 2508 2509

more:
	dout("try_write out_kvec_bytes %d\n", con->out_kvec_bytes);

	/* open the socket first? */
2510 2511 2512
	if (con->state == CON_STATE_PREOPEN) {
		BUG_ON(con->sock);
		con->state = CON_STATE_CONNECTING;
2513

2514
		con_out_kvec_reset(con);
2515
		prepare_write_banner(con);
2516
		prepare_read_banner(con);
S
Sage Weil 已提交
2517

2518
		BUG_ON(con->in_msg);
S
Sage Weil 已提交
2519 2520 2521
		con->in_tag = CEPH_MSGR_TAG_READY;
		dout("try_write initiating connect on %p new state %lu\n",
		     con, con->state);
2522 2523
		ret = ceph_tcp_connect(con);
		if (ret < 0) {
S
Sage Weil 已提交
2524 2525 2526 2527 2528 2529 2530
			con->error_msg = "connect error";
			goto out;
		}
	}

more_kvec:
	/* kvec data queued? */
I
Ilya Dryomov 已提交
2531 2532
	if (con->out_kvec_left) {
		ret = write_partial_kvec(con);
S
Sage Weil 已提交
2533
		if (ret <= 0)
2534
			goto out;
S
Sage Weil 已提交
2535
	}
I
Ilya Dryomov 已提交
2536 2537
	if (con->out_skip) {
		ret = write_partial_skip(con);
S
Sage Weil 已提交
2538
		if (ret <= 0)
2539
			goto out;
S
Sage Weil 已提交
2540 2541 2542 2543
	}

	/* msg pages? */
	if (con->out_msg) {
2544 2545 2546 2547 2548 2549
		if (con->out_msg_done) {
			ceph_msg_put(con->out_msg);
			con->out_msg = NULL;   /* we're done with this one */
			goto do_next;
		}

2550
		ret = write_partial_message_data(con);
S
Sage Weil 已提交
2551 2552 2553
		if (ret == 1)
			goto more_kvec;  /* we need to send the footer, too! */
		if (ret == 0)
2554
			goto out;
S
Sage Weil 已提交
2555
		if (ret < 0) {
2556
			dout("try_write write_partial_message_data err %d\n",
S
Sage Weil 已提交
2557
			     ret);
2558
			goto out;
S
Sage Weil 已提交
2559 2560 2561
		}
	}

2562
do_next:
2563
	if (con->state == CON_STATE_OPEN) {
2564 2565 2566 2567
		if (con_flag_test_and_clear(con, CON_FLAG_KEEPALIVE_PENDING)) {
			prepare_write_keepalive(con);
			goto more;
		}
S
Sage Weil 已提交
2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
		/* is anything else pending? */
		if (!list_empty(&con->out_queue)) {
			prepare_write_message(con);
			goto more;
		}
		if (con->in_seq > con->in_seq_acked) {
			prepare_write_ack(con);
			goto more;
		}
	}

	/* Nothing to do! */
2580
	con_flag_clear(con, CON_FLAG_WRITE_PENDING);
S
Sage Weil 已提交
2581 2582 2583
	dout("try_write nothing else to write.\n");
	ret = 0;
out:
2584
	dout("try_write done on %p ret %d\n", con, ret);
S
Sage Weil 已提交
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
	return ret;
}



/*
 * Read what we can from the socket.
 */
static int try_read(struct ceph_connection *con)
{
	int ret = -1;

2597 2598 2599 2600 2601
more:
	dout("try_read start on %p state %lu\n", con, con->state);
	if (con->state != CON_STATE_CONNECTING &&
	    con->state != CON_STATE_NEGOTIATING &&
	    con->state != CON_STATE_OPEN)
S
Sage Weil 已提交
2602 2603
		return 0;

2604
	BUG_ON(!con->sock);
2605

S
Sage Weil 已提交
2606 2607
	dout("try_read tag %d in_base_pos %d\n", (int)con->in_tag,
	     con->in_base_pos);
2608

2609
	if (con->state == CON_STATE_CONNECTING) {
2610 2611 2612
		dout("try_read connecting\n");
		ret = read_partial_banner(con);
		if (ret <= 0)
2613
			goto out;
2614 2615 2616 2617
		ret = process_banner(con);
		if (ret < 0)
			goto out;

2618
		con->state = CON_STATE_NEGOTIATING;
2619

2620 2621 2622 2623 2624
		/*
		 * Received banner is good, exchange connection info.
		 * Do not reset out_kvec, as sending our banner raced
		 * with receiving peer banner after connect completed.
		 */
2625 2626 2627 2628 2629 2630
		ret = prepare_write_connect(con);
		if (ret < 0)
			goto out;
		prepare_read_connect(con);

		/* Send connection info before awaiting response */
2631 2632 2633
		goto out;
	}

2634
	if (con->state == CON_STATE_NEGOTIATING) {
2635
		dout("try_read negotiating\n");
S
Sage Weil 已提交
2636 2637 2638
		ret = read_partial_connect(con);
		if (ret <= 0)
			goto out;
2639 2640 2641
		ret = process_connect(con);
		if (ret < 0)
			goto out;
S
Sage Weil 已提交
2642 2643 2644
		goto more;
	}

2645
	WARN_ON(con->state != CON_STATE_OPEN);
2646

S
Sage Weil 已提交
2647 2648 2649 2650 2651 2652
	if (con->in_base_pos < 0) {
		/*
		 * skipping + discarding content.
		 *
		 * FIXME: there must be a better way to do this!
		 */
A
Alex Elder 已提交
2653 2654 2655
		static char buf[SKIP_BUF_SIZE];
		int skip = min((int) sizeof (buf), -con->in_base_pos);

S
Sage Weil 已提交
2656 2657 2658
		dout("skipping %d / %d bytes\n", skip, -con->in_base_pos);
		ret = ceph_tcp_recvmsg(con->sock, buf, skip);
		if (ret <= 0)
2659
			goto out;
S
Sage Weil 已提交
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669
		con->in_base_pos += ret;
		if (con->in_base_pos)
			goto more;
	}
	if (con->in_tag == CEPH_MSGR_TAG_READY) {
		/*
		 * what's next?
		 */
		ret = ceph_tcp_recvmsg(con->sock, &con->in_tag, 1);
		if (ret <= 0)
2670
			goto out;
S
Sage Weil 已提交
2671 2672 2673 2674 2675 2676 2677 2678
		dout("try_read got tag %d\n", (int)con->in_tag);
		switch (con->in_tag) {
		case CEPH_MSGR_TAG_MSG:
			prepare_read_message(con);
			break;
		case CEPH_MSGR_TAG_ACK:
			prepare_read_ack(con);
			break;
2679 2680 2681
		case CEPH_MSGR_TAG_KEEPALIVE2_ACK:
			prepare_read_keepalive_ack(con);
			break;
S
Sage Weil 已提交
2682
		case CEPH_MSGR_TAG_CLOSE:
2683 2684
			con_close_socket(con);
			con->state = CON_STATE_CLOSED;
2685
			goto out;
S
Sage Weil 已提交
2686 2687 2688 2689 2690 2691 2692 2693 2694
		default:
			goto bad_tag;
		}
	}
	if (con->in_tag == CEPH_MSGR_TAG_MSG) {
		ret = read_partial_message(con);
		if (ret <= 0) {
			switch (ret) {
			case -EBADMSG:
2695
				con->error_msg = "bad crc/signature";
2696 2697
				/* fall through */
			case -EBADE:
S
Sage Weil 已提交
2698
				ret = -EIO;
2699
				break;
S
Sage Weil 已提交
2700 2701
			case -EIO:
				con->error_msg = "io error";
2702
				break;
S
Sage Weil 已提交
2703
			}
2704
			goto out;
S
Sage Weil 已提交
2705 2706 2707 2708
		}
		if (con->in_tag == CEPH_MSGR_TAG_READY)
			goto more;
		process_message(con);
2709 2710
		if (con->state == CON_STATE_OPEN)
			prepare_read_tag(con);
S
Sage Weil 已提交
2711 2712
		goto more;
	}
2713 2714 2715 2716 2717 2718
	if (con->in_tag == CEPH_MSGR_TAG_ACK ||
	    con->in_tag == CEPH_MSGR_TAG_SEQ) {
		/*
		 * the final handshake seq exchange is semantically
		 * equivalent to an ACK
		 */
S
Sage Weil 已提交
2719 2720
		ret = read_partial_ack(con);
		if (ret <= 0)
2721
			goto out;
S
Sage Weil 已提交
2722 2723 2724
		process_ack(con);
		goto more;
	}
2725 2726 2727 2728 2729 2730
	if (con->in_tag == CEPH_MSGR_TAG_KEEPALIVE2_ACK) {
		ret = read_keepalive_ack(con);
		if (ret <= 0)
			goto out;
		goto more;
	}
S
Sage Weil 已提交
2731 2732

out:
2733
	dout("try_read done on %p ret %d\n", con, ret);
S
Sage Weil 已提交
2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744
	return ret;

bad_tag:
	pr_err("try_read bad con->in_tag = %d\n", (int)con->in_tag);
	con->error_msg = "protocol error, garbage tag";
	ret = -1;
	goto out;
}


/*
A
Alex Elder 已提交
2745 2746 2747
 * Atomically queue work on a connection after the specified delay.
 * Bump @con reference to avoid races with connection teardown.
 * Returns 0 if work was queued, or an error code otherwise.
S
Sage Weil 已提交
2748
 */
A
Alex Elder 已提交
2749
static int queue_con_delay(struct ceph_connection *con, unsigned long delay)
S
Sage Weil 已提交
2750 2751
{
	if (!con->ops->get(con)) {
A
Alex Elder 已提交
2752 2753
		dout("%s %p ref count 0\n", __func__, con);
		return -ENOENT;
S
Sage Weil 已提交
2754 2755
	}

A
Alex Elder 已提交
2756 2757
	if (!queue_delayed_work(ceph_msgr_wq, &con->work, delay)) {
		dout("%s %p - already queued\n", __func__, con);
S
Sage Weil 已提交
2758
		con->ops->put(con);
A
Alex Elder 已提交
2759
		return -EBUSY;
S
Sage Weil 已提交
2760
	}
A
Alex Elder 已提交
2761 2762 2763 2764 2765 2766 2767 2768

	dout("%s %p %lu\n", __func__, con, delay);
	return 0;
}

static void queue_con(struct ceph_connection *con)
{
	(void) queue_con_delay(con, 0);
S
Sage Weil 已提交
2769 2770
}

2771 2772 2773 2774 2775 2776 2777 2778
static void cancel_con(struct ceph_connection *con)
{
	if (cancel_delayed_work(&con->work)) {
		dout("%s %p\n", __func__, con);
		con->ops->put(con);
	}
}

2779 2780
static bool con_sock_closed(struct ceph_connection *con)
{
2781
	if (!con_flag_test_and_clear(con, CON_FLAG_SOCK_CLOSED))
2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796
		return false;

#define CASE(x)								\
	case CON_STATE_ ## x:						\
		con->error_msg = "socket closed (con state " #x ")";	\
		break;

	switch (con->state) {
	CASE(CLOSED);
	CASE(PREOPEN);
	CASE(CONNECTING);
	CASE(NEGOTIATING);
	CASE(OPEN);
	CASE(STANDBY);
	default:
2797
		pr_warn("%s con %p unrecognized state %lu\n",
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
			__func__, con, con->state);
		con->error_msg = "unrecognized con state";
		BUG();
		break;
	}
#undef CASE

	return true;
}

2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
static bool con_backoff(struct ceph_connection *con)
{
	int ret;

	if (!con_flag_test_and_clear(con, CON_FLAG_BACKOFF))
		return false;

	ret = queue_con_delay(con, round_jiffies_relative(con->delay));
	if (ret) {
		dout("%s: con %p FAILED to back off %lu\n", __func__,
			con, con->delay);
		BUG_ON(ret == -ENOENT);
		con_flag_set(con, CON_FLAG_BACKOFF);
	}

	return true;
}

2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842
/* Finish fault handling; con->mutex must *not* be held here */

static void con_fault_finish(struct ceph_connection *con)
{
	/*
	 * in case we faulted due to authentication, invalidate our
	 * current tickets so that we can get new ones.
	 */
	if (con->auth_retry && con->ops->invalidate_authorizer) {
		dout("calling invalidate_authorizer()\n");
		con->ops->invalidate_authorizer(con);
	}

	if (con->ops->fault)
		con->ops->fault(con);
}

S
Sage Weil 已提交
2843 2844 2845
/*
 * Do some work on a connection.  Drop a connection ref when we're done.
 */
2846
static void ceph_con_workfn(struct work_struct *work)
S
Sage Weil 已提交
2847 2848 2849
{
	struct ceph_connection *con = container_of(work, struct ceph_connection,
						   work.work);
2850
	bool fault;
S
Sage Weil 已提交
2851

S
Sage Weil 已提交
2852
	mutex_lock(&con->mutex);
2853 2854
	while (true) {
		int ret;
S
Sage Weil 已提交
2855

2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
		if ((fault = con_sock_closed(con))) {
			dout("%s: con %p SOCK_CLOSED\n", __func__, con);
			break;
		}
		if (con_backoff(con)) {
			dout("%s: con %p BACKOFF\n", __func__, con);
			break;
		}
		if (con->state == CON_STATE_STANDBY) {
			dout("%s: con %p STANDBY\n", __func__, con);
			break;
		}
		if (con->state == CON_STATE_CLOSED) {
			dout("%s: con %p CLOSED\n", __func__, con);
			BUG_ON(con->sock);
			break;
		}
		if (con->state == CON_STATE_PREOPEN) {
			dout("%s: con %p PREOPEN\n", __func__, con);
			BUG_ON(con->sock);
		}
2877

2878 2879 2880 2881
		ret = try_read(con);
		if (ret < 0) {
			if (ret == -EAGAIN)
				continue;
2882 2883
			if (!con->error_msg)
				con->error_msg = "socket error on read";
2884 2885 2886 2887 2888 2889 2890 2891
			fault = true;
			break;
		}

		ret = try_write(con);
		if (ret < 0) {
			if (ret == -EAGAIN)
				continue;
2892 2893
			if (!con->error_msg)
				con->error_msg = "socket error on write";
2894 2895 2896 2897
			fault = true;
		}

		break;	/* If we make it to here, we're done */
2898
	}
2899 2900
	if (fault)
		con_fault(con);
S
Sage Weil 已提交
2901
	mutex_unlock(&con->mutex);
2902

2903 2904 2905 2906
	if (fault)
		con_fault_finish(con);

	con->ops->put(con);
S
Sage Weil 已提交
2907 2908 2909 2910 2911 2912
}

/*
 * Generic error/fault handler.  A retry mechanism is used with
 * exponential backoff
 */
2913
static void con_fault(struct ceph_connection *con)
S
Sage Weil 已提交
2914 2915
{
	dout("fault %p state %lu to peer %s\n",
2916
	     con, con->state, ceph_pr_addr(&con->peer_addr.in_addr));
S
Sage Weil 已提交
2917

2918 2919 2920 2921
	pr_warn("%s%lld %s %s\n", ENTITY_NAME(con->peer_name),
		ceph_pr_addr(&con->peer_addr.in_addr), con->error_msg);
	con->error_msg = NULL;

2922
	WARN_ON(con->state != CON_STATE_CONNECTING &&
2923 2924
	       con->state != CON_STATE_NEGOTIATING &&
	       con->state != CON_STATE_OPEN);
2925

S
Sage Weil 已提交
2926
	con_close_socket(con);
S
Sage Weil 已提交
2927

2928
	if (con_flag_test(con, CON_FLAG_LOSSYTX)) {
2929 2930
		dout("fault on LOSSYTX channel, marking CLOSED\n");
		con->state = CON_STATE_CLOSED;
2931
		return;
2932 2933
	}

S
Sage Weil 已提交
2934
	if (con->in_msg) {
2935
		BUG_ON(con->in_msg->con != con);
S
Sage Weil 已提交
2936 2937 2938
		ceph_msg_put(con->in_msg);
		con->in_msg = NULL;
	}
S
Sage Weil 已提交
2939

2940 2941
	/* Requeue anything that hasn't been acked */
	list_splice_init(&con->out_sent, &con->out_queue);
2942

S
Sage Weil 已提交
2943 2944 2945
	/* If there are no messages queued or keepalive pending, place
	 * the connection in a STANDBY state */
	if (list_empty(&con->out_queue) &&
2946
	    !con_flag_test(con, CON_FLAG_KEEPALIVE_PENDING)) {
S
Sage Weil 已提交
2947
		dout("fault %p setting STANDBY clearing WRITE_PENDING\n", con);
2948
		con_flag_clear(con, CON_FLAG_WRITE_PENDING);
2949
		con->state = CON_STATE_STANDBY;
2950 2951
	} else {
		/* retry after a delay. */
2952
		con->state = CON_STATE_PREOPEN;
2953 2954 2955 2956
		if (con->delay == 0)
			con->delay = BASE_DELAY_INTERVAL;
		else if (con->delay < MAX_DELAY_INTERVAL)
			con->delay *= 2;
2957
		con_flag_set(con, CON_FLAG_BACKOFF);
A
Alex Elder 已提交
2958
		queue_con(con);
S
Sage Weil 已提交
2959 2960 2961 2962 2963 2964
	}
}



/*
2965
 * initialize a new messenger instance
S
Sage Weil 已提交
2966
 */
2967
void ceph_messenger_init(struct ceph_messenger *msgr,
2968
			 struct ceph_entity_addr *myaddr)
S
Sage Weil 已提交
2969 2970 2971 2972 2973 2974 2975
{
	spin_lock_init(&msgr->global_seq_lock);

	if (myaddr)
		msgr->inst.addr = *myaddr;

	/* select a random nonce */
2976
	msgr->inst.addr.type = 0;
S
Sage Weil 已提交
2977
	get_random_bytes(&msgr->inst.addr.nonce, sizeof(msgr->inst.addr.nonce));
2978
	encode_my_addr(msgr);
S
Sage Weil 已提交
2979

2980
	atomic_set(&msgr->stopping, 0);
2981
	write_pnet(&msgr->net, get_net(current->nsproxy->net_ns));
S
Sage Weil 已提交
2982

2983
	dout("%s %p\n", __func__, msgr);
S
Sage Weil 已提交
2984
}
2985
EXPORT_SYMBOL(ceph_messenger_init);
S
Sage Weil 已提交
2986

2987 2988 2989 2990 2991 2992
void ceph_messenger_fini(struct ceph_messenger *msgr)
{
	put_net(read_pnet(&msgr->net));
}
EXPORT_SYMBOL(ceph_messenger_fini);

2993 2994 2995 2996 2997 2998 2999 3000 3001
static void msg_con_set(struct ceph_msg *msg, struct ceph_connection *con)
{
	if (msg->con)
		msg->con->ops->put(msg->con);

	msg->con = con ? con->ops->get(con) : NULL;
	BUG_ON(msg->con != con);
}

S
Sage Weil 已提交
3002 3003 3004
static void clear_standby(struct ceph_connection *con)
{
	/* come back from STANDBY? */
3005
	if (con->state == CON_STATE_STANDBY) {
S
Sage Weil 已提交
3006
		dout("clear_standby %p and ++connect_seq\n", con);
3007
		con->state = CON_STATE_PREOPEN;
S
Sage Weil 已提交
3008
		con->connect_seq++;
3009 3010
		WARN_ON(con_flag_test(con, CON_FLAG_WRITE_PENDING));
		WARN_ON(con_flag_test(con, CON_FLAG_KEEPALIVE_PENDING));
S
Sage Weil 已提交
3011 3012 3013
	}
}

S
Sage Weil 已提交
3014 3015 3016 3017 3018 3019
/*
 * Queue up an outgoing message on the given connection.
 */
void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg)
{
	/* set src+dst */
3020
	msg->hdr.src = con->msgr->inst.name;
3021
	BUG_ON(msg->front.iov_len != le32_to_cpu(msg->hdr.front_len));
3022 3023
	msg->needs_out_seq = true;

3024
	mutex_lock(&con->mutex);
3025

3026
	if (con->state == CON_STATE_CLOSED) {
3027 3028 3029 3030 3031 3032
		dout("con_send %p closed, dropping %p\n", con, msg);
		ceph_msg_put(msg);
		mutex_unlock(&con->mutex);
		return;
	}

3033
	msg_con_set(msg, con);
3034

S
Sage Weil 已提交
3035 3036 3037 3038 3039 3040 3041 3042
	BUG_ON(!list_empty(&msg->list_head));
	list_add_tail(&msg->list_head, &con->out_queue);
	dout("----- %p to %s%lld %d=%s len %d+%d+%d -----\n", msg,
	     ENTITY_NAME(con->peer_name), le16_to_cpu(msg->hdr.type),
	     ceph_msg_type_name(le16_to_cpu(msg->hdr.type)),
	     le32_to_cpu(msg->hdr.front_len),
	     le32_to_cpu(msg->hdr.middle_len),
	     le32_to_cpu(msg->hdr.data_len));
3043 3044

	clear_standby(con);
3045
	mutex_unlock(&con->mutex);
S
Sage Weil 已提交
3046 3047 3048

	/* if there wasn't anything waiting to send before, queue
	 * new work */
3049
	if (con_flag_test_and_set(con, CON_FLAG_WRITE_PENDING) == 0)
S
Sage Weil 已提交
3050 3051
		queue_con(con);
}
3052
EXPORT_SYMBOL(ceph_con_send);
S
Sage Weil 已提交
3053 3054 3055 3056

/*
 * Revoke a message that was previously queued for send
 */
3057
void ceph_msg_revoke(struct ceph_msg *msg)
S
Sage Weil 已提交
3058
{
3059 3060
	struct ceph_connection *con = msg->con;

3061 3062
	if (!con) {
		dout("%s msg %p null con\n", __func__, msg);
3063
		return;		/* Message not in our possession */
3064
	}
3065

3066
	mutex_lock(&con->mutex);
S
Sage Weil 已提交
3067
	if (!list_empty(&msg->list_head)) {
3068
		dout("%s %p msg %p - was on queue\n", __func__, con, msg);
S
Sage Weil 已提交
3069 3070
		list_del_init(&msg->list_head);
		msg->hdr.seq = 0;
3071

S
Sage Weil 已提交
3072
		ceph_msg_put(msg);
S
Sage Weil 已提交
3073 3074
	}
	if (con->out_msg == msg) {
I
Ilya Dryomov 已提交
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
		BUG_ON(con->out_skip);
		/* footer */
		if (con->out_msg_done) {
			con->out_skip += con_out_kvec_skip(con);
		} else {
			BUG_ON(!msg->data_length);
			if (con->peer_features & CEPH_FEATURE_MSG_AUTH)
				con->out_skip += sizeof(msg->footer);
			else
				con->out_skip += sizeof(msg->old_footer);
S
Sage Weil 已提交
3085
		}
I
Ilya Dryomov 已提交
3086 3087 3088 3089 3090 3091 3092 3093 3094
		/* data, middle, front */
		if (msg->data_length)
			con->out_skip += msg->cursor.total_resid;
		if (msg->middle)
			con->out_skip += con_out_kvec_skip(con);
		con->out_skip += con_out_kvec_skip(con);

		dout("%s %p msg %p - was sending, will write %d skip %d\n",
		     __func__, con, msg, con->out_kvec_bytes, con->out_skip);
S
Sage Weil 已提交
3095
		msg->hdr.seq = 0;
I
Ilya Dryomov 已提交
3096
		con->out_msg = NULL;
3097
		ceph_msg_put(msg);
S
Sage Weil 已提交
3098
	}
I
Ilya Dryomov 已提交
3099

3100
	mutex_unlock(&con->mutex);
S
Sage Weil 已提交
3101 3102
}

3103
/*
3104
 * Revoke a message that we may be reading data into
3105
 */
3106
void ceph_msg_revoke_incoming(struct ceph_msg *msg)
3107
{
3108
	struct ceph_connection *con = msg->con;
3109

3110
	if (!con) {
3111 3112 3113 3114
		dout("%s msg %p null con\n", __func__, msg);
		return;		/* Message not in our possession */
	}

3115
	mutex_lock(&con->mutex);
3116
	if (con->in_msg == msg) {
3117 3118 3119
		unsigned int front_len = le32_to_cpu(con->in_hdr.front_len);
		unsigned int middle_len = le32_to_cpu(con->in_hdr.middle_len);
		unsigned int data_len = le32_to_cpu(con->in_hdr.data_len);
3120 3121

		/* skip rest of message */
3122 3123
		dout("%s %p msg %p revoked\n", __func__, con, msg);
		con->in_base_pos = con->in_base_pos -
3124
				sizeof(struct ceph_msg_header) -
3125 3126 3127
				front_len -
				middle_len -
				data_len -
3128 3129 3130 3131
				sizeof(struct ceph_msg_footer);
		ceph_msg_put(con->in_msg);
		con->in_msg = NULL;
		con->in_tag = CEPH_MSGR_TAG_READY;
3132
		con->in_seq++;
3133
	} else {
3134 3135
		dout("%s %p in_msg %p msg %p no-op\n",
		     __func__, con, con->in_msg, msg);
3136 3137 3138 3139
	}
	mutex_unlock(&con->mutex);
}

S
Sage Weil 已提交
3140 3141 3142 3143 3144
/*
 * Queue a keepalive byte to ensure the tcp connection is alive.
 */
void ceph_con_keepalive(struct ceph_connection *con)
{
S
Sage Weil 已提交
3145
	dout("con_keepalive %p\n", con);
3146
	mutex_lock(&con->mutex);
S
Sage Weil 已提交
3147
	clear_standby(con);
3148
	mutex_unlock(&con->mutex);
3149 3150
	if (con_flag_test_and_set(con, CON_FLAG_KEEPALIVE_PENDING) == 0 &&
	    con_flag_test_and_set(con, CON_FLAG_WRITE_PENDING) == 0)
S
Sage Weil 已提交
3151 3152
		queue_con(con);
}
3153
EXPORT_SYMBOL(ceph_con_keepalive);
S
Sage Weil 已提交
3154

3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168
bool ceph_con_keepalive_expired(struct ceph_connection *con,
			       unsigned long interval)
{
	if (interval > 0 &&
	    (con->peer_features & CEPH_FEATURE_MSGR_KEEPALIVE2)) {
		struct timespec now = CURRENT_TIME;
		struct timespec ts;
		jiffies_to_timespec(interval, &ts);
		ts = timespec_add(con->last_keepalive_ack, ts);
		return timespec_compare(&now, &ts) >= 0;
	}
	return false;
}

3169
static struct ceph_msg_data *ceph_msg_data_create(enum ceph_msg_data_type type)
A
Alex Elder 已提交
3170
{
3171 3172 3173 3174 3175
	struct ceph_msg_data *data;

	if (WARN_ON(!ceph_msg_data_type_valid(type)))
		return NULL;

3176
	data = kmem_cache_zalloc(ceph_msg_data_cache, GFP_NOFS);
3177 3178
	if (data)
		data->type = type;
3179
	INIT_LIST_HEAD(&data->links);
3180 3181 3182 3183 3184 3185 3186 3187 3188

	return data;
}

static void ceph_msg_data_destroy(struct ceph_msg_data *data)
{
	if (!data)
		return;

3189
	WARN_ON(!list_empty(&data->links));
Y
Yan, Zheng 已提交
3190
	if (data->type == CEPH_MSG_DATA_PAGELIST)
3191
		ceph_pagelist_release(data->pagelist);
3192
	kmem_cache_free(ceph_msg_data_cache, data);
A
Alex Elder 已提交
3193 3194
}

3195
void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages,
3196
		size_t length, size_t alignment)
3197
{
3198 3199
	struct ceph_msg_data *data;

3200 3201
	BUG_ON(!pages);
	BUG_ON(!length);
3202 3203 3204 3205 3206 3207

	data = ceph_msg_data_create(CEPH_MSG_DATA_PAGES);
	BUG_ON(!data);
	data->pages = pages;
	data->length = length;
	data->alignment = alignment & ~PAGE_MASK;
3208

3209 3210
	list_add_tail(&data->links, &msg->data);
	msg->data_length += length;
3211
}
3212
EXPORT_SYMBOL(ceph_msg_data_add_pages);
S
Sage Weil 已提交
3213

3214
void ceph_msg_data_add_pagelist(struct ceph_msg *msg,
3215 3216
				struct ceph_pagelist *pagelist)
{
3217 3218
	struct ceph_msg_data *data;

3219 3220
	BUG_ON(!pagelist);
	BUG_ON(!pagelist->length);
3221

3222 3223 3224 3225
	data = ceph_msg_data_create(CEPH_MSG_DATA_PAGELIST);
	BUG_ON(!data);
	data->pagelist = pagelist;

3226 3227
	list_add_tail(&data->links, &msg->data);
	msg->data_length += pagelist->length;
3228
}
3229
EXPORT_SYMBOL(ceph_msg_data_add_pagelist);
3230

3231
#ifdef	CONFIG_BLOCK
3232
void ceph_msg_data_add_bio(struct ceph_msg *msg, struct bio *bio,
A
Alex Elder 已提交
3233
		size_t length)
3234
{
3235 3236
	struct ceph_msg_data *data;

3237
	BUG_ON(!bio);
3238

3239 3240 3241
	data = ceph_msg_data_create(CEPH_MSG_DATA_BIO);
	BUG_ON(!data);
	data->bio = bio;
A
Alex Elder 已提交
3242
	data->bio_length = length;
3243

3244 3245
	list_add_tail(&data->links, &msg->data);
	msg->data_length += length;
3246
}
3247
EXPORT_SYMBOL(ceph_msg_data_add_bio);
3248
#endif	/* CONFIG_BLOCK */
3249

S
Sage Weil 已提交
3250 3251 3252 3253
/*
 * construct a new message with given type, size
 * the new msg has a ref count of 1.
 */
3254 3255
struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
			      bool can_fail)
S
Sage Weil 已提交
3256 3257 3258
{
	struct ceph_msg *m;

3259
	m = kmem_cache_zalloc(ceph_msg_cache, flags);
S
Sage Weil 已提交
3260 3261 3262 3263
	if (m == NULL)
		goto out;

	m->hdr.type = cpu_to_le16(type);
3264
	m->hdr.priority = cpu_to_le16(CEPH_MSG_PRIO_DEFAULT);
S
Sage Weil 已提交
3265
	m->hdr.front_len = cpu_to_le32(front_len);
3266

3267 3268
	INIT_LIST_HEAD(&m->list_head);
	kref_init(&m->kref);
3269
	INIT_LIST_HEAD(&m->data);
3270

S
Sage Weil 已提交
3271 3272
	/* front */
	if (front_len) {
3273
		m->front.iov_base = ceph_kvmalloc(front_len, flags);
S
Sage Weil 已提交
3274
		if (m->front.iov_base == NULL) {
3275
			dout("ceph_msg_new can't allocate %d bytes\n",
S
Sage Weil 已提交
3276 3277 3278 3279 3280 3281
			     front_len);
			goto out2;
		}
	} else {
		m->front.iov_base = NULL;
	}
3282
	m->front_alloc_len = m->front.iov_len = front_len;
S
Sage Weil 已提交
3283

S
Sage Weil 已提交
3284
	dout("ceph_msg_new %p front %d\n", m, front_len);
S
Sage Weil 已提交
3285 3286 3287 3288 3289
	return m;

out2:
	ceph_msg_put(m);
out:
3290 3291 3292
	if (!can_fail) {
		pr_err("msg_new can't create type %d front %d\n", type,
		       front_len);
3293
		WARN_ON(1);
3294 3295 3296 3297
	} else {
		dout("msg_new can't create type %d front %d\n", type,
		     front_len);
	}
3298
	return NULL;
S
Sage Weil 已提交
3299
}
3300
EXPORT_SYMBOL(ceph_msg_new);
S
Sage Weil 已提交
3301 3302 3303 3304 3305 3306 3307 3308

/*
 * Allocate "middle" portion of a message, if it is needed and wasn't
 * allocated by alloc_msg.  This allows us to read a small fixed-size
 * per-type header in the front and then gracefully fail (i.e.,
 * propagate the error to the caller based on info in the front) when
 * the middle is too large.
 */
3309
static int ceph_alloc_middle(struct ceph_connection *con, struct ceph_msg *msg)
S
Sage Weil 已提交
3310 3311 3312 3313 3314 3315 3316 3317 3318
{
	int type = le16_to_cpu(msg->hdr.type);
	int middle_len = le32_to_cpu(msg->hdr.middle_len);

	dout("alloc_middle %p type %d %s middle_len %d\n", msg, type,
	     ceph_msg_type_name(type), middle_len);
	BUG_ON(!middle_len);
	BUG_ON(msg->middle);

S
Sage Weil 已提交
3319
	msg->middle = ceph_buffer_new(middle_len, GFP_NOFS);
S
Sage Weil 已提交
3320 3321 3322 3323 3324
	if (!msg->middle)
		return -ENOMEM;
	return 0;
}

3325
/*
A
Alex Elder 已提交
3326 3327 3328 3329
 * Allocate a message for receiving an incoming message on a
 * connection, and save the result in con->in_msg.  Uses the
 * connection's private alloc_msg op if available.
 *
3330 3331 3332 3333 3334 3335 3336 3337 3338
 * Returns 0 on success, or a negative error code.
 *
 * On success, if we set *skip = 1:
 *  - the next message should be skipped and ignored.
 *  - con->in_msg == NULL
 * or if we set *skip = 0:
 *  - con->in_msg is non-null.
 * On error (ENOMEM, EAGAIN, ...),
 *  - con->in_msg == NULL
3339
 */
3340
static int ceph_con_in_msg_alloc(struct ceph_connection *con, int *skip)
3341
{
3342
	struct ceph_msg_header *hdr = &con->in_hdr;
3343
	int middle_len = le32_to_cpu(hdr->middle_len);
3344
	struct ceph_msg *msg;
3345
	int ret = 0;
3346

A
Alex Elder 已提交
3347
	BUG_ON(con->in_msg != NULL);
3348
	BUG_ON(!con->ops->alloc_msg);
3349

3350 3351 3352 3353 3354
	mutex_unlock(&con->mutex);
	msg = con->ops->alloc_msg(con, hdr, skip);
	mutex_lock(&con->mutex);
	if (con->state != CON_STATE_OPEN) {
		if (msg)
3355
			ceph_msg_put(msg);
3356 3357
		return -EAGAIN;
	}
3358 3359
	if (msg) {
		BUG_ON(*skip);
3360
		msg_con_set(msg, con);
3361 3362 3363 3364 3365 3366 3367 3368 3369 3370
		con->in_msg = msg;
	} else {
		/*
		 * Null message pointer means either we should skip
		 * this message or we couldn't allocate memory.  The
		 * former is not an error.
		 */
		if (*skip)
			return 0;

3371
		con->error_msg = "error allocating memory for incoming message";
3372
		return -ENOMEM;
3373
	}
A
Alex Elder 已提交
3374
	memcpy(&con->in_msg->hdr, &con->in_hdr, sizeof(con->in_hdr));
3375

A
Alex Elder 已提交
3376 3377
	if (middle_len && !con->in_msg->middle) {
		ret = ceph_alloc_middle(con, con->in_msg);
3378
		if (ret < 0) {
A
Alex Elder 已提交
3379 3380
			ceph_msg_put(con->in_msg);
			con->in_msg = NULL;
3381 3382
		}
	}
3383

3384
	return ret;
3385 3386
}

S
Sage Weil 已提交
3387 3388 3389 3390

/*
 * Free a generically kmalloc'd message.
 */
3391
static void ceph_msg_free(struct ceph_msg *m)
S
Sage Weil 已提交
3392
{
3393
	dout("%s %p\n", __func__, m);
I
Ilya Dryomov 已提交
3394
	kvfree(m->front.iov_base);
3395
	kmem_cache_free(ceph_msg_cache, m);
S
Sage Weil 已提交
3396 3397
}

3398
static void ceph_msg_release(struct kref *kref)
S
Sage Weil 已提交
3399 3400
{
	struct ceph_msg *m = container_of(kref, struct ceph_msg, kref);
3401
	struct ceph_msg_data *data, *next;
S
Sage Weil 已提交
3402

3403
	dout("%s %p\n", __func__, m);
S
Sage Weil 已提交
3404 3405
	WARN_ON(!list_empty(&m->list_head));

3406 3407
	msg_con_set(m, NULL);

S
Sage Weil 已提交
3408 3409 3410 3411
	/* drop middle, data, if any */
	if (m->middle) {
		ceph_buffer_put(m->middle);
		m->middle = NULL;
S
Sage Weil 已提交
3412
	}
3413

3414 3415
	list_for_each_entry_safe(data, next, &m->data, links) {
		list_del_init(&data->links);
3416 3417
		ceph_msg_data_destroy(data);
	}
A
Alex Elder 已提交
3418
	m->data_length = 0;
3419

S
Sage Weil 已提交
3420 3421 3422
	if (m->pool)
		ceph_msgpool_put(m->pool, m);
	else
3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439
		ceph_msg_free(m);
}

struct ceph_msg *ceph_msg_get(struct ceph_msg *msg)
{
	dout("%s %p (was %d)\n", __func__, msg,
	     atomic_read(&msg->kref.refcount));
	kref_get(&msg->kref);
	return msg;
}
EXPORT_SYMBOL(ceph_msg_get);

void ceph_msg_put(struct ceph_msg *msg)
{
	dout("%s %p (was %d)\n", __func__, msg,
	     atomic_read(&msg->kref.refcount));
	kref_put(&msg->kref, ceph_msg_release);
S
Sage Weil 已提交
3440
}
3441
EXPORT_SYMBOL(ceph_msg_put);
3442 3443 3444

void ceph_msg_dump(struct ceph_msg *msg)
{
3445 3446
	pr_debug("msg_dump %p (front_alloc_len %d length %zd)\n", msg,
		 msg->front_alloc_len, msg->data_length);
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461
	print_hex_dump(KERN_DEBUG, "header: ",
		       DUMP_PREFIX_OFFSET, 16, 1,
		       &msg->hdr, sizeof(msg->hdr), true);
	print_hex_dump(KERN_DEBUG, " front: ",
		       DUMP_PREFIX_OFFSET, 16, 1,
		       msg->front.iov_base, msg->front.iov_len, true);
	if (msg->middle)
		print_hex_dump(KERN_DEBUG, "middle: ",
			       DUMP_PREFIX_OFFSET, 16, 1,
			       msg->middle->vec.iov_base,
			       msg->middle->vec.iov_len, true);
	print_hex_dump(KERN_DEBUG, "footer: ",
		       DUMP_PREFIX_OFFSET, 16, 1,
		       &msg->footer, sizeof(msg->footer), true);
}
3462
EXPORT_SYMBOL(ceph_msg_dump);