message.c 9.7 KB
Newer Older
A
Andy Grover 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
/*
 * Copyright (c) 2006 Oracle.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
34
#include <linux/slab.h>
35
#include <linux/export.h>
A
Andy Grover 已提交
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51

#include "rds.h"

static unsigned int	rds_exthdr_size[__RDS_EXTHDR_MAX] = {
[RDS_EXTHDR_NONE]	= 0,
[RDS_EXTHDR_VERSION]	= sizeof(struct rds_ext_header_version),
[RDS_EXTHDR_RDMA]	= sizeof(struct rds_ext_header_rdma),
[RDS_EXTHDR_RDMA_DEST]	= sizeof(struct rds_ext_header_rdma_dest),
};


void rds_message_addref(struct rds_message *rm)
{
	rdsdebug("addref rm %p ref %d\n", rm, atomic_read(&rm->m_refcount));
	atomic_inc(&rm->m_refcount);
}
A
Andy Grover 已提交
52
EXPORT_SYMBOL_GPL(rds_message_addref);
A
Andy Grover 已提交
53 54 55 56 57 58 59 60 61 62 63

/*
 * This relies on dma_map_sg() not touching sg[].page during merging.
 */
static void rds_message_purge(struct rds_message *rm)
{
	unsigned long i;

	if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))
		return;

64 65
	for (i = 0; i < rm->data.op_nents; i++) {
		rdsdebug("putting data page %p\n", (void *)sg_page(&rm->data.op_sg[i]));
A
Andy Grover 已提交
66
		/* XXX will have to put_page for page refs */
67
		__free_page(sg_page(&rm->data.op_sg[i]));
A
Andy Grover 已提交
68
	}
69
	rm->data.op_nents = 0;
A
Andy Grover 已提交
70

A
Andy Grover 已提交
71 72 73 74
	if (rm->rdma.op_active)
		rds_rdma_free_op(&rm->rdma);
	if (rm->rdma.op_rdma_mr)
		rds_mr_put(rm->rdma.op_rdma_mr);
75 76 77 78 79

	if (rm->atomic.op_active)
		rds_atomic_free_op(&rm->atomic);
	if (rm->atomic.op_rdma_mr)
		rds_mr_put(rm->atomic.op_rdma_mr);
A
Andy Grover 已提交
80 81 82 83 84
}

void rds_message_put(struct rds_message *rm)
{
	rdsdebug("put rm %p ref %d\n", rm, atomic_read(&rm->m_refcount));
C
Cong Wang 已提交
85
	WARN(!atomic_read(&rm->m_refcount), "danger refcount zero on %p\n", rm);
A
Andy Grover 已提交
86 87 88 89 90 91 92 93
	if (atomic_dec_and_test(&rm->m_refcount)) {
		BUG_ON(!list_empty(&rm->m_sock_item));
		BUG_ON(!list_empty(&rm->m_conn_item));
		rds_message_purge(rm);

		kfree(rm);
	}
}
A
Andy Grover 已提交
94
EXPORT_SYMBOL_GPL(rds_message_put);
A
Andy Grover 已提交
95 96 97 98 99 100 101 102 103 104

void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
				 __be16 dport, u64 seq)
{
	hdr->h_flags = 0;
	hdr->h_sport = sport;
	hdr->h_dport = dport;
	hdr->h_sequence = cpu_to_be64(seq);
	hdr->h_exthdr[0] = RDS_EXTHDR_NONE;
}
A
Andy Grover 已提交
105
EXPORT_SYMBOL_GPL(rds_message_populate_header);
A
Andy Grover 已提交
106

107 108
int rds_message_add_extension(struct rds_header *hdr, unsigned int type,
			      const void *data, unsigned int len)
A
Andy Grover 已提交
109 110 111 112 113 114 115 116
{
	unsigned int ext_len = sizeof(u8) + len;
	unsigned char *dst;

	/* For now, refuse to add more than one extension header */
	if (hdr->h_exthdr[0] != RDS_EXTHDR_NONE)
		return 0;

117
	if (type >= __RDS_EXTHDR_MAX || len != rds_exthdr_size[type])
A
Andy Grover 已提交
118 119 120 121 122 123 124 125 126 127 128 129
		return 0;

	if (ext_len >= RDS_HEADER_EXT_SPACE)
		return 0;
	dst = hdr->h_exthdr;

	*dst++ = type;
	memcpy(dst, data, len);

	dst[len] = RDS_EXTHDR_NONE;
	return 1;
}
A
Andy Grover 已提交
130
EXPORT_SYMBOL_GPL(rds_message_add_extension);
A
Andy Grover 已提交
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185

/*
 * If a message has extension headers, retrieve them here.
 * Call like this:
 *
 * unsigned int pos = 0;
 *
 * while (1) {
 *	buflen = sizeof(buffer);
 *	type = rds_message_next_extension(hdr, &pos, buffer, &buflen);
 *	if (type == RDS_EXTHDR_NONE)
 *		break;
 *	...
 * }
 */
int rds_message_next_extension(struct rds_header *hdr,
		unsigned int *pos, void *buf, unsigned int *buflen)
{
	unsigned int offset, ext_type, ext_len;
	u8 *src = hdr->h_exthdr;

	offset = *pos;
	if (offset >= RDS_HEADER_EXT_SPACE)
		goto none;

	/* Get the extension type and length. For now, the
	 * length is implied by the extension type. */
	ext_type = src[offset++];

	if (ext_type == RDS_EXTHDR_NONE || ext_type >= __RDS_EXTHDR_MAX)
		goto none;
	ext_len = rds_exthdr_size[ext_type];
	if (offset + ext_len > RDS_HEADER_EXT_SPACE)
		goto none;

	*pos = offset + ext_len;
	if (ext_len < *buflen)
		*buflen = ext_len;
	memcpy(buf, src + offset, *buflen);
	return ext_type;

none:
	*pos = RDS_HEADER_EXT_SPACE;
	*buflen = 0;
	return RDS_EXTHDR_NONE;
}

int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset)
{
	struct rds_ext_header_rdma_dest ext_hdr;

	ext_hdr.h_rdma_rkey = cpu_to_be32(r_key);
	ext_hdr.h_rdma_offset = cpu_to_be32(offset);
	return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr, sizeof(ext_hdr));
}
A
Andy Grover 已提交
186
EXPORT_SYMBOL_GPL(rds_message_add_rdma_dest_extension);
A
Andy Grover 已提交
187

188 189 190 191 192 193
/*
 * Each rds_message is allocated with extra space for the scatterlist entries
 * rds ops will need. This is to minimize memory allocation count. Then, each rds op
 * can grab SGs when initializing its part of the rds_message.
 */
struct rds_message *rds_message_alloc(unsigned int extra_len, gfp_t gfp)
A
Andy Grover 已提交
194 195 196
{
	struct rds_message *rm;

197 198 199
	if (extra_len > KMALLOC_MAX_SIZE - sizeof(struct rds_message))
		return NULL;

200
	rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp);
A
Andy Grover 已提交
201 202 203
	if (!rm)
		goto out;

204 205 206
	rm->m_used_sgs = 0;
	rm->m_total_sgs = extra_len / sizeof(struct scatterlist);

A
Andy Grover 已提交
207 208 209 210
	atomic_set(&rm->m_refcount, 1);
	INIT_LIST_HEAD(&rm->m_sock_item);
	INIT_LIST_HEAD(&rm->m_conn_item);
	spin_lock_init(&rm->m_rs_lock);
C
Chris Mason 已提交
211
	init_waitqueue_head(&rm->m_flush_wait);
A
Andy Grover 已提交
212 213 214 215 216

out:
	return rm;
}

217 218 219 220 221 222 223 224 225
/*
 * RDS ops use this to grab SG entries from the rm's sg pool.
 */
struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents)
{
	struct scatterlist *sg_first = (struct scatterlist *) &rm[1];
	struct scatterlist *sg_ret;

	WARN_ON(rm->m_used_sgs + nents > rm->m_total_sgs);
226
	WARN_ON(!nents);
227

228 229 230
	if (rm->m_used_sgs + nents > rm->m_total_sgs)
		return NULL;

231
	sg_ret = &sg_first[rm->m_used_sgs];
232
	sg_init_table(sg_ret, nents);
233 234 235 236 237
	rm->m_used_sgs += nents;

	return sg_ret;
}

A
Andy Grover 已提交
238 239 240 241
struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len)
{
	struct rds_message *rm;
	unsigned int i;
242 243
	int num_sgs = ceil(total_len, PAGE_SIZE);
	int extra_bytes = num_sgs * sizeof(struct scatterlist);
A
Andy Grover 已提交
244

245
	rm = rds_message_alloc(extra_bytes, GFP_NOWAIT);
246
	if (!rm)
A
Andy Grover 已提交
247 248 249 250
		return ERR_PTR(-ENOMEM);

	set_bit(RDS_MSG_PAGEVEC, &rm->m_flags);
	rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);
251 252
	rm->data.op_nents = ceil(total_len, PAGE_SIZE);
	rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);
253 254
	if (!rm->data.op_sg) {
		rds_message_put(rm);
255
		return ERR_PTR(-ENOMEM);
256
	}
A
Andy Grover 已提交
257

258 259
	for (i = 0; i < rm->data.op_nents; ++i) {
		sg_set_page(&rm->data.op_sg[i],
A
Andy Grover 已提交
260 261 262 263 264 265 266
				virt_to_page(page_addrs[i]),
				PAGE_SIZE, 0);
	}

	return rm;
}

267
int rds_message_copy_from_user(struct rds_message *rm, struct iovec *first_iov,
A
Andy Grover 已提交
268 269 270 271 272 273 274
					       size_t total_len)
{
	unsigned long to_copy;
	unsigned long iov_off;
	unsigned long sg_off;
	struct iovec *iov;
	struct scatterlist *sg;
275
	int ret = 0;
A
Andy Grover 已提交
276 277 278 279 280 281

	rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);

	/*
	 * now allocate and copy in the data payload.
	 */
282
	sg = rm->data.op_sg;
A
Andy Grover 已提交
283 284 285 286 287
	iov = first_iov;
	iov_off = 0;
	sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */

	while (total_len) {
288
		if (!sg_page(sg)) {
A
Andy Grover 已提交
289 290 291 292
			ret = rds_page_remainder_alloc(sg, total_len,
						       GFP_HIGHUSER);
			if (ret)
				goto out;
293
			rm->data.op_nents++;
A
Andy Grover 已提交
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
			sg_off = 0;
		}

		while (iov_off == iov->iov_len) {
			iov_off = 0;
			iov++;
		}

		to_copy = min(iov->iov_len - iov_off, sg->length - sg_off);
		to_copy = min_t(size_t, to_copy, total_len);

		rdsdebug("copying %lu bytes from user iov [%p, %zu] + %lu to "
			 "sg [%p, %u, %u] + %lu\n",
			 to_copy, iov->iov_base, iov->iov_len, iov_off,
			 (void *)sg_page(sg), sg->offset, sg->length, sg_off);

		ret = rds_page_copy_from_user(sg_page(sg), sg->offset + sg_off,
					      iov->iov_base + iov_off,
					      to_copy);
		if (ret)
			goto out;

		iov_off += to_copy;
		total_len -= to_copy;
		sg_off += to_copy;

		if (sg_off == sg->length)
			sg++;
	}

out:
325
	return ret;
A
Andy Grover 已提交
326 327
}

328
int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
A
Andy Grover 已提交
329 330 331 332 333 334 335 336 337 338 339 340
{
	struct rds_message *rm;
	struct scatterlist *sg;
	unsigned long to_copy;
	unsigned long vec_off;
	int copied;
	int ret;
	u32 len;

	rm = container_of(inc, struct rds_message, m_inc);
	len = be32_to_cpu(rm->m_inc.i_hdr.h_len);

341
	sg = rm->data.op_sg;
A
Andy Grover 已提交
342 343 344
	vec_off = 0;
	copied = 0;

345 346
	while (iov_iter_count(to) && copied < len) {
		to_copy = min(iov_iter_count(to), sg->length - vec_off);
A
Andy Grover 已提交
347 348
		to_copy = min_t(unsigned long, to_copy, len - copied);

349 350 351 352 353
		rds_stats_add(s_copy_to_user, to_copy);
		ret = copy_page_to_iter(sg_page(sg), sg->offset + vec_off,
					to_copy, to);
		if (ret != to_copy)
			return -EFAULT;
A
Andy Grover 已提交
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372

		vec_off += to_copy;
		copied += to_copy;

		if (vec_off == sg->length) {
			vec_off = 0;
			sg++;
		}
	}

	return copied;
}

/*
 * If the message is still on the send queue, wait until the transport
 * is done with it. This is particularly important for RDMA operations.
 */
void rds_message_wait(struct rds_message *rm)
{
C
Chris Mason 已提交
373
	wait_event_interruptible(rm->m_flush_wait,
A
Andy Grover 已提交
374 375 376 377 378 379
			!test_bit(RDS_MSG_MAPPED, &rm->m_flags));
}

void rds_message_unmapped(struct rds_message *rm)
{
	clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
C
Chris Mason 已提交
380
	wake_up_interruptible(&rm->m_flush_wait);
A
Andy Grover 已提交
381
}
A
Andy Grover 已提交
382
EXPORT_SYMBOL_GPL(rds_message_unmapped);
A
Andy Grover 已提交
383