xfs_buf_item.c 30.4 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_fs.h"
L
Linus Torvalds 已提交
20
#include "xfs_types.h"
21
#include "xfs_bit.h"
L
Linus Torvalds 已提交
22 23 24
#include "xfs_log.h"
#include "xfs_trans.h"
#include "xfs_sb.h"
25
#include "xfs_ag.h"
L
Linus Torvalds 已提交
26
#include "xfs_mount.h"
27
#include "xfs_buf_item.h"
L
Linus Torvalds 已提交
28 29
#include "xfs_trans_priv.h"
#include "xfs_error.h"
30
#include "xfs_trace.h"
L
Linus Torvalds 已提交
31 32 33 34


kmem_zone_t	*xfs_buf_item_zone;

35 36 37 38 39
static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip)
{
	return container_of(lip, struct xfs_buf_log_item, bli_item);
}

40
STATIC void	xfs_buf_do_callbacks(struct xfs_buf *bp);
L
Linus Torvalds 已提交
41 42 43 44 45 46 47 48 49 50 51

/*
 * This returns the number of log iovecs needed to log the
 * given buf log item.
 *
 * It calculates this as 1 iovec for the buf log format structure
 * and 1 for each stretch of non-contiguous chunks to be logged.
 * Contiguous chunks are logged in a single iovec.
 *
 * If the XFS_BLI_STALE flag has been set, then log nothing.
 */
52
STATIC uint
53 54 55
xfs_buf_item_size_segment(
	struct xfs_buf_log_item	*bip,
	struct xfs_buf_log_format *blfp)
L
Linus Torvalds 已提交
56
{
57 58 59 60
	struct xfs_buf		*bp = bip->bli_buf;
	uint			nvecs;
	int			next_bit;
	int			last_bit;
L
Linus Torvalds 已提交
61

62 63 64 65 66 67 68 69 70
	last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
	if (last_bit == -1)
		return 0;

	/*
	 * initial count for a dirty buffer is 2 vectors - the format structure
	 * and the first dirty region.
	 */
	nvecs = 2;
L
Linus Torvalds 已提交
71 72 73 74 75 76 77 78

	while (last_bit != -1) {
		/*
		 * This takes the bit number to start looking from and
		 * returns the next set bit from there.  It returns -1
		 * if there are no more bits set or the start bit is
		 * beyond the end of the bitmap.
		 */
79 80
		next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
					last_bit + 1);
L
Linus Torvalds 已提交
81 82 83 84 85 86
		/*
		 * If we run out of bits, leave the loop,
		 * else if we find a new set of bits bump the number of vecs,
		 * else keep scanning the current set of bits.
		 */
		if (next_bit == -1) {
87
			break;
L
Linus Torvalds 已提交
88 89 90
		} else if (next_bit != last_bit + 1) {
			last_bit = next_bit;
			nvecs++;
91 92 93
		} else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) !=
			   (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) +
			    XFS_BLF_CHUNK)) {
L
Linus Torvalds 已提交
94 95 96 97 98 99 100 101 102 103 104
			last_bit = next_bit;
			nvecs++;
		} else {
			last_bit++;
		}
	}

	return nvecs;
}

/*
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
 * This returns the number of log iovecs needed to log the given buf log item.
 *
 * It calculates this as 1 iovec for the buf log format structure and 1 for each
 * stretch of non-contiguous chunks to be logged.  Contiguous chunks are logged
 * in a single iovec.
 *
 * Discontiguous buffers need a format structure per region that that is being
 * logged. This makes the changes in the buffer appear to log recovery as though
 * they came from separate buffers, just like would occur if multiple buffers
 * were used instead of a single discontiguous buffer. This enables
 * discontiguous buffers to be in-memory constructs, completely transparent to
 * what ends up on disk.
 *
 * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log
 * format structures.
L
Linus Torvalds 已提交
120
 */
121 122 123
STATIC uint
xfs_buf_item_size(
	struct xfs_log_item	*lip)
L
Linus Torvalds 已提交
124
{
125
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
126 127 128 129 130 131 132 133 134 135 136
	uint			nvecs;
	int			i;

	ASSERT(atomic_read(&bip->bli_refcount) > 0);
	if (bip->bli_flags & XFS_BLI_STALE) {
		/*
		 * The buffer is stale, so all we need to log
		 * is the buf log format structure with the
		 * cancel flag in it.
		 */
		trace_xfs_buf_item_size_stale(bip);
137
		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
		return bip->bli_format_count;
	}

	ASSERT(bip->bli_flags & XFS_BLI_LOGGED);

	/*
	 * the vector count is based on the number of buffer vectors we have
	 * dirty bits in. This will only be greater than one when we have a
	 * compound buffer with more than one segment dirty. Hence for compound
	 * buffers we need to track which segment the dirty bits correspond to,
	 * and when we move from one segment to the next increment the vector
	 * count for the extra buf log format structure that will need to be
	 * written.
	 */
	nvecs = 0;
	for (i = 0; i < bip->bli_format_count; i++) {
		nvecs += xfs_buf_item_size_segment(bip, &bip->bli_formats[i]);
	}

	trace_xfs_buf_item_size(bip);
	return nvecs;
}

static struct xfs_log_iovec *
xfs_buf_item_format_segment(
	struct xfs_buf_log_item	*bip,
	struct xfs_log_iovec	*vecp,
	uint			offset,
	struct xfs_buf_log_format *blfp)
{
168
	struct xfs_buf	*bp = bip->bli_buf;
L
Linus Torvalds 已提交
169 170 171 172 173 174 175 176
	uint		base_size;
	uint		nvecs;
	int		first_bit;
	int		last_bit;
	int		next_bit;
	uint		nbits;
	uint		buffer_offset;

177
	/* copy the flags across from the base format item */
178
	blfp->blf_flags = bip->__bli_format.blf_flags;
L
Linus Torvalds 已提交
179 180

	/*
181 182 183
	 * Base size is the actual size of the ondisk structure - it reflects
	 * the actual size of the dirty bitmap rather than the size of the in
	 * memory structure.
L
Linus Torvalds 已提交
184
	 */
185
	base_size = offsetof(struct xfs_buf_log_format, blf_data_map) +
186
			(blfp->blf_map_size * sizeof(blfp->blf_data_map[0]));
187 188 189 190 191 192 193 194 195 196 197

	nvecs = 0;
	first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
	if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) {
		/*
		 * If the map is not be dirty in the transaction, mark
		 * the size as zero and do not advance the vector pointer.
		 */
		goto out;
	}

198
	vecp->i_addr = blfp;
L
Linus Torvalds 已提交
199
	vecp->i_len = base_size;
200
	vecp->i_type = XLOG_REG_TYPE_BFORMAT;
L
Linus Torvalds 已提交
201 202 203 204 205 206 207 208 209
	vecp++;
	nvecs = 1;

	if (bip->bli_flags & XFS_BLI_STALE) {
		/*
		 * The buffer is stale, so all we need to log
		 * is the buf log format structure with the
		 * cancel flag in it.
		 */
210
		trace_xfs_buf_item_format_stale(bip);
211
		ASSERT(blfp->blf_flags & XFS_BLF_CANCEL);
212
		goto out;
L
Linus Torvalds 已提交
213 214 215 216 217
	}

	/*
	 * Fill in an iovec for each set of contiguous chunks.
	 */
218

L
Linus Torvalds 已提交
219 220 221 222 223 224 225 226 227
	last_bit = first_bit;
	nbits = 1;
	for (;;) {
		/*
		 * This takes the bit number to start looking from and
		 * returns the next set bit from there.  It returns -1
		 * if there are no more bits set or the start bit is
		 * beyond the end of the bitmap.
		 */
228 229
		next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
					(uint)last_bit + 1);
L
Linus Torvalds 已提交
230 231 232 233 234 235 236 237 238 239
		/*
		 * If we run out of bits fill in the last iovec and get
		 * out of the loop.
		 * Else if we start a new set of bits then fill in the
		 * iovec for the series we were looking at and start
		 * counting the bits in the new one.
		 * Else we're still in the same set of bits so just
		 * keep counting and scanning.
		 */
		if (next_bit == -1) {
240
			buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
L
Linus Torvalds 已提交
241
			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
242
			vecp->i_len = nbits * XFS_BLF_CHUNK;
243
			vecp->i_type = XLOG_REG_TYPE_BCHUNK;
L
Linus Torvalds 已提交
244 245 246
			nvecs++;
			break;
		} else if (next_bit != last_bit + 1) {
247
			buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
L
Linus Torvalds 已提交
248
			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
249
			vecp->i_len = nbits * XFS_BLF_CHUNK;
250
			vecp->i_type = XLOG_REG_TYPE_BCHUNK;
L
Linus Torvalds 已提交
251 252 253 254 255
			nvecs++;
			vecp++;
			first_bit = next_bit;
			last_bit = next_bit;
			nbits = 1;
256 257 258 259
		} else if (xfs_buf_offset(bp, offset +
					      (next_bit << XFS_BLF_SHIFT)) !=
			   (xfs_buf_offset(bp, offset +
					       (last_bit << XFS_BLF_SHIFT)) +
260
			    XFS_BLF_CHUNK)) {
261
			buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
L
Linus Torvalds 已提交
262
			vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
263
			vecp->i_len = nbits * XFS_BLF_CHUNK;
264
			vecp->i_type = XLOG_REG_TYPE_BCHUNK;
265 266
/*
 * You would think we need to bump the nvecs here too, but we do not
L
Linus Torvalds 已提交
267 268 269 270 271 272 273 274 275 276 277 278 279
 * this number is used by recovery, and it gets confused by the boundary
 * split here
 *			nvecs++;
 */
			vecp++;
			first_bit = next_bit;
			last_bit = next_bit;
			nbits = 1;
		} else {
			last_bit++;
			nbits++;
		}
	}
280 281
out:
	blfp->blf_size = nvecs;
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
	return vecp;
}

/*
 * This is called to fill in the vector of log iovecs for the
 * given log buf item.  It fills the first entry with a buf log
 * format structure, and the rest point to contiguous chunks
 * within the buffer.
 */
STATIC void
xfs_buf_item_format(
	struct xfs_log_item	*lip,
	struct xfs_log_iovec	*vecp)
{
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
	struct xfs_buf		*bp = bip->bli_buf;
	uint			offset = 0;
	int			i;

	ASSERT(atomic_read(&bip->bli_refcount) > 0);
	ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
	       (bip->bli_flags & XFS_BLI_STALE));

	/*
	 * If it is an inode buffer, transfer the in-memory state to the
	 * format flags and clear the in-memory state. We do not transfer
	 * this state if the inode buffer allocation has not yet been committed
	 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent
	 * correct replay of the inode allocation.
	 */
	if (bip->bli_flags & XFS_BLI_INODE_BUF) {
		if (!((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) &&
		      xfs_log_item_in_current_chkpt(lip)))
315
			bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF;
316 317 318 319 320 321 322 323
		bip->bli_flags &= ~XFS_BLI_INODE_BUF;
	}

	for (i = 0; i < bip->bli_format_count; i++) {
		vecp = xfs_buf_item_format_segment(bip, vecp, offset,
						&bip->bli_formats[i]);
		offset += bp->b_maps[i].bm_len;
	}
L
Linus Torvalds 已提交
324 325 326 327

	/*
	 * Check to make sure everything is consistent.
	 */
328
	trace_xfs_buf_item_format(bip);
L
Linus Torvalds 已提交
329 330 331
}

/*
332
 * This is called to pin the buffer associated with the buf log item in memory
333
 * so it cannot be written out.
334 335 336 337 338
 *
 * We also always take a reference to the buffer log item here so that the bli
 * is held while the item is pinned in memory. This means that we can
 * unconditionally drop the reference count a transaction holds when the
 * transaction is completed.
L
Linus Torvalds 已提交
339
 */
340
STATIC void
L
Linus Torvalds 已提交
341
xfs_buf_item_pin(
342
	struct xfs_log_item	*lip)
L
Linus Torvalds 已提交
343
{
344
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
L
Linus Torvalds 已提交
345 346 347 348

	ASSERT(atomic_read(&bip->bli_refcount) > 0);
	ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
	       (bip->bli_flags & XFS_BLI_STALE));
349

350
	trace_xfs_buf_item_pin(bip);
351 352 353

	atomic_inc(&bip->bli_refcount);
	atomic_inc(&bip->bli_buf->b_pin_count);
L
Linus Torvalds 已提交
354 355 356 357 358 359 360 361 362
}

/*
 * This is called to unpin the buffer associated with the buf log
 * item which was previously pinned with a call to xfs_buf_item_pin().
 *
 * Also drop the reference to the buf item for the current transaction.
 * If the XFS_BLI_STALE flag is set and we are the last reference,
 * then free up the buf log item and unlock the buffer.
363 364 365 366 367
 *
 * If the remove flag is set we are called from uncommit in the
 * forced-shutdown path.  If that is true and the reference count on
 * the log item is going to drop to zero we need to free the item's
 * descriptor in the transaction.
L
Linus Torvalds 已提交
368
 */
369
STATIC void
L
Linus Torvalds 已提交
370
xfs_buf_item_unpin(
371
	struct xfs_log_item	*lip,
372
	int			remove)
L
Linus Torvalds 已提交
373
{
374
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
375
	xfs_buf_t	*bp = bip->bli_buf;
376
	struct xfs_ail	*ailp = lip->li_ailp;
377
	int		stale = bip->bli_flags & XFS_BLI_STALE;
378
	int		freed;
L
Linus Torvalds 已提交
379

380
	ASSERT(bp->b_fspriv == bip);
L
Linus Torvalds 已提交
381
	ASSERT(atomic_read(&bip->bli_refcount) > 0);
382

383
	trace_xfs_buf_item_unpin(bip);
L
Linus Torvalds 已提交
384 385

	freed = atomic_dec_and_test(&bip->bli_refcount);
386 387 388

	if (atomic_dec_and_test(&bp->b_pin_count))
		wake_up_all(&bp->b_waiters);
389

L
Linus Torvalds 已提交
390 391
	if (freed && stale) {
		ASSERT(bip->bli_flags & XFS_BLI_STALE);
392
		ASSERT(xfs_buf_islocked(bp));
L
Linus Torvalds 已提交
393
		ASSERT(XFS_BUF_ISSTALE(bp));
394
		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
395

396 397
		trace_xfs_buf_item_unpin_stale(bip);

398 399
		if (remove) {
			/*
400 401 402 403 404
			 * If we are in a transaction context, we have to
			 * remove the log item from the transaction as we are
			 * about to release our reference to the buffer.  If we
			 * don't, the unlock that occurs later in
			 * xfs_trans_uncommit() will try to reference the
405 406
			 * buffer which we no longer have a hold on.
			 */
407 408
			if (lip->li_desc)
				xfs_trans_del_item(lip);
409 410 411 412 413

			/*
			 * Since the transaction no longer refers to the buffer,
			 * the buffer should no longer refer to the transaction.
			 */
414
			bp->b_transp = NULL;
415 416
		}

L
Linus Torvalds 已提交
417 418
		/*
		 * If we get called here because of an IO error, we may
419
		 * or may not have the item on the AIL. xfs_trans_ail_delete()
L
Linus Torvalds 已提交
420
		 * will take care of that situation.
421
		 * xfs_trans_ail_delete() drops the AIL lock.
L
Linus Torvalds 已提交
422 423
		 */
		if (bip->bli_flags & XFS_BLI_STALE_INODE) {
424
			xfs_buf_do_callbacks(bp);
425
			bp->b_fspriv = NULL;
426
			bp->b_iodone = NULL;
L
Linus Torvalds 已提交
427
		} else {
428
			spin_lock(&ailp->xa_lock);
429
			xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR);
L
Linus Torvalds 已提交
430
			xfs_buf_item_relse(bp);
431
			ASSERT(bp->b_fspriv == NULL);
L
Linus Torvalds 已提交
432 433
		}
		xfs_buf_relse(bp);
434
	} else if (freed && remove) {
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
		/*
		 * There are currently two references to the buffer - the active
		 * LRU reference and the buf log item. What we are about to do
		 * here - simulate a failed IO completion - requires 3
		 * references.
		 *
		 * The LRU reference is removed by the xfs_buf_stale() call. The
		 * buf item reference is removed by the xfs_buf_iodone()
		 * callback that is run by xfs_buf_do_callbacks() during ioend
		 * processing (via the bp->b_iodone callback), and then finally
		 * the ioend processing will drop the IO reference if the buffer
		 * is marked XBF_ASYNC.
		 *
		 * Hence we need to take an additional reference here so that IO
		 * completion processing doesn't free the buffer prematurely.
		 */
451
		xfs_buf_lock(bp);
452 453
		xfs_buf_hold(bp);
		bp->b_flags |= XBF_ASYNC;
454 455 456 457
		xfs_buf_ioerror(bp, EIO);
		XFS_BUF_UNDONE(bp);
		xfs_buf_stale(bp);
		xfs_buf_ioend(bp, 0);
L
Linus Torvalds 已提交
458 459 460
	}
}

461
STATIC uint
462 463 464
xfs_buf_item_push(
	struct xfs_log_item	*lip,
	struct list_head	*buffer_list)
L
Linus Torvalds 已提交
465
{
466 467
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
	struct xfs_buf		*bp = bip->bli_buf;
468
	uint			rval = XFS_ITEM_SUCCESS;
L
Linus Torvalds 已提交
469

470
	if (xfs_buf_ispinned(bp))
L
Linus Torvalds 已提交
471
		return XFS_ITEM_PINNED;
472 473 474 475 476 477 478 479 480 481
	if (!xfs_buf_trylock(bp)) {
		/*
		 * If we have just raced with a buffer being pinned and it has
		 * been marked stale, we could end up stalling until someone else
		 * issues a log force to unpin the stale buffer. Check for the
		 * race condition here so xfsaild recognizes the buffer is pinned
		 * and queues a log force to move it along.
		 */
		if (xfs_buf_ispinned(bp))
			return XFS_ITEM_PINNED;
L
Linus Torvalds 已提交
482
		return XFS_ITEM_LOCKED;
483
	}
L
Linus Torvalds 已提交
484 485

	ASSERT(!(bip->bli_flags & XFS_BLI_STALE));
486 487 488 489 490 491 492

	trace_xfs_buf_item_push(bip);

	if (!xfs_buf_delwri_queue(bp, buffer_list))
		rval = XFS_ITEM_FLUSHING;
	xfs_buf_unlock(bp);
	return rval;
L
Linus Torvalds 已提交
493 494 495
}

/*
496 497 498
 * Release the buffer associated with the buf log item.  If there is no dirty
 * logged data associated with the buffer recorded in the buf log item, then
 * free the buf log item and remove the reference to it in the buffer.
L
Linus Torvalds 已提交
499
 *
500 501
 * This call ignores the recursion count.  It is only called when the buffer
 * should REALLY be unlocked, regardless of the recursion count.
L
Linus Torvalds 已提交
502
 *
503 504 505 506 507 508 509 510 511 512
 * We unconditionally drop the transaction's reference to the log item. If the
 * item was logged, then another reference was taken when it was pinned, so we
 * can safely drop the transaction reference now.  This also allows us to avoid
 * potential races with the unpin code freeing the bli by not referencing the
 * bli after we've dropped the reference count.
 *
 * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item
 * if necessary but do not unlock the buffer.  This is for support of
 * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't
 * free the item.
L
Linus Torvalds 已提交
513
 */
514
STATIC void
L
Linus Torvalds 已提交
515
xfs_buf_item_unlock(
516
	struct xfs_log_item	*lip)
L
Linus Torvalds 已提交
517
{
518 519
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
	struct xfs_buf		*bp = bip->bli_buf;
520
	int			aborted, clean, i;
521
	uint			hold;
L
Linus Torvalds 已提交
522

523
	/* Clear the buffer's association with this transaction. */
524
	bp->b_transp = NULL;
L
Linus Torvalds 已提交
525 526

	/*
527 528 529 530
	 * If this is a transaction abort, don't return early.  Instead, allow
	 * the brelse to happen.  Normally it would be done for stale
	 * (cancelled) buffers at unpin time, but we'll never go through the
	 * pin/unpin cycle if we abort inside commit.
L
Linus Torvalds 已提交
531
	 */
532
	aborted = (lip->li_flags & XFS_LI_ABORTED) != 0;
L
Linus Torvalds 已提交
533 534

	/*
535 536 537 538 539 540 541 542 543 544 545 546
	 * Before possibly freeing the buf item, determine if we should
	 * release the buffer at the end of this routine.
	 */
	hold = bip->bli_flags & XFS_BLI_HOLD;

	/* Clear the per transaction state. */
	bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD);

	/*
	 * If the buf item is marked stale, then don't do anything.  We'll
	 * unlock the buffer and free the buf item when the buffer is unpinned
	 * for the last time.
L
Linus Torvalds 已提交
547 548
	 */
	if (bip->bli_flags & XFS_BLI_STALE) {
549
		trace_xfs_buf_item_unlock_stale(bip);
550
		ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
551 552
		if (!aborted) {
			atomic_dec(&bip->bli_refcount);
L
Linus Torvalds 已提交
553
			return;
554
		}
L
Linus Torvalds 已提交
555 556
	}

557
	trace_xfs_buf_item_unlock(bip);
L
Linus Torvalds 已提交
558 559

	/*
560
	 * If the buf item isn't tracking any data, free it, otherwise drop the
561 562 563 564
	 * reference we hold to it. If we are aborting the transaction, this may
	 * be the only reference to the buf item, so we free it anyway
	 * regardless of whether it is dirty or not. A dirty abort implies a
	 * shutdown, anyway.
L
Linus Torvalds 已提交
565
	 */
566 567 568 569 570 571 572 573 574
	clean = 1;
	for (i = 0; i < bip->bli_format_count; i++) {
		if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map,
			     bip->bli_formats[i].blf_map_size)) {
			clean = 0;
			break;
		}
	}
	if (clean)
L
Linus Torvalds 已提交
575
		xfs_buf_item_relse(bp);
576 577 578 579 580 581
	else if (aborted) {
		if (atomic_dec_and_test(&bip->bli_refcount)) {
			ASSERT(XFS_FORCED_SHUTDOWN(lip->li_mountp));
			xfs_buf_item_relse(bp);
		}
	} else
582
		atomic_dec(&bip->bli_refcount);
L
Linus Torvalds 已提交
583

584
	if (!hold)
L
Linus Torvalds 已提交
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
		xfs_buf_relse(bp);
}

/*
 * This is called to find out where the oldest active copy of the
 * buf log item in the on disk log resides now that the last log
 * write of it completed at the given lsn.
 * We always re-log all the dirty data in a buffer, so usually the
 * latest copy in the on disk log is the only one that matters.  For
 * those cases we simply return the given lsn.
 *
 * The one exception to this is for buffers full of newly allocated
 * inodes.  These buffers are only relogged with the XFS_BLI_INODE_BUF
 * flag set, indicating that only the di_next_unlinked fields from the
 * inodes in the buffers will be replayed during recovery.  If the
 * original newly allocated inode images have not yet been flushed
 * when the buffer is so relogged, then we need to make sure that we
 * keep the old images in the 'active' portion of the log.  We do this
 * by returning the original lsn of that transaction here rather than
 * the current one.
 */
606
STATIC xfs_lsn_t
L
Linus Torvalds 已提交
607
xfs_buf_item_committed(
608
	struct xfs_log_item	*lip,
L
Linus Torvalds 已提交
609 610
	xfs_lsn_t		lsn)
{
611 612
	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);

613 614
	trace_xfs_buf_item_committed(bip);

615 616 617
	if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0)
		return lip->li_lsn;
	return lsn;
L
Linus Torvalds 已提交
618 619
}

620
STATIC void
621 622 623
xfs_buf_item_committing(
	struct xfs_log_item	*lip,
	xfs_lsn_t		commit_lsn)
L
Linus Torvalds 已提交
624 625 626 627 628 629
{
}

/*
 * This is the ops vector shared by all buf log items.
 */
630
static const struct xfs_item_ops xfs_buf_item_ops = {
631 632 633 634 635 636 637 638
	.iop_size	= xfs_buf_item_size,
	.iop_format	= xfs_buf_item_format,
	.iop_pin	= xfs_buf_item_pin,
	.iop_unpin	= xfs_buf_item_unpin,
	.iop_unlock	= xfs_buf_item_unlock,
	.iop_committed	= xfs_buf_item_committed,
	.iop_push	= xfs_buf_item_push,
	.iop_committing = xfs_buf_item_committing
L
Linus Torvalds 已提交
639 640
};

641 642 643 644 645 646 647 648 649
STATIC int
xfs_buf_item_get_format(
	struct xfs_buf_log_item	*bip,
	int			count)
{
	ASSERT(bip->bli_formats == NULL);
	bip->bli_format_count = count;

	if (count == 1) {
650
		bip->bli_formats = &bip->__bli_format;
651 652 653 654 655 656 657 658 659 660 661 662 663 664
		return 0;
	}

	bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format),
				KM_SLEEP);
	if (!bip->bli_formats)
		return ENOMEM;
	return 0;
}

STATIC void
xfs_buf_item_free_format(
	struct xfs_buf_log_item	*bip)
{
665
	if (bip->bli_formats != &bip->__bli_format) {
666 667 668 669
		kmem_free(bip->bli_formats);
		bip->bli_formats = NULL;
	}
}
L
Linus Torvalds 已提交
670 671 672 673 674 675 676 677 678 679 680 681 682

/*
 * Allocate a new buf log item to go with the given buffer.
 * Set the buffer's b_fsprivate field to point to the new
 * buf log item.  If there are other item's attached to the
 * buffer (see xfs_buf_attach_iodone() below), then put the
 * buf log item at the front.
 */
void
xfs_buf_item_init(
	xfs_buf_t	*bp,
	xfs_mount_t	*mp)
{
683
	xfs_log_item_t		*lip = bp->b_fspriv;
L
Linus Torvalds 已提交
684 685 686
	xfs_buf_log_item_t	*bip;
	int			chunks;
	int			map_size;
687 688
	int			error;
	int			i;
L
Linus Torvalds 已提交
689 690 691 692 693 694 695

	/*
	 * Check to see if there is already a buf log item for
	 * this buffer.  If there is, it is guaranteed to be
	 * the first.  If we do already have one, there is
	 * nothing to do here so return.
	 */
696
	ASSERT(bp->b_target->bt_mount == mp);
697 698
	if (lip != NULL && lip->li_type == XFS_LI_BUF)
		return;
L
Linus Torvalds 已提交
699

700
	bip = kmem_zone_zalloc(xfs_buf_item_zone, KM_SLEEP);
701
	xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
L
Linus Torvalds 已提交
702
	bip->bli_buf = bp;
703
	xfs_buf_hold(bp);
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726

	/*
	 * chunks is the number of XFS_BLF_CHUNK size pieces the buffer
	 * can be divided into. Make sure not to truncate any pieces.
	 * map_size is the size of the bitmap needed to describe the
	 * chunks of the buffer.
	 *
	 * Discontiguous buffer support follows the layout of the underlying
	 * buffer. This makes the implementation as simple as possible.
	 */
	error = xfs_buf_item_get_format(bip, bp->b_map_count);
	ASSERT(error == 0);

	for (i = 0; i < bip->bli_format_count; i++) {
		chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),
				      XFS_BLF_CHUNK);
		map_size = DIV_ROUND_UP(chunks, NBWORD);

		bip->bli_formats[i].blf_type = XFS_LI_BUF;
		bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn;
		bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len;
		bip->bli_formats[i].blf_map_size = map_size;
	}
L
Linus Torvalds 已提交
727 728 729 730 731 732 733 734 735 736

#ifdef XFS_TRANS_DEBUG
	/*
	 * Allocate the arrays for tracking what needs to be logged
	 * and what our callers request to be logged.  bli_orig
	 * holds a copy of the original, clean buffer for comparison
	 * against, and bli_logged keeps a 1 bit flag per byte in
	 * the buffer to indicate which bytes the callers have asked
	 * to have logged.
	 */
737 738 739
	bip->bli_orig = kmem_alloc(BBTOB(bp->b_length), KM_SLEEP);
	memcpy(bip->bli_orig, bp->b_addr, BBTOB(bp->b_length));
	bip->bli_logged = kmem_zalloc(BBTOB(bp->b_length) / NBBY, KM_SLEEP);
L
Linus Torvalds 已提交
740 741 742 743 744 745
#endif

	/*
	 * Put the buf item into the list of items attached to the
	 * buffer at the front.
	 */
746 747 748
	if (bp->b_fspriv)
		bip->bli_item.li_bio_list = bp->b_fspriv;
	bp->b_fspriv = bip;
L
Linus Torvalds 已提交
749 750 751 752 753 754 755 756
}


/*
 * Mark bytes first through last inclusive as dirty in the buf
 * item's bitmap.
 */
void
757 758
xfs_buf_item_log_segment(
	struct xfs_buf_log_item	*bip,
L
Linus Torvalds 已提交
759
	uint			first,
760 761
	uint			last,
	uint			*map)
L
Linus Torvalds 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774 775
{
	uint		first_bit;
	uint		last_bit;
	uint		bits_to_set;
	uint		bits_set;
	uint		word_num;
	uint		*wordp;
	uint		bit;
	uint		end_bit;
	uint		mask;

	/*
	 * Convert byte offsets to bit numbers.
	 */
776 777
	first_bit = first >> XFS_BLF_SHIFT;
	last_bit = last >> XFS_BLF_SHIFT;
L
Linus Torvalds 已提交
778 779 780 781 782 783 784 785 786 787 788

	/*
	 * Calculate the total number of bits to be set.
	 */
	bits_to_set = last_bit - first_bit + 1;

	/*
	 * Get a pointer to the first word in the bitmap
	 * to set a bit in.
	 */
	word_num = first_bit >> BIT_TO_WORD_SHIFT;
789
	wordp = &map[word_num];
L
Linus Torvalds 已提交
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833

	/*
	 * Calculate the starting bit in the first word.
	 */
	bit = first_bit & (uint)(NBWORD - 1);

	/*
	 * First set any bits in the first word of our range.
	 * If it starts at bit 0 of the word, it will be
	 * set below rather than here.  That is what the variable
	 * bit tells us. The variable bits_set tracks the number
	 * of bits that have been set so far.  End_bit is the number
	 * of the last bit to be set in this word plus one.
	 */
	if (bit) {
		end_bit = MIN(bit + bits_to_set, (uint)NBWORD);
		mask = ((1 << (end_bit - bit)) - 1) << bit;
		*wordp |= mask;
		wordp++;
		bits_set = end_bit - bit;
	} else {
		bits_set = 0;
	}

	/*
	 * Now set bits a whole word at a time that are between
	 * first_bit and last_bit.
	 */
	while ((bits_to_set - bits_set) >= NBWORD) {
		*wordp |= 0xffffffff;
		bits_set += NBWORD;
		wordp++;
	}

	/*
	 * Finally, set any bits left to be set in one last partial word.
	 */
	end_bit = bits_to_set - bits_set;
	if (end_bit) {
		mask = (1 << end_bit) - 1;
		*wordp |= mask;
	}
}

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
/*
 * Mark bytes first through last inclusive as dirty in the buf
 * item's bitmap.
 */
void
xfs_buf_item_log(
	xfs_buf_log_item_t	*bip,
	uint			first,
	uint			last)
{
	int			i;
	uint			start;
	uint			end;
	struct xfs_buf		*bp = bip->bli_buf;

	/*
	 * Mark the item as having some dirty data for
	 * quick reference in xfs_buf_item_dirty.
	 */
	bip->bli_flags |= XFS_BLI_DIRTY;

	/*
	 * walk each buffer segment and mark them dirty appropriately.
	 */
	start = 0;
	for (i = 0; i < bip->bli_format_count; i++) {
		if (start > last)
			break;
		end = start + BBTOB(bp->b_maps[i].bm_len);
		if (first > end) {
			start += BBTOB(bp->b_maps[i].bm_len);
			continue;
		}
		if (first < start)
			first = start;
		if (end > last)
			end = last;

		xfs_buf_item_log_segment(bip, first, end,
					 &bip->bli_formats[i].blf_data_map[0]);

		start += bp->b_maps[i].bm_len;
	}
}

L
Linus Torvalds 已提交
879 880 881 882 883 884 885 886 887 888 889 890

/*
 * Return 1 if the buffer has some data that has been logged (at any
 * point, not just the current transaction) and 0 if not.
 */
uint
xfs_buf_item_dirty(
	xfs_buf_log_item_t	*bip)
{
	return (bip->bli_flags & XFS_BLI_DIRTY);
}

891 892 893 894 895 896 897 898 899
STATIC void
xfs_buf_item_free(
	xfs_buf_log_item_t	*bip)
{
#ifdef XFS_TRANS_DEBUG
	kmem_free(bip->bli_orig);
	kmem_free(bip->bli_logged);
#endif /* XFS_TRANS_DEBUG */

900
	xfs_buf_item_free_format(bip);
901 902 903
	kmem_zone_free(xfs_buf_item_zone, bip);
}

L
Linus Torvalds 已提交
904 905 906 907 908 909 910 911 912 913 914 915 916
/*
 * This is called when the buf log item is no longer needed.  It should
 * free the buf log item associated with the given buffer and clear
 * the buffer's pointer to the buf log item.  If there are no more
 * items in the list, clear the b_iodone field of the buffer (see
 * xfs_buf_attach_iodone() below).
 */
void
xfs_buf_item_relse(
	xfs_buf_t	*bp)
{
	xfs_buf_log_item_t	*bip;

917 918
	trace_xfs_buf_item_relse(bp, _RET_IP_);

919 920
	bip = bp->b_fspriv;
	bp->b_fspriv = bip->bli_item.li_bio_list;
921 922
	if (bp->b_fspriv == NULL)
		bp->b_iodone = NULL;
923

924 925
	xfs_buf_rele(bp);
	xfs_buf_item_free(bip);
L
Linus Torvalds 已提交
926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
}


/*
 * Add the given log item with its callback to the list of callbacks
 * to be called when the buffer's I/O completes.  If it is not set
 * already, set the buffer's b_iodone() routine to be
 * xfs_buf_iodone_callbacks() and link the log item into the list of
 * items rooted at b_fsprivate.  Items are always added as the second
 * entry in the list if there is a first, because the buf item code
 * assumes that the buf log item is first.
 */
void
xfs_buf_attach_iodone(
	xfs_buf_t	*bp,
	void		(*cb)(xfs_buf_t *, xfs_log_item_t *),
	xfs_log_item_t	*lip)
{
	xfs_log_item_t	*head_lip;

946
	ASSERT(xfs_buf_islocked(bp));
L
Linus Torvalds 已提交
947 948

	lip->li_cb = cb;
949 950
	head_lip = bp->b_fspriv;
	if (head_lip) {
L
Linus Torvalds 已提交
951 952 953
		lip->li_bio_list = head_lip->li_bio_list;
		head_lip->li_bio_list = lip;
	} else {
954
		bp->b_fspriv = lip;
L
Linus Torvalds 已提交
955 956
	}

957 958 959
	ASSERT(bp->b_iodone == NULL ||
	       bp->b_iodone == xfs_buf_iodone_callbacks);
	bp->b_iodone = xfs_buf_iodone_callbacks;
L
Linus Torvalds 已提交
960 961
}

962 963 964 965 966 967 968 969 970 971 972 973
/*
 * We can have many callbacks on a buffer. Running the callbacks individually
 * can cause a lot of contention on the AIL lock, so we allow for a single
 * callback to be able to scan the remaining lip->li_bio_list for other items
 * of the same type and callback to be processed in the first call.
 *
 * As a result, the loop walking the callback list below will also modify the
 * list. it removes the first item from the list and then runs the callback.
 * The loop then restarts from the new head of the list. This allows the
 * callback to scan and modify the list attached to the buffer and we don't
 * have to care about maintaining a next item pointer.
 */
L
Linus Torvalds 已提交
974 975
STATIC void
xfs_buf_do_callbacks(
976
	struct xfs_buf		*bp)
L
Linus Torvalds 已提交
977
{
978
	struct xfs_log_item	*lip;
L
Linus Torvalds 已提交
979

980 981
	while ((lip = bp->b_fspriv) != NULL) {
		bp->b_fspriv = lip->li_bio_list;
L
Linus Torvalds 已提交
982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
		ASSERT(lip->li_cb != NULL);
		/*
		 * Clear the next pointer so we don't have any
		 * confusion if the item is added to another buf.
		 * Don't touch the log item after calling its
		 * callback, because it could have freed itself.
		 */
		lip->li_bio_list = NULL;
		lip->li_cb(bp, lip);
	}
}

/*
 * This is the iodone() function for buffers which have had callbacks
 * attached to them by xfs_buf_attach_iodone().  It should remove each
 * log item from the buffer's list and call the callback of each in turn.
 * When done, the buffer's fsprivate field is set to NULL and the buffer
 * is unlocked with a call to iodone().
 */
void
xfs_buf_iodone_callbacks(
1003
	struct xfs_buf		*bp)
L
Linus Torvalds 已提交
1004
{
1005 1006 1007 1008
	struct xfs_log_item	*lip = bp->b_fspriv;
	struct xfs_mount	*mp = lip->li_mountp;
	static ulong		lasttime;
	static xfs_buftarg_t	*lasttarg;
L
Linus Torvalds 已提交
1009

1010
	if (likely(!xfs_buf_geterror(bp)))
1011
		goto do_callbacks;
L
Linus Torvalds 已提交
1012

1013 1014 1015 1016 1017
	/*
	 * If we've already decided to shutdown the filesystem because of
	 * I/O errors, there's no point in giving this a retry.
	 */
	if (XFS_FORCED_SHUTDOWN(mp)) {
1018 1019
		xfs_buf_stale(bp);
		XFS_BUF_DONE(bp);
1020 1021 1022
		trace_xfs_buf_item_iodone(bp, _RET_IP_);
		goto do_callbacks;
	}
L
Linus Torvalds 已提交
1023

1024
	if (bp->b_target != lasttarg ||
1025 1026
	    time_after(jiffies, (lasttime + 5*HZ))) {
		lasttime = jiffies;
1027
		xfs_buf_ioerror_alert(bp, __func__);
1028
	}
1029
	lasttarg = bp->b_target;
L
Linus Torvalds 已提交
1030

1031
	/*
L
Lucas De Marchi 已提交
1032
	 * If the write was asynchronous then no one will be looking for the
1033 1034
	 * error.  Clear the error state and write the buffer out again.
	 *
1035 1036 1037 1038 1039 1040
	 * XXX: This helps against transient write errors, but we need to find
	 * a way to shut the filesystem down if the writes keep failing.
	 *
	 * In practice we'll shut the filesystem down soon as non-transient
	 * erorrs tend to affect the whole device and a failing log write
	 * will make us give up.  But we really ought to do better here.
1041 1042
	 */
	if (XFS_BUF_ISASYNC(bp)) {
1043 1044 1045 1046
		ASSERT(bp->b_iodone != NULL);

		trace_xfs_buf_item_iodone_async(bp, _RET_IP_);

1047
		xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */
1048 1049

		if (!XFS_BUF_ISSTALE(bp)) {
1050
			bp->b_flags |= XBF_WRITE | XBF_ASYNC | XBF_DONE;
1051
			xfs_buf_iorequest(bp);
1052 1053
		} else {
			xfs_buf_relse(bp);
L
Linus Torvalds 已提交
1054
		}
1055

L
Linus Torvalds 已提交
1056 1057
		return;
	}
1058

1059 1060 1061 1062
	/*
	 * If the write of the buffer was synchronous, we want to make
	 * sure to return the error to the caller of xfs_bwrite().
	 */
1063
	xfs_buf_stale(bp);
L
Linus Torvalds 已提交
1064
	XFS_BUF_DONE(bp);
1065 1066 1067

	trace_xfs_buf_error_relse(bp, _RET_IP_);

1068
do_callbacks:
1069
	xfs_buf_do_callbacks(bp);
1070
	bp->b_fspriv = NULL;
1071
	bp->b_iodone = NULL;
1072
	xfs_buf_ioend(bp, 0);
L
Linus Torvalds 已提交
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
}

/*
 * This is the iodone() function for buffers which have been
 * logged.  It is called when they are eventually flushed out.
 * It should remove the buf item from the AIL, and free the buf item.
 * It is called by xfs_buf_iodone_callbacks() above which will take
 * care of cleaning up the buffer itself.
 */
void
xfs_buf_iodone(
1084 1085
	struct xfs_buf		*bp,
	struct xfs_log_item	*lip)
L
Linus Torvalds 已提交
1086
{
1087
	struct xfs_ail		*ailp = lip->li_ailp;
L
Linus Torvalds 已提交
1088

1089
	ASSERT(BUF_ITEM(lip)->bli_buf == bp);
L
Linus Torvalds 已提交
1090

1091
	xfs_buf_rele(bp);
L
Linus Torvalds 已提交
1092 1093 1094 1095 1096 1097

	/*
	 * If we are forcibly shutting down, this may well be
	 * off the AIL already. That's because we simulate the
	 * log-committed callbacks to unpin these buffers. Or we may never
	 * have put this item on AIL because of the transaction was
1098
	 * aborted forcibly. xfs_trans_ail_delete() takes care of these.
L
Linus Torvalds 已提交
1099 1100 1101
	 *
	 * Either way, AIL is useless if we're forcing a shutdown.
	 */
1102
	spin_lock(&ailp->xa_lock);
1103
	xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE);
1104
	xfs_buf_item_free(BUF_ITEM(lip));
L
Linus Torvalds 已提交
1105
}