xfs_reflink.c 44.1 KB
Newer Older
D
Dave Chinner 已提交
1
// SPDX-License-Identifier: GPL-2.0+
D
Darrick J. Wong 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Copyright (C) 2016 Oracle.  All Rights Reserved.
 * Author: Darrick J. Wong <darrick.wong@oracle.com>
 */
#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"
#include "xfs_defer.h"
#include "xfs_inode.h"
#include "xfs_trans.h"
#include "xfs_bmap.h"
#include "xfs_bmap_util.h"
#include "xfs_trace.h"
#include "xfs_icache.h"
20
#include "xfs_btree.h"
D
Darrick J. Wong 已提交
21 22 23 24 25 26 27 28
#include "xfs_refcount_btree.h"
#include "xfs_refcount.h"
#include "xfs_bmap_btree.h"
#include "xfs_trans_space.h"
#include "xfs_bit.h"
#include "xfs_alloc.h"
#include "xfs_quota.h"
#include "xfs_reflink.h"
29
#include "xfs_iomap.h"
30
#include "xfs_ag.h"
31
#include "xfs_ag_resv.h"
D
Darrick J. Wong 已提交
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57

/*
 * Copy on Write of Shared Blocks
 *
 * XFS must preserve "the usual" file semantics even when two files share
 * the same physical blocks.  This means that a write to one file must not
 * alter the blocks in a different file; the way that we'll do that is
 * through the use of a copy-on-write mechanism.  At a high level, that
 * means that when we want to write to a shared block, we allocate a new
 * block, write the data to the new block, and if that succeeds we map the
 * new block into the file.
 *
 * XFS provides a "delayed allocation" mechanism that defers the allocation
 * of disk blocks to dirty-but-not-yet-mapped file blocks as long as
 * possible.  This reduces fragmentation by enabling the filesystem to ask
 * for bigger chunks less often, which is exactly what we want for CoW.
 *
 * The delalloc mechanism begins when the kernel wants to make a block
 * writable (write_begin or page_mkwrite).  If the offset is not mapped, we
 * create a delalloc mapping, which is a regular in-core extent, but without
 * a real startblock.  (For delalloc mappings, the startblock encodes both
 * a flag that this is a delalloc mapping, and a worst-case estimate of how
 * many blocks might be required to put the mapping into the BMBT.)  delalloc
 * mappings are a reservation against the free space in the filesystem;
 * adjacent mappings can also be combined into fewer larger mappings.
 *
58 59 60 61 62 63 64 65
 * As an optimization, the CoW extent size hint (cowextsz) creates
 * outsized aligned delalloc reservations in the hope of landing out of
 * order nearby CoW writes in a single extent on disk, thereby reducing
 * fragmentation and improving future performance.
 *
 * D: --RRRRRRSSSRRRRRRRR--- (data fork)
 * C: ------DDDDDDD--------- (CoW fork)
 *
D
Darrick J. Wong 已提交
66
 * When dirty pages are being written out (typically in writepage), the
67 68 69 70 71 72 73
 * delalloc reservations are converted into unwritten mappings by
 * allocating blocks and replacing the delalloc mapping with real ones.
 * A delalloc mapping can be replaced by several unwritten ones if the
 * free space is fragmented.
 *
 * D: --RRRRRRSSSRRRRRRRR---
 * C: ------UUUUUUU---------
D
Darrick J. Wong 已提交
74 75 76 77 78 79 80 81 82 83 84 85 86 87
 *
 * We want to adapt the delalloc mechanism for copy-on-write, since the
 * write paths are similar.  The first two steps (creating the reservation
 * and allocating the blocks) are exactly the same as delalloc except that
 * the mappings must be stored in a separate CoW fork because we do not want
 * to disturb the mapping in the data fork until we're sure that the write
 * succeeded.  IO completion in this case is the process of removing the old
 * mapping from the data fork and moving the new mapping from the CoW fork to
 * the data fork.  This will be discussed shortly.
 *
 * For now, unaligned directio writes will be bounced back to the page cache.
 * Block-aligned directio writes will use the same mechanism as buffered
 * writes.
 *
88 89 90 91 92 93 94 95
 * Just prior to submitting the actual disk write requests, we convert
 * the extents representing the range of the file actually being written
 * (as opposed to extra pieces created for the cowextsize hint) to real
 * extents.  This will become important in the next step:
 *
 * D: --RRRRRRSSSRRRRRRRR---
 * C: ------UUrrUUU---------
 *
D
Darrick J. Wong 已提交
96 97 98 99 100 101
 * CoW remapping must be done after the data block write completes,
 * because we don't want to destroy the old data fork map until we're sure
 * the new block has been written.  Since the new mappings are kept in a
 * separate fork, we can simply iterate these mappings to find the ones
 * that cover the file blocks that we just CoW'd.  For each extent, simply
 * unmap the corresponding range in the data fork, map the new range into
102 103 104 105 106 107 108 109 110
 * the data fork, and remove the extent from the CoW fork.  Because of
 * the presence of the cowextsize hint, however, we must be careful
 * only to remap the blocks that we've actually written out --  we must
 * never remap delalloc reservations nor CoW staging blocks that have
 * yet to be written.  This corresponds exactly to the real extents in
 * the CoW fork:
 *
 * D: --RRRRRRrrSRRRRRRRR---
 * C: ------UU--UUU---------
D
Darrick J. Wong 已提交
111 112 113 114 115 116 117 118 119
 *
 * Since the remapping operation can be applied to an arbitrary file
 * range, we record the need for the remap step as a flag in the ioend
 * instead of declaring a new IO type.  This is required for direct io
 * because we only have ioend for the whole dio, and we have to be able to
 * remember the presence of unwritten blocks and CoW blocks with a single
 * ioend structure.  Better yet, the more ground we can cover with one
 * ioend, the better.
 */
120 121 122 123 124 125 126 127 128 129 130

/*
 * Given an AG extent, find the lowest-numbered run of shared blocks
 * within that range and return the range in fbno/flen.  If
 * find_end_of_shared is true, return the longest contiguous extent of
 * shared blocks.  If there are no shared extents, fbno and flen will
 * be set to NULLAGBLOCK and 0, respectively.
 */
int
xfs_reflink_find_shared(
	struct xfs_mount	*mp,
131
	struct xfs_trans	*tp,
132 133 134 135 136 137 138 139 140 141 142
	xfs_agnumber_t		agno,
	xfs_agblock_t		agbno,
	xfs_extlen_t		aglen,
	xfs_agblock_t		*fbno,
	xfs_extlen_t		*flen,
	bool			find_end_of_shared)
{
	struct xfs_buf		*agbp;
	struct xfs_btree_cur	*cur;
	int			error;

143
	error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp);
144 145 146
	if (error)
		return error;

147
	cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agbp->b_pag);
148 149 150 151

	error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen,
			find_end_of_shared);

152
	xfs_btree_del_cursor(cur, error);
153

154
	xfs_trans_brelse(tp, agbp);
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
	return error;
}

/*
 * Trim the mapping to the next block where there's a change in the
 * shared/unshared status.  More specifically, this means that we
 * find the lowest-numbered extent of shared blocks that coincides with
 * the given block mapping.  If the shared extent overlaps the start of
 * the mapping, trim the mapping to the end of the shared extent.  If
 * the shared region intersects the mapping, trim the mapping to the
 * start of the shared extent.  If there are no shared regions that
 * overlap, just return the original extent.
 */
int
xfs_reflink_trim_around_shared(
	struct xfs_inode	*ip,
	struct xfs_bmbt_irec	*irec,
172
	bool			*shared)
173 174 175 176 177 178 179 180 181
{
	xfs_agnumber_t		agno;
	xfs_agblock_t		agbno;
	xfs_extlen_t		aglen;
	xfs_agblock_t		fbno;
	xfs_extlen_t		flen;
	int			error = 0;

	/* Holes, unwritten, and delalloc extents cannot be shared */
182
	if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) {
183 184 185 186 187 188 189 190 191 192
		*shared = false;
		return 0;
	}

	trace_xfs_reflink_trim_around_shared(ip, irec);

	agno = XFS_FSB_TO_AGNO(ip->i_mount, irec->br_startblock);
	agbno = XFS_FSB_TO_AGBNO(ip->i_mount, irec->br_startblock);
	aglen = irec->br_blockcount;

193
	error = xfs_reflink_find_shared(ip->i_mount, NULL, agno, agbno,
194 195 196 197
			aglen, &fbno, &flen, true);
	if (error)
		return error;

198
	*shared = false;
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	if (fbno == NULLAGBLOCK) {
		/* No shared blocks at all. */
		return 0;
	} else if (fbno == agbno) {
		/*
		 * The start of this extent is shared.  Truncate the
		 * mapping at the end of the shared region so that a
		 * subsequent iteration starts at the start of the
		 * unshared region.
		 */
		irec->br_blockcount = flen;
		*shared = true;
		return 0;
	} else {
		/*
		 * There's a shared extent midway through this extent.
		 * Truncate the mapping at the start of the shared
		 * extent so that a subsequent iteration starts at the
		 * start of the shared region.
		 */
		irec->br_blockcount = fbno - agbno;
		return 0;
	}
}

224 225
int
xfs_bmap_trim_cow(
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
	struct xfs_inode	*ip,
	struct xfs_bmbt_irec	*imap,
	bool			*shared)
{
	/* We can't update any real extents in always COW mode. */
	if (xfs_is_always_cow_inode(ip) &&
	    !isnullstartblock(imap->br_startblock)) {
		*shared = true;
		return 0;
	}

	/* Trim the mapping to the nearest shared extent boundary. */
	return xfs_reflink_trim_around_shared(ip, imap, shared);
}

241 242 243 244 245
static int
xfs_reflink_convert_cow_locked(
	struct xfs_inode	*ip,
	xfs_fileoff_t		offset_fsb,
	xfs_filblks_t		count_fsb)
246
{
247 248 249 250
	struct xfs_iext_cursor	icur;
	struct xfs_bmbt_irec	got;
	struct xfs_btree_cur	*dummy_cur = NULL;
	int			dummy_logflags;
251
	int			error = 0;
252

253
	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
254 255
		return 0;

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
	do {
		if (got.br_startoff >= offset_fsb + count_fsb)
			break;
		if (got.br_state == XFS_EXT_NORM)
			continue;
		if (WARN_ON_ONCE(isnullstartblock(got.br_startblock)))
			return -EIO;

		xfs_trim_extent(&got, offset_fsb, count_fsb);
		if (!got.br_blockcount)
			continue;

		got.br_state = XFS_EXT_NORM;
		error = xfs_bmap_add_extent_unwritten_real(NULL, ip,
				XFS_COW_FORK, &icur, &dummy_cur, &got,
				&dummy_logflags);
		if (error)
			return error;
	} while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got));

	return error;
277 278 279 280 281 282 283 284 285 286 287 288
}

/* Convert all of the unwritten CoW extents in a file's range to real ones. */
int
xfs_reflink_convert_cow(
	struct xfs_inode	*ip,
	xfs_off_t		offset,
	xfs_off_t		count)
{
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = XFS_B_TO_FSB(mp, offset + count);
289
	xfs_filblks_t		count_fsb = end_fsb - offset_fsb;
290
	int			error;
291

292
	ASSERT(count != 0);
293

294
	xfs_ilock(ip, XFS_ILOCK_EXCL);
295
	error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb);
296 297 298 299
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return error;
}

300 301 302 303 304 305 306 307 308
/*
 * Find the extent that maps the given range in the COW fork. Even if the extent
 * is not shared we might have a preallocation for it in the COW fork. If so we
 * use it that rather than trigger a new allocation.
 */
static int
xfs_find_trim_cow_extent(
	struct xfs_inode	*ip,
	struct xfs_bmbt_irec	*imap,
309
	struct xfs_bmbt_irec	*cmap,
310 311 312 313 314 315 316 317 318 319 320 321 322
	bool			*shared,
	bool			*found)
{
	xfs_fileoff_t		offset_fsb = imap->br_startoff;
	xfs_filblks_t		count_fsb = imap->br_blockcount;
	struct xfs_iext_cursor	icur;

	*found = false;

	/*
	 * If we don't find an overlapping extent, trim the range we need to
	 * allocate to fit the hole we found.
	 */
323 324 325
	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, cmap))
		cmap->br_startoff = offset_fsb + count_fsb;
	if (cmap->br_startoff > offset_fsb) {
326
		xfs_trim_extent(imap, imap->br_startoff,
327
				cmap->br_startoff - imap->br_startoff);
328
		return xfs_bmap_trim_cow(ip, imap, shared);
329
	}
330 331

	*shared = true;
332 333
	if (isnullstartblock(cmap->br_startblock)) {
		xfs_trim_extent(imap, cmap->br_startoff, cmap->br_blockcount);
334 335 336 337
		return 0;
	}

	/* real extent found - no need to allocate */
338
	xfs_trim_extent(cmap, offset_fsb, count_fsb);
339 340 341 342
	*found = true;
	return 0;
}

343
/* Allocate all CoW reservations covering a range of blocks in a file. */
344 345
int
xfs_reflink_allocate_cow(
346
	struct xfs_inode	*ip,
347
	struct xfs_bmbt_irec	*imap,
348
	struct xfs_bmbt_irec	*cmap,
349
	bool			*shared,
350
	uint			*lockmode,
351
	bool			convert_now)
352 353
{
	struct xfs_mount	*mp = ip->i_mount;
354 355
	xfs_fileoff_t		offset_fsb = imap->br_startoff;
	xfs_filblks_t		count_fsb = imap->br_blockcount;
356
	struct xfs_trans	*tp;
357
	int			nimaps, error = 0;
358
	bool			found;
359
	xfs_filblks_t		resaligned;
360
	xfs_extlen_t		resblks = 0;
361

362
	ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL));
363 364 365 366
	if (!ip->i_cowfp) {
		ASSERT(!xfs_is_reflink_inode(ip));
		xfs_ifork_init_cow(ip);
	}
367

368
	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found);
369 370 371 372
	if (error || !*shared)
		return error;
	if (found)
		goto convert;
373

374 375 376
	resaligned = xfs_aligned_fsb_count(imap->br_startoff,
		imap->br_blockcount, xfs_get_cowextsz_hint(ip));
	resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
377

378
	xfs_iunlock(ip, *lockmode);
379
	*lockmode = 0;
380

381 382
	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0,
			false, &tp);
383 384
	if (error)
		return error;
385

386
	*lockmode = XFS_ILOCK_EXCL;
387

388 389 390
	/*
	 * Check for an overlapping extent again now that we dropped the ilock.
	 */
391
	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found);
392 393 394 395 396
	if (error || !*shared)
		goto out_trans_cancel;
	if (found) {
		xfs_trans_cancel(tp);
		goto convert;
397 398
	}

399
	/* Allocate the entire reservation as unwritten blocks. */
400
	nimaps = 1;
401
	error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount,
402 403
			XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap,
			&nimaps);
404
	if (error)
405
		goto out_trans_cancel;
406

407
	xfs_inode_set_cowblocks_tag(ip);
408
	error = xfs_trans_commit(tp);
409
	if (error)
410
		return error;
411 412 413 414 415 416 417

	/*
	 * Allocation succeeded but the requested range was not even partially
	 * satisfied?  Bail out!
	 */
	if (nimaps == 0)
		return -ENOSPC;
418
convert:
419
	xfs_trim_extent(cmap, offset_fsb, count_fsb);
420 421 422 423 424
	/*
	 * COW fork extents are supposed to remain unwritten until we're ready
	 * to initiate a disk write.  For direct I/O we are going to write the
	 * data and need the conversion, but for buffered writes we're done.
	 */
425
	if (!convert_now || cmap->br_state == XFS_EXT_NORM)
426
		return 0;
427
	trace_xfs_reflink_convert_cow(ip, cmap);
428
	return xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb);
429 430 431

out_trans_cancel:
	xfs_trans_cancel(tp);
432
	return error;
433 434
}

435
/*
436 437 438 439
 * Cancel CoW reservations for some block range of an inode.
 *
 * If cancel_real is true this function cancels all COW fork extents for the
 * inode; if cancel_real is false, real extents are not cleared.
440 441 442
 *
 * Caller must have already joined the inode to the current transaction. The
 * inode will be joined to the transaction returned to the caller.
443 444 445 446 447 448
 */
int
xfs_reflink_cancel_cow_blocks(
	struct xfs_inode		*ip,
	struct xfs_trans		**tpp,
	xfs_fileoff_t			offset_fsb,
449 450
	xfs_fileoff_t			end_fsb,
	bool				cancel_real)
451
{
452
	struct xfs_ifork		*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
453
	struct xfs_bmbt_irec		got, del;
454
	struct xfs_iext_cursor		icur;
455
	int				error = 0;
456

457
	if (!xfs_inode_has_cow_data(ip))
458
		return 0;
459
	if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got))
460
		return 0;
461

462 463
	/* Walk backwards until we're out of the I/O range... */
	while (got.br_startoff + got.br_blockcount > offset_fsb) {
464 465
		del = got;
		xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb);
466 467 468 469 470 471 472

		/* Extent delete may have bumped ext forward */
		if (!del.br_blockcount) {
			xfs_iext_prev(ifp, &icur);
			goto next_extent;
		}

473
		trace_xfs_reflink_cancel_cow(ip, &del);
474

475 476
		if (isnullstartblock(del.br_startblock)) {
			error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK,
477
					&icur, &got, &del);
478 479
			if (error)
				break;
480
		} else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) {
481
			ASSERT((*tpp)->t_firstblock == NULLFSBLOCK);
482

483
			/* Free the CoW orphan record. */
484 485
			xfs_refcount_free_cow_extent(*tpp, del.br_startblock,
					del.br_blockcount);
486

487 488
			xfs_bmap_add_free(*tpp, del.br_startblock,
					  del.br_blockcount, NULL);
489 490

			/* Roll the transaction */
491
			error = xfs_defer_finish(tpp);
492
			if (error)
493 494 495
				break;

			/* Remove the mapping from the CoW fork. */
496
			xfs_bmap_del_extent_cow(ip, &icur, &got, &del);
497 498

			/* Remove the quota reservation */
499 500
			error = xfs_quota_unreserve_blkres(ip,
					del.br_blockcount);
501 502
			if (error)
				break;
503 504 505
		} else {
			/* Didn't do anything, push cursor back. */
			xfs_iext_prev(ifp, &icur);
506
		}
507 508
next_extent:
		if (!xfs_iext_get_extent(ifp, &icur, &got))
509
			break;
510 511
	}

512 513 514
	/* clear tag if cow fork is emptied */
	if (!ifp->if_bytes)
		xfs_inode_clear_cowblocks_tag(ip);
515 516 517 518
	return error;
}

/*
519 520 521 522
 * Cancel CoW reservations for some byte range of an inode.
 *
 * If cancel_real is true this function cancels all COW fork extents for the
 * inode; if cancel_real is false, real extents are not cleared.
523 524 525 526 527
 */
int
xfs_reflink_cancel_cow_range(
	struct xfs_inode	*ip,
	xfs_off_t		offset,
528 529
	xfs_off_t		count,
	bool			cancel_real)
530 531 532 533 534 535 536
{
	struct xfs_trans	*tp;
	xfs_fileoff_t		offset_fsb;
	xfs_fileoff_t		end_fsb;
	int			error;

	trace_xfs_reflink_cancel_cow_range(ip, offset, count);
537
	ASSERT(ip->i_cowfp);
538 539 540 541 542 543 544 545 546

	offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
	if (count == NULLFILEOFF)
		end_fsb = NULLFILEOFF;
	else
		end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count);

	/* Start a rolling transaction to remove the mappings */
	error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write,
C
Christoph Hellwig 已提交
547
			0, 0, 0, &tp);
548 549 550 551 552 553 554
	if (error)
		goto out;

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, ip, 0);

	/* Scrape out the old CoW reservations */
555 556
	error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb,
			cancel_real);
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
	if (error)
		goto out_cancel;

	error = xfs_trans_commit(tp);

	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return error;

out_cancel:
	xfs_trans_cancel(tp);
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
out:
	trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_);
	return error;
}

/*
574 575 576 577 578 579 580 581
 * Remap part of the CoW fork into the data fork.
 *
 * We aim to remap the range starting at @offset_fsb and ending at @end_fsb
 * into the data fork; this function will remap what it can (at the end of the
 * range) and update @end_fsb appropriately.  Each remap gets its own
 * transaction because we can end up merging and splitting bmbt blocks for
 * every remap operation and we'd like to keep the block reservation
 * requirements as low as possible.
582
 */
583 584 585 586 587
STATIC int
xfs_reflink_end_cow_extent(
	struct xfs_inode	*ip,
	xfs_fileoff_t		offset_fsb,
	xfs_fileoff_t		*end_fsb)
588
{
589 590 591 592 593 594 595 596
	struct xfs_bmbt_irec	got, del;
	struct xfs_iext_cursor	icur;
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_trans	*tp;
	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
	xfs_filblks_t		rlen;
	unsigned int		resblks;
	int			error;
597

598
	/* No COW extents?  That's easy! */
599 600
	if (ifp->if_bytes == 0) {
		*end_fsb = offset_fsb;
601
		return 0;
602
	}
603

604 605
	resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0,
C
Christoph Hellwig 已提交
606
			XFS_TRANS_RESERVE, &tp);
607 608
	if (error)
		return error;
609

610
	/*
611 612 613
	 * Lock the inode.  We have to ijoin without automatic unlock because
	 * the lead transaction is the refcountbt record deletion; the data
	 * fork update follows as a deferred log item.
614
	 */
615 616 617
	xfs_ilock(ip, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, ip, 0);

618 619 620 621 622
	error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
			XFS_IEXT_REFLINK_END_COW_CNT);
	if (error)
		goto out_cancel;

623 624 625 626 627
	/*
	 * In case of racing, overlapping AIO writes no COW extents might be
	 * left by the time I/O completes for the loser of the race.  In that
	 * case we are done.
	 */
628 629 630
	if (!xfs_iext_lookup_extent_before(ip, ifp, end_fsb, &icur, &got) ||
	    got.br_startoff + got.br_blockcount <= offset_fsb) {
		*end_fsb = offset_fsb;
631
		goto out_cancel;
632
	}
633

634 635 636 637 638 639 640 641
	/*
	 * Structure copy @got into @del, then trim @del to the range that we
	 * were asked to remap.  We preserve @got for the eventual CoW fork
	 * deletion; from now on @del represents the mapping that we're
	 * actually remapping.
	 */
	del = got;
	xfs_trim_extent(&del, offset_fsb, *end_fsb - offset_fsb);
642

643
	ASSERT(del.br_blockcount > 0);
644

645 646 647 648 649
	/*
	 * Only remap real extents that contain data.  With AIO, speculative
	 * preallocations can leak into the range we are called upon, and we
	 * need to skip them.
	 */
650
	if (!xfs_bmap_is_written_extent(&got)) {
651 652 653
		*end_fsb = del.br_startoff;
		goto out_cancel;
	}
654

655 656 657 658 659
	/* Unmap the old blocks in the data fork. */
	rlen = del.br_blockcount;
	error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1);
	if (error)
		goto out_cancel;
660

661 662 663
	/* Trim the extent to whatever got unmapped. */
	xfs_trim_extent(&del, del.br_startoff + rlen, del.br_blockcount - rlen);
	trace_xfs_reflink_cow_remap(ip, &del);
664

665
	/* Free the CoW orphan record. */
666
	xfs_refcount_free_cow_extent(tp, del.br_startblock, del.br_blockcount);
667

668
	/* Map the new blocks into the data fork. */
669
	xfs_bmap_map_extent(tp, ip, &del);
670

671 672 673
	/* Charge this new data fork mapping to the on-disk quota. */
	xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT,
			(long)del.br_blockcount);
674

675 676
	/* Remove the mapping from the CoW fork. */
	xfs_bmap_del_extent_cow(ip, &icur, &got, &del);
677 678 679 680

	error = xfs_trans_commit(tp);
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	if (error)
681 682 683 684
		return error;

	/* Update the caller about how much progress we made. */
	*end_fsb = del.br_startoff;
685 686
	return 0;

687
out_cancel:
688 689
	xfs_trans_cancel(tp);
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
	return error;
}

/*
 * Remap parts of a file's data fork after a successful CoW.
 */
int
xfs_reflink_end_cow(
	struct xfs_inode		*ip,
	xfs_off_t			offset,
	xfs_off_t			count)
{
	xfs_fileoff_t			offset_fsb;
	xfs_fileoff_t			end_fsb;
	int				error = 0;

	trace_xfs_reflink_end_cow(ip, offset, count);

	offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
	end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count);

	/*
	 * Walk backwards until we're out of the I/O range.  The loop function
	 * repeatedly cycles the ILOCK to allocate one transaction per remapped
	 * extent.
	 *
716
	 * If we're being called by writeback then the pages will still
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	 * have PageWriteback set, which prevents races with reflink remapping
	 * and truncate.  Reflink remapping prevents races with writeback by
	 * taking the iolock and mmaplock before flushing the pages and
	 * remapping, which means there won't be any further writeback or page
	 * cache dirtying until the reflink completes.
	 *
	 * We should never have two threads issuing writeback for the same file
	 * region.  There are also have post-eof checks in the writeback
	 * preparation code so that we don't bother writing out pages that are
	 * about to be truncated.
	 *
	 * If we're being called as part of directio write completion, the dio
	 * count is still elevated, which reflink and truncate will wait for.
	 * Reflink remapping takes the iolock and mmaplock and waits for
	 * pending dio to finish, which should prevent any directio until the
	 * remap completes.  Multiple concurrent directio writes to the same
	 * region are handled by end_cow processing only occurring for the
	 * threads which succeed; the outcome of multiple overlapping direct
	 * writes is not well defined anyway.
	 *
	 * It's possible that a buffered write and a direct write could collide
	 * here (the buffered write stumbles in after the dio flushes and
	 * invalidates the page cache and immediately queues writeback), but we
	 * have never supported this 100%.  If either disk write succeeds the
	 * blocks will be remapped.
	 */
	while (end_fsb > offset_fsb && !error)
		error = xfs_reflink_end_cow_extent(ip, offset_fsb, &end_fsb);

	if (error)
		trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_);
748 749
	return error;
}
750 751 752 753 754 755 756 757

/*
 * Free leftover CoW reservations that didn't get cleaned out.
 */
int
xfs_reflink_recover_cow(
	struct xfs_mount	*mp)
{
758
	struct xfs_perag	*pag;
759 760 761 762 763 764
	xfs_agnumber_t		agno;
	int			error = 0;

	if (!xfs_sb_version_hasreflink(&mp->m_sb))
		return 0;

765
	for_each_perag(mp, agno, pag) {
766
		error = xfs_refcount_recover_cow_leftovers(mp, pag);
767 768
		if (error) {
			xfs_perag_put(pag);
769
			break;
770
		}
771 772 773 774
	}

	return error;
}
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874

/*
 * Reflinking (Block) Ranges of Two Files Together
 *
 * First, ensure that the reflink flag is set on both inodes.  The flag is an
 * optimization to avoid unnecessary refcount btree lookups in the write path.
 *
 * Now we can iteratively remap the range of extents (and holes) in src to the
 * corresponding ranges in dest.  Let drange and srange denote the ranges of
 * logical blocks in dest and src touched by the reflink operation.
 *
 * While the length of drange is greater than zero,
 *    - Read src's bmbt at the start of srange ("imap")
 *    - If imap doesn't exist, make imap appear to start at the end of srange
 *      with zero length.
 *    - If imap starts before srange, advance imap to start at srange.
 *    - If imap goes beyond srange, truncate imap to end at the end of srange.
 *    - Punch (imap start - srange start + imap len) blocks from dest at
 *      offset (drange start).
 *    - If imap points to a real range of pblks,
 *         > Increase the refcount of the imap's pblks
 *         > Map imap's pblks into dest at the offset
 *           (drange start + imap start - srange start)
 *    - Advance drange and srange by (imap start - srange start + imap len)
 *
 * Finally, if the reflink made dest longer, update both the in-core and
 * on-disk file sizes.
 *
 * ASCII Art Demonstration:
 *
 * Let's say we want to reflink this source file:
 *
 * ----SSSSSSS-SSSSS----SSSSSS (src file)
 *   <-------------------->
 *
 * into this destination file:
 *
 * --DDDDDDDDDDDDDDDDDDD--DDD (dest file)
 *        <-------------------->
 * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest.
 * Observe that the range has different logical offsets in either file.
 *
 * Consider that the first extent in the source file doesn't line up with our
 * reflink range.  Unmapping  and remapping are separate operations, so we can
 * unmap more blocks from the destination file than we remap.
 *
 * ----SSSSSSS-SSSSS----SSSSSS
 *   <------->
 * --DDDDD---------DDDDD--DDD
 *        <------->
 *
 * Now remap the source extent into the destination file:
 *
 * ----SSSSSSS-SSSSS----SSSSSS
 *   <------->
 * --DDDDD--SSSSSSSDDDDD--DDD
 *        <------->
 *
 * Do likewise with the second hole and extent in our range.  Holes in the
 * unmap range don't affect our operation.
 *
 * ----SSSSSSS-SSSSS----SSSSSS
 *            <---->
 * --DDDDD--SSSSSSS-SSSSS-DDD
 *                 <---->
 *
 * Finally, unmap and remap part of the third extent.  This will increase the
 * size of the destination file.
 *
 * ----SSSSSSS-SSSSS----SSSSSS
 *                  <----->
 * --DDDDD--SSSSSSS-SSSSS----SSS
 *                       <----->
 *
 * Once we update the destination file's i_size, we're done.
 */

/*
 * Ensure the reflink bit is set in both inodes.
 */
STATIC int
xfs_reflink_set_inode_flag(
	struct xfs_inode	*src,
	struct xfs_inode	*dest)
{
	struct xfs_mount	*mp = src->i_mount;
	int			error;
	struct xfs_trans	*tp;

	if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest))
		return 0;

	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp);
	if (error)
		goto out_error;

	/* Lock both files against IO */
	if (src->i_ino == dest->i_ino)
		xfs_ilock(src, XFS_ILOCK_EXCL);
	else
875
		xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL);
876 877 878 879

	if (!xfs_is_reflink_inode(src)) {
		trace_xfs_reflink_set_inode_flag(src);
		xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL);
880
		src->i_diflags2 |= XFS_DIFLAG2_REFLINK;
881 882 883 884 885 886 887 888 889 890 891
		xfs_trans_log_inode(tp, src, XFS_ILOG_CORE);
		xfs_ifork_init_cow(src);
	} else
		xfs_iunlock(src, XFS_ILOCK_EXCL);

	if (src->i_ino == dest->i_ino)
		goto commit_flags;

	if (!xfs_is_reflink_inode(dest)) {
		trace_xfs_reflink_set_inode_flag(dest);
		xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL);
892
		dest->i_diflags2 |= XFS_DIFLAG2_REFLINK;
893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909
		xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE);
		xfs_ifork_init_cow(dest);
	} else
		xfs_iunlock(dest, XFS_ILOCK_EXCL);

commit_flags:
	error = xfs_trans_commit(tp);
	if (error)
		goto out_error;
	return error;

out_error:
	trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_);
	return error;
}

/*
910
 * Update destination inode size & cowextsize hint, if necessary.
911
 */
912
int
913 914
xfs_reflink_update_dest(
	struct xfs_inode	*dest,
915
	xfs_off_t		newlen,
916
	xfs_extlen_t		cowextsize,
917
	unsigned int		remap_flags)
918 919 920 921 922
{
	struct xfs_mount	*mp = dest->i_mount;
	struct xfs_trans	*tp;
	int			error;

923
	if (newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0)
924 925 926 927 928 929 930 931 932
		return 0;

	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp);
	if (error)
		goto out_error;

	xfs_ilock(dest, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL);

933 934 935
	if (newlen > i_size_read(VFS_I(dest))) {
		trace_xfs_reflink_update_inode_size(dest, newlen);
		i_size_write(VFS_I(dest), newlen);
936
		dest->i_disk_size = newlen;
937 938 939
	}

	if (cowextsize) {
940
		dest->i_cowextsize = cowextsize;
941
		dest->i_diflags2 |= XFS_DIFLAG2_COWEXTSIZE;
942 943
	}

944 945 946 947 948 949 950 951 952 953 954 955
	xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE);

	error = xfs_trans_commit(tp);
	if (error)
		goto out_error;
	return error;

out_error:
	trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_);
	return error;
}

956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
/*
 * Do we have enough reserve in this AG to handle a reflink?  The refcount
 * btree already reserved all the space it needs, but the rmap btree can grow
 * infinitely, so we won't allow more reflinks when the AG is down to the
 * btree reserves.
 */
static int
xfs_reflink_ag_has_free_space(
	struct xfs_mount	*mp,
	xfs_agnumber_t		agno)
{
	struct xfs_perag	*pag;
	int			error = 0;

	if (!xfs_sb_version_hasrmapbt(&mp->m_sb))
		return 0;

	pag = xfs_perag_get(mp, agno);
974
	if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) ||
975 976 977 978 979 980
	    xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA))
		error = -ENOSPC;
	xfs_perag_put(pag);
	return error;
}

981
/*
982 983
 * Remap the given extent into the file.  The dmap blockcount will be set to
 * the number of blocks that were actually remapped.
984 985 986 987
 */
STATIC int
xfs_reflink_remap_extent(
	struct xfs_inode	*ip,
988
	struct xfs_bmbt_irec	*dmap,
989 990
	xfs_off_t		new_isize)
{
991
	struct xfs_bmbt_irec	smap;
992 993 994
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_trans	*tp;
	xfs_off_t		newlen;
995
	int64_t			qdelta = 0;
996
	unsigned int		resblks;
997
	bool			quota_reserved = true;
998 999
	bool			smap_real;
	bool			dmap_written = xfs_bmap_is_written_extent(dmap);
1000
	int			iext_delta = 0;
1001
	int			nimaps;
1002 1003
	int			error;

1004 1005 1006 1007 1008 1009 1010 1011 1012
	/*
	 * Start a rolling transaction to switch the mappings.
	 *
	 * Adding a written extent to the extent map can cause a bmbt split,
	 * and removing a mapped extent from the extent can cause a bmbt split.
	 * The two operations cannot both cause a split since they operate on
	 * the same index in the bmap btree, so we only need a reservation for
	 * one bmbt split if either thing is happening.  However, we haven't
	 * locked the inode yet, so we reserve assuming this is the case.
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
	 *
	 * The first allocation call tries to reserve enough space to handle
	 * mapping dmap into a sparse part of the file plus the bmbt split.  We
	 * haven't locked the inode or read the existing mapping yet, so we do
	 * not know for sure that we need the space.  This should succeed most
	 * of the time.
	 *
	 * If the first attempt fails, try again but reserving only enough
	 * space to handle a bmbt split.  This is the hard minimum requirement,
	 * and we revisit quota reservations later when we know more about what
	 * we're remapping.
1024
	 */
1025
	resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
1026 1027 1028 1029 1030 1031 1032
	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write,
			resblks + dmap->br_blockcount, 0, false, &tp);
	if (error == -EDQUOT || error == -ENOSPC) {
		quota_reserved = false;
		error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write,
				resblks, 0, false, &tp);
	}
1033 1034 1035
	if (error)
		goto out;

1036
	/*
1037 1038 1039
	 * Read what's currently mapped in the destination file into smap.
	 * If smap isn't a hole, we will have to remove it before we can add
	 * dmap to the destination file.
1040
	 */
1041 1042 1043
	nimaps = 1;
	error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount,
			&smap, &nimaps, 0);
1044 1045
	if (error)
		goto out_cancel;
1046 1047
	ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff);
	smap_real = xfs_bmap_is_real_extent(&smap);
1048

1049 1050 1051 1052 1053 1054
	/*
	 * We can only remap as many blocks as the smaller of the two extent
	 * maps, because we can only remap one extent at a time.
	 */
	dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount);
	ASSERT(dmap->br_blockcount == smap.br_blockcount);
1055

1056 1057
	trace_xfs_reflink_remap_extent_dest(ip, &smap);

1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	/*
	 * Two extents mapped to the same physical block must not have
	 * different states; that's filesystem corruption.  Move on to the next
	 * extent if they're both holes or both the same physical extent.
	 */
	if (dmap->br_startblock == smap.br_startblock) {
		if (dmap->br_state != smap.br_state)
			error = -EFSCORRUPTED;
		goto out_cancel;
	}

	/* If both extents are unwritten, leave them alone. */
	if (dmap->br_state == XFS_EXT_UNWRITTEN &&
	    smap.br_state == XFS_EXT_UNWRITTEN)
		goto out_cancel;

1074 1075 1076 1077
	/* No reflinking if the AG of the dest mapping is low on space. */
	if (dmap_written) {
		error = xfs_reflink_ag_has_free_space(mp,
				XFS_FSB_TO_AGNO(mp, dmap->br_startblock));
1078
		if (error)
1079
			goto out_cancel;
1080
	}
1081

1082
	/*
1083
	 * Increase quota reservation if we think the quota block counter for
1084 1085 1086 1087
	 * this file could increase.
	 *
	 * If we are mapping a written extent into the file, we need to have
	 * enough quota block count reservation to handle the blocks in that
1088 1089 1090
	 * extent.  We log only the delta to the quota block counts, so if the
	 * extent we're unmapping also has blocks allocated to it, we don't
	 * need a quota reservation for the extent itself.
1091 1092 1093 1094 1095 1096 1097
	 *
	 * Note that if we're replacing a delalloc reservation with a written
	 * extent, we have to take the full quota reservation because removing
	 * the delalloc reservation gives the block count back to the quota
	 * count.  This is suboptimal, but the VFS flushed the dest range
	 * before we started.  That should have removed all the delalloc
	 * reservations, but we code defensively.
1098 1099 1100 1101 1102
	 *
	 * xfs_trans_alloc_inode above already tried to grab an even larger
	 * quota reservation, and kicked off a blockgc scan if it couldn't.
	 * If we can't get a potentially smaller quota reservation now, we're
	 * done.
1103
	 */
1104
	if (!quota_reserved && !smap_real && dmap_written) {
1105 1106
		error = xfs_trans_reserve_quota_nblks(tp, ip,
				dmap->br_blockcount, 0, false);
1107 1108 1109
		if (error)
			goto out_cancel;
	}
1110

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
	if (smap_real)
		++iext_delta;

	if (dmap_written)
		++iext_delta;

	error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta);
	if (error)
		goto out_cancel;

1121
	if (smap_real) {
1122
		/*
1123 1124
		 * If the extent we're unmapping is backed by storage (written
		 * or not), unmap the extent and drop its refcount.
1125
		 */
1126 1127 1128 1129 1130
		xfs_bmap_unmap_extent(tp, ip, &smap);
		xfs_refcount_decrease_extent(tp, &smap);
		qdelta -= smap.br_blockcount;
	} else if (smap.br_startblock == DELAYSTARTBLOCK) {
		xfs_filblks_t	len = smap.br_blockcount;
1131

1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
		/*
		 * If the extent we're unmapping is a delalloc reservation,
		 * we can use the regular bunmapi function to release the
		 * incore state.  Dropping the delalloc reservation takes care
		 * of the quota reservation for us.
		 */
		error = __xfs_bunmapi(NULL, ip, smap.br_startoff, &len, 0, 1);
		if (error)
			goto out_cancel;
		ASSERT(len == 0);
	}
1143

1144 1145 1146 1147 1148 1149 1150 1151 1152
	/*
	 * If the extent we're sharing is backed by written storage, increase
	 * its refcount and map it into the file.
	 */
	if (dmap_written) {
		xfs_refcount_increase_extent(tp, dmap);
		xfs_bmap_map_extent(tp, ip, dmap);
		qdelta += dmap->br_blockcount;
	}
1153

1154
	xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, qdelta);
1155

1156 1157 1158 1159 1160 1161
	/* Update dest isize if needed. */
	newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount);
	newlen = min_t(xfs_off_t, newlen, new_isize);
	if (newlen > i_size_read(VFS_I(ip))) {
		trace_xfs_reflink_update_inode_size(ip, newlen);
		i_size_write(VFS_I(ip), newlen);
1162
		ip->i_disk_size = newlen;
1163
		xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
1164 1165
	}

1166
	/* Commit everything and unlock. */
1167
	error = xfs_trans_commit(tp);
1168
	goto out_unlock;
1169 1170 1171

out_cancel:
	xfs_trans_cancel(tp);
1172
out_unlock:
1173 1174
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
out:
1175 1176
	if (error)
		trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_);
1177 1178 1179
	return error;
}

1180
/* Remap a range of one file to the other. */
1181
int
1182 1183
xfs_reflink_remap_blocks(
	struct xfs_inode	*src,
1184
	loff_t			pos_in,
1185
	struct xfs_inode	*dest,
1186
	loff_t			pos_out,
1187 1188
	loff_t			remap_len,
	loff_t			*remapped)
1189 1190
{
	struct xfs_bmbt_irec	imap;
1191 1192 1193
	struct xfs_mount	*mp = src->i_mount;
	xfs_fileoff_t		srcoff = XFS_B_TO_FSBT(mp, pos_in);
	xfs_fileoff_t		destoff = XFS_B_TO_FSBT(mp, pos_out);
1194
	xfs_filblks_t		len;
1195
	xfs_filblks_t		remapped_len = 0;
1196
	xfs_off_t		new_isize = pos_out + remap_len;
1197 1198
	int			nimaps;
	int			error = 0;
1199

1200 1201
	len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len),
			XFS_MAX_FILEOFF);
1202

1203
	trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff);
1204

1205 1206
	while (len > 0) {
		unsigned int	lock_mode;
1207

1208 1209
		/* Read extent from the source file */
		nimaps = 1;
1210
		lock_mode = xfs_ilock_data_map_shared(src);
1211
		error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0);
1212
		xfs_iunlock(src, lock_mode);
1213
		if (error)
1214
			break;
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
		/*
		 * The caller supposedly flushed all dirty pages in the source
		 * file range, which means that writeback should have allocated
		 * or deleted all delalloc reservations in that range.  If we
		 * find one, that's a good sign that something is seriously
		 * wrong here.
		 */
		ASSERT(nimaps == 1 && imap.br_startoff == srcoff);
		if (imap.br_startblock == DELAYSTARTBLOCK) {
			ASSERT(imap.br_startblock != DELAYSTARTBLOCK);
			error = -EFSCORRUPTED;
			break;
		}
1228

1229
		trace_xfs_reflink_remap_extent_src(src, &imap);
1230

1231 1232 1233
		/* Remap into the destination file at the given offset. */
		imap.br_startoff = destoff;
		error = xfs_reflink_remap_extent(dest, &imap, new_isize);
1234
		if (error)
1235
			break;
1236 1237 1238

		if (fatal_signal_pending(current)) {
			error = -EINTR;
1239
			break;
1240 1241 1242
		}

		/* Advance drange/srange */
1243 1244 1245 1246
		srcoff += imap.br_blockcount;
		destoff += imap.br_blockcount;
		len -= imap.br_blockcount;
		remapped_len += imap.br_blockcount;
1247 1248
	}

1249 1250
	if (error)
		trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_);
1251 1252
	*remapped = min_t(loff_t, remap_len,
			  XFS_FSB_TO_B(src->i_mount, remapped_len));
1253 1254 1255
	return error;
}

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
/*
 * If we're reflinking to a point past the destination file's EOF, we must
 * zero any speculative post-EOF preallocations that sit between the old EOF
 * and the destination file offset.
 */
static int
xfs_reflink_zero_posteof(
	struct xfs_inode	*ip,
	loff_t			pos)
{
	loff_t			isize = i_size_read(VFS_I(ip));

	if (pos <= isize)
		return 0;

	trace_xfs_zero_eof(ip, isize, pos - isize);
	return iomap_zero_range(VFS_I(ip), isize, pos - isize, NULL,
1273
			&xfs_buffered_write_iomap_ops);
1274 1275
}

1276
/*
1277
 * Prepare two files for range cloning.  Upon a successful return both inodes
1278 1279 1280
 * will have the iolock and mmaplock held, the page cache of the out file will
 * be truncated, and any leases on the out file will have been broken.  This
 * function borrows heavily from xfs_file_aio_write_checks.
1281 1282 1283 1284
 *
 * The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't
 * checked that the bytes beyond EOF physically match. Hence we cannot use the
 * EOF block in the source dedupe range because it's not a complete block match,
1285
 * hence can introduce a corruption into the file that has it's block replaced.
1286
 *
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
 * In similar fashion, the VFS file cloning also allows partial EOF blocks to be
 * "block aligned" for the purposes of cloning entire files.  However, if the
 * source file range includes the EOF block and it lands within the existing EOF
 * of the destination file, then we can expose stale data from beyond the source
 * file EOF in the destination file.
 *
 * XFS doesn't support partial block sharing, so in both cases we have check
 * these cases ourselves. For dedupe, we can simply round the length to dedupe
 * down to the previous whole block and ignore the partial EOF block. While this
 * means we can't dedupe the last block of a file, this is an acceptible
 * tradeoff for simplicity on implementation.
 *
 * For cloning, we want to share the partial EOF block if it is also the new EOF
 * block of the destination file. If the partial EOF block lies inside the
 * existing destination EOF, then we have to abort the clone to avoid exposing
 * stale data in the destination file. Hence we reject these clone attempts with
 * -EINVAL in this case.
1304
 */
1305
int
1306
xfs_reflink_remap_prep(
1307 1308 1309 1310
	struct file		*file_in,
	loff_t			pos_in,
	struct file		*file_out,
	loff_t			pos_out,
1311
	loff_t			*len,
1312
	unsigned int		remap_flags)
1313
{
1314 1315 1316 1317
	struct inode		*inode_in = file_inode(file_in);
	struct xfs_inode	*src = XFS_I(inode_in);
	struct inode		*inode_out = file_inode(file_out);
	struct xfs_inode	*dest = XFS_I(inode_out);
1318
	int			ret;
1319

1320
	/* Lock both files against IO */
1321
	ret = xfs_ilock2_io_mmap(src, dest);
1322 1323
	if (ret)
		return ret;
1324

1325
	/* Check file eligibility and prepare for block sharing. */
1326
	ret = -EINVAL;
1327 1328
	/* Don't reflink realtime inodes */
	if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest))
1329 1330 1331 1332 1333 1334
		goto out_unlock;

	/* Don't share DAX file data for now. */
	if (IS_DAX(inode_in) || IS_DAX(inode_out))
		goto out_unlock;

1335
	ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out,
1336
			len, remap_flags);
1337
	if (ret || *len == 0)
1338 1339
		goto out_unlock;

1340
	/* Attach dquots to dest inode before changing block map */
1341
	ret = xfs_qm_dqattach(dest);
1342 1343 1344
	if (ret)
		goto out_unlock;

1345
	/*
1346 1347
	 * Zero existing post-eof speculative preallocations in the destination
	 * file.
1348
	 */
1349 1350 1351
	ret = xfs_reflink_zero_posteof(dest, pos_out);
	if (ret)
		goto out_unlock;
1352

1353
	/* Set flags and remap blocks. */
1354 1355 1356
	ret = xfs_reflink_set_inode_flag(src, dest);
	if (ret)
		goto out_unlock;
1357

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
	/*
	 * If pos_out > EOF, we may have dirtied blocks between EOF and
	 * pos_out. In that case, we need to extend the flush and unmap to cover
	 * from EOF to the end of the copy length.
	 */
	if (pos_out > XFS_ISIZE(dest)) {
		loff_t	flen = *len + (pos_out - XFS_ISIZE(dest));
		ret = xfs_flush_unmap_range(dest, XFS_ISIZE(dest), flen);
	} else {
		ret = xfs_flush_unmap_range(dest, pos_out, *len);
	}
	if (ret)
		goto out_unlock;
1371

1372
	return 0;
1373
out_unlock:
1374
	xfs_iunlock2_io_mmap(src, dest);
1375 1376 1377
	return ret;
}

1378
/* Does this inode need the reflink flag? */
1379
int
1380 1381 1382 1383
xfs_reflink_inode_has_shared_extents(
	struct xfs_trans		*tp,
	struct xfs_inode		*ip,
	bool				*has_shared)
1384
{
1385 1386 1387 1388 1389 1390 1391 1392
	struct xfs_bmbt_irec		got;
	struct xfs_mount		*mp = ip->i_mount;
	struct xfs_ifork		*ifp;
	xfs_agnumber_t			agno;
	xfs_agblock_t			agbno;
	xfs_extlen_t			aglen;
	xfs_agblock_t			rbno;
	xfs_extlen_t			rlen;
1393
	struct xfs_iext_cursor		icur;
1394 1395
	bool				found;
	int				error;
1396

1397
	ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK);
1398 1399 1400
	error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
	if (error)
		return error;
1401

1402
	*has_shared = false;
1403
	found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got);
1404 1405 1406 1407 1408 1409 1410
	while (found) {
		if (isnullstartblock(got.br_startblock) ||
		    got.br_state != XFS_EXT_NORM)
			goto next;
		agno = XFS_FSB_TO_AGNO(mp, got.br_startblock);
		agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock);
		aglen = got.br_blockcount;
1411

1412
		error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen,
1413 1414 1415 1416
				&rbno, &rlen, false);
		if (error)
			return error;
		/* Is there still a shared block here? */
1417 1418
		if (rbno != NULLAGBLOCK) {
			*has_shared = true;
1419
			return 0;
1420
		}
1421
next:
1422
		found = xfs_iext_next_extent(ifp, &icur, &got);
1423 1424
	}

1425 1426 1427
	return 0;
}

1428 1429 1430 1431 1432 1433
/*
 * Clear the inode reflink flag if there are no shared extents.
 *
 * The caller is responsible for joining the inode to the transaction passed in.
 * The inode will be joined to the transaction that is returned to the caller.
 */
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
int
xfs_reflink_clear_inode_flag(
	struct xfs_inode	*ip,
	struct xfs_trans	**tpp)
{
	bool			needs_flag;
	int			error = 0;

	ASSERT(xfs_is_reflink_inode(ip));

	error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag);
	if (error || needs_flag)
		return error;

1448 1449 1450 1451
	/*
	 * We didn't find any shared blocks so turn off the reflink flag.
	 * First, get rid of any leftover CoW mappings.
	 */
D
Darrick J. Wong 已提交
1452 1453
	error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, XFS_MAX_FILEOFF,
			true);
1454 1455 1456 1457 1458
	if (error)
		return error;

	/* Clear the inode flag. */
	trace_xfs_reflink_unset_inode_flag(ip);
1459
	ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK;
1460
	xfs_inode_clear_cowblocks_tag(ip);
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
	xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE);

	return error;
}

/*
 * Clear the inode reflink flag if there are no shared extents and the size
 * hasn't changed.
 */
STATIC int
xfs_reflink_try_clear_inode_flag(
1472
	struct xfs_inode	*ip)
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
{
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_trans	*tp;
	int			error = 0;

	/* Start a rolling transaction to remove the mappings */
	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp);
	if (error)
		return error;

	xfs_ilock(ip, XFS_ILOCK_EXCL);
	xfs_trans_ijoin(tp, ip, 0);

	error = xfs_reflink_clear_inode_flag(ip, &tp);
	if (error)
		goto cancel;

	error = xfs_trans_commit(tp);
	if (error)
		goto out;

	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return 0;
cancel:
	xfs_trans_cancel(tp);
out:
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return error;
}

/*
 * Pre-COW all shared blocks within a given byte range of a file and turn off
 * the reflink flag if we unshare all of the file's blocks.
 */
int
xfs_reflink_unshare(
	struct xfs_inode	*ip,
	xfs_off_t		offset,
	xfs_off_t		len)
{
1513
	struct inode		*inode = VFS_I(ip);
1514 1515 1516 1517 1518 1519 1520
	int			error;

	if (!xfs_is_reflink_inode(ip))
		return 0;

	trace_xfs_reflink_unshare(ip, offset, len);

1521
	inode_dio_wait(inode);
1522

1523 1524
	error = iomap_file_unshare(inode, offset, len,
			&xfs_buffered_write_iomap_ops);
1525
	if (error)
1526
		goto out;
1527

1528 1529
	error = filemap_write_and_wait_range(inode->i_mapping, offset,
			offset + len - 1);
1530 1531 1532
	if (error)
		goto out;

1533 1534 1535 1536
	/* Turn off the reflink flag if possible. */
	error = xfs_reflink_try_clear_inode_flag(ip);
	if (error)
		goto out;
1537 1538 1539 1540 1541 1542
	return 0;

out:
	trace_xfs_reflink_unshare_error(ip, error, _RET_IP_);
	return error;
}