xfs_iomap.c 35.1 KB
Newer Older
D
Dave Chinner 已提交
1
// SPDX-License-Identifier: GPL-2.0
L
Linus Torvalds 已提交
2
/*
3
 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
4
 * Copyright (c) 2016-2018 Christoph Hellwig.
5
 * All Rights Reserved.
L
Linus Torvalds 已提交
6 7 8
 */
#include "xfs.h"
#include "xfs_fs.h"
9
#include "xfs_shared.h"
10 11 12
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
L
Linus Torvalds 已提交
13 14
#include "xfs_mount.h"
#include "xfs_inode.h"
15
#include "xfs_btree.h"
16
#include "xfs_bmap_btree.h"
L
Linus Torvalds 已提交
17
#include "xfs_bmap.h"
D
Dave Chinner 已提交
18
#include "xfs_bmap_util.h"
19
#include "xfs_errortag.h"
L
Linus Torvalds 已提交
20
#include "xfs_error.h"
21
#include "xfs_trans.h"
L
Linus Torvalds 已提交
22
#include "xfs_trans_space.h"
23
#include "xfs_inode_item.h"
L
Linus Torvalds 已提交
24
#include "xfs_iomap.h"
C
Christoph Hellwig 已提交
25
#include "xfs_trace.h"
26
#include "xfs_quota.h"
27 28
#include "xfs_dquot_item.h"
#include "xfs_dquot.h"
29
#include "xfs_reflink.h"
L
Linus Torvalds 已提交
30 31


32 33
#define XFS_ALLOC_ALIGN(mp, off) \
	(((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
L
Linus Torvalds 已提交
34

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
static int
xfs_alert_fsblock_zero(
	xfs_inode_t	*ip,
	xfs_bmbt_irec_t	*imap)
{
	xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
			"Access to block zero in inode %llu "
			"start_block: %llx start_off: %llx "
			"blkcnt: %llx extent-state: %x",
		(unsigned long long)ip->i_ino,
		(unsigned long long)imap->br_startblock,
		(unsigned long long)imap->br_startoff,
		(unsigned long long)imap->br_blockcount,
		imap->br_state);
	return -EFSCORRUPTED;
}

int
53 54 55
xfs_bmbt_to_iomap(
	struct xfs_inode	*ip,
	struct iomap		*iomap,
56
	struct xfs_bmbt_irec	*imap,
57
	u16			flags)
58 59
{
	struct xfs_mount	*mp = ip->i_mount;
60
	struct xfs_buftarg	*target = xfs_inode_buftarg(ip);
61

62
	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock)))
63 64
		return xfs_alert_fsblock_zero(ip, imap);

65
	if (imap->br_startblock == HOLESTARTBLOCK) {
66
		iomap->addr = IOMAP_NULL_ADDR;
67
		iomap->type = IOMAP_HOLE;
68 69
	} else if (imap->br_startblock == DELAYSTARTBLOCK ||
		   isnullstartblock(imap->br_startblock)) {
70
		iomap->addr = IOMAP_NULL_ADDR;
71 72
		iomap->type = IOMAP_DELALLOC;
	} else {
73
		iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock));
74 75 76 77 78 79 80
		if (imap->br_state == XFS_EXT_UNWRITTEN)
			iomap->type = IOMAP_UNWRITTEN;
		else
			iomap->type = IOMAP_MAPPED;
	}
	iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
	iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
81 82
	iomap->bdev = target->bt_bdev;
	iomap->dax_dev = target->bt_daxdev;
83
	iomap->flags = flags;
84 85 86 87 88

	if (xfs_ipincount(ip) &&
	    (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
		iomap->flags |= IOMAP_F_DIRTY;
	return 0;
89 90
}

91 92 93 94 95 96 97
static void
xfs_hole_to_iomap(
	struct xfs_inode	*ip,
	struct iomap		*iomap,
	xfs_fileoff_t		offset_fsb,
	xfs_fileoff_t		end_fsb)
{
98 99
	struct xfs_buftarg	*target = xfs_inode_buftarg(ip);

100 101 102 103
	iomap->addr = IOMAP_NULL_ADDR;
	iomap->type = IOMAP_HOLE;
	iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb);
	iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb);
104 105
	iomap->bdev = target->bt_bdev;
	iomap->dax_dev = target->bt_daxdev;
106 107
}

108 109 110 111 112 113 114 115 116 117 118
static inline xfs_fileoff_t
xfs_iomap_end_fsb(
	struct xfs_mount	*mp,
	loff_t			offset,
	loff_t			count)
{
	ASSERT(offset <= mp->m_super->s_maxbytes);
	return min(XFS_B_TO_FSB(mp, offset + count),
		   XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
}

119
static xfs_extlen_t
120
xfs_eof_alignment(
121
	struct xfs_inode	*ip)
122
{
123 124
	struct xfs_mount	*mp = ip->i_mount;
	xfs_extlen_t		align = 0;
125

126 127 128 129 130 131 132 133 134 135 136 137 138 139
	if (!XFS_IS_REALTIME_INODE(ip)) {
		/*
		 * Round up the allocation request to a stripe unit
		 * (m_dalign) boundary if the file size is >= stripe unit
		 * size, and we are allocating past the allocation eof.
		 *
		 * If mounted with the "-o swalloc" option the alignment is
		 * increased from the strip unit size to the stripe width.
		 */
		if (mp->m_swidth && (mp->m_flags & XFS_MOUNT_SWALLOC))
			align = mp->m_swidth;
		else if (mp->m_dalign)
			align = mp->m_dalign;

140 141
		if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
			align = 0;
142
	}
143

144 145 146
	return align;
}

147 148 149 150
/*
 * Check if last_fsb is outside the last extent, and if so grow it to the next
 * stripe unit boundary.
 */
151
xfs_fileoff_t
152 153
xfs_iomap_eof_align_last_fsb(
	struct xfs_inode	*ip,
154
	xfs_fileoff_t		end_fsb)
155
{
156 157
	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK);
	xfs_extlen_t		extsz = xfs_get_extsz_hint(ip);
158
	xfs_extlen_t		align = xfs_eof_alignment(ip);
159 160 161 162
	struct xfs_bmbt_irec	irec;
	struct xfs_iext_cursor	icur;

	ASSERT(ifp->if_flags & XFS_IFEXTENTS);
163

164 165 166 167 168 169 170 171 172 173
	/*
	 * Always round up the allocation request to the extent hint boundary.
	 */
	if (extsz) {
		if (align)
			align = roundup_64(align, extsz);
		else
			align = extsz;
	}

174
	if (align) {
175 176 177 178 179 180
		xfs_fileoff_t	aligned_end_fsb = roundup_64(end_fsb, align);

		xfs_iext_last(ifp, &icur);
		if (!xfs_iext_get_extent(ifp, &icur, &irec) ||
		    aligned_end_fsb >= irec.br_startoff + irec.br_blockcount)
			return aligned_end_fsb;
181
	}
182 183

	return end_fsb;
184 185
}

C
Christoph Hellwig 已提交
186
int
L
Linus Torvalds 已提交
187
xfs_iomap_write_direct(
188 189 190 191
	struct xfs_inode	*ip,
	xfs_fileoff_t		offset_fsb,
	xfs_fileoff_t		count_fsb,
	struct xfs_bmbt_irec	*imap)
L
Linus Torvalds 已提交
192
{
193 194 195 196 197 198 199 200 201 202
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_trans	*tp;
	xfs_filblks_t		resaligned;
	int			nimaps;
	int			quota_flag;
	uint			qblocks, resblks;
	unsigned int		resrtextents = 0;
	int			error;
	int			bmapi_flags = XFS_BMAPI_PREALLOC;
	uint			tflags = 0;
L
Linus Torvalds 已提交
203

204 205
	ASSERT(count_fsb > 0);

206 207 208
	resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb,
					   xfs_get_extsz_hint(ip));
	if (unlikely(XFS_IS_REALTIME_INODE(ip))) {
209 210
		resrtextents = qblocks = resaligned;
		resrtextents /= mp->m_sb.sb_rextsize;
211 212 213
		resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
		quota_flag = XFS_QMOPT_RES_RTBLKS;
	} else {
214
		resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
215 216
		quota_flag = XFS_QMOPT_RES_REGBLKS;
	}
L
Linus Torvalds 已提交
217

218
	error = xfs_qm_dqattach(ip);
219 220 221
	if (error)
		return error;

222 223 224 225 226 227
	/*
	 * For DAX, we do not allocate unwritten extents, but instead we zero
	 * the block before we commit the transaction.  Ideally we'd like to do
	 * this outside the transaction context, but if we commit and then crash
	 * we may not have zeroed the blocks and this will be exposed on
	 * recovery of the allocation. Hence we must zero before commit.
228
	 *
229 230 231
	 * Further, if we are mapping unwritten extents here, we need to zero
	 * and convert them to written so that we don't need an unwritten extent
	 * callback for DAX. This also means that we need to be able to dip into
232 233
	 * the reserve block pool for bmbt block allocation if there is no space
	 * left but we need to do unwritten extent conversion.
234 235 236
	 */
	if (IS_DAX(VFS_I(ip))) {
		bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
237
		if (imap->br_state == XFS_EXT_UNWRITTEN) {
238
			tflags |= XFS_TRANS_RESERVE;
239
			resblks = qblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
240
		}
241
	}
242 243 244
	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, resrtextents,
			tflags, &tp);
	if (error)
E
Eric Sandeen 已提交
245
		return error;
246

247
	xfs_ilock(ip, XFS_ILOCK_EXCL);
L
Linus Torvalds 已提交
248

C
Christoph Hellwig 已提交
249
	error = xfs_trans_reserve_quota_nblks(tp, ip, qblocks, 0, quota_flag);
250
	if (error)
251
		goto out_trans_cancel;
L
Linus Torvalds 已提交
252

253
	xfs_trans_ijoin(tp, ip, 0);
L
Linus Torvalds 已提交
254 255

	/*
256 257
	 * From this point onwards we overwrite the imap pointer that the
	 * caller gave to us.
L
Linus Torvalds 已提交
258
	 */
259
	nimaps = 1;
260 261
	error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0,
				imap, &nimaps);
262
	if (error)
263
		goto out_trans_cancel;
L
Linus Torvalds 已提交
264 265

	/*
266
	 * Complete the transaction
L
Linus Torvalds 已提交
267
	 */
268
	error = xfs_trans_commit(tp);
269
	if (error)
270
		goto out_unlock;
L
Linus Torvalds 已提交
271

272 273 274
	/*
	 * Copy any maps to caller's array and return any error.
	 */
L
Linus Torvalds 已提交
275
	if (nimaps == 0) {
D
Dave Chinner 已提交
276
		error = -ENOSPC;
277
		goto out_unlock;
278 279
	}

280
	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock)))
281
		error = xfs_alert_fsblock_zero(ip, imap);
L
Linus Torvalds 已提交
282

283
out_unlock:
284
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
285
	return error;
L
Linus Torvalds 已提交
286

287
out_trans_cancel:
288
	xfs_trans_cancel(tp);
289
	goto out_unlock;
L
Linus Torvalds 已提交
290 291
}

292 293
STATIC bool
xfs_quota_need_throttle(
294 295 296
	struct xfs_inode	*ip,
	xfs_dqtype_t		type,
	xfs_fsblock_t		alloc_blocks)
297
{
298
	struct xfs_dquot	*dq = xfs_inode_dquot(ip, type);
299 300 301 302 303 304 305 306 307

	if (!dq || !xfs_this_quota_on(ip->i_mount, type))
		return false;

	/* no hi watermark, no throttle */
	if (!dq->q_prealloc_hi_wmark)
		return false;

	/* under the lo watermark, no throttle */
308
	if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark)
309 310 311 312 313 314 315
		return false;

	return true;
}

STATIC void
xfs_quota_calc_throttle(
316 317 318 319 320
	struct xfs_inode	*ip,
	xfs_dqtype_t		type,
	xfs_fsblock_t		*qblocks,
	int			*qshift,
	int64_t			*qfreesp)
321
{
322 323 324
	struct xfs_dquot	*dq = xfs_inode_dquot(ip, type);
	int64_t			freesp;
	int			shift = 0;
325

326
	/* no dq, or over hi wmark, squash the prealloc completely */
327
	if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) {
328
		*qblocks = 0;
329
		*qfreesp = 0;
330 331 332
		return;
	}

333
	freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved;
334 335 336 337 338 339 340 341
	if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) {
		shift = 2;
		if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT])
			shift += 2;
		if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT])
			shift += 2;
	}

342 343 344
	if (freesp < *qfreesp)
		*qfreesp = freesp;

345 346 347 348 349 350 351
	/* only overwrite the throttle values if we are more aggressive */
	if ((freesp >> shift) < (*qblocks >> *qshift)) {
		*qblocks = freesp;
		*qshift = shift;
	}
}

352 353
/*
 * If we don't have a user specified preallocation size, dynamically increase
354
 * the preallocation size as the size of the file grows.  Cap the maximum size
355
 * at a single extent or less if the filesystem is near full. The closer the
356
 * filesystem is to being full, the smaller the maximum preallocation.
357 358 359
 */
STATIC xfs_fsblock_t
xfs_iomap_prealloc_size(
360
	struct xfs_inode	*ip,
361
	int			whichfork,
362 363
	loff_t			offset,
	loff_t			count,
364
	struct xfs_iext_cursor	*icur)
365
{
366 367
	struct xfs_iext_cursor	ncur = *icur;
	struct xfs_bmbt_irec	prev, got;
368
	struct xfs_mount	*mp = ip->i_mount;
369
	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, whichfork);
370
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
371
	int64_t			freesp;
372
	xfs_fsblock_t		qblocks;
373
	xfs_fsblock_t		alloc_blocks = 0;
374 375 376
	xfs_extlen_t		plen;
	int			shift = 0;
	int			qshift = 0;
377

378 379 380 381 382 383 384
	/*
	 * As an exception we don't do any preallocation at all if the file is
	 * smaller than the minimum preallocation and we are using the default
	 * dynamic preallocation scheme, as it is likely this is the only write
	 * to the file that is going to be done.
	 */
	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))
385 386 387
		return 0;

	/*
388 389
	 * Use the minimum preallocation size for small files or if we are
	 * writing right after a hole.
390
	 */
391
	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
392
	    !xfs_iext_prev_extent(ifp, &ncur, &prev) ||
393
	    prev.br_startoff + prev.br_blockcount < offset_fsb)
394
		return mp->m_allocsize_blocks;
395

396
	/*
397 398 399
	 * Take the size of the preceding data extents as the basis for the
	 * preallocation size. Note that we don't care if the previous extents
	 * are written or not.
400
	 */
401 402 403 404 405 406 407 408 409 410
	plen = prev.br_blockcount;
	while (xfs_iext_prev_extent(ifp, &ncur, &got)) {
		if (plen > MAXEXTLEN / 2 ||
		    isnullstartblock(got.br_startblock) ||
		    got.br_startoff + got.br_blockcount != prev.br_startoff ||
		    got.br_startblock + got.br_blockcount != prev.br_startblock)
			break;
		plen += got.br_blockcount;
		prev = got;
	}
411 412 413 414 415 416 417 418

	/*
	 * If the size of the extents is greater than half the maximum extent
	 * length, then use the current offset as the basis.  This ensures that
	 * for large files the preallocation size always extends to MAXEXTLEN
	 * rather than falling short due to things like stripe unit/width
	 * alignment of real extents.
	 */
419 420
	alloc_blocks = plen * 2;
	if (alloc_blocks > MAXEXTLEN)
421
		alloc_blocks = XFS_B_TO_FSB(mp, offset);
422
	qblocks = alloc_blocks;
423

424 425 426
	/*
	 * MAXEXTLEN is not a power of two value but we round the prealloc down
	 * to the nearest power of two value after throttling. To prevent the
427 428 429
	 * round down from unconditionally reducing the maximum supported
	 * prealloc size, we round up first, apply appropriate throttling,
	 * round down and cap the value to MAXEXTLEN.
430 431 432
	 */
	alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(MAXEXTLEN),
				       alloc_blocks);
433

434
	freesp = percpu_counter_read_positive(&mp->m_fdblocks);
435 436 437 438 439 440 441 442 443 444
	if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) {
		shift = 2;
		if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT])
			shift++;
		if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT])
			shift++;
		if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT])
			shift++;
		if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT])
			shift++;
445
	}
446 447

	/*
448 449
	 * Check each quota to cap the prealloc size, provide a shift value to
	 * throttle with and adjust amount of available space.
450
	 */
451 452
	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
		xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
453
					&freesp);
454 455
	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
		xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
456
					&freesp);
457 458
	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
		xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
459
					&freesp);
460 461 462 463 464 465 466 467

	/*
	 * The final prealloc size is set to the minimum of free space available
	 * in each of the quotas and the overall filesystem.
	 *
	 * The shift throttle value is set to the maximum value as determined by
	 * the global low free space values and per-quota low free space values.
	 */
D
Dave Chinner 已提交
468 469
	alloc_blocks = min(alloc_blocks, qblocks);
	shift = max(shift, qshift);
470

471 472
	if (shift)
		alloc_blocks >>= shift;
473 474 475 476 477 478 479 480
	/*
	 * rounddown_pow_of_two() returns an undefined result if we pass in
	 * alloc_blocks = 0.
	 */
	if (alloc_blocks)
		alloc_blocks = rounddown_pow_of_two(alloc_blocks);
	if (alloc_blocks > MAXEXTLEN)
		alloc_blocks = MAXEXTLEN;
481 482 483 484 485 486 487 488 489

	/*
	 * If we are still trying to allocate more space than is
	 * available, squash the prealloc hard. This can happen if we
	 * have a large file on a small filesystem and the above
	 * lowspace thresholds are smaller than MAXEXTLEN.
	 */
	while (alloc_blocks && alloc_blocks >= freesp)
		alloc_blocks >>= 4;
490 491
	if (alloc_blocks < mp->m_allocsize_blocks)
		alloc_blocks = mp->m_allocsize_blocks;
492
	trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
493
				      mp->m_allocsize_blocks);
494 495 496
	return alloc_blocks;
}

L
Linus Torvalds 已提交
497 498 499
int
xfs_iomap_write_unwritten(
	xfs_inode_t	*ip,
500
	xfs_off_t	offset,
501 502
	xfs_off_t	count,
	bool		update_isize)
L
Linus Torvalds 已提交
503 504 505 506 507
{
	xfs_mount_t	*mp = ip->i_mount;
	xfs_fileoff_t	offset_fsb;
	xfs_filblks_t	count_fsb;
	xfs_filblks_t	numblks_fsb;
508 509 510
	int		nimaps;
	xfs_trans_t	*tp;
	xfs_bmbt_irec_t imap;
511
	struct inode	*inode = VFS_I(ip);
512
	xfs_fsize_t	i_size;
513
	uint		resblks;
L
Linus Torvalds 已提交
514 515
	int		error;

C
Christoph Hellwig 已提交
516
	trace_xfs_unwritten_convert(ip, offset, count);
L
Linus Torvalds 已提交
517 518 519 520 521

	offset_fsb = XFS_B_TO_FSBT(mp, offset);
	count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
	count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);

522 523 524 525 526 527 528 529 530 531
	/*
	 * Reserve enough blocks in this transaction for two complete extent
	 * btree splits.  We may be converting the middle part of an unwritten
	 * extent and in this case we will insert two new extents in the btree
	 * each of which could cause a full split.
	 *
	 * This reservation amount will be used in the first call to
	 * xfs_bmbt_split() to select an AG with enough space to satisfy the
	 * rest of the operation.
	 */
532
	resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
L
Linus Torvalds 已提交
533

534 535 536 537 538
	/* Attach dquots so that bmbt splits are accounted correctly. */
	error = xfs_qm_dqattach(ip);
	if (error)
		return error;

539
	do {
L
Linus Torvalds 已提交
540
		/*
541
		 * Set up a transaction to convert the range of extents
L
Linus Torvalds 已提交
542 543
		 * from unwritten to real. Do allocations in a loop until
		 * we have covered the range passed in.
544
		 *
545 546 547
		 * Note that we can't risk to recursing back into the filesystem
		 * here as we might be asked to write out the same inode that we
		 * complete here and might deadlock on the iolock.
L
Linus Torvalds 已提交
548
		 */
549
		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0,
C
Christoph Hellwig 已提交
550
				XFS_TRANS_RESERVE, &tp);
551
		if (error)
E
Eric Sandeen 已提交
552
			return error;
L
Linus Torvalds 已提交
553 554

		xfs_ilock(ip, XFS_ILOCK_EXCL);
555
		xfs_trans_ijoin(tp, ip, 0);
L
Linus Torvalds 已提交
556

557
		error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0,
558
				XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_FORCE_RES);
559 560 561
		if (error)
			goto error_on_bmapi_transaction;

L
Linus Torvalds 已提交
562 563 564 565
		/*
		 * Modify the unwritten extent state of the buffer.
		 */
		nimaps = 1;
566
		error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
567 568
					XFS_BMAPI_CONVERT, resblks, &imap,
					&nimaps);
L
Linus Torvalds 已提交
569 570 571
		if (error)
			goto error_on_bmapi_transaction;

572 573 574 575 576 577 578 579
		/*
		 * Log the updated inode size as we go.  We have to be careful
		 * to only log it up to the actual write offset if it is
		 * halfway into a block.
		 */
		i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
		if (i_size > offset + count)
			i_size = offset + count;
580 581
		if (update_isize && i_size > i_size_read(inode))
			i_size_write(inode, i_size);
582 583 584 585 586 587
		i_size = xfs_new_eof(ip, i_size);
		if (i_size) {
			ip->i_d.di_size = i_size;
			xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
		}

588
		error = xfs_trans_commit(tp);
L
Linus Torvalds 已提交
589 590
		xfs_iunlock(ip, XFS_ILOCK_EXCL);
		if (error)
E
Eric Sandeen 已提交
591
			return error;
592

593
		if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock)))
594
			return xfs_alert_fsblock_zero(ip, &imap);
L
Linus Torvalds 已提交
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610

		if ((numblks_fsb = imap.br_blockcount) == 0) {
			/*
			 * The numblks_fsb value should always get
			 * smaller, otherwise the loop is stuck.
			 */
			ASSERT(imap.br_blockcount);
			break;
		}
		offset_fsb += numblks_fsb;
		count_fsb -= numblks_fsb;
	} while (count_fsb > 0);

	return 0;

error_on_bmapi_transaction:
611
	xfs_trans_cancel(tp);
L
Linus Torvalds 已提交
612
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
E
Eric Sandeen 已提交
613
	return error;
L
Linus Torvalds 已提交
614
}
615

616 617 618
static inline bool
imap_needs_alloc(
	struct inode		*inode,
619
	unsigned		flags,
620 621
	struct xfs_bmbt_irec	*imap,
	int			nimaps)
622
{
623 624 625 626 627 628 629 630 631 632 633
	/* don't allocate blocks when just zeroing */
	if (flags & IOMAP_ZERO)
		return false;
	if (!nimaps ||
	    imap->br_startblock == HOLESTARTBLOCK ||
	    imap->br_startblock == DELAYSTARTBLOCK)
		return true;
	/* we convert unwritten extents before copying the data for DAX */
	if (IS_DAX(inode) && imap->br_state == XFS_EXT_UNWRITTEN)
		return true;
	return false;
634 635
}

636
static inline bool
637 638 639
imap_needs_cow(
	struct xfs_inode	*ip,
	unsigned int		flags,
640 641
	struct xfs_bmbt_irec	*imap,
	int			nimaps)
642
{
643 644 645 646 647 648 649 650 651 652 653 654
	if (!xfs_is_cow_inode(ip))
		return false;

	/* when zeroing we don't have to COW holes or unwritten extents */
	if (flags & IOMAP_ZERO) {
		if (!nimaps ||
		    imap->br_startblock == HOLESTARTBLOCK ||
		    imap->br_state == XFS_EXT_UNWRITTEN)
			return false;
	}

	return true;
655 656
}

657 658 659 660 661
static int
xfs_ilock_for_iomap(
	struct xfs_inode	*ip,
	unsigned		flags,
	unsigned		*lockmode)
C
Christoph Hellwig 已提交
662
{
663
	unsigned		mode = XFS_ILOCK_SHARED;
664
	bool			is_write = flags & (IOMAP_WRITE | IOMAP_ZERO);
665

C
Christoph Hellwig 已提交
666
	/*
667 668
	 * COW writes may allocate delalloc space or convert unwritten COW
	 * extents, so we need to make sure to take the lock exclusively here.
C
Christoph Hellwig 已提交
669
	 */
670
	if (xfs_is_cow_inode(ip) && is_write)
671
		mode = XFS_ILOCK_EXCL;
672 673

	/*
674 675
	 * Extents not yet cached requires exclusive access, don't block.  This
	 * is an opencoded xfs_ilock_data_map_shared() call but with
676 677
	 * non-blocking behaviour.
	 */
678 679 680 681 682 683
	if (!(ip->i_df.if_flags & XFS_IFEXTENTS)) {
		if (flags & IOMAP_NOWAIT)
			return -EAGAIN;
		mode = XFS_ILOCK_EXCL;
	}

684
relock:
685 686 687 688 689 690 691
	if (flags & IOMAP_NOWAIT) {
		if (!xfs_ilock_nowait(ip, mode))
			return -EAGAIN;
	} else {
		xfs_ilock(ip, mode);
	}

692 693 694 695 696
	/*
	 * The reflink iflag could have changed since the earlier unlocked
	 * check, so if we got ILOCK_SHARED for a write and but we're now a
	 * reflink inode we have to switch to ILOCK_EXCL and relock.
	 */
697
	if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) {
698 699 700 701 702
		xfs_iunlock(ip, mode);
		mode = XFS_ILOCK_EXCL;
		goto relock;
	}

703 704
	*lockmode = mode;
	return 0;
C
Christoph Hellwig 已提交
705 706
}

707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
/*
 * Check that the imap we are going to return to the caller spans the entire
 * range that the caller requested for the IO.
 */
static bool
imap_spans_range(
	struct xfs_bmbt_irec	*imap,
	xfs_fileoff_t		offset_fsb,
	xfs_fileoff_t		end_fsb)
{
	if (imap->br_startoff > offset_fsb)
		return false;
	if (imap->br_startoff + imap->br_blockcount < end_fsb)
		return false;
	return true;
}

724
static int
725
xfs_direct_write_iomap_begin(
726 727 728 729
	struct inode		*inode,
	loff_t			offset,
	loff_t			length,
	unsigned		flags,
730 731
	struct iomap		*iomap,
	struct iomap		*srcmap)
732 733 734
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
735
	struct xfs_bmbt_irec	imap, cmap;
736 737
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, length);
738
	int			nimaps = 1, error = 0;
739
	bool			shared = false;
740
	u16			iomap_flags = 0;
741
	unsigned		lockmode;
742

743 744
	ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO));

745 746 747
	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

748 749 750 751 752 753 754 755
	/*
	 * Writes that span EOF might trigger an IO size update on completion,
	 * so consider them to be dirty for the purposes of O_DSYNC even if
	 * there is no other metadata changes pending or have been made here.
	 */
	if (offset + length > i_size_read(inode))
		iomap_flags |= IOMAP_F_DIRTY;

756 757 758
	error = xfs_ilock_for_iomap(ip, flags, &lockmode);
	if (error)
		return error;
G
Goldwyn Rodrigues 已提交
759

760
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
761
			       &nimaps, 0);
762 763
	if (error)
		goto out_unlock;
764

765
	if (imap_needs_cow(ip, flags, &imap, nimaps)) {
766 767 768 769
		error = -EAGAIN;
		if (flags & IOMAP_NOWAIT)
			goto out_unlock;

770
		/* may drop and re-acquire the ilock */
771
		error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared,
772
				&lockmode, flags & IOMAP_DIRECT);
773 774
		if (error)
			goto out_unlock;
775 776
		if (shared)
			goto out_found_cow;
777 778
		end_fsb = imap.br_startoff + imap.br_blockcount;
		length = XFS_FSB_TO_B(mp, end_fsb) - offset;
779 780
	}

781 782
	if (imap_needs_alloc(inode, flags, &imap, nimaps))
		goto allocate_blocks;
783

784 785 786 787 788 789 790 791 792 793 794 795
	/*
	 * NOWAIT IO needs to span the entire requested IO with a single map so
	 * that we avoid partial IO failures due to the rest of the IO range not
	 * covered by this map triggering an EAGAIN condition when it is
	 * subsequently mapped and aborting the IO.
	 */
	if ((flags & IOMAP_NOWAIT) &&
	    !imap_spans_range(&imap, offset_fsb, end_fsb)) {
		error = -EAGAIN;
		goto out_unlock;
	}

796 797 798
	xfs_iunlock(ip, lockmode);
	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
	return xfs_bmbt_to_iomap(ip, iomap, &imap, iomap_flags);
799

800 801 802
allocate_blocks:
	error = -EAGAIN;
	if (flags & IOMAP_NOWAIT)
803
		goto out_unlock;
804

805 806 807 808 809 810 811 812 813 814
	/*
	 * We cap the maximum length we map to a sane size  to keep the chunks
	 * of work done where somewhat symmetric with the work writeback does.
	 * This is a completely arbitrary number pulled out of thin air as a
	 * best guess for initial testing.
	 *
	 * Note that the values needs to be less than 32-bits wide until the
	 * lower level functions are updated.
	 */
	length = min_t(loff_t, length, 1024 * PAGE_SIZE);
815
	end_fsb = xfs_iomap_end_fsb(mp, offset, length);
816

817 818 819 820 821 822 823 824
	if (offset + length > XFS_ISIZE(ip))
		end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb);
	else if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
		end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
	xfs_iunlock(ip, lockmode);

	error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb,
			&imap);
825 826 827
	if (error)
		return error;

828
	trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap);
829
	return xfs_bmbt_to_iomap(ip, iomap, &imap, iomap_flags | IOMAP_F_NEW);
830

831 832 833 834 835 836 837 838 839 840 841
out_found_cow:
	xfs_iunlock(ip, lockmode);
	length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount);
	trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap);
	if (imap.br_startblock != HOLESTARTBLOCK) {
		error = xfs_bmbt_to_iomap(ip, srcmap, &imap, 0);
		if (error)
			return error;
	}
	return xfs_bmbt_to_iomap(ip, iomap, &cmap, IOMAP_F_SHARED);

842 843 844
out_unlock:
	xfs_iunlock(ip, lockmode);
	return error;
845 846
}

847 848 849 850
const struct iomap_ops xfs_direct_write_iomap_ops = {
	.iomap_begin		= xfs_direct_write_iomap_begin,
};

851
static int
852
xfs_buffered_write_iomap_begin(
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
	struct inode		*inode,
	loff_t			offset,
	loff_t			count,
	unsigned		flags,
	struct iomap		*iomap,
	struct iomap		*srcmap)
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, count);
	struct xfs_bmbt_irec	imap, cmap;
	struct xfs_iext_cursor	icur, ccur;
	xfs_fsblock_t		prealloc_blocks = 0;
	bool			eof = false, cow_eof = false, shared = false;
868
	int			allocfork = XFS_DATA_FORK;
869 870
	int			error = 0;

871 872 873 874 875
	/* we can't use delayed allocations when using extent size hints */
	if (xfs_get_extsz_hint(ip))
		return xfs_direct_write_iomap_begin(inode, offset, count,
				flags, iomap, srcmap);

876 877 878 879
	ASSERT(!XFS_IS_REALTIME_INODE(ip));

	xfs_ilock(ip, XFS_ILOCK_EXCL);

880
	if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
881
	    XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
882 883 884 885 886 887 888 889 890 891 892 893 894
		error = -EFSCORRUPTED;
		goto out_unlock;
	}

	XFS_STATS_INC(mp, xs_blk_mapw);

	if (!(ip->i_df.if_flags & XFS_IFEXTENTS)) {
		error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
		if (error)
			goto out_unlock;
	}

	/*
895
	 * Search the data fork first to look up our source mapping.  We
896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946
	 * always need the data fork map, as we have to return it to the
	 * iomap code so that the higher level write code can read data in to
	 * perform read-modify-write cycles for unaligned writes.
	 */
	eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap);
	if (eof)
		imap.br_startoff = end_fsb; /* fake hole until the end */

	/* We never need to allocate blocks for zeroing a hole. */
	if ((flags & IOMAP_ZERO) && imap.br_startoff > offset_fsb) {
		xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff);
		goto out_unlock;
	}

	/*
	 * Search the COW fork extent list even if we did not find a data fork
	 * extent.  This serves two purposes: first this implements the
	 * speculative preallocation using cowextsize, so that we also unshare
	 * block adjacent to shared blocks instead of just the shared blocks
	 * themselves.  Second the lookup in the extent list is generally faster
	 * than going out to the shared extent tree.
	 */
	if (xfs_is_cow_inode(ip)) {
		if (!ip->i_cowfp) {
			ASSERT(!xfs_is_reflink_inode(ip));
			xfs_ifork_init_cow(ip);
		}
		cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb,
				&ccur, &cmap);
		if (!cow_eof && cmap.br_startoff <= offset_fsb) {
			trace_xfs_reflink_cow_found(ip, &cmap);
			goto found_cow;
		}
	}

	if (imap.br_startoff <= offset_fsb) {
		/*
		 * For reflink files we may need a delalloc reservation when
		 * overwriting shared extents.   This includes zeroing of
		 * existing extents that contain data.
		 */
		if (!xfs_is_cow_inode(ip) ||
		    ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) {
			trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
					&imap);
			goto found_imap;
		}

		xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);

		/* Trim the mapping to the nearest shared extent boundary. */
947
		error = xfs_bmap_trim_cow(ip, &imap, &shared);
948 949 950 951 952 953 954 955 956 957 958 959 960 961
		if (error)
			goto out_unlock;

		/* Not shared?  Just report the (potentially capped) extent. */
		if (!shared) {
			trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
					&imap);
			goto found_imap;
		}

		/*
		 * Fork all the shared blocks from our write offset until the
		 * end of the extent.
		 */
962
		allocfork = XFS_COW_FORK;
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
		end_fsb = imap.br_startoff + imap.br_blockcount;
	} else {
		/*
		 * We cap the maximum length we map here to MAX_WRITEBACK_PAGES
		 * pages to keep the chunks of work done where somewhat
		 * symmetric with the work writeback does.  This is a completely
		 * arbitrary number pulled out of thin air.
		 *
		 * Note that the values needs to be less than 32-bits wide until
		 * the lower level functions are updated.
		 */
		count = min_t(loff_t, count, 1024 * PAGE_SIZE);
		end_fsb = xfs_iomap_end_fsb(mp, offset, count);

		if (xfs_is_always_cow_inode(ip))
978
			allocfork = XFS_COW_FORK;
979 980 981 982 983 984
	}

	error = xfs_qm_dqattach_locked(ip, false);
	if (error)
		goto out_unlock;

985 986 987 988 989 990 991 992 993 994
	if (eof && offset + count > XFS_ISIZE(ip)) {
		/*
		 * Determine the initial size of the preallocation.
		 * We clean up any extra preallocation when the file is closed.
		 */
		if (mp->m_flags & XFS_MOUNT_ALLOCSIZE)
			prealloc_blocks = mp->m_allocsize_blocks;
		else
			prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
						offset, count, &icur);
995 996 997 998 999
		if (prealloc_blocks) {
			xfs_extlen_t	align;
			xfs_off_t	end_offset;
			xfs_fileoff_t	p_end_fsb;

1000
			end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1);
1001 1002 1003
			p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
					prealloc_blocks;

1004
			align = xfs_eof_alignment(ip);
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
			if (align)
				p_end_fsb = roundup_64(p_end_fsb, align);

			p_end_fsb = min(p_end_fsb,
				XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
			ASSERT(p_end_fsb > offset_fsb);
			prealloc_blocks = p_end_fsb - end_fsb;
		}
	}

retry:
1016
	error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1017
			end_fsb - offset_fsb, prealloc_blocks,
1018 1019 1020
			allocfork == XFS_DATA_FORK ? &imap : &cmap,
			allocfork == XFS_DATA_FORK ? &icur : &ccur,
			allocfork == XFS_DATA_FORK ? eof : cow_eof);
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	switch (error) {
	case 0:
		break;
	case -ENOSPC:
	case -EDQUOT:
		/* retry without any preallocation */
		trace_xfs_delalloc_enospc(ip, offset, count);
		if (prealloc_blocks) {
			prealloc_blocks = 0;
			goto retry;
		}
1032
		fallthrough;
1033 1034 1035 1036
	default:
		goto out_unlock;
	}

1037 1038
	if (allocfork == XFS_COW_FORK) {
		trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap);
1039 1040 1041 1042 1043 1044 1045 1046
		goto found_cow;
	}

	/*
	 * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
	 * them out if the write happens to fail.
	 */
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1047
	trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap);
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	return xfs_bmbt_to_iomap(ip, iomap, &imap, IOMAP_F_NEW);

found_imap:
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return xfs_bmbt_to_iomap(ip, iomap, &imap, 0);

found_cow:
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	if (imap.br_startoff <= offset_fsb) {
		error = xfs_bmbt_to_iomap(ip, srcmap, &imap, 0);
		if (error)
			return error;
1060
		return xfs_bmbt_to_iomap(ip, iomap, &cmap, IOMAP_F_SHARED);
1061
	}
1062 1063 1064

	xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb);
	return xfs_bmbt_to_iomap(ip, iomap, &cmap, 0);
1065 1066 1067 1068 1069 1070

out_unlock:
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
	return error;
}

1071
static int
1072 1073
xfs_buffered_write_iomap_end(
	struct inode		*inode,
1074 1075
	loff_t			offset,
	loff_t			length,
1076
	ssize_t			written,
1077
	unsigned		flags,
1078
	struct iomap		*iomap)
1079
{
1080
	struct xfs_inode	*ip = XFS_I(inode);
1081 1082 1083 1084 1085
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		start_fsb;
	xfs_fileoff_t		end_fsb;
	int			error = 0;

1086 1087 1088
	if (iomap->type != IOMAP_DELALLOC)
		return 0;

1089 1090 1091 1092
	/*
	 * Behave as if the write failed if drop writes is enabled. Set the NEW
	 * flag to force delalloc cleanup.
	 */
1093
	if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_DROP_WRITES)) {
1094
		iomap->flags |= IOMAP_F_NEW;
1095
		written = 0;
1096
	}
1097

1098 1099 1100 1101 1102 1103 1104 1105 1106
	/*
	 * start_fsb refers to the first unused block after a short write. If
	 * nothing was written, round offset down to point at the first block in
	 * the range.
	 */
	if (unlikely(!written))
		start_fsb = XFS_B_TO_FSBT(mp, offset);
	else
		start_fsb = XFS_B_TO_FSB(mp, offset + written);
1107 1108 1109
	end_fsb = XFS_B_TO_FSB(mp, offset + length);

	/*
1110 1111
	 * Trim delalloc blocks if they were allocated by this write and we
	 * didn't manage to write the whole range.
1112 1113 1114 1115 1116
	 *
	 * We don't need to care about racing delalloc as we hold i_mutex
	 * across the reserve/allocate/unreserve calls. If there are delalloc
	 * blocks in the range, they are ours.
	 */
1117
	if ((iomap->flags & IOMAP_F_NEW) && start_fsb < end_fsb) {
1118 1119 1120
		truncate_pagecache_range(VFS_I(ip), XFS_FSB_TO_B(mp, start_fsb),
					 XFS_FSB_TO_B(mp, end_fsb) - 1);

1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
		error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
					       end_fsb - start_fsb);
		if (error && !XFS_FORCED_SHUTDOWN(mp)) {
			xfs_alert(mp, "%s: unable to clean up ino %lld",
				__func__, ip->i_ino);
			return error;
		}
	}

	return 0;
}

1133 1134 1135
const struct iomap_ops xfs_buffered_write_iomap_ops = {
	.iomap_begin		= xfs_buffered_write_iomap_begin,
	.iomap_end		= xfs_buffered_write_iomap_end,
1136
};
1137

1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
static int
xfs_read_iomap_begin(
	struct inode		*inode,
	loff_t			offset,
	loff_t			length,
	unsigned		flags,
	struct iomap		*iomap,
	struct iomap		*srcmap)
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_bmbt_irec	imap;
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = xfs_iomap_end_fsb(mp, offset, length);
	int			nimaps = 1, error = 0;
	bool			shared = false;
	unsigned		lockmode;

	ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO)));

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

	error = xfs_ilock_for_iomap(ip, flags, &lockmode);
	if (error)
		return error;
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
			       &nimaps, 0);
	if (!error && (flags & IOMAP_REPORT))
		error = xfs_reflink_trim_around_shared(ip, &imap, &shared);
	xfs_iunlock(ip, lockmode);

	if (error)
		return error;
	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
	return xfs_bmbt_to_iomap(ip, iomap, &imap, shared ? IOMAP_F_SHARED : 0);
}

const struct iomap_ops xfs_read_iomap_ops = {
	.iomap_begin		= xfs_read_iomap_begin,
};

1180 1181 1182 1183 1184 1185
static int
xfs_seek_iomap_begin(
	struct inode		*inode,
	loff_t			offset,
	loff_t			length,
	unsigned		flags,
1186 1187
	struct iomap		*iomap,
	struct iomap		*srcmap)
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = XFS_B_TO_FSB(mp, offset + length);
	xfs_fileoff_t		cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF;
	struct xfs_iext_cursor	icur;
	struct xfs_bmbt_irec	imap, cmap;
	int			error = 0;
	unsigned		lockmode;

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

	lockmode = xfs_ilock_data_map_shared(ip);
	if (!(ip->i_df.if_flags & XFS_IFEXTENTS)) {
		error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
		if (error)
			goto out_unlock;
	}

	if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) {
		/*
		 * If we found a data extent we are done.
		 */
		if (imap.br_startoff <= offset_fsb)
			goto done;
		data_fsb = imap.br_startoff;
	} else {
		/*
		 * Fake a hole until the end of the file.
		 */
1220
		data_fsb = xfs_iomap_end_fsb(mp, offset, length);
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
	}

	/*
	 * If a COW fork extent covers the hole, report it - capped to the next
	 * data fork extent:
	 */
	if (xfs_inode_has_cow_data(ip) &&
	    xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
		cow_fsb = cmap.br_startoff;
	if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
		if (data_fsb < cow_fsb + cmap.br_blockcount)
			end_fsb = min(end_fsb, data_fsb);
		xfs_trim_extent(&cmap, offset_fsb, end_fsb);
1234
		error = xfs_bmbt_to_iomap(ip, iomap, &cmap, IOMAP_F_SHARED);
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
		/*
		 * This is a COW extent, so we must probe the page cache
		 * because there could be dirty page cache being backed
		 * by this extent.
		 */
		iomap->type = IOMAP_UNWRITTEN;
		goto out_unlock;
	}

	/*
	 * Else report a hole, capped to the next found data or COW extent.
	 */
	if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb)
		imap.br_blockcount = cow_fsb - offset_fsb;
	else
		imap.br_blockcount = data_fsb - offset_fsb;
	imap.br_startoff = offset_fsb;
	imap.br_startblock = HOLESTARTBLOCK;
	imap.br_state = XFS_EXT_NORM;
done:
	xfs_trim_extent(&imap, offset_fsb, end_fsb);
1256
	error = xfs_bmbt_to_iomap(ip, iomap, &imap, 0);
1257 1258 1259 1260 1261 1262 1263 1264 1265
out_unlock:
	xfs_iunlock(ip, lockmode);
	return error;
}

const struct iomap_ops xfs_seek_iomap_ops = {
	.iomap_begin		= xfs_seek_iomap_begin,
};

1266 1267 1268 1269 1270 1271
static int
xfs_xattr_iomap_begin(
	struct inode		*inode,
	loff_t			offset,
	loff_t			length,
	unsigned		flags,
1272 1273
	struct iomap		*iomap,
	struct iomap		*srcmap)
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
{
	struct xfs_inode	*ip = XFS_I(inode);
	struct xfs_mount	*mp = ip->i_mount;
	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
	xfs_fileoff_t		end_fsb = XFS_B_TO_FSB(mp, offset + length);
	struct xfs_bmbt_irec	imap;
	int			nimaps = 1, error = 0;
	unsigned		lockmode;

	if (XFS_FORCED_SHUTDOWN(mp))
		return -EIO;

1286
	lockmode = xfs_ilock_attr_map_shared(ip);
1287 1288

	/* if there are no attribute fork or extents, return ENOENT */
1289
	if (!XFS_IFORK_Q(ip) || !ip->i_afp->if_nextents) {
1290 1291 1292 1293
		error = -ENOENT;
		goto out_unlock;
	}

1294
	ASSERT(ip->i_afp->if_format != XFS_DINODE_FMT_LOCAL);
1295
	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
1296
			       &nimaps, XFS_BMAPI_ATTRFORK);
1297 1298 1299
out_unlock:
	xfs_iunlock(ip, lockmode);

1300 1301 1302
	if (error)
		return error;
	ASSERT(nimaps);
1303
	return xfs_bmbt_to_iomap(ip, iomap, &imap, 0);
1304 1305
}

1306
const struct iomap_ops xfs_xattr_iomap_ops = {
1307 1308
	.iomap_begin		= xfs_xattr_iomap_begin,
};