xfs_icache.c 42.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7
/*
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
 */
#include "xfs.h"
#include "xfs_fs.h"
8
#include "xfs_shared.h"
9
#include "xfs_format.h"
10 11
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
12 13 14
#include "xfs_sb.h"
#include "xfs_mount.h"
#include "xfs_inode.h"
15 16
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
17
#include "xfs_inode_item.h"
C
Christoph Hellwig 已提交
18
#include "xfs_quota.h"
19
#include "xfs_trace.h"
20
#include "xfs_icache.h"
21
#include "xfs_bmap_util.h"
22 23
#include "xfs_dquot_item.h"
#include "xfs_dquot.h"
24
#include "xfs_reflink.h"
25
#include "xfs_ialloc.h"
26

27
#include <linux/iversion.h>
28

D
Dave Chinner 已提交
29 30 31
/*
 * Allocate and initialise an xfs_inode.
 */
32
struct xfs_inode *
D
Dave Chinner 已提交
33 34 35 36 37 38 39 40 41 42 43
xfs_inode_alloc(
	struct xfs_mount	*mp,
	xfs_ino_t		ino)
{
	struct xfs_inode	*ip;

	/*
	 * if this didn't occur in transactions, we could use
	 * KM_MAYFAIL and return NULL here on ENOMEM. Set the
	 * code up to do this anyway.
	 */
44
	ip = kmem_zone_alloc(xfs_inode_zone, 0);
D
Dave Chinner 已提交
45 46 47
	if (!ip)
		return NULL;
	if (inode_init_always(mp->m_super, VFS_I(ip))) {
48
		kmem_cache_free(xfs_inode_zone, ip);
D
Dave Chinner 已提交
49 50 51
		return NULL;
	}

52 53 54
	/* VFS doesn't initialise i_mode! */
	VFS_I(ip)->i_mode = 0;

55
	XFS_STATS_INC(mp, vn_active);
D
Dave Chinner 已提交
56 57 58 59 60 61 62 63 64
	ASSERT(atomic_read(&ip->i_pincount) == 0);
	ASSERT(!xfs_isiflocked(ip));
	ASSERT(ip->i_ino == 0);

	/* initialise the xfs inode */
	ip->i_ino = ino;
	ip->i_mount = mp;
	memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
	ip->i_afp = NULL;
65
	ip->i_cowfp = NULL;
66
	memset(&ip->i_df, 0, sizeof(ip->i_df));
D
Dave Chinner 已提交
67 68
	ip->i_flags = 0;
	ip->i_delayed_blks = 0;
69
	memset(&ip->i_d, 0, sizeof(ip->i_d));
70 71
	ip->i_sick = 0;
	ip->i_checked = 0;
72 73 74
	INIT_WORK(&ip->i_ioend_work, xfs_end_io);
	INIT_LIST_HEAD(&ip->i_ioend_list);
	spin_lock_init(&ip->i_ioend_lock);
D
Dave Chinner 已提交
75 76 77 78 79 80 81 82 83 84 85

	return ip;
}

STATIC void
xfs_inode_free_callback(
	struct rcu_head		*head)
{
	struct inode		*inode = container_of(head, struct inode, i_rcu);
	struct xfs_inode	*ip = XFS_I(inode);

86
	switch (VFS_I(ip)->i_mode & S_IFMT) {
D
Dave Chinner 已提交
87 88 89
	case S_IFREG:
	case S_IFDIR:
	case S_IFLNK:
90
		xfs_idestroy_fork(&ip->i_df);
D
Dave Chinner 已提交
91 92 93
		break;
	}

94 95 96 97 98 99 100 101
	if (ip->i_afp) {
		xfs_idestroy_fork(ip->i_afp);
		kmem_cache_free(xfs_ifork_zone, ip->i_afp);
	}
	if (ip->i_cowfp) {
		xfs_idestroy_fork(ip->i_cowfp);
		kmem_cache_free(xfs_ifork_zone, ip->i_cowfp);
	}
D
Dave Chinner 已提交
102
	if (ip->i_itemp) {
103 104
		ASSERT(!test_bit(XFS_LI_IN_AIL,
				 &ip->i_itemp->ili_item.li_flags));
D
Dave Chinner 已提交
105 106 107 108
		xfs_inode_item_destroy(ip);
		ip->i_itemp = NULL;
	}

109
	kmem_cache_free(xfs_inode_zone, ip);
110 111
}

112 113 114 115 116 117 118 119 120 121 122
static void
__xfs_inode_free(
	struct xfs_inode	*ip)
{
	/* asserts to verify all state is correct here */
	ASSERT(atomic_read(&ip->i_pincount) == 0);
	XFS_STATS_DEC(ip->i_mount, vn_active);

	call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
}

123 124 125 126
void
xfs_inode_free(
	struct xfs_inode	*ip)
{
127 128
	ASSERT(!xfs_isiflocked(ip));

D
Dave Chinner 已提交
129 130 131 132 133 134 135 136 137 138 139
	/*
	 * Because we use RCU freeing we need to ensure the inode always
	 * appears to be reclaimed with an invalid inode number when in the
	 * free state. The ip->i_flags_lock provides the barrier against lookup
	 * races.
	 */
	spin_lock(&ip->i_flags_lock);
	ip->i_flags = XFS_IRECLAIM;
	ip->i_ino = 0;
	spin_unlock(&ip->i_flags_lock);

140
	__xfs_inode_free(ip);
D
Dave Chinner 已提交
141 142
}

143
/*
144 145
 * Queue background inode reclaim work if there are reclaimable inodes and there
 * isn't reclaim work already scheduled or in progress.
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
 */
static void
xfs_reclaim_work_queue(
	struct xfs_mount        *mp)
{

	rcu_read_lock();
	if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
		queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
			msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
	}
	rcu_read_unlock();
}

static void
xfs_perag_set_reclaim_tag(
	struct xfs_perag	*pag)
{
	struct xfs_mount	*mp = pag->pag_mount;

166
	lockdep_assert_held(&pag->pag_ici_lock);
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	if (pag->pag_ici_reclaimable++)
		return;

	/* propagate the reclaim tag up into the perag radix tree */
	spin_lock(&mp->m_perag_lock);
	radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno,
			   XFS_ICI_RECLAIM_TAG);
	spin_unlock(&mp->m_perag_lock);

	/* schedule periodic background inode reclaim */
	xfs_reclaim_work_queue(mp);

	trace_xfs_perag_set_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
}

static void
xfs_perag_clear_reclaim_tag(
	struct xfs_perag	*pag)
{
	struct xfs_mount	*mp = pag->pag_mount;

188
	lockdep_assert_held(&pag->pag_ici_lock);
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
	if (--pag->pag_ici_reclaimable)
		return;

	/* clear the reclaim tag from the perag radix tree */
	spin_lock(&mp->m_perag_lock);
	radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno,
			     XFS_ICI_RECLAIM_TAG);
	spin_unlock(&mp->m_perag_lock);
	trace_xfs_perag_clear_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
}


/*
 * We set the inode flag atomically with the radix tree tag.
 * Once we get tag lookups on the radix tree, this inode flag
 * can go away.
 */
void
xfs_inode_set_reclaim_tag(
	struct xfs_inode	*ip)
{
	struct xfs_mount	*mp = ip->i_mount;
	struct xfs_perag	*pag;

	pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
	spin_lock(&pag->pag_ici_lock);
	spin_lock(&ip->i_flags_lock);

	radix_tree_tag_set(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino),
			   XFS_ICI_RECLAIM_TAG);
	xfs_perag_set_reclaim_tag(pag);
	__xfs_iflags_set(ip, XFS_IRECLAIMABLE);

	spin_unlock(&ip->i_flags_lock);
	spin_unlock(&pag->pag_ici_lock);
	xfs_perag_put(pag);
}

STATIC void
xfs_inode_clear_reclaim_tag(
	struct xfs_perag	*pag,
	xfs_ino_t		ino)
{
	radix_tree_tag_clear(&pag->pag_ici_root,
			     XFS_INO_TO_AGINO(pag->pag_mount, ino),
			     XFS_ICI_RECLAIM_TAG);
	xfs_perag_clear_reclaim_tag(pag);
}

238 239 240 241 242 243 244 245
static void
xfs_inew_wait(
	struct xfs_inode	*ip)
{
	wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
	DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);

	do {
246
		prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
247 248 249 250
		if (!xfs_iflags_test(ip, XFS_INEW))
			break;
		schedule();
	} while (true);
251
	finish_wait(wq, &wait.wq_entry);
252 253
}

254 255 256 257
/*
 * When we recycle a reclaimable inode, we need to re-initialise the VFS inode
 * part of the structure. This is made more complex by the fact we store
 * information about the on-disk values in the VFS inode and so we can't just
258
 * overwrite the values unconditionally. Hence we save the parameters we
259
 * need to retain across reinitialisation, and rewrite them into the VFS inode
260
 * after reinitialisation even if it fails.
261 262 263 264 265 266 267
 */
static int
xfs_reinit_inode(
	struct xfs_mount	*mp,
	struct inode		*inode)
{
	int		error;
268
	uint32_t	nlink = inode->i_nlink;
269
	uint32_t	generation = inode->i_generation;
270
	uint64_t	version = inode_peek_iversion(inode);
271
	umode_t		mode = inode->i_mode;
272
	dev_t		dev = inode->i_rdev;
273 274
	kuid_t		uid = inode->i_uid;
	kgid_t		gid = inode->i_gid;
275 276 277

	error = inode_init_always(mp->m_super, inode);

278
	set_nlink(inode, nlink);
279
	inode->i_generation = generation;
280
	inode_set_iversion_queried(inode, version);
281
	inode->i_mode = mode;
282
	inode->i_rdev = dev;
283 284
	inode->i_uid = uid;
	inode->i_gid = gid;
285 286 287
	return error;
}

288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
/*
 * If we are allocating a new inode, then check what was returned is
 * actually a free, empty inode. If we are not allocating an inode,
 * then check we didn't find a free inode.
 *
 * Returns:
 *	0		if the inode free state matches the lookup context
 *	-ENOENT		if the inode is free and we are not allocating
 *	-EFSCORRUPTED	if there is any state mismatch at all
 */
static int
xfs_iget_check_free_state(
	struct xfs_inode	*ip,
	int			flags)
{
	if (flags & XFS_IGET_CREATE) {
		/* should be a free inode */
		if (VFS_I(ip)->i_mode != 0) {
			xfs_warn(ip->i_mount,
"Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)",
				ip->i_ino, VFS_I(ip)->i_mode);
			return -EFSCORRUPTED;
		}

		if (ip->i_d.di_nblocks != 0) {
			xfs_warn(ip->i_mount,
"Corruption detected! Free inode 0x%llx has blocks allocated!",
				ip->i_ino);
			return -EFSCORRUPTED;
		}
		return 0;
	}

	/* should be an allocated inode */
	if (VFS_I(ip)->i_mode == 0)
		return -ENOENT;

	return 0;
}

D
Dave Chinner 已提交
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
/*
 * Check the validity of the inode we just found it the cache
 */
static int
xfs_iget_cache_hit(
	struct xfs_perag	*pag,
	struct xfs_inode	*ip,
	xfs_ino_t		ino,
	int			flags,
	int			lock_flags) __releases(RCU)
{
	struct inode		*inode = VFS_I(ip);
	struct xfs_mount	*mp = ip->i_mount;
	int			error;

	/*
	 * check for re-use of an inode within an RCU grace period due to the
	 * radix tree nodes not being updated yet. We monitor for this by
	 * setting the inode number to zero before freeing the inode structure.
	 * If the inode has been reallocated and set up, then the inode number
	 * will not match, so check for that, too.
	 */
	spin_lock(&ip->i_flags_lock);
	if (ip->i_ino != ino) {
		trace_xfs_iget_skip(ip);
353
		XFS_STATS_INC(mp, xs_ig_frecycle);
354
		error = -EAGAIN;
D
Dave Chinner 已提交
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
		goto out_error;
	}


	/*
	 * If we are racing with another cache hit that is currently
	 * instantiating this inode or currently recycling it out of
	 * reclaimabe state, wait for the initialisation to complete
	 * before continuing.
	 *
	 * XXX(hch): eventually we should do something equivalent to
	 *	     wait_on_inode to wait for these flags to be cleared
	 *	     instead of polling for it.
	 */
	if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) {
		trace_xfs_iget_skip(ip);
371
		XFS_STATS_INC(mp, xs_ig_frecycle);
372
		error = -EAGAIN;
D
Dave Chinner 已提交
373 374 375 376
		goto out_error;
	}

	/*
377 378
	 * Check the inode free state is valid. This also detects lookup
	 * racing with unlinks.
D
Dave Chinner 已提交
379
	 */
380 381
	error = xfs_iget_check_free_state(ip, flags);
	if (error)
D
Dave Chinner 已提交
382 383 384 385 386 387 388 389 390
		goto out_error;

	/*
	 * If IRECLAIMABLE is set, we've torn down the VFS inode already.
	 * Need to carefully get it back into useable state.
	 */
	if (ip->i_flags & XFS_IRECLAIMABLE) {
		trace_xfs_iget_reclaim(ip);

391 392 393 394 395
		if (flags & XFS_IGET_INCORE) {
			error = -EAGAIN;
			goto out_error;
		}

D
Dave Chinner 已提交
396 397 398 399 400 401 402 403 404 405 406
		/*
		 * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode
		 * from stomping over us while we recycle the inode.  We can't
		 * clear the radix tree reclaimable tag yet as it requires
		 * pag_ici_lock to be held exclusive.
		 */
		ip->i_flags |= XFS_IRECLAIM;

		spin_unlock(&ip->i_flags_lock);
		rcu_read_unlock();

407
		ASSERT(!rwsem_is_locked(&inode->i_rwsem));
408
		error = xfs_reinit_inode(mp, inode);
D
Dave Chinner 已提交
409
		if (error) {
410
			bool wake;
D
Dave Chinner 已提交
411 412 413 414 415 416
			/*
			 * Re-initializing the inode failed, and we are in deep
			 * trouble.  Try to re-add it to the reclaim list.
			 */
			rcu_read_lock();
			spin_lock(&ip->i_flags_lock);
417
			wake = !!__xfs_iflags_test(ip, XFS_INEW);
D
Dave Chinner 已提交
418
			ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
419 420
			if (wake)
				wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
D
Dave Chinner 已提交
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
			ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
			trace_xfs_iget_reclaim_fail(ip);
			goto out_error;
		}

		spin_lock(&pag->pag_ici_lock);
		spin_lock(&ip->i_flags_lock);

		/*
		 * Clear the per-lifetime state in the inode as we are now
		 * effectively a new inode and need to return to the initial
		 * state before reuse occurs.
		 */
		ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
		ip->i_flags |= XFS_INEW;
436
		xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
D
Dave Chinner 已提交
437
		inode->i_state = I_NEW;
438 439
		ip->i_sick = 0;
		ip->i_checked = 0;
D
Dave Chinner 已提交
440 441 442 443 444 445 446

		spin_unlock(&ip->i_flags_lock);
		spin_unlock(&pag->pag_ici_lock);
	} else {
		/* If the VFS inode is being torn down, pause and try again. */
		if (!igrab(inode)) {
			trace_xfs_iget_skip(ip);
447
			error = -EAGAIN;
D
Dave Chinner 已提交
448 449 450 451 452 453 454 455 456 457 458 459
			goto out_error;
		}

		/* We've got a live one. */
		spin_unlock(&ip->i_flags_lock);
		rcu_read_unlock();
		trace_xfs_iget_hit(ip);
	}

	if (lock_flags != 0)
		xfs_ilock(ip, lock_flags);

460
	if (!(flags & XFS_IGET_INCORE))
461
		xfs_iflags_clear(ip, XFS_ISTALE);
462
	XFS_STATS_INC(mp, xs_ig_found);
D
Dave Chinner 已提交
463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489

	return 0;

out_error:
	spin_unlock(&ip->i_flags_lock);
	rcu_read_unlock();
	return error;
}


static int
xfs_iget_cache_miss(
	struct xfs_mount	*mp,
	struct xfs_perag	*pag,
	xfs_trans_t		*tp,
	xfs_ino_t		ino,
	struct xfs_inode	**ipp,
	int			flags,
	int			lock_flags)
{
	struct xfs_inode	*ip;
	int			error;
	xfs_agino_t		agino = XFS_INO_TO_AGINO(mp, ino);
	int			iflags;

	ip = xfs_inode_alloc(mp, ino);
	if (!ip)
490
		return -ENOMEM;
D
Dave Chinner 已提交
491

492
	error = xfs_imap(mp, tp, ip->i_ino, &ip->i_imap, flags);
D
Dave Chinner 已提交
493 494 495
	if (error)
		goto out_destroy;

496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
	/*
	 * For version 5 superblocks, if we are initialising a new inode and we
	 * are not utilising the XFS_MOUNT_IKEEP inode cluster mode, we can
	 * simply build the new inode core with a random generation number.
	 *
	 * For version 4 (and older) superblocks, log recovery is dependent on
	 * the di_flushiter field being initialised from the current on-disk
	 * value and hence we must also read the inode off disk even when
	 * initializing new inodes.
	 */
	if (xfs_sb_version_has_v3inode(&mp->m_sb) &&
	    (flags & XFS_IGET_CREATE) && !(mp->m_flags & XFS_MOUNT_IKEEP)) {
		VFS_I(ip)->i_generation = prandom_u32();
	} else {
		struct xfs_dinode	*dip;
		struct xfs_buf		*bp;

		error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0);
		if (error)
			goto out_destroy;

		error = xfs_inode_from_disk(ip, dip);
		if (!error)
			xfs_buf_set_ref(bp, XFS_INO_REF);
		xfs_trans_brelse(tp, bp);

		if (error)
			goto out_destroy;
	}

D
Dave Chinner 已提交
526 527
	trace_xfs_iget_miss(ip);

528
	/*
529 530
	 * Check the inode free state is valid. This also detects lookup
	 * racing with unlinks.
531
	 */
532 533
	error = xfs_iget_check_free_state(ip, flags);
	if (error)
D
Dave Chinner 已提交
534 535 536 537 538 539 540 541 542
		goto out_destroy;

	/*
	 * Preload the radix tree so we can insert safely under the
	 * write spinlock. Note that we cannot sleep inside the preload
	 * region. Since we can be called from transaction context, don't
	 * recurse into the file system.
	 */
	if (radix_tree_preload(GFP_NOFS)) {
543
		error = -EAGAIN;
D
Dave Chinner 已提交
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
		goto out_destroy;
	}

	/*
	 * Because the inode hasn't been added to the radix-tree yet it can't
	 * be found by another thread, so we can do the non-sleeping lock here.
	 */
	if (lock_flags) {
		if (!xfs_ilock_nowait(ip, lock_flags))
			BUG();
	}

	/*
	 * These values must be set before inserting the inode into the radix
	 * tree as the moment it is inserted a concurrent lookup (allowed by the
	 * RCU locking mechanism) can find it and that lookup must see that this
	 * is an inode currently under construction (i.e. that XFS_INEW is set).
	 * The ip->i_flags_lock that protects the XFS_INEW flag forms the
	 * memory barrier that ensures this detection works correctly at lookup
	 * time.
	 */
	iflags = XFS_INEW;
	if (flags & XFS_IGET_DONTCACHE)
567
		d_mark_dontcache(VFS_I(ip));
568 569
	ip->i_udquot = NULL;
	ip->i_gdquot = NULL;
570
	ip->i_pdquot = NULL;
D
Dave Chinner 已提交
571 572 573 574 575 576 577
	xfs_iflags_set(ip, iflags);

	/* insert the new inode */
	spin_lock(&pag->pag_ici_lock);
	error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
	if (unlikely(error)) {
		WARN_ON(error != -EEXIST);
578
		XFS_STATS_INC(mp, xs_ig_dup);
579
		error = -EAGAIN;
D
Dave Chinner 已提交
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
		goto out_preload_end;
	}
	spin_unlock(&pag->pag_ici_lock);
	radix_tree_preload_end();

	*ipp = ip;
	return 0;

out_preload_end:
	spin_unlock(&pag->pag_ici_lock);
	radix_tree_preload_end();
	if (lock_flags)
		xfs_iunlock(ip, lock_flags);
out_destroy:
	__destroy_inode(VFS_I(ip));
	xfs_inode_free(ip);
	return error;
}

/*
600 601 602
 * Look up an inode by number in the given file system.  The inode is looked up
 * in the cache held in each AG.  If the inode is found in the cache, initialise
 * the vfs inode if necessary.
D
Dave Chinner 已提交
603
 *
604 605
 * If it is not in core, read it in from the file system's device, add it to the
 * cache and initialise the vfs inode.
D
Dave Chinner 已提交
606 607
 *
 * The inode is locked according to the value of the lock_flags parameter.
608 609
 * Inode lookup is only done during metadata operations and not as part of the
 * data IO path. Hence we only allow locking of the XFS_ILOCK during lookup.
D
Dave Chinner 已提交
610 611 612
 */
int
xfs_iget(
613 614 615 616 617 618
	struct xfs_mount	*mp,
	struct xfs_trans	*tp,
	xfs_ino_t		ino,
	uint			flags,
	uint			lock_flags,
	struct xfs_inode	**ipp)
D
Dave Chinner 已提交
619
{
620 621 622 623
	struct xfs_inode	*ip;
	struct xfs_perag	*pag;
	xfs_agino_t		agino;
	int			error;
D
Dave Chinner 已提交
624 625 626 627 628

	ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);

	/* reject inode numbers outside existing AGs */
	if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
629
		return -EINVAL;
D
Dave Chinner 已提交
630

631
	XFS_STATS_INC(mp, xs_ig_attempts);
632

D
Dave Chinner 已提交
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
	/* get the perag structure and ensure that it's inode capable */
	pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
	agino = XFS_INO_TO_AGINO(mp, ino);

again:
	error = 0;
	rcu_read_lock();
	ip = radix_tree_lookup(&pag->pag_ici_root, agino);

	if (ip) {
		error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
		if (error)
			goto out_error_or_again;
	} else {
		rcu_read_unlock();
648
		if (flags & XFS_IGET_INCORE) {
649
			error = -ENODATA;
650 651
			goto out_error_or_again;
		}
652
		XFS_STATS_INC(mp, xs_ig_missed);
D
Dave Chinner 已提交
653 654 655 656 657 658 659 660 661 662 663

		error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
							flags, lock_flags);
		if (error)
			goto out_error_or_again;
	}
	xfs_perag_put(pag);

	*ipp = ip;

	/*
664
	 * If we have a real type for an on-disk inode, we can setup the inode
D
Dave Chinner 已提交
665 666
	 * now.	 If it's a new inode being created, xfs_ialloc will handle it.
	 */
667
	if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
668
		xfs_setup_existing_inode(ip);
D
Dave Chinner 已提交
669 670 671
	return 0;

out_error_or_again:
672
	if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
D
Dave Chinner 已提交
673 674 675 676 677 678 679
		delay(1);
		goto again;
	}
	xfs_perag_put(pag);
	return error;
}

680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
/*
 * "Is this a cached inode that's also allocated?"
 *
 * Look up an inode by number in the given file system.  If the inode is
 * in cache and isn't in purgatory, return 1 if the inode is allocated
 * and 0 if it is not.  For all other cases (not in cache, being torn
 * down, etc.), return a negative error code.
 *
 * The caller has to prevent inode allocation and freeing activity,
 * presumably by locking the AGI buffer.   This is to ensure that an
 * inode cannot transition from allocated to freed until the caller is
 * ready to allow that.  If the inode is in an intermediate state (new,
 * reclaimable, or being reclaimed), -EAGAIN will be returned; if the
 * inode is not in the cache, -ENOENT will be returned.  The caller must
 * deal with these scenarios appropriately.
 *
 * This is a specialized use case for the online scrubber; if you're
 * reading this, you probably want xfs_iget.
 */
int
xfs_icache_inode_is_allocated(
	struct xfs_mount	*mp,
	struct xfs_trans	*tp,
	xfs_ino_t		ino,
	bool			*inuse)
{
	struct xfs_inode	*ip;
	int			error;

	error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
	if (error)
		return error;

	*inuse = !!(VFS_I(ip)->i_mode);
714
	xfs_irele(ip);
715 716 717
	return 0;
}

718 719 720 721 722 723 724 725
/*
 * The inode lookup is done in batches to keep the amount of lock traffic and
 * radix tree lookups to a minimum. The batch size is a trade off between
 * lookup reduction and stack usage. This is in the reclaim path, so we can't
 * be too greedy.
 */
#define XFS_LOOKUP_BATCH	32

726 727 728 729 730 731
/*
 * Decide if the given @ip is eligible to be a part of the inode walk, and
 * grab it if so.  Returns true if it's ready to go or false if we should just
 * ignore it.
 */
STATIC bool
732
xfs_inode_walk_ag_grab(
733 734
	struct xfs_inode	*ip,
	int			flags)
735 736
{
	struct inode		*inode = VFS_I(ip);
737
	bool			newinos = !!(flags & XFS_INODE_WALK_INEW_WAIT);
738

739 740
	ASSERT(rcu_read_lock_held());

741
	/* Check for stale RCU freed inode */
742 743 744 745 746
	spin_lock(&ip->i_flags_lock);
	if (!ip->i_ino)
		goto out_unlock_noent;

	/* avoid new or reclaimable inodes. Leave for reclaim code to flush */
747 748
	if ((!newinos && __xfs_iflags_test(ip, XFS_INEW)) ||
	    __xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM))
749 750 751
		goto out_unlock_noent;
	spin_unlock(&ip->i_flags_lock);

752 753
	/* nothing to sync during shutdown */
	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
754
		return false;
755 756 757

	/* If we can't grab the inode, it must on it's way to reclaim. */
	if (!igrab(inode))
758
		return false;
759 760

	/* inode is valid */
761
	return true;
762 763 764

out_unlock_noent:
	spin_unlock(&ip->i_flags_lock);
765
	return false;
766 767
}

768 769 770 771
/*
 * For a given per-AG structure @pag, grab, @execute, and rele all incore
 * inodes with the given radix tree @tag.
 */
772
STATIC int
773
xfs_inode_walk_ag(
D
Dave Chinner 已提交
774
	struct xfs_perag	*pag,
775
	int			iter_flags,
776
	int			(*execute)(struct xfs_inode *ip, void *args),
777
	void			*args,
778
	int			tag)
779
{
780
	struct xfs_mount	*mp = pag->pag_mount;
781 782 783
	uint32_t		first_index;
	int			last_error = 0;
	int			skipped;
784
	bool			done;
785
	int			nr_found;
786 787

restart:
788
	done = false;
789 790
	skipped = 0;
	first_index = 0;
791
	nr_found = 0;
792
	do {
793
		struct xfs_inode *batch[XFS_LOOKUP_BATCH];
794
		int		error = 0;
795
		int		i;
796

797
		rcu_read_lock();
798

799
		if (tag == XFS_ICI_NO_TAG)
800
			nr_found = radix_tree_gang_lookup(&pag->pag_ici_root,
801 802
					(void **)batch, first_index,
					XFS_LOOKUP_BATCH);
803 804 805 806 807 808
		else
			nr_found = radix_tree_gang_lookup_tag(
					&pag->pag_ici_root,
					(void **) batch, first_index,
					XFS_LOOKUP_BATCH, tag);

809
		if (!nr_found) {
810
			rcu_read_unlock();
811
			break;
812
		}
813

814
		/*
815 816
		 * Grab the inodes before we drop the lock. if we found
		 * nothing, nr == 0 and the loop will be skipped.
817
		 */
818 819 820
		for (i = 0; i < nr_found; i++) {
			struct xfs_inode *ip = batch[i];

821
			if (done || !xfs_inode_walk_ag_grab(ip, iter_flags))
822 823 824
				batch[i] = NULL;

			/*
825 826 827 828 829 830 831 832 833 834
			 * Update the index for the next lookup. Catch
			 * overflows into the next AG range which can occur if
			 * we have inodes in the last block of the AG and we
			 * are currently pointing to the last inode.
			 *
			 * Because we may see inodes that are from the wrong AG
			 * due to RCU freeing and reallocation, only update the
			 * index if it lies in this AG. It was a race that lead
			 * us to see this inode, so another lookup from the
			 * same index will not find it again.
835
			 */
836 837
			if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
				continue;
838 839
			first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
			if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
840
				done = true;
841
		}
842 843

		/* unlock now we've grabbed the inodes. */
844
		rcu_read_unlock();
845

846 847 848
		for (i = 0; i < nr_found; i++) {
			if (!batch[i])
				continue;
849
			if ((iter_flags & XFS_INODE_WALK_INEW_WAIT) &&
850 851
			    xfs_iflags_test(batch[i], XFS_INEW))
				xfs_inew_wait(batch[i]);
852
			error = execute(batch[i], args);
853
			xfs_irele(batch[i]);
854
			if (error == -EAGAIN) {
855 856 857
				skipped++;
				continue;
			}
858
			if (error && last_error != -EFSCORRUPTED)
859
				last_error = error;
860
		}
861 862

		/* bail out if the filesystem is corrupted.  */
863
		if (error == -EFSCORRUPTED)
864 865
			break;

866 867
		cond_resched();

868
	} while (nr_found && !done);
869 870 871 872 873 874 875 876

	if (skipped) {
		delay(1);
		goto restart;
	}
	return last_error;
}

877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
/* Fetch the next (possibly tagged) per-AG structure. */
static inline struct xfs_perag *
xfs_inode_walk_get_perag(
	struct xfs_mount	*mp,
	xfs_agnumber_t		agno,
	int			tag)
{
	if (tag == XFS_ICI_NO_TAG)
		return xfs_perag_get(mp, agno);
	return xfs_perag_get_tag(mp, agno, tag);
}

/*
 * Call the @execute function on all incore inodes matching the radix tree
 * @tag.
 */
int
894
xfs_inode_walk(
895 896 897 898 899 900 901 902 903 904 905 906 907 908
	struct xfs_mount	*mp,
	int			iter_flags,
	int			(*execute)(struct xfs_inode *ip, void *args),
	void			*args,
	int			tag)
{
	struct xfs_perag	*pag;
	int			error = 0;
	int			last_error = 0;
	xfs_agnumber_t		ag;

	ag = 0;
	while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) {
		ag = pag->pag_agno + 1;
909
		error = xfs_inode_walk_ag(pag, iter_flags, execute, args, tag);
910 911 912 913 914 915 916 917 918 919
		xfs_perag_put(pag);
		if (error) {
			last_error = error;
			if (error == -EFSCORRUPTED)
				break;
		}
	}
	return last_error;
}

920 921
/*
 * Background scanning to trim post-EOF preallocated space. This is queued
922
 * based on the 'speculative_prealloc_lifetime' tunable (5m by default).
923
 */
924
void
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
xfs_queue_eofblocks(
	struct xfs_mount *mp)
{
	rcu_read_lock();
	if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG))
		queue_delayed_work(mp->m_eofblocks_workqueue,
				   &mp->m_eofblocks_work,
				   msecs_to_jiffies(xfs_eofb_secs * 1000));
	rcu_read_unlock();
}

void
xfs_eofblocks_worker(
	struct work_struct *work)
{
	struct xfs_mount *mp = container_of(to_delayed_work(work),
				struct xfs_mount, m_eofblocks_work);
942 943 944

	if (!sb_start_write_trylock(mp->m_super))
		return;
945
	xfs_icache_free_eofblocks(mp, NULL);
946 947
	sb_end_write(mp->m_super);

948 949 950
	xfs_queue_eofblocks(mp);
}

951 952 953 954 955
/*
 * Background scanning to trim preallocated CoW space. This is queued
 * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default).
 * (We'll just piggyback on the post-EOF prealloc space workqueue.)
 */
956
void
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
xfs_queue_cowblocks(
	struct xfs_mount *mp)
{
	rcu_read_lock();
	if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG))
		queue_delayed_work(mp->m_eofblocks_workqueue,
				   &mp->m_cowblocks_work,
				   msecs_to_jiffies(xfs_cowb_secs * 1000));
	rcu_read_unlock();
}

void
xfs_cowblocks_worker(
	struct work_struct *work)
{
	struct xfs_mount *mp = container_of(to_delayed_work(work),
				struct xfs_mount, m_cowblocks_work);
974 975 976

	if (!sb_start_write_trylock(mp->m_super))
		return;
977
	xfs_icache_free_cowblocks(mp, NULL);
978 979
	sb_end_write(mp->m_super);

980 981 982
	xfs_queue_cowblocks(mp);
}

983 984
/*
 * Grab the inode for reclaim exclusively.
985 986 987 988 989 990 991 992 993 994 995 996 997 998
 *
 * We have found this inode via a lookup under RCU, so the inode may have
 * already been freed, or it may be in the process of being recycled by
 * xfs_iget(). In both cases, the inode will have XFS_IRECLAIM set. If the inode
 * has been fully recycled by the time we get the i_flags_lock, XFS_IRECLAIMABLE
 * will not be set. Hence we need to check for both these flag conditions to
 * avoid inodes that are no longer reclaim candidates.
 *
 * Note: checking for other state flags here, under the i_flags_lock or not, is
 * racy and should be avoided. Those races should be resolved only after we have
 * ensured that we are able to reclaim this inode and the world can see that we
 * are going to reclaim it.
 *
 * Return true if we grabbed it, false otherwise.
999
 */
1000
static bool
1001
xfs_reclaim_inode_grab(
1002
	struct xfs_inode	*ip)
1003
{
1004 1005
	ASSERT(rcu_read_lock_held());

1006
	spin_lock(&ip->i_flags_lock);
1007 1008 1009
	if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
	    __xfs_iflags_test(ip, XFS_IRECLAIM)) {
		/* not a reclaim candidate. */
1010
		spin_unlock(&ip->i_flags_lock);
1011
		return false;
1012 1013 1014
	}
	__xfs_iflags_set(ip, XFS_IRECLAIM);
	spin_unlock(&ip->i_flags_lock);
1015
	return true;
1016 1017
}

1018
/*
1019 1020 1021 1022 1023
 * Inode reclaim is non-blocking, so the default action if progress cannot be
 * made is to "requeue" the inode for reclaim by unlocking it and clearing the
 * XFS_IRECLAIM flag.  If we are in a shutdown state, we don't care about
 * blocking anymore and hence we can wait for the inode to be able to reclaim
 * it.
1024
 *
1025 1026 1027 1028
 * We do no IO here - if callers require inodes to be cleaned they must push the
 * AIL first to trigger writeback of dirty inodes.  This enables writeback to be
 * done in the background in a non-blocking manner, and enables memory reclaim
 * to make progress without blocking.
1029
 */
1030
static void
1031
xfs_reclaim_inode(
1032
	struct xfs_inode	*ip,
1033
	struct xfs_perag	*pag)
1034
{
1035
	xfs_ino_t		ino = ip->i_ino; /* for radix_tree_delete */
1036

1037
	if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
1038
		goto out;
1039 1040
	if (!xfs_iflock_nowait(ip))
		goto out_iunlock;
1041

1042 1043
	if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
		xfs_iunpin_wait(ip);
1044
		/* xfs_iflush_abort() drops the flush lock */
1045
		xfs_iflush_abort(ip);
1046 1047
		goto reclaim;
	}
1048 1049 1050
	if (xfs_ipincount(ip))
		goto out_ifunlock;
	if (!xfs_inode_clean(ip))
1051 1052
		goto out_ifunlock;

1053
	xfs_ifunlock(ip);
1054
reclaim:
1055 1056
	ASSERT(!xfs_isiflocked(ip));

1057 1058 1059
	/*
	 * Because we use RCU freeing we need to ensure the inode always appears
	 * to be reclaimed with an invalid inode number when in the free state.
1060
	 * We do this as early as possible under the ILOCK so that
1061 1062 1063 1064 1065
	 * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to
	 * detect races with us here. By doing this, we guarantee that once
	 * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that
	 * it will see either a valid inode that will serialise correctly, or it
	 * will see an invalid inode that it can skip.
1066 1067 1068 1069 1070 1071
	 */
	spin_lock(&ip->i_flags_lock);
	ip->i_flags = XFS_IRECLAIM;
	ip->i_ino = 0;
	spin_unlock(&ip->i_flags_lock);

1072
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1073

1074
	XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
1075 1076 1077 1078 1079 1080 1081
	/*
	 * Remove the inode from the per-AG radix tree.
	 *
	 * Because radix_tree_delete won't complain even if the item was never
	 * added to the tree assert that it's been there before to catch
	 * problems with the inode life time early on.
	 */
1082
	spin_lock(&pag->pag_ici_lock);
1083
	if (!radix_tree_delete(&pag->pag_ici_root,
1084
				XFS_INO_TO_AGINO(ip->i_mount, ino)))
1085
		ASSERT(0);
1086
	xfs_perag_clear_reclaim_tag(pag);
1087
	spin_unlock(&pag->pag_ici_lock);
1088 1089 1090 1091 1092 1093 1094

	/*
	 * Here we do an (almost) spurious inode lock in order to coordinate
	 * with inode cache radix tree lookups.  This is because the lookup
	 * can reference the inodes in the cache without taking references.
	 *
	 * We make that OK here by ensuring that we wait until the inode is
1095
	 * unlocked after the lookup before we go ahead and free it.
1096
	 */
1097
	xfs_ilock(ip, XFS_ILOCK_EXCL);
1098
	xfs_qm_dqdetach(ip);
1099
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1100
	ASSERT(xfs_inode_clean(ip));
1101

1102
	__xfs_inode_free(ip);
1103
	return;
1104 1105 1106

out_ifunlock:
	xfs_ifunlock(ip);
1107
out_iunlock:
1108
	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1109
out:
1110
	xfs_iflags_clear(ip, XFS_IRECLAIM);
1111 1112
}

1113 1114 1115 1116 1117
/*
 * Walk the AGs and reclaim the inodes in them. Even if the filesystem is
 * corrupted, we still want to try to reclaim all the inodes. If we don't,
 * then a shut down during filesystem unmount reclaim walk leak all the
 * unreclaimed inodes.
1118 1119 1120 1121
 *
 * Returns non-zero if any AGs or inodes were skipped in the reclaim pass
 * so that callers that want to block until all dirty inodes are written back
 * and reclaimed can sanely loop.
1122
 */
1123
static void
1124 1125 1126 1127 1128
xfs_reclaim_inodes_ag(
	struct xfs_mount	*mp,
	int			*nr_to_scan)
{
	struct xfs_perag	*pag;
1129
	xfs_agnumber_t		ag = 0;
1130 1131 1132 1133

	while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
		unsigned long	first_index = 0;
		int		done = 0;
1134
		int		nr_found = 0;
1135 1136 1137

		ag = pag->pag_agno + 1;

1138
		first_index = READ_ONCE(pag->pag_ici_reclaim_cursor);
1139
		do {
1140 1141
			struct xfs_inode *batch[XFS_LOOKUP_BATCH];
			int	i;
1142

1143
			rcu_read_lock();
1144 1145 1146 1147
			nr_found = radix_tree_gang_lookup_tag(
					&pag->pag_ici_root,
					(void **)batch, first_index,
					XFS_LOOKUP_BATCH,
1148 1149
					XFS_ICI_RECLAIM_TAG);
			if (!nr_found) {
1150
				done = 1;
1151
				rcu_read_unlock();
1152 1153 1154 1155
				break;
			}

			/*
1156 1157
			 * Grab the inodes before we drop the lock. if we found
			 * nothing, nr == 0 and the loop will be skipped.
1158
			 */
1159 1160 1161
			for (i = 0; i < nr_found; i++) {
				struct xfs_inode *ip = batch[i];

1162
				if (done || !xfs_reclaim_inode_grab(ip))
1163 1164 1165 1166 1167 1168 1169 1170
					batch[i] = NULL;

				/*
				 * Update the index for the next lookup. Catch
				 * overflows into the next AG range which can
				 * occur if we have inodes in the last block of
				 * the AG and we are currently pointing to the
				 * last inode.
1171 1172 1173 1174 1175 1176 1177
				 *
				 * Because we may see inodes that are from the
				 * wrong AG due to RCU freeing and
				 * reallocation, only update the index if it
				 * lies in this AG. It was a race that lead us
				 * to see this inode, so another lookup from
				 * the same index will not find it again.
1178
				 */
1179 1180 1181
				if (XFS_INO_TO_AGNO(mp, ip->i_ino) !=
								pag->pag_agno)
					continue;
1182 1183 1184 1185
				first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
				if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
					done = 1;
			}
1186

1187
			/* unlock now we've grabbed the inodes. */
1188
			rcu_read_unlock();
1189 1190

			for (i = 0; i < nr_found; i++) {
1191 1192
				if (batch[i])
					xfs_reclaim_inode(batch[i], pag);
1193 1194 1195
			}

			*nr_to_scan -= XFS_LOOKUP_BATCH;
1196
			cond_resched();
1197
		} while (nr_found && !done && *nr_to_scan > 0);
1198

1199 1200 1201
		if (done)
			first_index = 0;
		WRITE_ONCE(pag->pag_ici_reclaim_cursor, first_index);
1202 1203 1204 1205
		xfs_perag_put(pag);
	}
}

1206
void
1207
xfs_reclaim_inodes(
1208
	struct xfs_mount	*mp)
1209
{
1210 1211
	int		nr_to_scan = INT_MAX;

1212
	while (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
1213
		xfs_ail_push_all_sync(mp->m_ail);
1214 1215
		xfs_reclaim_inodes_ag(mp, &nr_to_scan);
	};
1216 1217 1218
}

/*
1219 1220 1221 1222 1223
 * The shrinker infrastructure determines how many inodes we should scan for
 * reclaim. We want as many clean inodes ready to reclaim as possible, so we
 * push the AIL here. We also want to proactively free up memory if we can to
 * minimise the amount of work memory reclaim has to do so we kick the
 * background reclaim if it isn't already scheduled.
1224
 */
1225
long
1226 1227 1228
xfs_reclaim_inodes_nr(
	struct xfs_mount	*mp,
	int			nr_to_scan)
1229
{
1230
	/* kick background reclaimer and push the AIL */
1231
	xfs_reclaim_work_queue(mp);
1232
	xfs_ail_push_all(mp->m_ail);
1233

1234
	xfs_reclaim_inodes_ag(mp, &nr_to_scan);
1235
	return 0;
1236
}
1237

1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
/*
 * Return the number of reclaimable inodes in the filesystem for
 * the shrinker to determine how much to reclaim.
 */
int
xfs_reclaim_inodes_count(
	struct xfs_mount	*mp)
{
	struct xfs_perag	*pag;
	xfs_agnumber_t		ag = 0;
	int			reclaimable = 0;
1249

1250 1251
	while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
		ag = pag->pag_agno + 1;
1252 1253
		reclaimable += pag->pag_ici_reclaimable;
		xfs_perag_put(pag);
1254 1255 1256 1257
	}
	return reclaimable;
}

1258
STATIC bool
1259 1260 1261 1262
xfs_inode_match_id(
	struct xfs_inode	*ip,
	struct xfs_eofblocks	*eofb)
{
1263 1264
	if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
	    !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
1265
		return false;
1266

1267 1268
	if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
	    !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
1269
		return false;
1270

1271
	if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
1272
	    ip->i_d.di_projid != eofb->eof_prid)
1273
		return false;
1274

1275
	return true;
1276 1277
}

1278 1279 1280 1281
/*
 * A union-based inode filtering algorithm. Process the inode if any of the
 * criteria match. This is for global/internal scans only.
 */
1282
STATIC bool
1283 1284 1285 1286 1287 1288
xfs_inode_match_id_union(
	struct xfs_inode	*ip,
	struct xfs_eofblocks	*eofb)
{
	if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
	    uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
1289
		return true;
1290 1291 1292

	if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
	    gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
1293
		return true;
1294 1295

	if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
1296
	    ip->i_d.di_projid == eofb->eof_prid)
1297
		return true;
1298

1299
	return false;
1300 1301
}

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
/*
 * Is this inode @ip eligible for eof/cow block reclamation, given some
 * filtering parameters @eofb?  The inode is eligible if @eofb is null or
 * if the predicate functions match.
 */
static bool
xfs_inode_matches_eofb(
	struct xfs_inode	*ip,
	struct xfs_eofblocks	*eofb)
{
1312
	bool			match;
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331

	if (!eofb)
		return true;

	if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
		match = xfs_inode_match_id_union(ip, eofb);
	else
		match = xfs_inode_match_id(ip, eofb);
	if (!match)
		return false;

	/* skip the inode if the file size is too small */
	if ((eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE) &&
	    XFS_ISIZE(ip) < eofb->eof_min_file_size)
		return false;

	return true;
}

1332 1333 1334 1335
/*
 * This is a fast pass over the inode cache to try to get reclaim moving on as
 * many inodes as possible in a short period of time. It kicks itself every few
 * seconds, as well as being kicked by the inode cache shrinker when memory
1336
 * goes low.
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
 */
void
xfs_reclaim_worker(
	struct work_struct *work)
{
	struct xfs_mount *mp = container_of(to_delayed_work(work),
					struct xfs_mount, m_reclaim_work);
	int		nr_to_scan = INT_MAX;

	xfs_reclaim_inodes_ag(mp, &nr_to_scan);
	xfs_reclaim_work_queue(mp);
}

1350 1351 1352 1353 1354
STATIC int
xfs_inode_free_eofblocks(
	struct xfs_inode	*ip,
	void			*args)
{
1355 1356 1357 1358 1359
	struct xfs_eofblocks	*eofb = args;
	bool			wait;
	int			ret;

	wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC);
1360

1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
	if (!xfs_can_free_eofblocks(ip, false)) {
		/* inode could be preallocated or append-only */
		trace_xfs_inode_free_eofblocks_invalid(ip);
		xfs_inode_clear_eofblocks_tag(ip);
		return 0;
	}

	/*
	 * If the mapping is dirty the operation can block and wait for some
	 * time. Unless we are waiting, skip it.
	 */
1372
	if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
1373 1374
		return 0;

1375 1376
	if (!xfs_inode_matches_eofb(ip, eofb))
		return 0;
1377

1378 1379 1380 1381
	/*
	 * If the caller is waiting, return -EAGAIN to keep the background
	 * scanner moving and revisit the inode in a subsequent pass.
	 */
1382
	if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
1383 1384 1385
		if (wait)
			return -EAGAIN;
		return 0;
1386
	}
1387

1388
	ret = xfs_free_eofblocks(ip);
1389
	xfs_iunlock(ip, XFS_IOLOCK_EXCL);
1390 1391 1392 1393

	return ret;
}

1394 1395 1396 1397 1398
int
xfs_icache_free_eofblocks(
	struct xfs_mount	*mp,
	struct xfs_eofblocks	*eofb)
{
1399
	return xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb,
1400
			XFS_ICI_EOFBLOCKS_TAG);
1401 1402
}

1403 1404 1405 1406 1407 1408
/*
 * Run eofblocks scans on the quotas applicable to the inode. For inodes with
 * multiple quotas, we don't know exactly which quota caused an allocation
 * failure. We make a best effort by including each quota under low free space
 * conditions (less than 1% free space) in the scan.
 */
1409 1410 1411 1412 1413
static int
__xfs_inode_free_quota_eofblocks(
	struct xfs_inode	*ip,
	int			(*execute)(struct xfs_mount *mp,
					   struct xfs_eofblocks	*eofb))
1414 1415 1416 1417 1418 1419
{
	int scan = 0;
	struct xfs_eofblocks eofb = {0};
	struct xfs_dquot *dq;

	/*
1420
	 * Run a sync scan to increase effectiveness and use the union filter to
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443
	 * cover all applicable quotas in a single scan.
	 */
	eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;

	if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
		dq = xfs_inode_dquot(ip, XFS_DQ_USER);
		if (dq && xfs_dquot_lowsp(dq)) {
			eofb.eof_uid = VFS_I(ip)->i_uid;
			eofb.eof_flags |= XFS_EOF_FLAGS_UID;
			scan = 1;
		}
	}

	if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
		dq = xfs_inode_dquot(ip, XFS_DQ_GROUP);
		if (dq && xfs_dquot_lowsp(dq)) {
			eofb.eof_gid = VFS_I(ip)->i_gid;
			eofb.eof_flags |= XFS_EOF_FLAGS_GID;
			scan = 1;
		}
	}

	if (scan)
1444
		execute(ip->i_mount, &eofb);
1445 1446 1447 1448

	return scan;
}

1449 1450 1451 1452 1453 1454 1455
int
xfs_inode_free_quota_eofblocks(
	struct xfs_inode *ip)
{
	return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks);
}

1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
static inline unsigned long
xfs_iflag_for_tag(
	int		tag)
{
	switch (tag) {
	case XFS_ICI_EOFBLOCKS_TAG:
		return XFS_IEOFBLOCKS;
	case XFS_ICI_COWBLOCKS_TAG:
		return XFS_ICOWBLOCKS;
	default:
		ASSERT(0);
		return 0;
	}
}

1471
static void
1472
__xfs_inode_set_blocks_tag(
1473 1474 1475 1476 1477
	xfs_inode_t	*ip,
	void		(*execute)(struct xfs_mount *mp),
	void		(*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
				  int error, unsigned long caller_ip),
	int		tag)
1478 1479 1480 1481 1482
{
	struct xfs_mount *mp = ip->i_mount;
	struct xfs_perag *pag;
	int tagged;

1483 1484 1485 1486
	/*
	 * Don't bother locking the AG and looking up in the radix trees
	 * if we already know that we have the tag set.
	 */
1487
	if (ip->i_flags & xfs_iflag_for_tag(tag))
1488 1489
		return;
	spin_lock(&ip->i_flags_lock);
1490
	ip->i_flags |= xfs_iflag_for_tag(tag);
1491 1492
	spin_unlock(&ip->i_flags_lock);

1493 1494 1495
	pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
	spin_lock(&pag->pag_ici_lock);

1496
	tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
1497
	radix_tree_tag_set(&pag->pag_ici_root,
1498
			   XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
1499 1500 1501 1502 1503
	if (!tagged) {
		/* propagate the eofblocks tag up into the perag radix tree */
		spin_lock(&ip->i_mount->m_perag_lock);
		radix_tree_tag_set(&ip->i_mount->m_perag_tree,
				   XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
1504
				   tag);
1505
		spin_unlock(&ip->i_mount->m_perag_lock);
1506 1507

		/* kick off background trimming */
1508
		execute(ip->i_mount);
1509

1510
		set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
1511 1512 1513 1514 1515 1516 1517
	}

	spin_unlock(&pag->pag_ici_lock);
	xfs_perag_put(pag);
}

void
1518
xfs_inode_set_eofblocks_tag(
1519
	xfs_inode_t	*ip)
1520 1521
{
	trace_xfs_inode_set_eofblocks_tag(ip);
1522
	return __xfs_inode_set_blocks_tag(ip, xfs_queue_eofblocks,
1523 1524 1525 1526 1527
			trace_xfs_perag_set_eofblocks,
			XFS_ICI_EOFBLOCKS_TAG);
}

static void
1528
__xfs_inode_clear_blocks_tag(
1529 1530 1531 1532
	xfs_inode_t	*ip,
	void		(*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
				    int error, unsigned long caller_ip),
	int		tag)
1533 1534 1535 1536
{
	struct xfs_mount *mp = ip->i_mount;
	struct xfs_perag *pag;

1537
	spin_lock(&ip->i_flags_lock);
1538
	ip->i_flags &= ~xfs_iflag_for_tag(tag);
1539 1540
	spin_unlock(&ip->i_flags_lock);

1541 1542 1543 1544
	pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
	spin_lock(&pag->pag_ici_lock);

	radix_tree_tag_clear(&pag->pag_ici_root,
1545 1546
			     XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
	if (!radix_tree_tagged(&pag->pag_ici_root, tag)) {
1547 1548 1549 1550
		/* clear the eofblocks tag from the perag radix tree */
		spin_lock(&ip->i_mount->m_perag_lock);
		radix_tree_tag_clear(&ip->i_mount->m_perag_tree,
				     XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
1551
				     tag);
1552
		spin_unlock(&ip->i_mount->m_perag_lock);
1553
		clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
1554 1555 1556 1557 1558 1559
	}

	spin_unlock(&pag->pag_ici_lock);
	xfs_perag_put(pag);
}

1560 1561 1562 1563 1564
void
xfs_inode_clear_eofblocks_tag(
	xfs_inode_t	*ip)
{
	trace_xfs_inode_clear_eofblocks_tag(ip);
1565
	return __xfs_inode_clear_blocks_tag(ip,
1566 1567 1568 1569
			trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG);
}

/*
1570 1571 1572
 * Set ourselves up to free CoW blocks from this file.  If it's already clean
 * then we can bail out quickly, but otherwise we must back off if the file
 * is undergoing some kind of write.
1573
 */
1574 1575
static bool
xfs_prep_free_cowblocks(
1576
	struct xfs_inode	*ip)
1577
{
1578 1579 1580 1581
	/*
	 * Just clear the tag if we have an empty cow fork or none at all. It's
	 * possible the inode was fully unshared since it was originally tagged.
	 */
1582
	if (!xfs_inode_has_cow_data(ip)) {
1583 1584
		trace_xfs_inode_free_cowblocks_invalid(ip);
		xfs_inode_clear_cowblocks_tag(ip);
1585
		return false;
1586 1587 1588 1589 1590 1591
	}

	/*
	 * If the mapping is dirty or under writeback we cannot touch the
	 * CoW fork.  Leave it alone if we're in the midst of a directio.
	 */
1592 1593
	if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
	    mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
1594 1595
	    mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
	    atomic_read(&VFS_I(ip)->i_dio_count))
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
		return false;

	return true;
}

/*
 * Automatic CoW Reservation Freeing
 *
 * These functions automatically garbage collect leftover CoW reservations
 * that were made on behalf of a cowextsize hint when we start to run out
 * of quota or when the reservations sit around for too long.  If the file
 * has dirty pages or is undergoing writeback, its CoW reservations will
 * be retained.
 *
 * The actual garbage collection piggybacks off the same code that runs
 * the speculative EOF preallocation garbage collector.
 */
STATIC int
xfs_inode_free_cowblocks(
	struct xfs_inode	*ip,
	void			*args)
{
	struct xfs_eofblocks	*eofb = args;
	int			ret = 0;

1621
	if (!xfs_prep_free_cowblocks(ip))
1622 1623
		return 0;

1624 1625
	if (!xfs_inode_matches_eofb(ip, eofb))
		return 0;
1626 1627

	/* Free the CoW blocks */
1628 1629
	xfs_ilock(ip, XFS_IOLOCK_EXCL);
	xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
1630

1631 1632 1633 1634
	/*
	 * Check again, nobody else should be able to dirty blocks or change
	 * the reflink iflag now that we have the first two locks held.
	 */
1635
	if (xfs_prep_free_cowblocks(ip))
1636
		ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
1637

1638 1639
	xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
	xfs_iunlock(ip, XFS_IOLOCK_EXCL);
1640 1641 1642 1643 1644 1645 1646 1647 1648

	return ret;
}

int
xfs_icache_free_cowblocks(
	struct xfs_mount	*mp,
	struct xfs_eofblocks	*eofb)
{
1649
	return xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb,
1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
			XFS_ICI_COWBLOCKS_TAG);
}

int
xfs_inode_free_quota_cowblocks(
	struct xfs_inode *ip)
{
	return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks);
}

void
xfs_inode_set_cowblocks_tag(
	xfs_inode_t	*ip)
{
1664
	trace_xfs_inode_set_cowblocks_tag(ip);
1665
	return __xfs_inode_set_blocks_tag(ip, xfs_queue_cowblocks,
1666
			trace_xfs_perag_set_cowblocks,
1667 1668 1669 1670 1671 1672 1673
			XFS_ICI_COWBLOCKS_TAG);
}

void
xfs_inode_clear_cowblocks_tag(
	xfs_inode_t	*ip)
{
1674
	trace_xfs_inode_clear_cowblocks_tag(ip);
1675
	return __xfs_inode_clear_blocks_tag(ip,
1676
			trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG);
1677
}
1678 1679 1680

/* Disable post-EOF and CoW block auto-reclamation. */
void
1681
xfs_stop_block_reaping(
1682 1683 1684 1685 1686 1687 1688 1689
	struct xfs_mount	*mp)
{
	cancel_delayed_work_sync(&mp->m_eofblocks_work);
	cancel_delayed_work_sync(&mp->m_cowblocks_work);
}

/* Enable post-EOF and CoW block auto-reclamation. */
void
1690
xfs_start_block_reaping(
1691 1692 1693 1694 1695
	struct xfs_mount	*mp)
{
	xfs_queue_eofblocks(mp);
	xfs_queue_cowblocks(mp);
}
新手
引导
客服 返回
顶部