xfs_mount.c 51.0 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
L
Linus Torvalds 已提交
4
 *
5 6
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
L
Linus Torvalds 已提交
7 8
 * published by the Free Software Foundation.
 *
9 10 11 12
 * This program is distributed in the hope that it would be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
L
Linus Torvalds 已提交
13
 *
14 15 16
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software Foundation,
 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
L
Linus Torvalds 已提交
17 18
 */
#include "xfs.h"
19
#include "xfs_fs.h"
20
#include "xfs_shared.h"
21 22 23
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
24 25
#include "xfs_bit.h"
#include "xfs_inum.h"
L
Linus Torvalds 已提交
26 27 28
#include "xfs_sb.h"
#include "xfs_ag.h"
#include "xfs_mount.h"
29
#include "xfs_da_format.h"
L
Linus Torvalds 已提交
30
#include "xfs_inode.h"
31
#include "xfs_dir2.h"
32
#include "xfs_ialloc.h"
L
Linus Torvalds 已提交
33 34 35
#include "xfs_alloc.h"
#include "xfs_rtalloc.h"
#include "xfs_bmap.h"
36 37 38
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
#include "xfs_log.h"
L
Linus Torvalds 已提交
39 40 41
#include "xfs_error.h"
#include "xfs_quota.h"
#include "xfs_fsops.h"
C
Christoph Hellwig 已提交
42
#include "xfs_trace.h"
43
#include "xfs_icache.h"
44
#include "xfs_dinode.h"
C
Christoph Hellwig 已提交
45

L
Linus Torvalds 已提交
46

47
#ifdef HAVE_PERCPU_SB
48
STATIC void	xfs_icsb_balance_counter(xfs_mount_t *, xfs_sb_field_t,
49 50 51
						int);
STATIC void	xfs_icsb_balance_counter_locked(xfs_mount_t *, xfs_sb_field_t,
						int);
52
STATIC void	xfs_icsb_disable_counter(xfs_mount_t *, xfs_sb_field_t);
53 54
#else

55 56
#define xfs_icsb_balance_counter(mp, a, b)		do { } while (0)
#define xfs_icsb_balance_counter_locked(mp, a, b)	do { } while (0)
57 58
#endif

C
Christoph Hellwig 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
static DEFINE_MUTEX(xfs_uuid_table_mutex);
static int xfs_uuid_table_size;
static uuid_t *xfs_uuid_table;

/*
 * See if the UUID is unique among mounted XFS filesystems.
 * Mount fails if UUID is nil or a FS with the same UUID is already mounted.
 */
STATIC int
xfs_uuid_mount(
	struct xfs_mount	*mp)
{
	uuid_t			*uuid = &mp->m_sb.sb_uuid;
	int			hole, i;

	if (mp->m_flags & XFS_MOUNT_NOUUID)
		return 0;

	if (uuid_is_nil(uuid)) {
78
		xfs_warn(mp, "Filesystem has nil UUID - can't mount");
D
Dave Chinner 已提交
79
		return -EINVAL;
C
Christoph Hellwig 已提交
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	}

	mutex_lock(&xfs_uuid_table_mutex);
	for (i = 0, hole = -1; i < xfs_uuid_table_size; i++) {
		if (uuid_is_nil(&xfs_uuid_table[i])) {
			hole = i;
			continue;
		}
		if (uuid_equal(uuid, &xfs_uuid_table[i]))
			goto out_duplicate;
	}

	if (hole < 0) {
		xfs_uuid_table = kmem_realloc(xfs_uuid_table,
			(xfs_uuid_table_size + 1) * sizeof(*xfs_uuid_table),
			xfs_uuid_table_size  * sizeof(*xfs_uuid_table),
			KM_SLEEP);
		hole = xfs_uuid_table_size++;
	}
	xfs_uuid_table[hole] = *uuid;
	mutex_unlock(&xfs_uuid_table_mutex);

	return 0;

 out_duplicate:
	mutex_unlock(&xfs_uuid_table_mutex);
106
	xfs_warn(mp, "Filesystem has duplicate UUID %pU - can't mount", uuid);
D
Dave Chinner 已提交
107
	return -EINVAL;
C
Christoph Hellwig 已提交
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
}

STATIC void
xfs_uuid_unmount(
	struct xfs_mount	*mp)
{
	uuid_t			*uuid = &mp->m_sb.sb_uuid;
	int			i;

	if (mp->m_flags & XFS_MOUNT_NOUUID)
		return;

	mutex_lock(&xfs_uuid_table_mutex);
	for (i = 0; i < xfs_uuid_table_size; i++) {
		if (uuid_is_nil(&xfs_uuid_table[i]))
			continue;
		if (!uuid_equal(uuid, &xfs_uuid_table[i]))
			continue;
		memset(&xfs_uuid_table[i], 0, sizeof(uuid_t));
		break;
	}
	ASSERT(i < xfs_uuid_table_size);
	mutex_unlock(&xfs_uuid_table_mutex);
}


D
Dave Chinner 已提交
134 135 136 137 138 139 140 141 142 143
STATIC void
__xfs_free_perag(
	struct rcu_head	*head)
{
	struct xfs_perag *pag = container_of(head, struct xfs_perag, rcu_head);

	ASSERT(atomic_read(&pag->pag_ref) == 0);
	kmem_free(pag);
}

L
Linus Torvalds 已提交
144
/*
D
Dave Chinner 已提交
145
 * Free up the per-ag resources associated with the mount structure.
L
Linus Torvalds 已提交
146
 */
C
Christoph Hellwig 已提交
147
STATIC void
148
xfs_free_perag(
149
	xfs_mount_t	*mp)
L
Linus Torvalds 已提交
150
{
151 152 153 154 155 156 157
	xfs_agnumber_t	agno;
	struct xfs_perag *pag;

	for (agno = 0; agno < mp->m_sb.sb_agcount; agno++) {
		spin_lock(&mp->m_perag_lock);
		pag = radix_tree_delete(&mp->m_perag_tree, agno);
		spin_unlock(&mp->m_perag_lock);
D
Dave Chinner 已提交
158
		ASSERT(pag);
159
		ASSERT(atomic_read(&pag->pag_ref) == 0);
D
Dave Chinner 已提交
160
		call_rcu(&pag->rcu_head, __xfs_free_perag);
L
Linus Torvalds 已提交
161 162 163
	}
}

164 165 166 167 168 169 170 171 172 173 174 175 176 177
/*
 * Check size of device based on the (data/realtime) block count.
 * Note: this check is used by the growfs code as well as mount.
 */
int
xfs_sb_validate_fsb_count(
	xfs_sb_t	*sbp,
	__uint64_t	nblocks)
{
	ASSERT(PAGE_SHIFT >= sbp->sb_blocklog);
	ASSERT(sbp->sb_blocklog >= BBSHIFT);

#if XFS_BIG_BLKNOS     /* Limited by ULONG_MAX of page cache index */
	if (nblocks >> (PAGE_CACHE_SHIFT - sbp->sb_blocklog) > ULONG_MAX)
D
Dave Chinner 已提交
178
		return -EFBIG;
179 180
#else                  /* Limited by UINT_MAX of sectors */
	if (nblocks << (sbp->sb_blocklog - BBSHIFT) > UINT_MAX)
D
Dave Chinner 已提交
181
		return -EFBIG;
182 183 184
#endif
	return 0;
}
L
Linus Torvalds 已提交
185

186
int
187 188
xfs_initialize_perag(
	xfs_mount_t	*mp,
189 190
	xfs_agnumber_t	agcount,
	xfs_agnumber_t	*maxagi)
L
Linus Torvalds 已提交
191
{
192
	xfs_agnumber_t	index;
193
	xfs_agnumber_t	first_initialised = 0;
L
Linus Torvalds 已提交
194 195 196 197
	xfs_perag_t	*pag;
	xfs_agino_t	agino;
	xfs_ino_t	ino;
	xfs_sb_t	*sbp = &mp->m_sb;
198
	int		error = -ENOMEM;
L
Linus Torvalds 已提交
199

200 201 202 203 204 205 206 207 208 209 210
	/*
	 * Walk the current per-ag tree so we don't try to initialise AGs
	 * that already exist (growfs case). Allocate and insert all the
	 * AGs we don't find ready for initialisation.
	 */
	for (index = 0; index < agcount; index++) {
		pag = xfs_perag_get(mp, index);
		if (pag) {
			xfs_perag_put(pag);
			continue;
		}
211 212
		if (!first_initialised)
			first_initialised = index;
213

214 215
		pag = kmem_zalloc(sizeof(*pag), KM_MAYFAIL);
		if (!pag)
216
			goto out_unwind;
217 218
		pag->pag_agno = index;
		pag->pag_mount = mp;
219
		spin_lock_init(&pag->pag_ici_lock);
220
		mutex_init(&pag->pag_ici_reclaim_lock);
221
		INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC);
222 223
		spin_lock_init(&pag->pag_buf_lock);
		pag->pag_buf_tree = RB_ROOT;
224

225
		if (radix_tree_preload(GFP_NOFS))
226
			goto out_unwind;
227

228 229 230 231
		spin_lock(&mp->m_perag_lock);
		if (radix_tree_insert(&mp->m_perag_tree, index, pag)) {
			BUG();
			spin_unlock(&mp->m_perag_lock);
232 233 234
			radix_tree_preload_end();
			error = -EEXIST;
			goto out_unwind;
235 236 237 238 239
		}
		spin_unlock(&mp->m_perag_lock);
		radix_tree_preload_end();
	}

240 241 242
	/*
	 * If we mount with the inode64 option, or no inode overflows
	 * the legacy 32-bit address space clear the inode32 option.
L
Linus Torvalds 已提交
243
	 */
244 245 246 247
	agino = XFS_OFFBNO_TO_AGINO(mp, sbp->sb_agblocks - 1, 0);
	ino = XFS_AGINO_TO_INO(mp, agcount - 1, agino);

	if ((mp->m_flags & XFS_MOUNT_SMALL_INUMS) && ino > XFS_MAXINUMBER_32)
L
Linus Torvalds 已提交
248
		mp->m_flags |= XFS_MOUNT_32BITINODES;
249
	else
L
Linus Torvalds 已提交
250 251
		mp->m_flags &= ~XFS_MOUNT_32BITINODES;

252 253 254 255
	if (mp->m_flags & XFS_MOUNT_32BITINODES)
		index = xfs_set_inode32(mp);
	else
		index = xfs_set_inode64(mp);
256

257 258 259
	if (maxagi)
		*maxagi = index;
	return 0;
260 261 262 263 264 265 266 267

out_unwind:
	kmem_free(pag);
	for (; index > first_initialised; index--) {
		pag = radix_tree_delete(&mp->m_perag_tree, index);
		kmem_free(pag);
	}
	return error;
L
Linus Torvalds 已提交
268 269 270 271 272 273 274 275
}

/*
 * xfs_readsb
 *
 * Does the initial read of the superblock.
 */
int
276 277 278
xfs_readsb(
	struct xfs_mount *mp,
	int		flags)
L
Linus Torvalds 已提交
279 280
{
	unsigned int	sector_size;
281 282
	struct xfs_buf	*bp;
	struct xfs_sb	*sbp = &mp->m_sb;
L
Linus Torvalds 已提交
283
	int		error;
284
	int		loud = !(flags & XFS_MFSI_QUIET);
285
	const struct xfs_buf_ops *buf_ops;
L
Linus Torvalds 已提交
286 287 288 289

	ASSERT(mp->m_sb_bp == NULL);
	ASSERT(mp->m_ddev_targp != NULL);

290 291 292 293 294 295 296 297 298 299
	/*
	 * For the initial read, we must guess at the sector
	 * size based on the block device.  It's enough to
	 * get the sb_sectsize out of the superblock and
	 * then reread with the proper length.
	 * We don't verify it yet, because it may not be complete.
	 */
	sector_size = xfs_getsize_buftarg(mp->m_ddev_targp);
	buf_ops = NULL;

L
Linus Torvalds 已提交
300 301 302 303 304
	/*
	 * Allocate a (locked) buffer to hold the superblock.
	 * This will be kept around at all times to optimize
	 * access to the superblock.
	 */
D
Dave Chinner 已提交
305
reread:
306
	bp = xfs_buf_read_uncached(mp->m_ddev_targp, XFS_SB_DADDR,
307
				   BTOBB(sector_size), 0, buf_ops);
D
Dave Chinner 已提交
308
	if (!bp) {
309 310
		if (loud)
			xfs_warn(mp, "SB buffer read failed");
D
Dave Chinner 已提交
311
		return -EIO;
L
Linus Torvalds 已提交
312
	}
313 314 315
	if (bp->b_error) {
		error = bp->b_error;
		if (loud)
316
			xfs_warn(mp, "SB validate failed with error %d.", error);
317
		/* bad CRC means corrupted metadata */
D
Dave Chinner 已提交
318 319
		if (error == -EFSBADCRC)
			error = -EFSCORRUPTED;
320 321
		goto release_buf;
	}
L
Linus Torvalds 已提交
322 323 324 325

	/*
	 * Initialize the mount structure from the superblock.
	 */
326 327 328 329 330 331 332 333 334 335
	xfs_sb_from_disk(sbp, XFS_BUF_TO_SBP(bp));
	xfs_sb_quota_from_disk(sbp);

	/*
	 * If we haven't validated the superblock, do so now before we try
	 * to check the sector size and reread the superblock appropriately.
	 */
	if (sbp->sb_magicnum != XFS_SB_MAGIC) {
		if (loud)
			xfs_warn(mp, "Invalid superblock magic number");
D
Dave Chinner 已提交
336
		error = -EINVAL;
337 338
		goto release_buf;
	}
339

L
Linus Torvalds 已提交
340 341 342
	/*
	 * We must be able to do sector-sized and sector-aligned IO.
	 */
343
	if (sector_size > sbp->sb_sectsize) {
344 345
		if (loud)
			xfs_warn(mp, "device supports %u byte sectors (not %u)",
346
				sector_size, sbp->sb_sectsize);
D
Dave Chinner 已提交
347
		error = -ENOSYS;
D
Dave Chinner 已提交
348
		goto release_buf;
L
Linus Torvalds 已提交
349 350
	}

351
	if (buf_ops == NULL) {
352 353 354 355
		/*
		 * Re-read the superblock so the buffer is correctly sized,
		 * and properly verified.
		 */
L
Linus Torvalds 已提交
356
		xfs_buf_relse(bp);
357
		sector_size = sbp->sb_sectsize;
358
		buf_ops = loud ? &xfs_sb_buf_ops : &xfs_sb_quiet_buf_ops;
D
Dave Chinner 已提交
359
		goto reread;
L
Linus Torvalds 已提交
360 361
	}

362 363
	/* Initialize per-cpu counters */
	xfs_icsb_reinit_counters(mp);
364

365 366 367
	/* no need to be quiet anymore, so reset the buf ops */
	bp->b_ops = &xfs_sb_buf_ops;

L
Linus Torvalds 已提交
368
	mp->m_sb_bp = bp;
D
Dave Chinner 已提交
369
	xfs_buf_unlock(bp);
L
Linus Torvalds 已提交
370 371
	return 0;

D
Dave Chinner 已提交
372 373
release_buf:
	xfs_buf_relse(bp);
L
Linus Torvalds 已提交
374 375 376 377
	return error;
}

/*
E
Eric Sandeen 已提交
378
 * Update alignment values based on mount options and sb values
L
Linus Torvalds 已提交
379
 */
E
Eric Sandeen 已提交
380
STATIC int
381
xfs_update_alignment(xfs_mount_t *mp)
L
Linus Torvalds 已提交
382 383 384
{
	xfs_sb_t	*sbp = &(mp->m_sb);

C
Christoph Hellwig 已提交
385
	if (mp->m_dalign) {
L
Linus Torvalds 已提交
386 387 388 389 390 391
		/*
		 * If stripe unit and stripe width are not multiples
		 * of the fs blocksize turn off alignment.
		 */
		if ((BBTOB(mp->m_dalign) & mp->m_blockmask) ||
		    (BBTOB(mp->m_swidth) & mp->m_blockmask)) {
J
Jie Liu 已提交
392 393 394
			xfs_warn(mp,
		"alignment check failed: sunit/swidth vs. blocksize(%d)",
				sbp->sb_blocksize);
D
Dave Chinner 已提交
395
			return -EINVAL;
L
Linus Torvalds 已提交
396 397 398 399 400 401
		} else {
			/*
			 * Convert the stripe unit and width to FSBs.
			 */
			mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign);
			if (mp->m_dalign && (sbp->sb_agblocks % mp->m_dalign)) {
402
				xfs_warn(mp,
J
Jie Liu 已提交
403 404
			"alignment check failed: sunit/swidth vs. agsize(%d)",
					 sbp->sb_agblocks);
D
Dave Chinner 已提交
405
				return -EINVAL;
L
Linus Torvalds 已提交
406 407 408
			} else if (mp->m_dalign) {
				mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth);
			} else {
J
Jie Liu 已提交
409 410 411
				xfs_warn(mp,
			"alignment check failed: sunit(%d) less than bsize(%d)",
					 mp->m_dalign, sbp->sb_blocksize);
D
Dave Chinner 已提交
412
				return -EINVAL;
L
Linus Torvalds 已提交
413 414 415 416 417 418 419
			}
		}

		/*
		 * Update superblock with new values
		 * and log changes
		 */
420
		if (xfs_sb_version_hasdalign(sbp)) {
L
Linus Torvalds 已提交
421 422
			if (sbp->sb_unit != mp->m_dalign) {
				sbp->sb_unit = mp->m_dalign;
423
				mp->m_update_flags |= XFS_SB_UNIT;
L
Linus Torvalds 已提交
424 425 426
			}
			if (sbp->sb_width != mp->m_swidth) {
				sbp->sb_width = mp->m_swidth;
427
				mp->m_update_flags |= XFS_SB_WIDTH;
L
Linus Torvalds 已提交
428
			}
429 430 431
		} else {
			xfs_warn(mp,
	"cannot change alignment: superblock does not support data alignment");
D
Dave Chinner 已提交
432
			return -EINVAL;
L
Linus Torvalds 已提交
433 434
		}
	} else if ((mp->m_flags & XFS_MOUNT_NOALIGN) != XFS_MOUNT_NOALIGN &&
435
		    xfs_sb_version_hasdalign(&mp->m_sb)) {
L
Linus Torvalds 已提交
436 437 438 439
			mp->m_dalign = sbp->sb_unit;
			mp->m_swidth = sbp->sb_width;
	}

E
Eric Sandeen 已提交
440 441
	return 0;
}
L
Linus Torvalds 已提交
442

E
Eric Sandeen 已提交
443 444 445 446 447 448 449 450
/*
 * Set the maximum inode count for this filesystem
 */
STATIC void
xfs_set_maxicount(xfs_mount_t *mp)
{
	xfs_sb_t	*sbp = &(mp->m_sb);
	__uint64_t	icount;
L
Linus Torvalds 已提交
451

E
Eric Sandeen 已提交
452 453 454 455
	if (sbp->sb_imax_pct) {
		/*
		 * Make sure the maximum inode count is a multiple
		 * of the units we allocate inodes in.
L
Linus Torvalds 已提交
456 457 458 459 460 461
		 */
		icount = sbp->sb_dblocks * sbp->sb_imax_pct;
		do_div(icount, 100);
		do_div(icount, mp->m_ialloc_blks);
		mp->m_maxicount = (icount * mp->m_ialloc_blks)  <<
				   sbp->sb_inopblog;
E
Eric Sandeen 已提交
462
	} else {
L
Linus Torvalds 已提交
463 464
		mp->m_maxicount = 0;
	}
E
Eric Sandeen 已提交
465 466 467 468 469 470 471 472 473 474 475 476 477
}

/*
 * Set the default minimum read and write sizes unless
 * already specified in a mount option.
 * We use smaller I/O sizes when the file system
 * is being used for NFS service (wsync mount option).
 */
STATIC void
xfs_set_rw_sizes(xfs_mount_t *mp)
{
	xfs_sb_t	*sbp = &(mp->m_sb);
	int		readio_log, writeio_log;
L
Linus Torvalds 已提交
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503

	if (!(mp->m_flags & XFS_MOUNT_DFLT_IOSIZE)) {
		if (mp->m_flags & XFS_MOUNT_WSYNC) {
			readio_log = XFS_WSYNC_READIO_LOG;
			writeio_log = XFS_WSYNC_WRITEIO_LOG;
		} else {
			readio_log = XFS_READIO_LOG_LARGE;
			writeio_log = XFS_WRITEIO_LOG_LARGE;
		}
	} else {
		readio_log = mp->m_readio_log;
		writeio_log = mp->m_writeio_log;
	}

	if (sbp->sb_blocklog > readio_log) {
		mp->m_readio_log = sbp->sb_blocklog;
	} else {
		mp->m_readio_log = readio_log;
	}
	mp->m_readio_blocks = 1 << (mp->m_readio_log - sbp->sb_blocklog);
	if (sbp->sb_blocklog > writeio_log) {
		mp->m_writeio_log = sbp->sb_blocklog;
	} else {
		mp->m_writeio_log = writeio_log;
	}
	mp->m_writeio_blocks = 1 << (mp->m_writeio_log - sbp->sb_blocklog);
E
Eric Sandeen 已提交
504
}
L
Linus Torvalds 已提交
505

506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
/*
 * precalculate the low space thresholds for dynamic speculative preallocation.
 */
void
xfs_set_low_space_thresholds(
	struct xfs_mount	*mp)
{
	int i;

	for (i = 0; i < XFS_LOWSP_MAX; i++) {
		__uint64_t space = mp->m_sb.sb_dblocks;

		do_div(space, 100);
		mp->m_low_space[i] = space * (i + 1);
	}
}


E
Eric Sandeen 已提交
524 525 526 527 528 529
/*
 * Set whether we're using inode alignment.
 */
STATIC void
xfs_set_inoalignment(xfs_mount_t *mp)
{
530
	if (xfs_sb_version_hasalign(&mp->m_sb) &&
L
Linus Torvalds 已提交
531 532 533 534 535 536 537 538 539 540 541 542 543 544
	    mp->m_sb.sb_inoalignmt >=
	    XFS_B_TO_FSBT(mp, mp->m_inode_cluster_size))
		mp->m_inoalign_mask = mp->m_sb.sb_inoalignmt - 1;
	else
		mp->m_inoalign_mask = 0;
	/*
	 * If we are using stripe alignment, check whether
	 * the stripe unit is a multiple of the inode alignment
	 */
	if (mp->m_dalign && mp->m_inoalign_mask &&
	    !(mp->m_dalign & mp->m_inoalign_mask))
		mp->m_sinoalign = mp->m_dalign;
	else
		mp->m_sinoalign = 0;
E
Eric Sandeen 已提交
545 546 547
}

/*
548
 * Check that the data (and log if separate) is an ok size.
E
Eric Sandeen 已提交
549 550
 */
STATIC int
C
Christoph Hellwig 已提交
551
xfs_check_sizes(xfs_mount_t *mp)
E
Eric Sandeen 已提交
552 553 554 555
{
	xfs_buf_t	*bp;
	xfs_daddr_t	d;

L
Linus Torvalds 已提交
556 557
	d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks);
	if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_dblocks) {
558
		xfs_warn(mp, "filesystem size mismatch detected");
D
Dave Chinner 已提交
559
		return -EFBIG;
L
Linus Torvalds 已提交
560
	}
561
	bp = xfs_buf_read_uncached(mp->m_ddev_targp,
562
					d - XFS_FSS_TO_BB(mp, 1),
563
					XFS_FSS_TO_BB(mp, 1), 0, NULL);
564
	if (!bp) {
565
		xfs_warn(mp, "last sector read failed");
D
Dave Chinner 已提交
566
		return -EIO;
L
Linus Torvalds 已提交
567
	}
568
	xfs_buf_relse(bp);
L
Linus Torvalds 已提交
569

C
Christoph Hellwig 已提交
570
	if (mp->m_logdev_targp != mp->m_ddev_targp) {
L
Linus Torvalds 已提交
571 572
		d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks);
		if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_logblocks) {
573
			xfs_warn(mp, "log size mismatch detected");
D
Dave Chinner 已提交
574
			return -EFBIG;
L
Linus Torvalds 已提交
575
		}
576
		bp = xfs_buf_read_uncached(mp->m_logdev_targp,
577
					d - XFS_FSB_TO_BB(mp, 1),
578
					XFS_FSB_TO_BB(mp, 1), 0, NULL);
579
		if (!bp) {
580
			xfs_warn(mp, "log device read failed");
D
Dave Chinner 已提交
581
			return -EIO;
E
Eric Sandeen 已提交
582
		}
583
		xfs_buf_relse(bp);
E
Eric Sandeen 已提交
584 585 586 587
	}
	return 0;
}

C
Christoph Hellwig 已提交
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
/*
 * Clear the quotaflags in memory and in the superblock.
 */
int
xfs_mount_reset_sbqflags(
	struct xfs_mount	*mp)
{
	int			error;
	struct xfs_trans	*tp;

	mp->m_qflags = 0;

	/*
	 * It is OK to look at sb_qflags here in mount path,
	 * without m_sb_lock.
	 */
	if (mp->m_sb.sb_qflags == 0)
		return 0;
	spin_lock(&mp->m_sb_lock);
	mp->m_sb.sb_qflags = 0;
	spin_unlock(&mp->m_sb_lock);

	/*
	 * If the fs is readonly, let the incore superblock run
	 * with quotas off but don't flush the update out to disk
	 */
	if (mp->m_flags & XFS_MOUNT_RDONLY)
		return 0;

	tp = xfs_trans_alloc(mp, XFS_TRANS_QM_SBCHANGE);
618
	error = xfs_trans_reserve(tp, &M_RES(mp)->tr_qm_sbchange, 0, 0);
C
Christoph Hellwig 已提交
619 620
	if (error) {
		xfs_trans_cancel(tp, 0);
621
		xfs_alert(mp, "%s: Superblock update failed!", __func__);
C
Christoph Hellwig 已提交
622 623 624 625 626 627 628
		return error;
	}

	xfs_mod_sb(tp, XFS_SB_QFLAGS);
	return xfs_trans_commit(tp, 0);
}

E
Eric Sandeen 已提交
629 630 631 632 633 634
__uint64_t
xfs_default_resblks(xfs_mount_t *mp)
{
	__uint64_t resblks;

	/*
635 636 637 638 639
	 * We default to 5% or 8192 fsbs of space reserved, whichever is
	 * smaller.  This is intended to cover concurrent allocation
	 * transactions when we initially hit enospc. These each require a 4
	 * block reservation. Hence by default we cover roughly 2000 concurrent
	 * allocation reservations.
E
Eric Sandeen 已提交
640 641 642
	 */
	resblks = mp->m_sb.sb_dblocks;
	do_div(resblks, 20);
643
	resblks = min_t(__uint64_t, resblks, 8192);
E
Eric Sandeen 已提交
644 645 646
	return resblks;
}

E
Eric Sandeen 已提交
647 648 649 650 651 652 653 654 655 656 657 658
/*
 * This function does the following on an initial mount of a file system:
 *	- reads the superblock from disk and init the mount struct
 *	- if we're a 32-bit kernel, do a size check on the superblock
 *		so we don't mount terabyte filesystems
 *	- init mount struct realtime fields
 *	- allocate inode hash table for fs
 *	- init directory manager
 *	- perform recovery and init the log manager
 */
int
xfs_mountfs(
C
Christoph Hellwig 已提交
659
	xfs_mount_t	*mp)
E
Eric Sandeen 已提交
660 661 662 663
{
	xfs_sb_t	*sbp = &(mp->m_sb);
	xfs_inode_t	*rip;
	__uint64_t	resblks;
C
Christoph Hellwig 已提交
664 665
	uint		quotamount = 0;
	uint		quotaflags = 0;
E
Eric Sandeen 已提交
666 667
	int		error = 0;

668
	xfs_sb_mount_common(mp, sbp);
E
Eric Sandeen 已提交
669

670
	/*
671 672 673 674 675
	 * Check for a mismatched features2 values.  Older kernels
	 * read & wrote into the wrong sb offset for sb_features2
	 * on some platforms due to xfs_sb_t not being 64bit size aligned
	 * when sb_features2 was added, which made older superblock
	 * reading/writing routines swap it as a 64-bit value.
676
	 *
677 678 679 680 681 682 683 684
	 * For backwards compatibility, we make both slots equal.
	 *
	 * If we detect a mismatched field, we OR the set bits into the
	 * existing features2 field in case it has already been modified; we
	 * don't want to lose any features.  We then update the bad location
	 * with the ORed value so that older kernels will see any features2
	 * flags, and mark the two fields as needing updates once the
	 * transaction subsystem is online.
685
	 */
686
	if (xfs_sb_has_mismatched_features2(sbp)) {
687
		xfs_warn(mp, "correcting sb_features alignment problem");
688
		sbp->sb_features2 |= sbp->sb_bad_features2;
689
		sbp->sb_bad_features2 = sbp->sb_features2;
690
		mp->m_update_flags |= XFS_SB_FEATURES2 | XFS_SB_BAD_FEATURES2;
691 692 693 694 695

		/*
		 * Re-check for ATTR2 in case it was found in bad_features2
		 * slot.
		 */
696 697
		if (xfs_sb_version_hasattr2(&mp->m_sb) &&
		   !(mp->m_flags & XFS_MOUNT_NOATTR2))
698
			mp->m_flags |= XFS_MOUNT_ATTR2;
699 700 701 702 703
	}

	if (xfs_sb_version_hasattr2(&mp->m_sb) &&
	   (mp->m_flags & XFS_MOUNT_NOATTR2)) {
		xfs_sb_version_removeattr2(&mp->m_sb);
704
		mp->m_update_flags |= XFS_SB_FEATURES2;
705

706 707
		/* update sb_versionnum for the clearing of the morebits */
		if (!sbp->sb_features2)
708
			mp->m_update_flags |= XFS_SB_VERSIONNUM;
709 710
	}

711 712 713 714 715 716
	/* always use v2 inodes by default now */
	if (!(mp->m_sb.sb_versionnum & XFS_SB_VERSION_NLINKBIT)) {
		mp->m_sb.sb_versionnum |= XFS_SB_VERSION_NLINKBIT;
		mp->m_update_flags |= XFS_SB_VERSIONNUM;
	}

E
Eric Sandeen 已提交
717 718 719 720 721 722
	/*
	 * Check if sb_agblocks is aligned at stripe boundary
	 * If sb_agblocks is NOT aligned turn off m_dalign since
	 * allocator alignment is within an ag, therefore ag has
	 * to be aligned at stripe boundary.
	 */
723
	error = xfs_update_alignment(mp);
E
Eric Sandeen 已提交
724
	if (error)
725
		goto out;
E
Eric Sandeen 已提交
726 727 728 729 730 731 732 733

	xfs_alloc_compute_maxlevels(mp);
	xfs_bmap_compute_maxlevels(mp, XFS_DATA_FORK);
	xfs_bmap_compute_maxlevels(mp, XFS_ATTR_FORK);
	xfs_ialloc_compute_maxlevels(mp);

	xfs_set_maxicount(mp);

C
Christoph Hellwig 已提交
734 735 736
	error = xfs_uuid_mount(mp);
	if (error)
		goto out;
L
Linus Torvalds 已提交
737

E
Eric Sandeen 已提交
738 739 740 741 742
	/*
	 * Set the minimum read and write sizes
	 */
	xfs_set_rw_sizes(mp);

743 744 745
	/* set the low space thresholds for dynamic preallocation */
	xfs_set_low_space_thresholds(mp);

E
Eric Sandeen 已提交
746 747 748 749
	/*
	 * Set the inode cluster size.
	 * This may still be overridden by the file system
	 * block size if it is larger than the chosen cluster size.
750 751 752 753 754
	 *
	 * For v5 filesystems, scale the cluster size with the inode size to
	 * keep a constant ratio of inode per cluster buffer, but only if mkfs
	 * has set the inode alignment value appropriately for larger cluster
	 * sizes.
E
Eric Sandeen 已提交
755 756
	 */
	mp->m_inode_cluster_size = XFS_INODE_BIG_CLUSTER_SIZE;
757 758 759 760 761 762 763
	if (xfs_sb_version_hascrc(&mp->m_sb)) {
		int	new_size = mp->m_inode_cluster_size;

		new_size *= mp->m_sb.sb_inodesize / XFS_DINODE_MIN_SIZE;
		if (mp->m_sb.sb_inoalignmt >= XFS_B_TO_FSBT(mp, new_size))
			mp->m_inode_cluster_size = new_size;
	}
E
Eric Sandeen 已提交
764 765 766 767 768 769 770

	/*
	 * Set inode alignment fields
	 */
	xfs_set_inoalignment(mp);

	/*
771
	 * Check that the data (and log if separate) is an ok size.
E
Eric Sandeen 已提交
772
	 */
C
Christoph Hellwig 已提交
773
	error = xfs_check_sizes(mp);
E
Eric Sandeen 已提交
774
	if (error)
775
		goto out_remove_uuid;
E
Eric Sandeen 已提交
776

L
Linus Torvalds 已提交
777 778 779
	/*
	 * Initialize realtime fields in the mount structure
	 */
E
Eric Sandeen 已提交
780 781
	error = xfs_rtmount_init(mp);
	if (error) {
782
		xfs_warn(mp, "RT mount failed");
783
		goto out_remove_uuid;
L
Linus Torvalds 已提交
784 785 786 787 788 789 790 791 792 793
	}

	/*
	 *  Copies the low order bits of the timestamp and the randomly
	 *  set "sequence" number out of a UUID.
	 */
	uuid_getnodeuniq(&sbp->sb_uuid, mp->m_fixedfsid);

	mp->m_dmevmask = 0;	/* not persistent; set after each mount */

794 795 796 797 798
	error = xfs_da_mount(mp);
	if (error) {
		xfs_warn(mp, "Failed dir/attr init: %d", error);
		goto out_remove_uuid;
	}
L
Linus Torvalds 已提交
799 800 801 802 803 804 805 806 807

	/*
	 * Initialize the precomputed transaction reservations values.
	 */
	xfs_trans_init(mp);

	/*
	 * Allocate and initialize the per-ag data.
	 */
808
	spin_lock_init(&mp->m_perag_lock);
809
	INIT_RADIX_TREE(&mp->m_perag_tree, GFP_ATOMIC);
810 811
	error = xfs_initialize_perag(mp, sbp->sb_agcount, &mp->m_maxagi);
	if (error) {
812
		xfs_warn(mp, "Failed per-ag init: %d", error);
813
		goto out_free_dir;
814
	}
L
Linus Torvalds 已提交
815

816
	if (!sbp->sb_logblocks) {
817
		xfs_warn(mp, "no log defined");
818
		XFS_ERROR_REPORT("xfs_mountfs", XFS_ERRLEVEL_LOW, mp);
D
Dave Chinner 已提交
819
		error = -EFSCORRUPTED;
820 821 822
		goto out_free_perag;
	}

L
Linus Torvalds 已提交
823 824 825
	/*
	 * log's mount-time initialization. Perform 1st part recovery if needed
	 */
826 827 828 829
	error = xfs_log_mount(mp, mp->m_logdev_targp,
			      XFS_FSB_TO_DADDR(mp, sbp->sb_logstart),
			      XFS_FSB_TO_BB(mp, sbp->sb_logblocks));
	if (error) {
830
		xfs_warn(mp, "log mount failed");
831
		goto out_fail_wait;
L
Linus Torvalds 已提交
832 833
	}

D
David Chinner 已提交
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
	/*
	 * Now the log is mounted, we know if it was an unclean shutdown or
	 * not. If it was, with the first phase of recovery has completed, we
	 * have consistent AG blocks on disk. We have not recovered EFIs yet,
	 * but they are recovered transactionally in the second recovery phase
	 * later.
	 *
	 * Hence we can safely re-initialise incore superblock counters from
	 * the per-ag data. These may not be correct if the filesystem was not
	 * cleanly unmounted, so we need to wait for recovery to finish before
	 * doing this.
	 *
	 * If the filesystem was cleanly unmounted, then we can trust the
	 * values in the superblock to be correct and we don't need to do
	 * anything here.
	 *
	 * If we are currently making the filesystem, the initialisation will
	 * fail as the perag data is in an undefined state.
	 */
	if (xfs_sb_version_haslazysbcount(&mp->m_sb) &&
	    !XFS_LAST_UNMOUNT_WAS_CLEAN(mp) &&
	     !mp->m_sb.sb_inprogress) {
		error = xfs_initialize_perag_data(mp, sbp->sb_agcount);
857
		if (error)
858
			goto out_log_dealloc;;
D
David Chinner 已提交
859
	}
860

L
Linus Torvalds 已提交
861 862 863 864
	/*
	 * Get and sanity-check the root inode.
	 * Save the pointer to it in the mount structure.
	 */
865
	error = xfs_iget(mp, NULL, sbp->sb_rootino, 0, XFS_ILOCK_EXCL, &rip);
L
Linus Torvalds 已提交
866
	if (error) {
867
		xfs_warn(mp, "failed to read root inode");
868
		goto out_log_dealloc;
L
Linus Torvalds 已提交
869 870 871 872
	}

	ASSERT(rip != NULL);

873
	if (unlikely(!S_ISDIR(rip->i_d.di_mode))) {
874
		xfs_warn(mp, "corrupted root inode %llu: not a directory",
875
			(unsigned long long)rip->i_ino);
L
Linus Torvalds 已提交
876 877 878
		xfs_iunlock(rip, XFS_ILOCK_EXCL);
		XFS_ERROR_REPORT("xfs_mountfs_int(2)", XFS_ERRLEVEL_LOW,
				 mp);
D
Dave Chinner 已提交
879
		error = -EFSCORRUPTED;
880
		goto out_rele_rip;
L
Linus Torvalds 已提交
881 882 883 884 885 886 887 888
	}
	mp->m_rootip = rip;	/* save it */

	xfs_iunlock(rip, XFS_ILOCK_EXCL);

	/*
	 * Initialize realtime inode pointers in the mount structure
	 */
E
Eric Sandeen 已提交
889 890
	error = xfs_rtmount_inodes(mp);
	if (error) {
L
Linus Torvalds 已提交
891 892 893
		/*
		 * Free up the root inode.
		 */
894
		xfs_warn(mp, "failed to read RT inodes");
895
		goto out_rele_rip;
L
Linus Torvalds 已提交
896 897 898
	}

	/*
899 900 901
	 * If this is a read-only mount defer the superblock updates until
	 * the next remount into writeable mode.  Otherwise we would never
	 * perform the update e.g. for the root filesystem.
L
Linus Torvalds 已提交
902
	 */
903 904
	if (mp->m_update_flags && !(mp->m_flags & XFS_MOUNT_RDONLY)) {
		error = xfs_mount_log_sb(mp, mp->m_update_flags);
905
		if (error) {
906
			xfs_warn(mp, "failed to write sb changes");
907
			goto out_rtunmount;
908 909
		}
	}
L
Linus Torvalds 已提交
910 911 912 913

	/*
	 * Initialise the XFS quota management subsystem for this mount
	 */
C
Christoph Hellwig 已提交
914 915 916 917 918 919 920 921 922 923 924 925 926
	if (XFS_IS_QUOTA_RUNNING(mp)) {
		error = xfs_qm_newmount(mp, &quotamount, &quotaflags);
		if (error)
			goto out_rtunmount;
	} else {
		ASSERT(!XFS_IS_QUOTA_ON(mp));

		/*
		 * If a file system had quotas running earlier, but decided to
		 * mount without -o uquota/pquota/gquota options, revoke the
		 * quotachecked license.
		 */
		if (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_ACCT) {
927
			xfs_notice(mp, "resetting quota flags");
C
Christoph Hellwig 已提交
928 929
			error = xfs_mount_reset_sbqflags(mp);
			if (error)
930
				goto out_rtunmount;
C
Christoph Hellwig 已提交
931 932
		}
	}
L
Linus Torvalds 已提交
933 934 935 936 937 938

	/*
	 * Finish recovering the file system.  This part needed to be
	 * delayed until after the root and real-time bitmap inodes
	 * were consistently read in.
	 */
C
Christoph Hellwig 已提交
939
	error = xfs_log_mount_finish(mp);
L
Linus Torvalds 已提交
940
	if (error) {
941
		xfs_warn(mp, "log mount finish failed");
942
		goto out_rtunmount;
L
Linus Torvalds 已提交
943 944 945 946 947
	}

	/*
	 * Complete the quota initialisation, post-log-replay component.
	 */
C
Christoph Hellwig 已提交
948 949 950 951 952 953 954
	if (quotamount) {
		ASSERT(mp->m_qflags == 0);
		mp->m_qflags = quotaflags;

		xfs_qm_mount_quotas(mp);
	}

955 956 957 958 959 960 961
	/*
	 * Now we are mounted, reserve a small amount of unused space for
	 * privileged transactions. This is needed so that transaction
	 * space required for critical operations can dip into this pool
	 * when at ENOSPC. This is needed for operations like create with
	 * attr, unwritten extent conversion at ENOSPC, etc. Data allocations
	 * are not allowed to use this reserved space.
962 963 964
	 *
	 * This may drive us straight to ENOSPC on mount, but that implies
	 * we were already there on the last unmount. Warn if this occurs.
965
	 */
E
Eric Sandeen 已提交
966 967 968 969
	if (!(mp->m_flags & XFS_MOUNT_RDONLY)) {
		resblks = xfs_default_resblks(mp);
		error = xfs_reserve_blocks(mp, &resblks, NULL);
		if (error)
970 971
			xfs_warn(mp,
	"Unable to allocate reserve blocks. Continuing without reserve pool.");
E
Eric Sandeen 已提交
972
	}
973

L
Linus Torvalds 已提交
974 975
	return 0;

976 977
 out_rtunmount:
	xfs_rtunmount_inodes(mp);
978
 out_rele_rip:
979
	IRELE(rip);
980
 out_log_dealloc:
981
	xfs_log_unmount(mp);
982 983 984 985
 out_fail_wait:
	if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp)
		xfs_wait_buftarg(mp->m_logdev_targp);
	xfs_wait_buftarg(mp->m_ddev_targp);
986
 out_free_perag:
987
	xfs_free_perag(mp);
988 989
 out_free_dir:
	xfs_da_unmount(mp);
990
 out_remove_uuid:
C
Christoph Hellwig 已提交
991
	xfs_uuid_unmount(mp);
992
 out:
L
Linus Torvalds 已提交
993 994 995 996 997 998 999
	return error;
}

/*
 * This flushes out the inodes,dquots and the superblock, unmounts the
 * log and makes sure that incore structures are freed.
 */
1000 1001 1002
void
xfs_unmountfs(
	struct xfs_mount	*mp)
L
Linus Torvalds 已提交
1003
{
1004 1005
	__uint64_t		resblks;
	int			error;
L
Linus Torvalds 已提交
1006

1007 1008
	cancel_delayed_work_sync(&mp->m_eofblocks_work);

C
Christoph Hellwig 已提交
1009
	xfs_qm_unmount_quotas(mp);
1010
	xfs_rtunmount_inodes(mp);
1011 1012
	IRELE(mp->m_rootip);

1013 1014
	/*
	 * We can potentially deadlock here if we have an inode cluster
M
Malcolm Parsons 已提交
1015
	 * that has been freed has its buffer still pinned in memory because
1016 1017 1018 1019 1020 1021 1022
	 * the transaction is still sitting in a iclog. The stale inodes
	 * on that buffer will have their flush locks held until the
	 * transaction hits the disk and the callbacks run. the inode
	 * flush takes the flush lock unconditionally and with nothing to
	 * push out the iclog we will never get that unlocked. hence we
	 * need to force the log first.
	 */
1023
	xfs_log_force(mp, XFS_LOG_SYNC);
1024 1025

	/*
1026 1027 1028 1029 1030 1031
	 * Flush all pending changes from the AIL.
	 */
	xfs_ail_push_all_sync(mp->m_ail);

	/*
	 * And reclaim all inodes.  At this point there should be no dirty
D
Dave Chinner 已提交
1032 1033 1034
	 * inodes and none should be pinned or locked, but use synchronous
	 * reclaim just to be sure. We can stop background inode reclaim
	 * here as well if it is still running.
1035
	 */
D
Dave Chinner 已提交
1036
	cancel_delayed_work_sync(&mp->m_reclaim_work);
1037
	xfs_reclaim_inodes(mp, SYNC_WAIT);
L
Linus Torvalds 已提交
1038

C
Christoph Hellwig 已提交
1039
	xfs_qm_unmount(mp);
1040

1041 1042 1043 1044
	/*
	 * Unreserve any blocks we have so that when we unmount we don't account
	 * the reserved free space as used. This is really only necessary for
	 * lazy superblock counting because it trusts the incore superblock
M
Malcolm Parsons 已提交
1045
	 * counters to be absolutely correct on clean unmount.
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
	 *
	 * We don't bother correcting this elsewhere for lazy superblock
	 * counting because on mount of an unclean filesystem we reconstruct the
	 * correct counter value and this is irrelevant.
	 *
	 * For non-lazy counter filesystems, this doesn't matter at all because
	 * we only every apply deltas to the superblock and hence the incore
	 * value does not matter....
	 */
	resblks = 0;
1056 1057
	error = xfs_reserve_blocks(mp, &resblks, NULL);
	if (error)
1058
		xfs_warn(mp, "Unable to free reserved block pool. "
1059 1060
				"Freespace may not be correct on next mount.");

1061
	error = xfs_log_sbcount(mp);
1062
	if (error)
1063
		xfs_warn(mp, "Unable to update superblock counters. "
1064
				"Freespace may not be correct on next mount.");
1065

1066
	xfs_log_unmount(mp);
1067
	xfs_da_unmount(mp);
C
Christoph Hellwig 已提交
1068
	xfs_uuid_unmount(mp);
L
Linus Torvalds 已提交
1069

C
Christoph Hellwig 已提交
1070
#if defined(DEBUG)
1071
	xfs_errortag_clearall(mp, 0);
L
Linus Torvalds 已提交
1072
#endif
1073
	xfs_free_perag(mp);
L
Linus Torvalds 已提交
1074 1075
}

D
David Chinner 已提交
1076 1077 1078
int
xfs_fs_writable(xfs_mount_t *mp)
{
J
Jan Kara 已提交
1079
	return !(mp->m_super->s_writers.frozen || XFS_FORCED_SHUTDOWN(mp) ||
1080
		(mp->m_flags & XFS_MOUNT_RDONLY));
D
David Chinner 已提交
1081 1082 1083
}

/*
1084 1085
 * xfs_log_sbcount
 *
1086
 * Sync the superblock counters to disk.
1087 1088
 *
 * Note this code can be called during the process of freezing, so
1089
 * we may need to use the transaction allocator which does not
1090
 * block when the transaction subsystem is in its frozen state.
D
David Chinner 已提交
1091 1092
 */
int
1093
xfs_log_sbcount(xfs_mount_t *mp)
D
David Chinner 已提交
1094 1095 1096 1097 1098 1099 1100
{
	xfs_trans_t	*tp;
	int		error;

	if (!xfs_fs_writable(mp))
		return 0;

1101
	xfs_icsb_sync_counters(mp, 0);
D
David Chinner 已提交
1102 1103 1104 1105 1106 1107 1108 1109

	/*
	 * we don't need to do this if we are updating the superblock
	 * counters on every modification.
	 */
	if (!xfs_sb_version_haslazysbcount(&mp->m_sb))
		return 0;

1110
	tp = _xfs_trans_alloc(mp, XFS_TRANS_SB_COUNT, KM_SLEEP);
1111
	error = xfs_trans_reserve(tp, &M_RES(mp)->tr_sb, 0, 0);
D
David Chinner 已提交
1112 1113 1114 1115 1116 1117
	if (error) {
		xfs_trans_cancel(tp, 0);
		return error;
	}

	xfs_mod_sb(tp, XFS_SB_IFREE | XFS_SB_ICOUNT | XFS_SB_FDBLOCKS);
1118
	xfs_trans_set_sync(tp);
1119 1120
	error = xfs_trans_commit(tp, 0);
	return error;
D
David Chinner 已提交
1121 1122
}

L
Linus Torvalds 已提交
1123
/*
1124
 * xfs_mod_incore_sb_unlocked() is a utility routine commonly used to apply
L
Linus Torvalds 已提交
1125 1126 1127 1128 1129
 * a delta to a specified field in the in-core superblock.  Simply
 * switch on the field indicated and apply the delta to that field.
 * Fields are not allowed to dip below zero, so if the delta would
 * do this do not apply it and return EINVAL.
 *
E
Eric Sandeen 已提交
1130
 * The m_sb_lock must be held when this routine is called.
L
Linus Torvalds 已提交
1131
 */
1132
STATIC int
1133 1134 1135 1136 1137
xfs_mod_incore_sb_unlocked(
	xfs_mount_t	*mp,
	xfs_sb_field_t	field,
	int64_t		delta,
	int		rsvd)
L
Linus Torvalds 已提交
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
{
	int		scounter;	/* short counter for 32 bit fields */
	long long	lcounter;	/* long counter for 64 bit fields */
	long long	res_used, rem;

	/*
	 * With the in-core superblock spin lock held, switch
	 * on the indicated field.  Apply the delta to the
	 * proper field.  If the fields value would dip below
	 * 0, then do not apply the delta and return EINVAL.
	 */
	switch (field) {
	case XFS_SBS_ICOUNT:
		lcounter = (long long)mp->m_sb.sb_icount;
		lcounter += delta;
		if (lcounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1155
			return -EINVAL;
L
Linus Torvalds 已提交
1156 1157
		}
		mp->m_sb.sb_icount = lcounter;
1158
		return 0;
L
Linus Torvalds 已提交
1159 1160 1161 1162 1163
	case XFS_SBS_IFREE:
		lcounter = (long long)mp->m_sb.sb_ifree;
		lcounter += delta;
		if (lcounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1164
			return -EINVAL;
L
Linus Torvalds 已提交
1165 1166
		}
		mp->m_sb.sb_ifree = lcounter;
1167
		return 0;
L
Linus Torvalds 已提交
1168
	case XFS_SBS_FDBLOCKS:
1169 1170
		lcounter = (long long)
			mp->m_sb.sb_fdblocks - XFS_ALLOC_SET_ASIDE(mp);
L
Linus Torvalds 已提交
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
		res_used = (long long)(mp->m_resblks - mp->m_resblks_avail);

		if (delta > 0) {		/* Putting blocks back */
			if (res_used > delta) {
				mp->m_resblks_avail += delta;
			} else {
				rem = delta - res_used;
				mp->m_resblks_avail = mp->m_resblks;
				lcounter += rem;
			}
		} else {				/* Taking blocks away */
			lcounter += delta;
1183 1184 1185 1186 1187
			if (lcounter >= 0) {
				mp->m_sb.sb_fdblocks = lcounter +
							XFS_ALLOC_SET_ASIDE(mp);
				return 0;
			}
L
Linus Torvalds 已提交
1188

1189 1190 1191 1192 1193
			/*
			 * We are out of blocks, use any available reserved
			 * blocks if were allowed to.
			 */
			if (!rsvd)
D
Dave Chinner 已提交
1194
				return -ENOSPC;
L
Linus Torvalds 已提交
1195

1196 1197 1198 1199
			lcounter = (long long)mp->m_resblks_avail + delta;
			if (lcounter >= 0) {
				mp->m_resblks_avail = lcounter;
				return 0;
L
Linus Torvalds 已提交
1200
			}
1201 1202 1203 1204
			printk_once(KERN_WARNING
				"Filesystem \"%s\": reserve blocks depleted! "
				"Consider increasing reserve pool size.",
				mp->m_fsname);
D
Dave Chinner 已提交
1205
			return -ENOSPC;
L
Linus Torvalds 已提交
1206 1207
		}

1208
		mp->m_sb.sb_fdblocks = lcounter + XFS_ALLOC_SET_ASIDE(mp);
1209
		return 0;
L
Linus Torvalds 已提交
1210 1211 1212 1213
	case XFS_SBS_FREXTENTS:
		lcounter = (long long)mp->m_sb.sb_frextents;
		lcounter += delta;
		if (lcounter < 0) {
D
Dave Chinner 已提交
1214
			return -ENOSPC;
L
Linus Torvalds 已提交
1215 1216
		}
		mp->m_sb.sb_frextents = lcounter;
1217
		return 0;
L
Linus Torvalds 已提交
1218 1219 1220 1221 1222
	case XFS_SBS_DBLOCKS:
		lcounter = (long long)mp->m_sb.sb_dblocks;
		lcounter += delta;
		if (lcounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1223
			return -EINVAL;
L
Linus Torvalds 已提交
1224 1225
		}
		mp->m_sb.sb_dblocks = lcounter;
1226
		return 0;
L
Linus Torvalds 已提交
1227 1228 1229 1230 1231
	case XFS_SBS_AGCOUNT:
		scounter = mp->m_sb.sb_agcount;
		scounter += delta;
		if (scounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1232
			return -EINVAL;
L
Linus Torvalds 已提交
1233 1234
		}
		mp->m_sb.sb_agcount = scounter;
1235
		return 0;
L
Linus Torvalds 已提交
1236 1237 1238 1239 1240
	case XFS_SBS_IMAX_PCT:
		scounter = mp->m_sb.sb_imax_pct;
		scounter += delta;
		if (scounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1241
			return -EINVAL;
L
Linus Torvalds 已提交
1242 1243
		}
		mp->m_sb.sb_imax_pct = scounter;
1244
		return 0;
L
Linus Torvalds 已提交
1245 1246 1247 1248 1249
	case XFS_SBS_REXTSIZE:
		scounter = mp->m_sb.sb_rextsize;
		scounter += delta;
		if (scounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1250
			return -EINVAL;
L
Linus Torvalds 已提交
1251 1252
		}
		mp->m_sb.sb_rextsize = scounter;
1253
		return 0;
L
Linus Torvalds 已提交
1254 1255 1256 1257 1258
	case XFS_SBS_RBMBLOCKS:
		scounter = mp->m_sb.sb_rbmblocks;
		scounter += delta;
		if (scounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1259
			return -EINVAL;
L
Linus Torvalds 已提交
1260 1261
		}
		mp->m_sb.sb_rbmblocks = scounter;
1262
		return 0;
L
Linus Torvalds 已提交
1263 1264 1265 1266 1267
	case XFS_SBS_RBLOCKS:
		lcounter = (long long)mp->m_sb.sb_rblocks;
		lcounter += delta;
		if (lcounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1268
			return -EINVAL;
L
Linus Torvalds 已提交
1269 1270
		}
		mp->m_sb.sb_rblocks = lcounter;
1271
		return 0;
L
Linus Torvalds 已提交
1272 1273 1274 1275 1276
	case XFS_SBS_REXTENTS:
		lcounter = (long long)mp->m_sb.sb_rextents;
		lcounter += delta;
		if (lcounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1277
			return -EINVAL;
L
Linus Torvalds 已提交
1278 1279
		}
		mp->m_sb.sb_rextents = lcounter;
1280
		return 0;
L
Linus Torvalds 已提交
1281 1282 1283 1284 1285
	case XFS_SBS_REXTSLOG:
		scounter = mp->m_sb.sb_rextslog;
		scounter += delta;
		if (scounter < 0) {
			ASSERT(0);
D
Dave Chinner 已提交
1286
			return -EINVAL;
L
Linus Torvalds 已提交
1287 1288
		}
		mp->m_sb.sb_rextslog = scounter;
1289
		return 0;
L
Linus Torvalds 已提交
1290 1291
	default:
		ASSERT(0);
D
Dave Chinner 已提交
1292
		return -EINVAL;
L
Linus Torvalds 已提交
1293 1294 1295 1296 1297 1298
	}
}

/*
 * xfs_mod_incore_sb() is used to change a field in the in-core
 * superblock structure by the specified delta.  This modification
E
Eric Sandeen 已提交
1299
 * is protected by the m_sb_lock.  Just use the xfs_mod_incore_sb_unlocked()
L
Linus Torvalds 已提交
1300 1301 1302
 * routine to do the work.
 */
int
1303
xfs_mod_incore_sb(
1304 1305 1306 1307
	struct xfs_mount	*mp,
	xfs_sb_field_t		field,
	int64_t			delta,
	int			rsvd)
L
Linus Torvalds 已提交
1308
{
1309
	int			status;
L
Linus Torvalds 已提交
1310

1311
#ifdef HAVE_PERCPU_SB
1312
	ASSERT(field < XFS_SBS_ICOUNT || field > XFS_SBS_FDBLOCKS);
1313
#endif
1314 1315 1316
	spin_lock(&mp->m_sb_lock);
	status = xfs_mod_incore_sb_unlocked(mp, field, delta, rsvd);
	spin_unlock(&mp->m_sb_lock);
1317

1318
	return status;
L
Linus Torvalds 已提交
1319 1320 1321
}

/*
1322
 * Change more than one field in the in-core superblock structure at a time.
L
Linus Torvalds 已提交
1323
 *
1324 1325 1326 1327 1328 1329 1330 1331
 * The fields and changes to those fields are specified in the array of
 * xfs_mod_sb structures passed in.  Either all of the specified deltas
 * will be applied or none of them will.  If any modified field dips below 0,
 * then all modifications will be backed out and EINVAL will be returned.
 *
 * Note that this function may not be used for the superblock values that
 * are tracked with the in-memory per-cpu counters - a direct call to
 * xfs_icsb_modify_counters is required for these.
L
Linus Torvalds 已提交
1332 1333
 */
int
1334 1335 1336 1337 1338
xfs_mod_incore_sb_batch(
	struct xfs_mount	*mp,
	xfs_mod_sb_t		*msb,
	uint			nmsb,
	int			rsvd)
L
Linus Torvalds 已提交
1339
{
1340
	xfs_mod_sb_t		*msbp;
1341
	int			error = 0;
L
Linus Torvalds 已提交
1342 1343

	/*
1344 1345 1346 1347
	 * Loop through the array of mod structures and apply each individually.
	 * If any fail, then back out all those which have already been applied.
	 * Do all of this within the scope of the m_sb_lock so that all of the
	 * changes will be atomic.
L
Linus Torvalds 已提交
1348
	 */
E
Eric Sandeen 已提交
1349
	spin_lock(&mp->m_sb_lock);
1350
	for (msbp = msb; msbp < (msb + nmsb); msbp++) {
1351 1352
		ASSERT(msbp->msb_field < XFS_SBS_ICOUNT ||
		       msbp->msb_field > XFS_SBS_FDBLOCKS);
1353

1354 1355 1356 1357
		error = xfs_mod_incore_sb_unlocked(mp, msbp->msb_field,
						   msbp->msb_delta, rsvd);
		if (error)
			goto unwind;
L
Linus Torvalds 已提交
1358
	}
1359 1360
	spin_unlock(&mp->m_sb_lock);
	return 0;
L
Linus Torvalds 已提交
1361

1362 1363 1364 1365 1366
unwind:
	while (--msbp >= msb) {
		error = xfs_mod_incore_sb_unlocked(mp, msbp->msb_field,
						   -msbp->msb_delta, rsvd);
		ASSERT(error == 0);
L
Linus Torvalds 已提交
1367
	}
E
Eric Sandeen 已提交
1368
	spin_unlock(&mp->m_sb_lock);
1369
	return error;
L
Linus Torvalds 已提交
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
}

/*
 * xfs_getsb() is called to obtain the buffer for the superblock.
 * The buffer is returned locked and read in from disk.
 * The buffer should be released with a call to xfs_brelse().
 *
 * If the flags parameter is BUF_TRYLOCK, then we'll only return
 * the superblock buffer if it can be locked without sleeping.
 * If it can't then we'll return NULL.
 */
1381
struct xfs_buf *
L
Linus Torvalds 已提交
1382
xfs_getsb(
1383 1384
	struct xfs_mount	*mp,
	int			flags)
L
Linus Torvalds 已提交
1385
{
1386
	struct xfs_buf		*bp = mp->m_sb_bp;
L
Linus Torvalds 已提交
1387

1388 1389
	if (!xfs_buf_trylock(bp)) {
		if (flags & XBF_TRYLOCK)
L
Linus Torvalds 已提交
1390
			return NULL;
1391
		xfs_buf_lock(bp);
L
Linus Torvalds 已提交
1392
	}
1393

1394
	xfs_buf_hold(bp);
L
Linus Torvalds 已提交
1395
	ASSERT(XFS_BUF_ISDONE(bp));
1396
	return bp;
L
Linus Torvalds 已提交
1397 1398 1399 1400 1401 1402 1403
}

/*
 * Used to free the superblock along various error paths.
 */
void
xfs_freesb(
D
Dave Chinner 已提交
1404
	struct xfs_mount	*mp)
L
Linus Torvalds 已提交
1405
{
D
Dave Chinner 已提交
1406
	struct xfs_buf		*bp = mp->m_sb_bp;
L
Linus Torvalds 已提交
1407

D
Dave Chinner 已提交
1408
	xfs_buf_lock(bp);
L
Linus Torvalds 已提交
1409
	mp->m_sb_bp = NULL;
D
Dave Chinner 已提交
1410
	xfs_buf_relse(bp);
L
Linus Torvalds 已提交
1411 1412 1413 1414
}

/*
 * Used to log changes to the superblock unit and width fields which could
1415 1416
 * be altered by the mount options, as well as any potential sb_features2
 * fixup. Only the first superblock is updated.
L
Linus Torvalds 已提交
1417
 */
1418
int
1419
xfs_mount_log_sb(
L
Linus Torvalds 已提交
1420 1421 1422 1423
	xfs_mount_t	*mp,
	__int64_t	fields)
{
	xfs_trans_t	*tp;
1424
	int		error;
L
Linus Torvalds 已提交
1425

1426
	ASSERT(fields & (XFS_SB_UNIT | XFS_SB_WIDTH | XFS_SB_UUID |
1427 1428
			 XFS_SB_FEATURES2 | XFS_SB_BAD_FEATURES2 |
			 XFS_SB_VERSIONNUM));
L
Linus Torvalds 已提交
1429 1430

	tp = xfs_trans_alloc(mp, XFS_TRANS_SB_UNIT);
1431
	error = xfs_trans_reserve(tp, &M_RES(mp)->tr_sb, 0, 0);
1432
	if (error) {
L
Linus Torvalds 已提交
1433
		xfs_trans_cancel(tp, 0);
1434
		return error;
L
Linus Torvalds 已提交
1435 1436
	}
	xfs_mod_sb(tp, fields);
1437 1438
	error = xfs_trans_commit(tp, 0);
	return error;
L
Linus Torvalds 已提交
1439
}
1440

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
/*
 * If the underlying (data/log/rt) device is readonly, there are some
 * operations that cannot proceed.
 */
int
xfs_dev_is_read_only(
	struct xfs_mount	*mp,
	char			*message)
{
	if (xfs_readonly_buftarg(mp->m_ddev_targp) ||
	    xfs_readonly_buftarg(mp->m_logdev_targp) ||
	    (mp->m_rtdev_targp && xfs_readonly_buftarg(mp->m_rtdev_targp))) {
1453 1454
		xfs_notice(mp, "%s required on read-only device.", message);
		xfs_notice(mp, "write access unavailable, cannot proceed.");
D
Dave Chinner 已提交
1455
		return -EROFS;
1456 1457 1458
	}
	return 0;
}
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482

#ifdef HAVE_PERCPU_SB
/*
 * Per-cpu incore superblock counters
 *
 * Simple concept, difficult implementation
 *
 * Basically, replace the incore superblock counters with a distributed per cpu
 * counter for contended fields (e.g.  free block count).
 *
 * Difficulties arise in that the incore sb is used for ENOSPC checking, and
 * hence needs to be accurately read when we are running low on space. Hence
 * there is a method to enable and disable the per-cpu counters based on how
 * much "stuff" is available in them.
 *
 * Basically, a counter is enabled if there is enough free resource to justify
 * running a per-cpu fast-path. If the per-cpu counter runs out (i.e. a local
 * ENOSPC), then we disable the counters to synchronise all callers and
 * re-distribute the available resources.
 *
 * If, once we redistributed the available resources, we still get a failure,
 * we disable the per-cpu counter and go through the slow path.
 *
 * The slow path is the current xfs_mod_incore_sb() function.  This means that
M
Malcolm Parsons 已提交
1483
 * when we disable a per-cpu counter, we need to drain its resources back to
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
 * the global superblock. We do this after disabling the counter to prevent
 * more threads from queueing up on the counter.
 *
 * Essentially, this means that we still need a lock in the fast path to enable
 * synchronisation between the global counters and the per-cpu counters. This
 * is not a problem because the lock will be local to a CPU almost all the time
 * and have little contention except when we get to ENOSPC conditions.
 *
 * Basically, this lock becomes a barrier that enables us to lock out the fast
 * path while we do things like enabling and disabling counters and
 * synchronising the counters.
 *
 * Locking rules:
 *
E
Eric Sandeen 已提交
1498
 * 	1. m_sb_lock before picking up per-cpu locks
1499
 * 	2. per-cpu locks always picked up via for_each_online_cpu() order
E
Eric Sandeen 已提交
1500
 * 	3. accurate counter sync requires m_sb_lock + per cpu locks
1501
 * 	4. modifying per-cpu counters requires holding per-cpu lock
E
Eric Sandeen 已提交
1502 1503
 * 	5. modifying global counters requires holding m_sb_lock
 *	6. enabling or disabling a counter requires holding the m_sb_lock 
1504 1505 1506 1507 1508 1509 1510
 *	   and _none_ of the per-cpu locks.
 *
 * Disabled counters are only ever re-enabled by a balance operation
 * that results in more free resources per CPU than a given threshold.
 * To ensure counters don't remain disabled, they are rebalanced when
 * the global resource goes above a higher threshold (i.e. some hysteresis
 * is present to prevent thrashing).
1511 1512
 */

1513
#ifdef CONFIG_HOTPLUG_CPU
1514 1515
/*
 * hot-plug CPU notifier support.
1516
 *
1517 1518 1519 1520
 * We need a notifier per filesystem as we need to be able to identify
 * the filesystem to balance the counters out. This is achieved by
 * having a notifier block embedded in the xfs_mount_t and doing pointer
 * magic to get the mount pointer from the notifier block address.
1521
 */
1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
STATIC int
xfs_icsb_cpu_notify(
	struct notifier_block *nfb,
	unsigned long action,
	void *hcpu)
{
	xfs_icsb_cnts_t *cntp;
	xfs_mount_t	*mp;

	mp = (xfs_mount_t *)container_of(nfb, xfs_mount_t, m_icsb_notifier);
	cntp = (xfs_icsb_cnts_t *)
			per_cpu_ptr(mp->m_sb_cnts, (unsigned long)hcpu);
	switch (action) {
	case CPU_UP_PREPARE:
1536
	case CPU_UP_PREPARE_FROZEN:
1537 1538
		/* Easy Case - initialize the area and locks, and
		 * then rebalance when online does everything else for us. */
1539
		memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
1540 1541
		break;
	case CPU_ONLINE:
1542
	case CPU_ONLINE_FROZEN:
1543
		xfs_icsb_lock(mp);
1544 1545 1546
		xfs_icsb_balance_counter(mp, XFS_SBS_ICOUNT, 0);
		xfs_icsb_balance_counter(mp, XFS_SBS_IFREE, 0);
		xfs_icsb_balance_counter(mp, XFS_SBS_FDBLOCKS, 0);
1547
		xfs_icsb_unlock(mp);
1548 1549
		break;
	case CPU_DEAD:
1550
	case CPU_DEAD_FROZEN:
1551 1552 1553
		/* Disable all the counters, then fold the dead cpu's
		 * count into the total on the global superblock and
		 * re-enable the counters. */
1554
		xfs_icsb_lock(mp);
E
Eric Sandeen 已提交
1555
		spin_lock(&mp->m_sb_lock);
1556 1557 1558 1559 1560 1561 1562 1563
		xfs_icsb_disable_counter(mp, XFS_SBS_ICOUNT);
		xfs_icsb_disable_counter(mp, XFS_SBS_IFREE);
		xfs_icsb_disable_counter(mp, XFS_SBS_FDBLOCKS);

		mp->m_sb.sb_icount += cntp->icsb_icount;
		mp->m_sb.sb_ifree += cntp->icsb_ifree;
		mp->m_sb.sb_fdblocks += cntp->icsb_fdblocks;

1564
		memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
1565

1566 1567 1568
		xfs_icsb_balance_counter_locked(mp, XFS_SBS_ICOUNT, 0);
		xfs_icsb_balance_counter_locked(mp, XFS_SBS_IFREE, 0);
		xfs_icsb_balance_counter_locked(mp, XFS_SBS_FDBLOCKS, 0);
E
Eric Sandeen 已提交
1569
		spin_unlock(&mp->m_sb_lock);
1570
		xfs_icsb_unlock(mp);
1571 1572 1573 1574 1575
		break;
	}

	return NOTIFY_OK;
}
1576
#endif /* CONFIG_HOTPLUG_CPU */
1577

1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
int
xfs_icsb_init_counters(
	xfs_mount_t	*mp)
{
	xfs_icsb_cnts_t *cntp;
	int		i;

	mp->m_sb_cnts = alloc_percpu(xfs_icsb_cnts_t);
	if (mp->m_sb_cnts == NULL)
		return -ENOMEM;

	for_each_online_cpu(i) {
		cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
1591
		memset(cntp, 0, sizeof(xfs_icsb_cnts_t));
1592
	}
1593 1594 1595

	mutex_init(&mp->m_icsb_mutex);

1596 1597 1598 1599 1600
	/*
	 * start with all counters disabled so that the
	 * initial balance kicks us off correctly
	 */
	mp->m_icsb_counters = -1;
1601 1602 1603 1604 1605 1606 1607

#ifdef CONFIG_HOTPLUG_CPU
	mp->m_icsb_notifier.notifier_call = xfs_icsb_cpu_notify;
	mp->m_icsb_notifier.priority = 0;
	register_hotcpu_notifier(&mp->m_icsb_notifier);
#endif /* CONFIG_HOTPLUG_CPU */

1608 1609 1610
	return 0;
}

1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
void
xfs_icsb_reinit_counters(
	xfs_mount_t	*mp)
{
	xfs_icsb_lock(mp);
	/*
	 * start with all counters disabled so that the
	 * initial balance kicks us off correctly
	 */
	mp->m_icsb_counters = -1;
1621 1622 1623
	xfs_icsb_balance_counter(mp, XFS_SBS_ICOUNT, 0);
	xfs_icsb_balance_counter(mp, XFS_SBS_IFREE, 0);
	xfs_icsb_balance_counter(mp, XFS_SBS_FDBLOCKS, 0);
1624 1625 1626
	xfs_icsb_unlock(mp);
}

C
Christoph Hellwig 已提交
1627
void
1628 1629 1630
xfs_icsb_destroy_counters(
	xfs_mount_t	*mp)
{
1631
	if (mp->m_sb_cnts) {
1632
		unregister_hotcpu_notifier(&mp->m_icsb_notifier);
1633
		free_percpu(mp->m_sb_cnts);
1634
	}
1635
	mutex_destroy(&mp->m_icsb_mutex);
1636 1637
}

1638
STATIC void
1639 1640 1641 1642 1643 1644 1645 1646
xfs_icsb_lock_cntr(
	xfs_icsb_cnts_t	*icsbp)
{
	while (test_and_set_bit(XFS_ICSB_FLAG_LOCK, &icsbp->icsb_flags)) {
		ndelay(1000);
	}
}

1647
STATIC void
1648 1649 1650 1651 1652 1653
xfs_icsb_unlock_cntr(
	xfs_icsb_cnts_t	*icsbp)
{
	clear_bit(XFS_ICSB_FLAG_LOCK, &icsbp->icsb_flags);
}

1654

1655
STATIC void
1656 1657 1658 1659 1660 1661 1662 1663
xfs_icsb_lock_all_counters(
	xfs_mount_t	*mp)
{
	xfs_icsb_cnts_t *cntp;
	int		i;

	for_each_online_cpu(i) {
		cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
1664
		xfs_icsb_lock_cntr(cntp);
1665 1666 1667
	}
}

1668
STATIC void
1669 1670 1671 1672 1673 1674 1675 1676
xfs_icsb_unlock_all_counters(
	xfs_mount_t	*mp)
{
	xfs_icsb_cnts_t *cntp;
	int		i;

	for_each_online_cpu(i) {
		cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
1677
		xfs_icsb_unlock_cntr(cntp);
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
	}
}

STATIC void
xfs_icsb_count(
	xfs_mount_t	*mp,
	xfs_icsb_cnts_t	*cnt,
	int		flags)
{
	xfs_icsb_cnts_t *cntp;
	int		i;

	memset(cnt, 0, sizeof(xfs_icsb_cnts_t));

	if (!(flags & XFS_ICSB_LAZY_COUNT))
		xfs_icsb_lock_all_counters(mp);

	for_each_online_cpu(i) {
		cntp = (xfs_icsb_cnts_t *)per_cpu_ptr(mp->m_sb_cnts, i);
		cnt->icsb_icount += cntp->icsb_icount;
		cnt->icsb_ifree += cntp->icsb_ifree;
		cnt->icsb_fdblocks += cntp->icsb_fdblocks;
	}

	if (!(flags & XFS_ICSB_LAZY_COUNT))
		xfs_icsb_unlock_all_counters(mp);
}

STATIC int
xfs_icsb_counter_disabled(
	xfs_mount_t	*mp,
	xfs_sb_field_t	field)
{
	ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));
	return test_bit(field, &mp->m_icsb_counters);
}

1715
STATIC void
1716 1717 1718 1719 1720 1721 1722 1723
xfs_icsb_disable_counter(
	xfs_mount_t	*mp,
	xfs_sb_field_t	field)
{
	xfs_icsb_cnts_t	cnt;

	ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));

1724 1725 1726 1727 1728 1729 1730 1731 1732
	/*
	 * If we are already disabled, then there is nothing to do
	 * here. We check before locking all the counters to avoid
	 * the expensive lock operation when being called in the
	 * slow path and the counter is already disabled. This is
	 * safe because the only time we set or clear this state is under
	 * the m_icsb_mutex.
	 */
	if (xfs_icsb_counter_disabled(mp, field))
1733
		return;
1734

1735 1736 1737 1738
	xfs_icsb_lock_all_counters(mp);
	if (!test_and_set_bit(field, &mp->m_icsb_counters)) {
		/* drain back to superblock */

1739
		xfs_icsb_count(mp, &cnt, XFS_ICSB_LAZY_COUNT);
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
		switch(field) {
		case XFS_SBS_ICOUNT:
			mp->m_sb.sb_icount = cnt.icsb_icount;
			break;
		case XFS_SBS_IFREE:
			mp->m_sb.sb_ifree = cnt.icsb_ifree;
			break;
		case XFS_SBS_FDBLOCKS:
			mp->m_sb.sb_fdblocks = cnt.icsb_fdblocks;
			break;
		default:
			BUG();
		}
	}

	xfs_icsb_unlock_all_counters(mp);
}

STATIC void
xfs_icsb_enable_counter(
	xfs_mount_t	*mp,
	xfs_sb_field_t	field,
	uint64_t	count,
	uint64_t	resid)
{
	xfs_icsb_cnts_t	*cntp;
	int		i;

	ASSERT((field >= XFS_SBS_ICOUNT) && (field <= XFS_SBS_FDBLOCKS));

	xfs_icsb_lock_all_counters(mp);
	for_each_online_cpu(i) {
		cntp = per_cpu_ptr(mp->m_sb_cnts, i);
		switch (field) {
		case XFS_SBS_ICOUNT:
			cntp->icsb_icount = count + resid;
			break;
		case XFS_SBS_IFREE:
			cntp->icsb_ifree = count + resid;
			break;
		case XFS_SBS_FDBLOCKS:
			cntp->icsb_fdblocks = count + resid;
			break;
		default:
			BUG();
			break;
		}
		resid = 0;
	}
	clear_bit(field, &mp->m_icsb_counters);
	xfs_icsb_unlock_all_counters(mp);
}

1793
void
1794
xfs_icsb_sync_counters_locked(
1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
	xfs_mount_t	*mp,
	int		flags)
{
	xfs_icsb_cnts_t	cnt;

	xfs_icsb_count(mp, &cnt, flags);

	if (!xfs_icsb_counter_disabled(mp, XFS_SBS_ICOUNT))
		mp->m_sb.sb_icount = cnt.icsb_icount;
	if (!xfs_icsb_counter_disabled(mp, XFS_SBS_IFREE))
		mp->m_sb.sb_ifree = cnt.icsb_ifree;
	if (!xfs_icsb_counter_disabled(mp, XFS_SBS_FDBLOCKS))
		mp->m_sb.sb_fdblocks = cnt.icsb_fdblocks;
}

/*
 * Accurate update of per-cpu counters to incore superblock
 */
1813
void
1814
xfs_icsb_sync_counters(
1815 1816
	xfs_mount_t	*mp,
	int		flags)
1817
{
1818 1819 1820
	spin_lock(&mp->m_sb_lock);
	xfs_icsb_sync_counters_locked(mp, flags);
	spin_unlock(&mp->m_sb_lock);
1821 1822 1823 1824 1825
}

/*
 * Balance and enable/disable counters as necessary.
 *
1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
 * Thresholds for re-enabling counters are somewhat magic.  inode counts are
 * chosen to be the same number as single on disk allocation chunk per CPU, and
 * free blocks is something far enough zero that we aren't going thrash when we
 * get near ENOSPC. We also need to supply a minimum we require per cpu to
 * prevent looping endlessly when xfs_alloc_space asks for more than will
 * be distributed to a single CPU but each CPU has enough blocks to be
 * reenabled.
 *
 * Note that we can be called when counters are already disabled.
 * xfs_icsb_disable_counter() optimises the counter locking in this case to
 * prevent locking every per-cpu counter needlessly.
1837
 */
1838 1839

#define XFS_ICSB_INO_CNTR_REENABLE	(uint64_t)64
1840
#define XFS_ICSB_FDBLK_CNTR_REENABLE(mp) \
1841
		(uint64_t)(512 + XFS_ALLOC_SET_ASIDE(mp))
1842
STATIC void
1843
xfs_icsb_balance_counter_locked(
1844 1845
	xfs_mount_t	*mp,
	xfs_sb_field_t  field,
1846
	int		min_per_cpu)
1847
{
1848
	uint64_t	count, resid;
1849
	int		weight = num_online_cpus();
1850
	uint64_t	min = (uint64_t)min_per_cpu;
1851 1852 1853 1854 1855 1856 1857 1858 1859

	/* disable counter and sync counter */
	xfs_icsb_disable_counter(mp, field);

	/* update counters  - first CPU gets residual*/
	switch (field) {
	case XFS_SBS_ICOUNT:
		count = mp->m_sb.sb_icount;
		resid = do_div(count, weight);
1860
		if (count < max(min, XFS_ICSB_INO_CNTR_REENABLE))
1861
			return;
1862 1863 1864 1865
		break;
	case XFS_SBS_IFREE:
		count = mp->m_sb.sb_ifree;
		resid = do_div(count, weight);
1866
		if (count < max(min, XFS_ICSB_INO_CNTR_REENABLE))
1867
			return;
1868 1869 1870 1871
		break;
	case XFS_SBS_FDBLOCKS:
		count = mp->m_sb.sb_fdblocks;
		resid = do_div(count, weight);
1872
		if (count < max(min, XFS_ICSB_FDBLK_CNTR_REENABLE(mp)))
1873
			return;
1874 1875 1876
		break;
	default:
		BUG();
1877
		count = resid = 0;	/* quiet, gcc */
1878 1879 1880 1881
		break;
	}

	xfs_icsb_enable_counter(mp, field, count, resid);
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
}

STATIC void
xfs_icsb_balance_counter(
	xfs_mount_t	*mp,
	xfs_sb_field_t  fields,
	int		min_per_cpu)
{
	spin_lock(&mp->m_sb_lock);
	xfs_icsb_balance_counter_locked(mp, fields, min_per_cpu);
	spin_unlock(&mp->m_sb_lock);
1893 1894
}

1895
int
1896
xfs_icsb_modify_counters(
1897 1898
	xfs_mount_t	*mp,
	xfs_sb_field_t	field,
1899
	int64_t		delta,
1900
	int		rsvd)
1901 1902 1903
{
	xfs_icsb_cnts_t	*icsbp;
	long long	lcounter;	/* long counter for 64 bit fields */
1904
	int		ret = 0;
1905

1906
	might_sleep();
1907
again:
1908 1909
	preempt_disable();
	icsbp = this_cpu_ptr(mp->m_sb_cnts);
1910 1911 1912 1913

	/*
	 * if the counter is disabled, go to slow path
	 */
1914 1915
	if (unlikely(xfs_icsb_counter_disabled(mp, field)))
		goto slow_path;
1916 1917 1918 1919 1920
	xfs_icsb_lock_cntr(icsbp);
	if (unlikely(xfs_icsb_counter_disabled(mp, field))) {
		xfs_icsb_unlock_cntr(icsbp);
		goto slow_path;
	}
1921 1922 1923 1924 1925 1926

	switch (field) {
	case XFS_SBS_ICOUNT:
		lcounter = icsbp->icsb_icount;
		lcounter += delta;
		if (unlikely(lcounter < 0))
1927
			goto balance_counter;
1928 1929 1930 1931 1932 1933 1934
		icsbp->icsb_icount = lcounter;
		break;

	case XFS_SBS_IFREE:
		lcounter = icsbp->icsb_ifree;
		lcounter += delta;
		if (unlikely(lcounter < 0))
1935
			goto balance_counter;
1936 1937 1938 1939 1940 1941
		icsbp->icsb_ifree = lcounter;
		break;

	case XFS_SBS_FDBLOCKS:
		BUG_ON((mp->m_resblks - mp->m_resblks_avail) != 0);

1942
		lcounter = icsbp->icsb_fdblocks - XFS_ALLOC_SET_ASIDE(mp);
1943 1944
		lcounter += delta;
		if (unlikely(lcounter < 0))
1945
			goto balance_counter;
1946
		icsbp->icsb_fdblocks = lcounter + XFS_ALLOC_SET_ASIDE(mp);
1947 1948 1949 1950 1951
		break;
	default:
		BUG();
		break;
	}
1952
	xfs_icsb_unlock_cntr(icsbp);
1953
	preempt_enable();
1954 1955 1956
	return 0;

slow_path:
1957
	preempt_enable();
1958

1959 1960 1961 1962 1963
	/*
	 * serialise with a mutex so we don't burn lots of cpu on
	 * the superblock lock. We still need to hold the superblock
	 * lock, however, when we modify the global structures.
	 */
1964
	xfs_icsb_lock(mp);
1965 1966 1967 1968 1969 1970 1971 1972

	/*
	 * Now running atomically.
	 *
	 * If the counter is enabled, someone has beaten us to rebalancing.
	 * Drop the lock and try again in the fast path....
	 */
	if (!(xfs_icsb_counter_disabled(mp, field))) {
1973
		xfs_icsb_unlock(mp);
1974 1975 1976
		goto again;
	}

1977 1978 1979 1980 1981
	/*
	 * The counter is currently disabled. Because we are
	 * running atomically here, we know a rebalance cannot
	 * be in progress. Hence we can go straight to operating
	 * on the global superblock. We do not call xfs_mod_incore_sb()
E
Eric Sandeen 已提交
1982
	 * here even though we need to get the m_sb_lock. Doing so
1983
	 * will cause us to re-enter this function and deadlock.
E
Eric Sandeen 已提交
1984
	 * Hence we get the m_sb_lock ourselves and then call
1985 1986 1987
	 * xfs_mod_incore_sb_unlocked() as the unlocked path operates
	 * directly on the global counters.
	 */
E
Eric Sandeen 已提交
1988
	spin_lock(&mp->m_sb_lock);
1989
	ret = xfs_mod_incore_sb_unlocked(mp, field, delta, rsvd);
E
Eric Sandeen 已提交
1990
	spin_unlock(&mp->m_sb_lock);
1991

1992 1993 1994 1995 1996 1997
	/*
	 * Now that we've modified the global superblock, we
	 * may be able to re-enable the distributed counters
	 * (e.g. lots of space just got freed). After that
	 * we are done.
	 */
D
Dave Chinner 已提交
1998
	if (ret != -ENOSPC)
1999
		xfs_icsb_balance_counter(mp, field, 0);
2000
	xfs_icsb_unlock(mp);
2001 2002
	return ret;

2003 2004
balance_counter:
	xfs_icsb_unlock_cntr(icsbp);
2005
	preempt_enable();
2006

2007 2008 2009 2010 2011 2012
	/*
	 * We may have multiple threads here if multiple per-cpu
	 * counters run dry at the same time. This will mean we can
	 * do more balances than strictly necessary but it is not
	 * the common slowpath case.
	 */
2013
	xfs_icsb_lock(mp);
2014 2015 2016 2017 2018 2019 2020 2021 2022

	/*
	 * running atomically.
	 *
	 * This will leave the counter in the correct state for future
	 * accesses. After the rebalance, we simply try again and our retry
	 * will either succeed through the fast path or slow path without
	 * another balance operation being required.
	 */
2023
	xfs_icsb_balance_counter(mp, field, delta);
2024
	xfs_icsb_unlock(mp);
2025
	goto again;
2026
}
2027

2028
#endif