i915_gem_shrinker.c 14.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright © 2008-2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#include <linux/oom.h>
#include <linux/shmem_fs.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/pci.h>
#include <linux/dma-buf.h>
31
#include <linux/vmalloc.h>
32 33 34 35 36 37 38 39 40 41 42
#include <drm/drmP.h>
#include <drm/i915_drm.h>

#include "i915_drv.h"
#include "i915_trace.h"

static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
{
	if (!mutex_is_locked(mutex))
		return false;

43
#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)
44 45 46 47 48 49 50
	return mutex->owner == task;
#else
	/* Since UP may be pre-empted, we cannot assume that we own the lock */
	return false;
#endif
}

51 52 53 54 55
static int num_vma_bound(struct drm_i915_gem_object *obj)
{
	struct i915_vma *vma;
	int count = 0;

56
	list_for_each_entry(vma, &obj->vma_list, obj_link) {
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
		if (drm_mm_node_allocated(&vma->node))
			count++;
		if (vma->pin_count)
			count++;
	}

	return count;
}

static bool swap_available(void)
{
	return get_nr_swap_pages() > 0;
}

static bool can_release_pages(struct drm_i915_gem_object *obj)
{
73 74 75 76
	/* Only shmemfs objects are backed by swap */
	if (!obj->base.filp)
		return false;

77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
	/* Only report true if by unbinding the object and putting its pages
	 * we can actually make forward progress towards freeing physical
	 * pages.
	 *
	 * If the pages are pinned for any other reason than being bound
	 * to the GPU, simply unbinding from the GPU is not going to succeed
	 * in releasing our pin count on the pages themselves.
	 */
	if (obj->pages_pin_count != num_vma_bound(obj))
		return false;

	/* We can only return physical pages to the system if we can either
	 * discard the contents (because the user has marked them as being
	 * purgeable) or if we can move their contents out to swap.
	 */
	return swap_available() || obj->madv == I915_MADV_DONTNEED;
}

95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
/**
 * i915_gem_shrink - Shrink buffer object caches
 * @dev_priv: i915 device
 * @target: amount of memory to make available, in pages
 * @flags: control flags for selecting cache types
 *
 * This function is the main interface to the shrinker. It will try to release
 * up to @target pages of main memory backing storage from buffer objects.
 * Selection of the specific caches can be done with @flags. This is e.g. useful
 * when purgeable objects should be removed from caches preferentially.
 *
 * Note that it's not guaranteed that released amount is actually available as
 * free system memory - the pages might still be in-used to due to other reasons
 * (like cpu mmaps) or the mm core has reused them before we could grab them.
 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
 *
 * Also note that any kind of pinning (both per-vma address space pins and
 * backing storage pins at the buffer object level) result in the shrinker code
 * having to skip the object.
 *
 * Returns:
 * The number of pages of backing storage actually released.
 */
119 120
unsigned long
i915_gem_shrink(struct drm_i915_private *dev_priv,
121
		unsigned long target, unsigned flags)
122 123 124 125 126 127 128 129 130 131 132
{
	const struct {
		struct list_head *list;
		unsigned int bit;
	} phases[] = {
		{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
		{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
		{ NULL, 0 },
	}, *phase;
	unsigned long count = 0;

133
	trace_i915_gem_shrink(dev_priv, target, flags);
134
	i915_gem_retire_requests(dev_priv);
135

136 137 138 139 140 141 142 143 144
	/*
	 * Unbinding of objects will require HW access; Let us not wake the
	 * device just to recover a little memory. If absolutely necessary,
	 * we will force the wake during oom-notifier.
	 */
	if ((flags & I915_SHRINK_BOUND) &&
	    !intel_runtime_pm_get_if_in_use(dev_priv))
		flags &= ~I915_SHRINK_BOUND;

145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
	/*
	 * As we may completely rewrite the (un)bound list whilst unbinding
	 * (due to retiring requests) we have to strictly process only
	 * one element of the list at the time, and recheck the list
	 * on every iteration.
	 *
	 * In particular, we must hold a reference whilst removing the
	 * object as we may end up waiting for and/or retiring the objects.
	 * This might release the final reference (held by the active list)
	 * and result in the object being freed from under us. This is
	 * similar to the precautions the eviction code must take whilst
	 * removing objects.
	 *
	 * Also note that although these lists do not hold a reference to
	 * the object we can safely grab one here: The final object
	 * unreferencing and the bound_list are both protected by the
	 * dev->struct_mutex and so we won't ever be able to observe an
	 * object on the bound_list with a reference count equals 0.
	 */
	for (phase = phases; phase->list; phase++) {
		struct list_head still_in_list;

		if ((flags & phase->bit) == 0)
			continue;

		INIT_LIST_HEAD(&still_in_list);
		while (count < target && !list_empty(phase->list)) {
			struct drm_i915_gem_object *obj;
			struct i915_vma *vma, *v;

			obj = list_first_entry(phase->list,
					       typeof(*obj), global_list);
			list_move_tail(&obj->global_list, &still_in_list);

			if (flags & I915_SHRINK_PURGEABLE &&
			    obj->madv != I915_MADV_DONTNEED)
				continue;

183 184 185 186
			if (flags & I915_SHRINK_VMAPS &&
			    !is_vmalloc_addr(obj->mapping))
				continue;

187 188 189
			if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
				continue;

190 191 192
			if (!can_release_pages(obj))
				continue;

193 194 195 196
			drm_gem_object_reference(&obj->base);

			/* For the unbound phase, this should be a no-op! */
			list_for_each_entry_safe(vma, v,
197
						 &obj->vma_list, obj_link)
198 199 200 201 202 203 204 205 206 207 208
				if (i915_vma_unbind(vma))
					break;

			if (i915_gem_object_put_pages(obj) == 0)
				count += obj->base.size >> PAGE_SHIFT;

			drm_gem_object_unreference(&obj->base);
		}
		list_splice(&still_in_list, phase->list);
	}

209 210 211
	if (flags & I915_SHRINK_BOUND)
		intel_runtime_pm_put(dev_priv);

212
	i915_gem_retire_requests(dev_priv);
213

214 215 216
	return count;
}

217
/**
218
 * i915_gem_shrink_all - Shrink buffer object caches completely
219 220 221 222 223 224 225 226 227 228 229 230
 * @dev_priv: i915 device
 *
 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
 * caches completely. It also first waits for and retires all outstanding
 * requests to also be able to release backing storage for active objects.
 *
 * This should only be used in code to intentionally quiescent the gpu or as a
 * last-ditch effort when memory seems to have run out.
 *
 * Returns:
 * The number of pages of backing storage actually released.
 */
231 232
unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
233
	return i915_gem_shrink(dev_priv, -1UL,
234 235 236
			       I915_SHRINK_BOUND |
			       I915_SHRINK_UNBOUND |
			       I915_SHRINK_ACTIVE);
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
}

static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
{
	if (!mutex_trylock(&dev->struct_mutex)) {
		if (!mutex_is_locked_by(&dev->struct_mutex, current))
			return false;

		if (to_i915(dev)->mm.shrinker_no_lock_stealing)
			return false;

		*unlock = false;
	} else
		*unlock = true;

	return true;
}

static unsigned long
i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
{
	struct drm_i915_private *dev_priv =
		container_of(shrinker, struct drm_i915_private, mm.shrinker);
260
	struct drm_device *dev = &dev_priv->drm;
261 262 263 264 265 266 267
	struct drm_i915_gem_object *obj;
	unsigned long count;
	bool unlock;

	if (!i915_gem_shrinker_lock(dev, &unlock))
		return 0;

268 269
	i915_gem_retire_requests(dev_priv);

270 271
	count = 0;
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
272
		if (can_release_pages(obj))
273 274 275
			count += obj->base.size >> PAGE_SHIFT;

	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
276
		if (!obj->active && can_release_pages(obj))
277 278 279 280 281 282 283 284 285 286 287 288 289 290
			count += obj->base.size >> PAGE_SHIFT;
	}

	if (unlock)
		mutex_unlock(&dev->struct_mutex);

	return count;
}

static unsigned long
i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
{
	struct drm_i915_private *dev_priv =
		container_of(shrinker, struct drm_i915_private, mm.shrinker);
291
	struct drm_device *dev = &dev_priv->drm;
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
	unsigned long freed;
	bool unlock;

	if (!i915_gem_shrinker_lock(dev, &unlock))
		return SHRINK_STOP;

	freed = i915_gem_shrink(dev_priv,
				sc->nr_to_scan,
				I915_SHRINK_BOUND |
				I915_SHRINK_UNBOUND |
				I915_SHRINK_PURGEABLE);
	if (freed < sc->nr_to_scan)
		freed += i915_gem_shrink(dev_priv,
					 sc->nr_to_scan - freed,
					 I915_SHRINK_BOUND |
					 I915_SHRINK_UNBOUND);
	if (unlock)
		mutex_unlock(&dev->struct_mutex);

	return freed;
}

314 315 316 317 318 319 320 321 322 323 324 325
struct shrinker_lock_uninterruptible {
	bool was_interruptible;
	bool unlock;
};

static bool
i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
				       struct shrinker_lock_uninterruptible *slu,
				       int timeout_ms)
{
	unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1;

326
	while (!i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) {
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
		schedule_timeout_killable(1);
		if (fatal_signal_pending(current))
			return false;
		if (--timeout == 0) {
			pr_err("Unable to lock GPU to purge memory.\n");
			return false;
		}
	}

	slu->was_interruptible = dev_priv->mm.interruptible;
	dev_priv->mm.interruptible = false;
	return true;
}

static void
i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
					 struct shrinker_lock_uninterruptible *slu)
{
	dev_priv->mm.interruptible = slu->was_interruptible;
	if (slu->unlock)
347
		mutex_unlock(&dev_priv->drm.struct_mutex);
348 349
}

350 351 352 353 354
static int
i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
{
	struct drm_i915_private *dev_priv =
		container_of(nb, struct drm_i915_private, mm.oom_notifier);
355
	struct shrinker_lock_uninterruptible slu;
356
	struct drm_i915_gem_object *obj;
357
	unsigned long unevictable, bound, unbound, freed_pages;
358

359
	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
360 361
		return NOTIFY_DONE;

362
	intel_runtime_pm_get(dev_priv);
363
	freed_pages = i915_gem_shrink_all(dev_priv);
364
	intel_runtime_pm_put(dev_priv);
365 366 367 368 369

	/* Because we may be allocating inside our own driver, we cannot
	 * assert that there are no objects with pinned pages that are not
	 * being pointed to by hardware.
	 */
370
	unbound = bound = unevictable = 0;
371
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
372 373
		if (!can_release_pages(obj))
			unevictable += obj->base.size >> PAGE_SHIFT;
374
		else
375
			unbound += obj->base.size >> PAGE_SHIFT;
376 377
	}
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
378 379
		if (!can_release_pages(obj))
			unevictable += obj->base.size >> PAGE_SHIFT;
380
		else
381
			bound += obj->base.size >> PAGE_SHIFT;
382 383
	}

384
	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
385 386

	if (freed_pages || unbound || bound)
387 388 389
		pr_info("Purging GPU memory, %lu pages freed, "
			"%lu pages still pinned.\n",
			freed_pages, unevictable);
390
	if (unbound || bound)
391
		pr_err("%lu and %lu pages still available in the "
392 393 394 395 396 397 398
		       "bound and unbound GPU page lists.\n",
		       bound, unbound);

	*(unsigned long *)ptr += freed_pages;
	return NOTIFY_DONE;
}

399 400 401 402 403
static int
i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
{
	struct drm_i915_private *dev_priv =
		container_of(nb, struct drm_i915_private, mm.vmap_notifier);
404
	struct shrinker_lock_uninterruptible slu;
405 406 407
	struct i915_vma *vma, *next;
	unsigned long freed_pages = 0;
	int ret;
408

409
	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
410 411
		return NOTIFY_DONE;

412
	/* Force everything onto the inactive lists */
413
	ret = i915_gem_wait_for_idle(dev_priv);
414 415 416
	if (ret)
		goto out;

417
	intel_runtime_pm_get(dev_priv);
418 419 420 421 422
	freed_pages += i915_gem_shrink(dev_priv, -1UL,
				       I915_SHRINK_BOUND |
				       I915_SHRINK_UNBOUND |
				       I915_SHRINK_ACTIVE |
				       I915_SHRINK_VMAPS);
423
	intel_runtime_pm_put(dev_priv);
424 425 426 427 428 429 430 431

	/* We also want to clear any cached iomaps as they wrap vmap */
	list_for_each_entry_safe(vma, next,
				 &dev_priv->ggtt.base.inactive_list, vm_link) {
		unsigned long count = vma->node.size >> PAGE_SHIFT;
		if (vma->iomap && i915_vma_unbind(vma) == 0)
			freed_pages += count;
	}
432

433
out:
434
	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
435 436 437 438 439

	*(unsigned long *)ptr += freed_pages;
	return NOTIFY_DONE;
}

440 441 442 443 444 445
/**
 * i915_gem_shrinker_init - Initialize i915 shrinker
 * @dev_priv: i915 device
 *
 * This function registers and sets up the i915 shrinker and OOM handler.
 */
446 447 448 449 450
void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
{
	dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
	dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
	dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
451
	WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
452 453

	dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
454
	WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
455 456 457

	dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
	WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
458 459 460 461 462 463 464 465 466 467
}

/**
 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
 * @dev_priv: i915 device
 *
 * This function unregisters the i915 shrinker and OOM handler.
 */
void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
{
468
	WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
469 470
	WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
	unregister_shrinker(&dev_priv->mm.shrinker);
471
}