drm/nouveau/mmu: implement new vmm frontend

These are the new priviledged interfaces to the VMM backends, and expose
some functionality that wasn't previously available.

It's now possible to allocate a chunk of address-space (even all of it),
without causing page tables to be allocated up-front, and then map into
it at arbitrary locations.  This is the basic primitive used to support
features such as sparse mapping, or to allow userspace control over its
own address-space, or HMM (where the GPU driver isn't in control of the
address-space layout).

Rather than being tied to a subtle combination of memory object and VMA
properties, arguments that control map flags (ro, kind, etc) are passed
explicitly at map time.

The compatibility hacks to implement the old frontend on top of the new
driver backends have been replaced with something similar to implement
the old frontend's interfaces on top of the new frontend.
Signed-off-by: NBen Skeggs <bskeggs@redhat.com>

drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h

 #ifndef __NVKM_FB_H__
 #define __NVKM_FB_H__
 #include <core/subdev.h>
-#include <core/memory.h>
-
-#include <subdev/mmu.h>
+#include <core/mm.h>
 
 /* memory type/access flags, do not match hardware values */
 #define NV_MEM_ACCESS_RO  1

drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h

 #ifndef __NVKM_MMU_H__
 #define __NVKM_MMU_H__
 #include <core/subdev.h>
-#include <core/memory.h>
-#include <core/mm.h>
-struct nvkm_gpuobj;
 struct nvkm_mem;
 
 struct nvkm_vm_pgt {
@@ -12,14 +9,25 @@ struct nvkm_vm_pgt {
 };
 
 struct nvkm_vma {
-	struct nvkm_memory *memory;
-	struct nvkm_tags *tags;
+	struct list_head head;
+	struct rb_node tree;
+	u64 addr;
+	u64 size:50;
+	bool mapref:1; /* PTs (de)referenced on (un)map (vs pre-allocated). */
+	bool sparse:1; /* Unmapped PDEs/PTEs will not trigger MMU faults. */
+#define NVKM_VMA_PAGE_NONE 7
+	u8   page:3; /* Requested page type (index, or NONE for automatic). */
+	u8   refd:3; /* Current page type (index, or NONE for unreferenced). */
+	bool used:1; /* Region allocated. */
+	bool part:1; /* Region was split from an allocated region by map(). */
+	bool user:1; /* Region user-allocated. */
+	bool busy:1; /* Region busy (for temporarily preventing user access). */
+	struct nvkm_memory *memory; /* Memory currently mapped into VMA. */
+	struct nvkm_tags *tags; /* Compression tag reference. */
+
+	struct nvkm_vma *node;
 	struct nvkm_vm *vm;
-	struct nvkm_mm_node *node;
-	union {
-		u64 offset;
-		u64 addr;
-	};
+	u64 offset;
 	u32 access;
 };
 
@@ -37,8 +45,9 @@ struct nvkm_vm {
 	struct nvkm_vmm_pt *pd;
 	struct list_head join;
 
-	struct nvkm_mm mm;
-	struct kref refcount;
+	struct list_head list;
+	struct rb_root free;
+	struct rb_root root;
 
 	bool bootstrapped;
 	atomic_t engref[NVKM_SUBDEV_NR];
@@ -57,9 +66,16 @@ void nvkm_vm_put(struct nvkm_vma *);
 void nvkm_vm_map(struct nvkm_vma *, struct nvkm_mem *);
 void nvkm_vm_map_at(struct nvkm_vma *, u64 offset, struct nvkm_mem *);
 void nvkm_vm_unmap(struct nvkm_vma *);
-void nvkm_vm_unmap_at(struct nvkm_vma *, u64 offset, u64 length);
 
+int nvkm_vmm_new(struct nvkm_device *, u64 addr, u64 size, void *argv, u32 argc,
+		 struct lock_class_key *, const char *name, struct nvkm_vmm **);
+struct nvkm_vmm *nvkm_vmm_ref(struct nvkm_vmm *);
+void nvkm_vmm_unref(struct nvkm_vmm **);
 int nvkm_vmm_boot(struct nvkm_vmm *);
+int nvkm_vmm_join(struct nvkm_vmm *, struct nvkm_memory *inst);
+void nvkm_vmm_part(struct nvkm_vmm *, struct nvkm_memory *inst);
+int nvkm_vmm_get(struct nvkm_vmm *, u8 page, u64 size, struct nvkm_vma **);
+void nvkm_vmm_put(struct nvkm_vmm *, struct nvkm_vma **);
 
 struct nvkm_vmm_map {
 	struct nvkm_memory *memory;
@@ -78,6 +94,12 @@ struct nvkm_vmm_map {
 	u64 ctag;
 };
 
+int nvkm_vmm_map(struct nvkm_vmm *, struct nvkm_vma *, void *argv, u32 argc,
+		 struct nvkm_vmm_map *);
+void nvkm_vmm_unmap(struct nvkm_vmm *, struct nvkm_vma *);
+
+struct nvkm_vmm *nvkm_uvmm_search(struct nvkm_client *, u64 handle);
+
 struct nvkm_mmu {
 	const struct nvkm_mmu_func *func;
 	struct nvkm_subdev subdev;

drivers/gpu/drm/nouveau/nouveau_mem.h

 #ifndef __NOUVEAU_MEM_H__
 #define __NOUVEAU_MEM_H__
+#include <core/memory.h>
 #include <subdev/fb.h>
+#include <subdev/mmu.h>
 
 #include <drm/ttm/ttm_bo_api.h>
 struct ttm_dma_tt;

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

@@ -48,6 +48,7 @@
 #include <core/tegra.h>
 #include <subdev/fb.h>
 #include <subdev/ltc.h>
+#include <subdev/mmu.h>
 
 struct gk20a_instobj {
 	struct nvkm_memory memory;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c

@@ -266,14 +266,14 @@ nvkm_vm_map_(const struct nvkm_vmm_page *page, struct nvkm_vma *vma, u64 delta,
 	}
 
 	mutex_lock(&vmm->mutex);
-	nvkm_vmm_ptes_map(vmm, page, ((u64)vma->node->offset << 12) + delta,
-				      (u64)vma->node->length << 12, map, fn);
+	nvkm_vmm_ptes_map(vmm, page, vma->node->addr + delta,
+				     vma->node->size, map, fn);
 	mutex_unlock(&vmm->mutex);
 
-	nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
-	nvkm_memory_unref(&vma->memory);
-	vma->memory = nvkm_memory_ref(map->memory);
-	vma->tags = map->tags;
+	nvkm_memory_tags_put(vma->node->memory, vmm->mmu->subdev.device, &vma->node->tags);
+	nvkm_memory_unref(&vma->node->memory);
+	vma->node->memory = nvkm_memory_ref(map->memory);
+	vma->node->tags = map->tags;
 }
 
 void
@@ -314,11 +314,9 @@ nvkm_mmu_ptc_init(struct nvkm_mmu *mmu)
 void
 nvkm_vm_map_at(struct nvkm_vma *vma, u64 delta, struct nvkm_mem *node)
 {
-	const struct nvkm_vmm_page *page = vma->vm->func->page;
+	const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page];
 	if (page->desc->func->unmap) {
 		struct nvkm_vmm_map map = { .mem = node->mem };
-		while (page->shift != vma->node->type)
-			page++;
 		nvkm_vm_map_(page, vma, delta, node, page->desc->func->mem, &map);
 		return;
 	}
@@ -328,11 +326,9 @@ static void
 nvkm_vm_map_sg_table(struct nvkm_vma *vma, u64 delta, u64 length,
 		     struct nvkm_mem *mem)
 {
-	const struct nvkm_vmm_page *page = vma->vm->func->page;
+	const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page];
 	if (page->desc->func->unmap) {
 		struct nvkm_vmm_map map = { .sgl = mem->sg->sgl };
-		while (page->shift != vma->node->type)
-			page++;
 		nvkm_vm_map_(page, vma, delta, mem, page->desc->func->sgl, &map);
 		return;
 	}
@@ -342,11 +338,9 @@ static void
 nvkm_vm_map_sg(struct nvkm_vma *vma, u64 delta, u64 length,
 	       struct nvkm_mem *mem)
 {
-	const struct nvkm_vmm_page *page = vma->vm->func->page;
+	const struct nvkm_vmm_page *page = &vma->vm->func->page[vma->node->page];
 	if (page->desc->func->unmap) {
 		struct nvkm_vmm_map map = { .dma = mem->pages };
-		while (page->shift != vma->node->type)
-			page++;
 		nvkm_vm_map_(page, vma, delta, mem, page->desc->func->dma, &map);
 		return;
 	}
@@ -364,67 +358,30 @@ nvkm_vm_map(struct nvkm_vma *vma, struct nvkm_mem *node)
 		nvkm_vm_map_at(vma, 0, node);
 }
 
-void
-nvkm_vm_unmap_at(struct nvkm_vma *vma, u64 delta, u64 length)
-{
-	struct nvkm_vm *vm = vma->vm;
-	if (vm->func->page->desc->func->unmap) {
-		const struct nvkm_vmm_page *page = vm->func->page;
-		while (page->shift != vma->node->type)
-			page++;
-		mutex_lock(&vm->mutex);
-		nvkm_vmm_ptes_unmap(vm, page, (vma->node->offset << 12) + delta,
-					       vma->node->length << 12, false);
-		mutex_unlock(&vm->mutex);
-		return;
-	}
-}
-
 void
 nvkm_vm_unmap(struct nvkm_vma *vma)
 {
-	nvkm_vm_unmap_at(vma, 0, (u64)vma->node->length << 12);
-
-	nvkm_memory_tags_put(vma->memory, vma->vm->mmu->subdev.device, &vma->tags);
-	nvkm_memory_unref(&vma->memory);
+	nvkm_vmm_unmap(vma->vm, vma->node);
 }
 
 int
 nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access,
 	    struct nvkm_vma *vma)
 {
-	u32 align = (1 << page_shift) >> 12;
-	u32 msize = size >> 12;
 	int ret;
 
 	mutex_lock(&vm->mutex);
-	ret = nvkm_mm_head(&vm->mm, 0, page_shift, msize, msize, align,
-			   &vma->node);
-	if (unlikely(ret != 0)) {
-		mutex_unlock(&vm->mutex);
-		return ret;
-	}
-
-	if (vm->func->page->desc->func->unmap) {
-		const struct nvkm_vmm_page *page = vm->func->page;
-		while (page->shift != page_shift)
-			page++;
-
-		ret = nvkm_vmm_ptes_get(vm, page, vma->node->offset << 12,
-						  vma->node->length << 12);
-		if (ret) {
-			nvkm_mm_free(&vm->mm, &vma->node);
-			mutex_unlock(&vm->mutex);
-			return ret;
-		}
-	}
+	ret = nvkm_vmm_get_locked(vm, true, false, false, page_shift, 0,
+				  size, &vma->node);
 	mutex_unlock(&vm->mutex);
+	if (ret)
+		return ret;
 
 	vma->memory = NULL;
 	vma->tags = NULL;
 	vma->vm = NULL;
 	nvkm_vm_ref(vm, &vma->vm, NULL);
-	vma->offset = (u64)vma->node->offset << 12;
+	vma->offset = vma->addr = vma->node->addr;
 	vma->access = access;
 	return 0;
 }
@@ -432,30 +389,7 @@ nvkm_vm_get(struct nvkm_vm *vm, u64 size, u32 page_shift, u32 access,
 void
 nvkm_vm_put(struct nvkm_vma *vma)
 {
-	struct nvkm_mmu *mmu;
-	struct nvkm_vm *vm;
-
-	if (unlikely(vma->node == NULL))
-		return;
-	vm = vma->vm;
-	mmu = vm->mmu;
-
-	nvkm_memory_tags_put(vma->memory, mmu->subdev.device, &vma->tags);
-	nvkm_memory_unref(&vma->memory);
-
-	mutex_lock(&vm->mutex);
-	if (vm->func->page->desc->func->unmap) {
-		const struct nvkm_vmm_page *page = vm->func->page;
-		while (page->shift != vma->node->type)
-			page++;
-
-		nvkm_vmm_ptes_put(vm, page, vma->node->offset << 12,
-					    vma->node->length << 12);
-	}
-
-	nvkm_mm_free(&vm->mm, &vma->node);
-	mutex_unlock(&vm->mutex);
-
+	nvkm_vmm_put(vma->vm, &vma->node);
 	nvkm_vm_ref(NULL, &vma->vm, NULL);
 }
 
@@ -465,26 +399,6 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size)
 	return nvkm_vmm_boot(vm);
 }
 
-static int
-nvkm_vm_legacy(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,
-	       u32 block, struct nvkm_vm *vm)
-{
-	u64 mm_length = (offset + length) - mm_offset;
-	int ret;
-
-	kref_init(&vm->refcount);
-
-	if (block > length)
-		block = length;
-
-	ret = nvkm_mm_init(&vm->mm, 0, mm_offset >> 12, mm_length >> 12,
-			   block >> 12);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 int
 nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset,
 	    struct lock_class_key *key, struct nvkm_vm **pvm)
@@ -501,46 +415,28 @@ nvkm_vm_new(struct nvkm_device *device, u64 offset, u64 length, u64 mm_offset,
 			return ret;
 		}
 
-		ret = nvkm_vm_legacy(mmu, offset, length, mm_offset,
-				     (*pvm)->func->page_block ?
-				     (*pvm)->func->page_block : 4096, *pvm);
-		if (ret)
-			nvkm_vm_ref(NULL, pvm, NULL);
-
 		return ret;
 	}
 
 	return -EINVAL;
 }
 
-static void
-nvkm_vm_del(struct kref *kref)
-{
-	struct nvkm_vm *vm = container_of(kref, typeof(*vm), refcount);
-
-	nvkm_mm_fini(&vm->mm);
-	if (vm->func)
-		nvkm_vmm_dtor(vm);
-	kfree(vm);
-}
-
 int
 nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_memory *inst)
 {
 	if (ref) {
-		if (ref->func->join && inst) {
-			int ret = ref->func->join(ref, inst);
+		if (inst) {
+			int ret = nvkm_vmm_join(ref, inst);
 			if (ret)
 				return ret;
 		}
 
-		kref_get(&ref->refcount);
+		nvkm_vmm_ref(ref);
 	}
 
 	if (*ptr) {
-		if ((*ptr)->func->part && inst)
-			(*ptr)->func->part(*ptr, inst);
-		kref_put(&(*ptr)->refcount, nvkm_vm_del);
+		nvkm_vmm_part(*ptr, inst);
+		nvkm_vmm_unref(ptr);
 	}
 
 	*ptr = ref;
@@ -553,8 +449,8 @@ nvkm_mmu_oneinit(struct nvkm_subdev *subdev)
 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
 
 	if (mmu->func->vmm.global) {
-		int ret = nvkm_vm_new(subdev->device, 0, mmu->limit, 0,
-				      NULL, &mmu->vmm);
+		int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL,
+				       "gart", &mmu->vmm);
 		if (ret)
 			return ret;
 	}
@@ -576,7 +472,7 @@ nvkm_mmu_dtor(struct nvkm_subdev *subdev)
 {
 	struct nvkm_mmu *mmu = nvkm_mmu(subdev);
 
-	nvkm_vm_ref(NULL, &mmu->vmm, NULL);
+	nvkm_vmm_unref(&mmu->vmm);
 
 	nvkm_mmu_ptc_fini(mmu);
 	return mmu;

drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h

@@ -153,16 +153,19 @@ int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
 		  u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
 		  const char *name, struct nvkm_vmm *);
-void nvkm_vmm_dtor(struct nvkm_vmm *);
-void nvkm_vmm_ptes_put(struct nvkm_vmm *, const struct nvkm_vmm_page *,
-		       u64 addr, u64 size);
-int nvkm_vmm_ptes_get(struct nvkm_vmm *, const struct nvkm_vmm_page *,
-		      u64 addr, u64 size);
+struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
+int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
+			bool sparse, u8 page, u8 align, u64 size,
+			struct nvkm_vma **pvma);
+void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
+void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
+void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
 void nvkm_vmm_ptes_map(struct nvkm_vmm *, const struct nvkm_vmm_page *,
 		       u64 addr, u64 size, struct nvkm_vmm_map *,
 		       nvkm_vmm_pte_func);
-void nvkm_vmm_ptes_unmap(struct nvkm_vmm *, const struct nvkm_vmm_page *,
-			 u64 addr, u64 size, bool sparse);
+
+struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
+void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *);
 
 int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
 		  u64, u64, void *, u32, struct lock_class_key *,

drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h

@@ -25,6 +25,7 @@
 
 #include <subdev/secboot.h>
 #include <subdev/mmu.h>
+struct nvkm_gpuobj;
 
 struct nvkm_secboot_func {
 	int (*oneinit)(struct nvkm_secboot *);