diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
index 6d38e21c5f6599ae16989edf5a9874301c5683c9..f26407869e20c522bb2a87785b0007ee4b4dfd06 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mmu.h
@@ -38,6 +38,8 @@ struct nvkm_vm {
 	struct nvkm_vm_pgt *pgt;
 	u32 fpde;
 	u32 lpde;
+
+	bool bootstrapped;
 };
 
 int  nvkm_vm_new(struct nvkm_device *, u64 offset, u64 length, u64 mm_offset,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
index 9f7b4e7532b7032d90beb4de5320364868e3c3db..8b588d1c776b84cce52f85fb1a4525f54a6453ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
@@ -168,10 +168,7 @@ gf100_bar_dtor(struct nvkm_bar *base)
 	nvkm_gpuobj_del(&bar->bar[1].pgd);
 	nvkm_memory_unref(&bar->bar[1].mem);
 
-	if (bar->bar[0].vm) {
-		nvkm_memory_unref(&bar->bar[0].vm->pgt[0].mem[0]);
-		nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd);
-	}
+	nvkm_vm_ref(NULL, &bar->bar[0].vm, bar->bar[0].pgd);
 	nvkm_gpuobj_del(&bar->bar[0].pgd);
 	nvkm_memory_unref(&bar->bar[0].mem);
 	return bar;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
index 1aa6b5390d7961feead80045d9f269a4e106db5d..b40c131af20e02bae2e5c5c5644dd0fc21f45eb7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
@@ -200,10 +200,7 @@ nv50_bar_dtor(struct nvkm_bar *base)
 	nvkm_gpuobj_del(&bar->bar1);
 	nvkm_vm_ref(NULL, &bar->bar1_vm, bar->pgd);
 	nvkm_gpuobj_del(&bar->bar2);
-	if (bar->bar2_vm) {
-		nvkm_memory_unref(&bar->bar2_vm->pgt[0].mem[0]);
-		nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd);
-	}
+	nvkm_vm_ref(NULL, &bar->bar2_vm, bar->pgd);
 	nvkm_gpuobj_del(&bar->pgd);
 	nvkm_gpuobj_del(&bar->pad);
 	nvkm_gpuobj_del(&bar->mem);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
index ad11db458fccedcd4199921fc931f516268b0816..d9f572db5c2908bbbce33517b47bd41e702e80b5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/base.c
@@ -357,6 +357,7 @@ nvkm_vm_boot(struct nvkm_vm *vm, u64 size)
 		vm->pgt[0].refcount[0] = 1;
 		vm->pgt[0].mem[0] = pgt;
 		nvkm_memory_boot(pgt, vm);
+		vm->bootstrapped = true;
 	}
 
 	return ret;
@@ -481,6 +482,8 @@ nvkm_vm_ref(struct nvkm_vm *ref, struct nvkm_vm **ptr, struct nvkm_gpuobj *pgd)
 	}
 
 	if (*ptr) {
+		if ((*ptr)->bootstrapped && pgd)
+			nvkm_memory_unref(&(*ptr)->pgt[0].mem[0]);
 		nvkm_vm_unlink(*ptr, pgd);
 		kref_put(&(*ptr)->refcount, nvkm_vm_del);
 	}