diff --git a/backends/cryptodev.c b/backends/cryptodev.c
index 4a49f9762f91415a26ced97990ad4155f97b84bc..6a66c27d68e4be5bb4cf6a5ddec8ed22a4ee1892 100644
--- a/backends/cryptodev.c
+++ b/backends/cryptodev.c
@@ -197,6 +197,22 @@ out:
     error_propagate(errp, local_err);
 }
 
+void cryptodev_backend_set_used(CryptoDevBackend *backend, bool used)
+{
+    backend->is_used = used;
+}
+
+bool cryptodev_backend_is_used(CryptoDevBackend *backend)
+{
+    return backend->is_used;
+}
+
+static bool
+cryptodev_backend_can_be_deleted(UserCreatable *uc, Error **errp)
+{
+    return !cryptodev_backend_is_used(CRYPTODEV_BACKEND(uc));
+}
+
 static void cryptodev_backend_instance_init(Object *obj)
 {
     object_property_add(obj, "queues", "int",
@@ -209,7 +225,9 @@ static void cryptodev_backend_instance_init(Object *obj)
 
 static void cryptodev_backend_finalize(Object *obj)
 {
+    CryptoDevBackend *backend = CRYPTODEV_BACKEND(obj);
 
+    cryptodev_backend_cleanup(backend, NULL);
 }
 
 static void
@@ -218,6 +236,7 @@ cryptodev_backend_class_init(ObjectClass *oc, void *data)
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
 
     ucc->complete = cryptodev_backend_complete;
+    ucc->can_be_deleted = cryptodev_backend_can_be_deleted;
 
     QTAILQ_INIT(&crypto_clients);
 }
diff --git a/hw/virtio/virtio-crypto.c b/hw/virtio/virtio-crypto.c
index f872c87cb9807816b8c6eaec7187510413c6de38..6318fcf01b76b69d6fbf33430953a562aa25dd7e 100644
--- a/hw/virtio/virtio-crypto.c
+++ b/hw/virtio/virtio-crypto.c
@@ -799,6 +799,7 @@ static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
     }
 
     virtio_crypto_init_config(vdev);
+    cryptodev_backend_set_used(vcrypto->cryptodev, true);
 }
 
 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
@@ -818,6 +819,7 @@ static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
     g_free(vcrypto->vqs);
 
     virtio_cleanup(vdev);
+    cryptodev_backend_set_used(vcrypto->cryptodev, false);
 }
 
 static const VMStateDescription vmstate_virtio_crypto = {
diff --git a/include/sysemu/cryptodev.h b/include/sysemu/cryptodev.h
index 84526c0d352c7c983f7ababa5dbdf29970f3aa0b..461389df8577a616806fcc31add530f2dd816c88 100644
--- a/include/sysemu/cryptodev.h
+++ b/include/sysemu/cryptodev.h
@@ -202,6 +202,8 @@ struct CryptoDevBackend {
     Object parent_obj;
 
     bool ready;
+    /* Tag the cryptodev backend is used by virtio-crypto or not */
+    bool is_used;
     CryptoDevBackendConf conf;
 };
 
@@ -295,4 +297,25 @@ int cryptodev_backend_crypto_operation(
                  void *opaque,
                  uint32_t queue_index, Error **errp);
 
+/**
+ * cryptodev_backend_set_used:
+ * @backend: the cryptodev backend object
+ * @used: ture or false
+ *
+ * Set the cryptodev backend is used by virtio-crypto or not
+ */
+void cryptodev_backend_set_used(CryptoDevBackend *backend, bool used);
+
+/**
+ * cryptodev_backend_is_used:
+ * @backend: the cryptodev backend object
+ *
+ * Return the status that the cryptodev backend is used
+ * by virtio-crypto or not
+ *
+ * Returns: true on used, or false on not used
+ */
+bool cryptodev_backend_is_used(CryptoDevBackend *backend);
+
+
 #endif /* CRYPTODEV_H */