diff --git a/modules/styles.py b/modules/styles.py
index 25ec504b33c5df54920e5797e7534d89c0432f65..bc7f070ffcd91d5f401789b6be5ba2d07bdd4144 100644
--- a/modules/styles.py
+++ b/modules/styles.py
@@ -63,5 +63,6 @@ def save_styles(path: str, styles: abc.Iterable[PromptStyle]) -> None:
         writer.writerows(style._asdict() for style in styles)
 
     # Always keep a backup file around
-    shutil.copy(path, path + ".bak")
+    if os.path.exists(path):
+        shutil.move(path, path + ".bak")
     shutil.move(temp_path, path)