diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc
index 6022caa3d1e38f1ce441fd852173265bba3534a4..c8bae1a02afeb37966fbb9bc9a48c0bedd8b4ab4 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system.cc
@@ -1164,7 +1164,13 @@ Status GcsFileSystem::DeleteRecursively(const string& dirname,
   for (const string& object : all_objects) {
     const string& full_path = JoinGcsPath(dirname, object);
     // Delete all objects including directory markers for subfolders.
-    if (!DeleteFile(full_path).ok()) {
+    // Since DeleteRecursively returns OK if individual file deletions fail,
+    // and therefore RetryingFileSystem won't pay attention to the failures,
+    // we need to make sure these failures are properly retried.
+    const auto& delete_file_status = RetryingUtils::DeleteWithRetries(
+        std::bind(&GcsFileSystem::DeleteFile, this, full_path),
+        initial_retry_delay_usec_);
+    if (!delete_file_status.ok()) {
       if (IsDirectory(full_path).ok()) {
         // The object is a directory marker.
         (*undeleted_dirs)++;
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index 7fb70acf118e419ba96171fe340a147f6cfb19fe..e84ec7cc666a854f3881584006b24fde4cd78fb4 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -1471,7 +1471,13 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
                            "Auth Token: fake_token\n"
                            "Delete: yes\n",
                            ""),
-       // Delete the object.
+       // Delete the object - fails and will be retried.
+       new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
+                           "/bucket/o/path%2Ffile1.txt\n"
+                           "Auth Token: fake_token\n"
+                           "Delete: yes\n",
+                           "", errors::Unavailable("500"), 500),
+       // Delete the object again.
        new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
                            "/bucket/o/path%2Ffile1.txt\n"
                            "Auth Token: fake_token\n"