diff --git a/src/rpc/virnetsocket.c b/src/rpc/virnetsocket.c
index fc8ce6c1232047b21ab3b15441bc2c1655dd0351..6a51dc840b9134e172940e810257a26995ae0886 100644
--- a/src/rpc/virnetsocket.c
+++ b/src/rpc/virnetsocket.c
@@ -690,7 +690,9 @@ int virNetSocketNewConnectUNIX(const char *path,
         }
         VIR_DEBUG("connect() failed: retries=%d errno=%d", retries, errno);
 
+        retries--;
         if (!spawnDaemon ||
+            retries == 0 ||
             (errno != ENOENT && errno != ECONNREFUSED)) {
             virReportSystemError(errno, _("Failed to connect socket to '%s'"),
                                  path);
@@ -700,7 +702,6 @@ int virNetSocketNewConnectUNIX(const char *path,
         if (virNetSocketForkDaemon(binary) < 0)
             goto cleanup;
 
-        retries--;
         usleep(5000);
     }