diff --git a/update-cache.c b/update-cache.c
index 1e4e62cc4c99e1372afa658198f504df28e53dd4..69b1afec94ef0a6716c85f128696c1da835433e9 100644
--- a/update-cache.c
+++ b/update-cache.c
@@ -53,14 +53,23 @@ static void fill_stat_cache_info(struct cache_entry *ce, struct stat *st)
 
 static int add_file_to_cache(char *path)
 {
-	int size, namelen, option;
+	int size, namelen, option, status;
 	struct cache_entry *ce;
 	struct stat st;
 	int fd;
 	char *target;
 
-	if (lstat(path, &st) < 0) {
-		if (errno == ENOENT || errno == ENOTDIR) {
+	status = lstat(path, &st);
+	if (status < 0 || S_ISDIR(st.st_mode)) {
+		/* When we used to have "path" and now we want to add
+		 * "path/file", we need a way to remove "path" before
+		 * being able to add "path/file".  However,
+		 * "git-update-cache --remove path" would not work.
+		 * --force-remove can be used but this is more user
+		 * friendly, especially since we can do the opposite
+		 * case just fine without --force-remove.
+		 */
+		if (status == 0 || (errno == ENOENT || errno == ENOTDIR)) {
 			if (allow_remove)
 				return remove_file_from_cache(path);
 		}