diff --git a/builtin/grep.c b/builtin/grep.c
index 1a6c02804c7f027b1da668f25b948f0d3e951519..4bc075422e3c17e7666ed90e4a35be020586be49 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -858,7 +858,8 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
 	}
 
 	parse_pathspec(&pathspec, 0,
-		       PATHSPEC_PREFER_CWD,
+		       PATHSPEC_PREFER_CWD |
+		       (opt.max_depth != -1 ? PATHSPEC_MAXDEPTH_VALID : 0),
 		       prefix, argv + i);
 	pathspec.max_depth = opt.max_depth;
 	pathspec.recursive = 1;
diff --git a/diff-lib.c b/diff-lib.c
index b6f4b21637fe66a614a0dd6581f90fc12e80a6f8..d4e17f76620a46a659c7affcb3cd270f86b5a957 100644
--- a/diff-lib.c
+++ b/diff-lib.c
@@ -476,7 +476,6 @@ static int diff_cache(struct rev_info *revs,
 	opts.dst_index = NULL;
 	opts.pathspec = &revs->diffopt.pathspec;
 	opts.pathspec->recursive = 1;
-	opts.pathspec->max_depth = -1;
 
 	init_tree_desc(&t, tree->buffer, tree->size);
 	return unpack_trees(1, &t, &opts);
diff --git a/dir.c b/dir.c
index 308028e5b16df77e22e87cf339a461676c7f4c1b..e28bc0d636d544a3305c9e6c71d666de568bc0fb 100644
--- a/dir.c
+++ b/dir.c
@@ -341,7 +341,9 @@ int match_pathspec_depth(const struct pathspec *ps,
 	int i, retval = 0;
 
 	if (!ps->nr) {
-		if (!ps->recursive || ps->max_depth == -1)
+		if (!ps->recursive ||
+		    !(ps->magic & PATHSPEC_MAXDEPTH) ||
+		    ps->max_depth == -1)
 			return MATCHED_RECURSIVELY;
 
 		if (within_depth(name, namelen, 0, ps->max_depth))
@@ -358,7 +360,9 @@ int match_pathspec_depth(const struct pathspec *ps,
 		if (seen && seen[i] == MATCHED_EXACTLY)
 			continue;
 		how = match_pathspec_item(ps->items+i, prefix, name, namelen);
-		if (ps->recursive && ps->max_depth != -1 &&
+		if (ps->recursive &&
+		    (ps->magic & PATHSPEC_MAXDEPTH) &&
+		    ps->max_depth != -1 &&
 		    how && how != MATCHED_FNMATCH) {
 			int len = ps->items[i].len;
 			if (name[len] == '/')
diff --git a/pathspec.c b/pathspec.c
index 6d99a3dcedbf7bec234094da37f1212cbd2fe368..06778fc75665a73e7c0b9c1429eb4e29be8b5c8a 100644
--- a/pathspec.c
+++ b/pathspec.c
@@ -267,6 +267,9 @@ void parse_pathspec(struct pathspec *pathspec,
 
 	memset(pathspec, 0, sizeof(*pathspec));
 
+	if (flags & PATHSPEC_MAXDEPTH_VALID)
+		pathspec->magic |= PATHSPEC_MAXDEPTH;
+
 	/* No arguments, no prefix -> no pathspec */
 	if (!entry && !prefix)
 		return;
@@ -322,8 +325,9 @@ void parse_pathspec(struct pathspec *pathspec,
 		pathspec->magic |= item[i].magic;
 	}
 
-	qsort(pathspec->items, pathspec->nr,
-	      sizeof(struct pathspec_item), pathspec_item_cmp);
+	if (pathspec->magic & PATHSPEC_MAXDEPTH)
+		qsort(pathspec->items, pathspec->nr,
+		      sizeof(struct pathspec_item), pathspec_item_cmp);
 }
 
 /*
diff --git a/pathspec.h b/pathspec.h
index d630e8b1f9eb24824f8876965358253fe00e3232..aa98597bda2e445d81cad15878bd8bb23c8250e3 100644
--- a/pathspec.h
+++ b/pathspec.h
@@ -3,7 +3,10 @@
 
 /* Pathspec magic */
 #define PATHSPEC_FROMTOP	(1<<0)
-#define PATHSPEC_ALL_MAGIC PATHSPEC_FROMTOP
+#define PATHSPEC_MAXDEPTH	(1<<1)
+#define PATHSPEC_ALL_MAGIC	  \
+	(PATHSPEC_FROMTOP	| \
+	 PATHSPEC_MAXDEPTH)
 
 #define PATHSPEC_ONESTAR 1	/* the pathspec pattern sastisfies GFNM_ONESTAR */
 
@@ -27,6 +30,7 @@ struct pathspec {
 /* parse_pathspec flags */
 #define PATHSPEC_PREFER_CWD (1<<0) /* No args means match cwd */
 #define PATHSPEC_PREFER_FULL (1<<1) /* No args means match everything */
+#define PATHSPEC_MAXDEPTH_VALID (1<<2) /* max_depth field is valid */
 
 extern int init_pathspec(struct pathspec *, const char **);
 extern void parse_pathspec(struct pathspec *pathspec,
diff --git a/tree-diff.c b/tree-diff.c
index ba01563a0241041cb401cfb03495e6067847248e..826512e84d0d99bd34d7d428e3d712d60d729eac 100644
--- a/tree-diff.c
+++ b/tree-diff.c
@@ -138,7 +138,6 @@ int diff_tree(struct tree_desc *t1, struct tree_desc *t2,
 
 	/* Enable recursion indefinitely */
 	opt->pathspec.recursive = DIFF_OPT_TST(opt, RECURSIVE);
-	opt->pathspec.max_depth = -1;
 
 	strbuf_init(&base, PATH_MAX);
 	strbuf_add(&base, base_str, baselen);
diff --git a/tree-walk.c b/tree-walk.c
index 72a96139f5dd782adb00f51aff268f52f0e403c1..d399ca9dcc69a21eaae31f2926b8480d22e86672 100644
--- a/tree-walk.c
+++ b/tree-walk.c
@@ -637,7 +637,9 @@ enum interesting tree_entry_interesting(const struct name_entry *entry,
 		entry_not_interesting : all_entries_not_interesting;
 
 	if (!ps->nr) {
-		if (!ps->recursive || ps->max_depth == -1)
+		if (!ps->recursive ||
+		    !(ps->magic & PATHSPEC_MAXDEPTH) ||
+		    ps->max_depth == -1)
 			return all_entries_interesting;
 		return within_depth(base->buf + base_offset, baselen,
 				    !!S_ISDIR(entry->mode),
@@ -658,7 +660,9 @@ enum interesting tree_entry_interesting(const struct name_entry *entry,
 			if (!match_dir_prefix(base_str, match, matchlen))
 				goto match_wildcards;
 
-			if (!ps->recursive || ps->max_depth == -1)
+			if (!ps->recursive ||
+			    !(ps->magic & PATHSPEC_MAXDEPTH) ||
+			    ps->max_depth == -1)
 				return all_entries_interesting;
 
 			return within_depth(base_str + matchlen + 1,