Merge branch 'np/index-pack'

* np/index-pack:
  remove .keep pack lock files when done with refs update
  have index-pack create .keep file more carefully
  improve fetch-pack's handling of kept packs
  git-fetch can use both --thin and --keep with fetch-pack now
  Teach receive-pack how to keep pack files based on object count.
  Allow pack header preprocessing before unpack-objects/index-pack.
  Remove unused variable in receive-pack.
  Revert "send-pack --keep: do not explode into loose objects on the receiving end."
  missing small substitution
  Teach git-index-pack how to keep a pack file.
  Only repack active packs by skipping over kept packs.
  Allow short pack names to git-pack-objects --unpacked=.
  send-pack --keep: do not explode into loose objects on the receiving end.
  index-pack: minor fixes to comment and function name
  enhance clone and fetch -k experience
  mimic unpack-objects when --stdin is used with index-pack
  add progress status to index-pack
  make index-pack able to complete thin packs.
  enable index-pack streaming capability

Documentation/config.txt

@@ -301,7 +301,16 @@ imap::
 	The configuration variables in the 'imap' section are described
 	in gitlink:git-imap-send[1].
 
-receive.denyNonFastforwads::
+receive.unpackLimit::
+	If the number of objects received in a push is below this
+	limit then the objects will be unpacked into loose object
+	files. However if the number of received objects equals or
+	exceeds this limit then the received pack will be stored as
+	a pack, after adding any missing delta bases.  Storing the
+	pack from a push can make the push operation complete faster,
+	especially on slow filesystems.
+
+receive.denyNonFastForwards::
 	If set to true, git-receive-pack will deny a ref update which is
 	not a fast forward. Use this to prevent such an update via a push,
 	even if that push is forced. This configuration variable is

Documentation/git-fetch-pack.txt

@@ -32,7 +32,8 @@ OPTIONS
 -k::
 	Do not invoke 'git-unpack-objects' on received data, but
 	create a single packfile out of it instead, and store it
-	in the object database.
+	in the object database. If provided twice then the pack is
+	locked against repacking.
 
 --exec=<git-upload-pack>::
 	Use this to specify the path to 'git-upload-pack' on the

Documentation/git-index-pack.txt

@@ -8,7 +8,8 @@ git-index-pack - Build pack index file for an existing packed archive
 
 SYNOPSIS
 --------
-'git-index-pack' [-o <index-file>] <pack-file>
+'git-index-pack' [-v] [-o <index-file>] <pack-file>
+'git-index-pack' --stdin [--fix-thin] [--keep] [-v] [-o <index-file>] [<pack-file>]
 
 
 DESCRIPTION
@@ -21,6 +22,9 @@ objects/pack/ directory of a git repository.
 
 OPTIONS
 -------
+-v::
+	Be verbose about what is going on, including progress status.
+
 -o <index-file>::
 	Write the generated pack index into the specified
 	file.  Without this option the name of pack index
@@ -29,6 +33,52 @@ OPTIONS
 	fails if the name of packed archive does not end
 	with .pack).
 
+--stdin::
+	When this flag is provided, the pack is read from stdin
+	instead and a copy is then written to <pack-file>. If
+	<pack-file> is not specified, the pack is written to
+	objects/pack/ directory of the current git repository with
+	a default name determined from the pack content.  If
+	<pack-file> is not specified consider using --keep to
+	prevent a race condition between this process and
+	gitlink::git-repack[1] .
+
+--fix-thin::
+	It is possible for gitlink:git-pack-objects[1] to build
+	"thin" pack, which records objects in deltified form based on
+	objects not included in the pack to reduce network traffic.
+	Those objects are expected to be present on the receiving end
+	and they must be included in the pack for that pack to be self
+	contained and indexable. Without this option any attempt to
+	index a thin pack will fail. This option only makes sense in
+	conjunction with --stdin.
+
+--keep::
+	Before moving the index into its final destination
+	create an empty .keep file for the associated pack file.
+	This option is usually necessary with --stdin to prevent a
+	simultaneous gitlink:git-repack[1] process from deleting
+	the newly constructed pack and index before refs can be
+	updated to use objects contained in the pack.
+
+--keep='why'::
+	Like --keep create a .keep file before moving the index into
+	its final destination, but rather than creating an empty file
+	place 'why' followed by an LF into the .keep file.  The 'why'
+	message can later be searched for within all .keep files to
+	locate any which have outlived their usefulness.
+
+
+Note
+----
+
+Once the index has been created, the list of object names is sorted
+and the SHA1 hash of that list is printed to stdout. If --stdin was
+also used then this is prefixed by either "pack\t", or "keep\t" if a
+new .keep file was successfully created. This is useful to remove a
+.keep file used as a lock to prevent the race with gitlink:git-repack[1]
+mentioned above.
+
 
 Author
 ------

Makefile

@@ -257,7 +257,7 @@ LIB_OBJS = \
 	quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \
 	server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \
 	tag.o tree.o usage.o config.o environment.o ctype.o copy.o \
-	fetch-clone.o revision.o pager.o tree-walk.o xdiff-interface.o \
+	revision.o pager.o tree-walk.o xdiff-interface.o \
 	write_or_die.o trace.o list-objects.o grep.o \
 	alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
 	color.o wt-status.o archive-zip.o archive-tar.o

builtin-unpack-objects.c

@@ -371,6 +371,21 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
 				recover = 1;
 				continue;
 			}
+			if (!strncmp(arg, "--pack_header=", 14)) {
+				struct pack_header *hdr;
+				char *c;
+
+				hdr = (struct pack_header *)buffer;
+				hdr->hdr_signature = htonl(PACK_SIGNATURE);
+				hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
+				if (*c != ',')
+					die("bad %s", arg);
+				hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
+				if (*c)
+					die("bad %s", arg);
+				len = sizeof(*hdr);
+				continue;
+			}
 			usage(unpack_usage);
 		}
 

cache.h

@@ -376,6 +376,7 @@ extern struct packed_git *parse_pack_index_file(const unsigned char *sha1,
 						char *idx_path);
 
 extern void prepare_packed_git(void);
+extern void reprepare_packed_git(void);
 extern void install_packed_git(struct packed_git *pack);
 
 extern struct packed_git *find_sha1_pack(const unsigned char *sha1, 
@@ -415,10 +416,6 @@ extern int copy_fd(int ifd, int ofd);
 extern void write_or_die(int fd, const void *buf, size_t count);
 extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
 
-/* Finish off pack transfer receiving end */
-extern int receive_unpack_pack(int fd[2], const char *me, int quiet, int);
-extern int receive_keep_pack(int fd[2], const char *me, int quiet, int);
-
 /* pager.c */
 extern void setup_pager(void);
 extern int pager_in_use;

fetch-clone.c

-#include "cache.h"
-#include "exec_cmd.h"
-#include "pkt-line.h"
-#include "sideband.h"
-#include <sys/wait.h>
-#include <sys/time.h>
-
-static int finish_pack(const char *pack_tmp_name, const char *me)
-{
-	int pipe_fd[2];
-	pid_t pid;
-	char idx[PATH_MAX];
-	char final[PATH_MAX];
-	char hash[41];
-	unsigned char sha1[20];
-	char *cp;
-	int err = 0;
-
-	if (pipe(pipe_fd) < 0)
-		die("%s: unable to set up pipe", me);
-
-	strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
-	cp = strrchr(idx, '/');
-	memcpy(cp, "/pidx", 5);
-
-	pid = fork();
-	if (pid < 0)
-		die("%s: unable to fork off git-index-pack", me);
-	if (!pid) {
-		close(0);
-		dup2(pipe_fd[1], 1);
-		close(pipe_fd[0]);
-		close(pipe_fd[1]);
-		execl_git_cmd("index-pack", "-o", idx, pack_tmp_name, NULL);
-		error("cannot exec git-index-pack <%s> <%s>",
-		      idx, pack_tmp_name);
-		exit(1);
-	}
-	close(pipe_fd[1]);
-	if (read(pipe_fd[0], hash, 40) != 40) {
-		error("%s: unable to read from git-index-pack", me);
-		err = 1;
-	}
-	close(pipe_fd[0]);
-
-	for (;;) {
-		int status, code;
-
-		if (waitpid(pid, &status, 0) < 0) {
-			if (errno == EINTR)
-				continue;
-			error("waitpid failed (%s)", strerror(errno));
-			goto error_die;
-		}
-		if (WIFSIGNALED(status)) {
-			int sig = WTERMSIG(status);
-			error("git-index-pack died of signal %d", sig);
-			goto error_die;
-		}
-		if (!WIFEXITED(status)) {
-			error("git-index-pack died of unnatural causes %d",
-			      status);
-			goto error_die;
-		}
-		code = WEXITSTATUS(status);
-		if (code) {
-			error("git-index-pack died with error code %d", code);
-			goto error_die;
-		}
-		if (err)
-			goto error_die;
-		break;
-	}
-	hash[40] = 0;
-	if (get_sha1_hex(hash, sha1)) {
-		error("git-index-pack reported nonsense '%s'", hash);
-		goto error_die;
-	}
-	/* Now we have pack in pack_tmp_name[], and
-	 * idx in idx[]; rename them to their final names.
-	 */
-	snprintf(final, sizeof(final),
-		 "%s/pack/pack-%s.pack", get_object_directory(), hash);
-	move_temp_to_file(pack_tmp_name, final);
-	chmod(final, 0444);
-	snprintf(final, sizeof(final),
-		 "%s/pack/pack-%s.idx", get_object_directory(), hash);
-	move_temp_to_file(idx, final);
-	chmod(final, 0444);
-	return 0;
-
- error_die:
-	unlink(idx);
-	unlink(pack_tmp_name);
-	exit(1);
-}
-
-static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
-{
-	pid_t side_pid;
-
-	if (!sideband) {
-		fd[0] = xd[0];
-		fd[1] = xd[1];
-		return 0;
-	}
-	/* xd[] is talking with upload-pack; subprocess reads from
-	 * xd[0], spits out band#2 to stderr, and feeds us band#1
-	 * through our fd[0].
-	 */
-	if (pipe(fd) < 0)
-		die("%s: unable to set up pipe", me);
-	side_pid = fork();
-	if (side_pid < 0)
-		die("%s: unable to fork off sideband demultiplexer", me);
-	if (!side_pid) {
-		/* subprocess */
-		close(fd[0]);
-		if (xd[0] != xd[1])
-			close(xd[1]);
-		if (recv_sideband(me, xd[0], fd[1], 2))
-			exit(1);
-		exit(0);
-	}
-	close(xd[0]);
-	close(fd[1]);
-	fd[1] = xd[1];
-	return side_pid;
-}
-
-int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
-{
-	int status;
-	pid_t pid, side_pid;
-	int fd[2];
-
-	side_pid = setup_sideband(sideband, me, fd, xd);
-	pid = fork();
-	if (pid < 0)
-		die("%s: unable to fork off git-unpack-objects", me);
-	if (!pid) {
-		dup2(fd[0], 0);
-		close(fd[0]);
-		close(fd[1]);
-		execl_git_cmd("unpack-objects", quiet ? "-q" : NULL, NULL);
-		die("git-unpack-objects exec failed");
-	}
-	close(fd[0]);
-	close(fd[1]);
-	while (waitpid(pid, &status, 0) < 0) {
-		if (errno != EINTR)
-			die("waiting for git-unpack-objects: %s",
-			    strerror(errno));
-	}
-	if (WIFEXITED(status)) {
-		int code = WEXITSTATUS(status);
-		if (code)
-			die("git-unpack-objects died with error code %d",
-			    code);
-		return 0;
-	}
-	if (WIFSIGNALED(status)) {
-		int sig = WTERMSIG(status);
-		die("git-unpack-objects died of signal %d", sig);
-	}
-	die("git-unpack-objects died of unnatural causes %d", status);
-}
-
-/*
- * We average out the download speed over this many "events", where
- * an event is a minimum of about half a second. That way, we get
- * a reasonably stable number.
- */
-#define NR_AVERAGE (4)
-
-/*
- * A "binary msec" is a power-of-two-msec, aka 1/1024th of a second.
- * Keeping the time in that format means that "bytes / msecs" means
- * the same as kB/s (modulo rounding).
- *
- * 1000512 is a magic number (usecs in a second, rounded up by half
- * of 1024, to make "rounding" come out right ;)
- */
-#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))
-
-int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)
-{
-	char tmpfile[PATH_MAX];
-	int ofd, ifd, fd[2];
-	unsigned long total;
-	static struct timeval prev_tv;
-	struct average {
-		unsigned long bytes;
-		unsigned long time;
-	} download[NR_AVERAGE] = { {0, 0}, };
-	unsigned long avg_bytes, avg_time;
-	int idx = 0;
-
-	setup_sideband(sideband, me, fd, xd);
-
-	ifd = fd[0];
-	snprintf(tmpfile, sizeof(tmpfile),
-		 "%s/pack/tmp-XXXXXX", get_object_directory());
-	ofd = mkstemp(tmpfile);
-	if (ofd < 0)
-		return error("unable to create temporary file %s", tmpfile);
-
-	gettimeofday(&prev_tv, NULL);
-	total = 0;
-	avg_bytes = 0;
-	avg_time = 0;
-	while (1) {
-		char buf[8192];
-		ssize_t sz, wsz, pos;
-		sz = read(ifd, buf, sizeof(buf));
-		if (sz == 0)
-			break;
-		if (sz < 0) {
-			if (errno != EINTR && errno != EAGAIN) {
-				error("error reading pack (%s)", strerror(errno));
-				close(ofd);
-				unlink(tmpfile);
-				return -1;
-			}
-			sz = 0;
-		}
-		pos = 0;
-		while (pos < sz) {
-			wsz = write(ofd, buf + pos, sz - pos);
-			if (wsz < 0) {
-				error("error writing pack (%s)",
-				      strerror(errno));
-				close(ofd);
-				unlink(tmpfile);
-				return -1;
-			}
-			pos += wsz;
-		}
-		total += sz;
-		if (!quiet) {
-			static unsigned long last;
-			struct timeval tv;
-			unsigned long diff = total - last;
-			/* not really "msecs", but a power-of-two millisec (1/1024th of a sec) */
-			unsigned long msecs;
-
-			gettimeofday(&tv, NULL);
-			msecs = tv.tv_sec - prev_tv.tv_sec;
-			msecs <<= 10;
-			msecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec);
-
-			if (msecs > 500) {
-				prev_tv = tv;
-				last = total;
-
-				/* Update averages ..*/
-				avg_bytes += diff;
-				avg_time += msecs;
-				avg_bytes -= download[idx].bytes;
-				avg_time -= download[idx].time;
-				download[idx].bytes = diff;
-				download[idx].time = msecs;
-				idx++;
-				if (idx >= NR_AVERAGE)
-					idx = 0;
-
-				fprintf(stderr, "%4lu.%03luMB  (%lu kB/s)      \r",
-					total >> 20,
-					1000*((total >> 10) & 1023)>>10,
-					avg_bytes / avg_time );
-			}
-		}
-	}
-	close(ofd);
-	return finish_pack(tmpfile, me);
-}

fetch-pack.c

@@ -3,6 +3,9 @@
 #include "pkt-line.h"
 #include "commit.h"
 #include "tag.h"
+#include "exec_cmd.h"
+#include "sideband.h"
+#include <sys/wait.h>
 
 static int keep_pack;
 static int quiet;
@@ -416,6 +419,103 @@ static int everything_local(struct ref **refs, int nr_match, char **match)
 	return retval;
 }
 
+static pid_t setup_sideband(int fd[2], int xd[2])
+{
+	pid_t side_pid;
+
+	if (!use_sideband) {
+		fd[0] = xd[0];
+		fd[1] = xd[1];
+		return 0;
+	}
+	/* xd[] is talking with upload-pack; subprocess reads from
+	 * xd[0], spits out band#2 to stderr, and feeds us band#1
+	 * through our fd[0].
+	 */
+	if (pipe(fd) < 0)
+		die("fetch-pack: unable to set up pipe");
+	side_pid = fork();
+	if (side_pid < 0)
+		die("fetch-pack: unable to fork off sideband demultiplexer");
+	if (!side_pid) {
+		/* subprocess */
+		close(fd[0]);
+		if (xd[0] != xd[1])
+			close(xd[1]);
+		if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
+			exit(1);
+		exit(0);
+	}
+	close(xd[0]);
+	close(fd[1]);
+	fd[1] = xd[1];
+	return side_pid;
+}
+
+static int get_pack(int xd[2], const char **argv)
+{
+	int status;
+	pid_t pid, side_pid;
+	int fd[2];
+
+	side_pid = setup_sideband(fd, xd);
+	pid = fork();
+	if (pid < 0)
+		die("fetch-pack: unable to fork off %s", argv[0]);
+	if (!pid) {
+		dup2(fd[0], 0);
+		close(fd[0]);
+		close(fd[1]);
+		execv_git_cmd(argv);
+		die("%s exec failed", argv[0]);
+	}
+	close(fd[0]);
+	close(fd[1]);
+	while (waitpid(pid, &status, 0) < 0) {
+		if (errno != EINTR)
+			die("waiting for %s: %s", argv[0], strerror(errno));
+	}
+	if (WIFEXITED(status)) {
+		int code = WEXITSTATUS(status);
+		if (code)
+			die("%s died with error code %d", argv[0], code);
+		return 0;
+	}
+	if (WIFSIGNALED(status)) {
+		int sig = WTERMSIG(status);
+		die("%s died of signal %d", argv[0], sig);
+	}
+	die("%s died of unnatural causes %d", argv[0], status);
+}
+
+static int explode_rx_pack(int xd[2])
+{
+	const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
+	return get_pack(xd, argv);
+}
+
+static int keep_rx_pack(int xd[2])
+{
+	const char *argv[6];
+	char keep_arg[256];
+	int n = 0;
+
+	argv[n++] = "index-pack";
+	argv[n++] = "--stdin";
+	if (!quiet)
+		argv[n++] = "-v";
+	if (use_thin_pack)
+		argv[n++] = "--fix-thin";
+	if (keep_pack > 1) {
+		int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
+		if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
+			strcpy(keep_arg + s, "localhost");
+		argv[n++] = keep_arg;
+	}
+	argv[n] = NULL;
+	return get_pack(xd, argv);
+}
+
 static int fetch_pack(int fd[2], int nr_match, char **match)
 {
 	struct ref *ref;
@@ -447,17 +547,13 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
 		goto all_done;
 	}
 	if (find_common(fd, sha1, ref) < 0)
-		if (!keep_pack)
+		if (keep_pack != 1)
 			/* When cloning, it is not unusual to have
 			 * no common commit.
 			 */
 			fprintf(stderr, "warning: no common commits\n");
 
-	if (keep_pack)
-		status = receive_keep_pack(fd, "git-fetch-pack", quiet, use_sideband);
-	else
-		status = receive_unpack_pack(fd, "git-fetch-pack", quiet, use_sideband);
-
+	status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 	if (status)
 		die("git-fetch-pack: fetch failed.");
 
@@ -494,7 +590,7 @@ int main(int argc, char **argv)
 				continue;
 			}
 			if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
-				keep_pack = 1;
+				keep_pack++;
 				continue;
 			}
 			if (!strcmp("--thin", arg)) {
@@ -518,8 +614,6 @@ int main(int argc, char **argv)
 	}
 	if (!dest)
 		usage(fetch_pack_usage);
-	if (keep_pack)
-		use_thin_pack = 0;
 	pid = git_connect(fd, dest, exec);
 	if (pid < 0)
 		return 1;

git-fetch.sh

@@ -20,7 +20,7 @@ verbose=
 update_head_ok=
 exec=
 upload_pack=
-keep=--thin
+keep=
 while case "$#" in 0) break ;; esac
 do
 	case "$1" in
@@ -51,7 +51,7 @@ do
 		verbose=Yes
 		;;
 	-k|--k|--ke|--kee|--keep)
-		keep=--keep
+		keep='-k -k'
 		;;
 	--reflog-action=*)
 		rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'`
@@ -368,9 +368,10 @@ fetch_main () {
       ;; # we are already done.
   *)
     ( : subshell because we muck with IFS
+      pack_lockfile=
       IFS=" 	$LF"
       (
-	  git-fetch-pack $exec $keep "$remote" $rref || echo failed "$remote"
+	  git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote"
       ) |
       while read sha1 remote_name
       do
@@ -378,6 +379,12 @@ fetch_main () {
 	  failed)
 		  echo >&2 "Fetch failure: $remote"
 		  exit 1 ;;
+	  # special line coming from index-pack with the pack name
+	  pack)
+		  continue ;;
+	  keep)
+		  pack_lockfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep"
+		  continue ;;
 	  esac
 	  found=
 	  single_force=
@@ -408,6 +415,7 @@ fetch_main () {
 	  append_fetch_head "$sha1" "$remote" \
 		  "$remote_name" "$remote_nick" "$local_name" "$not_for_merge"
       done
+      if [ "$pack_lockfile" ]; then rm -f "$pack_lockfile"; fi
     ) || exit ;;
   esac
 

git-repack.sh

@@ -45,11 +45,19 @@ case ",$all_into_one," in
 	args='--unpacked --incremental'
 	;;
 ,t,)
-	args=
-
-	# Redundancy check in all-into-one case is trivial.
-	existing=`test -d "$PACKDIR" && cd "$PACKDIR" && \
-	    find . -type f \( -name '*.pack' -o -name '*.idx' \) -print`
+	if [ -d "$PACKDIR" ]; then
+		for e in `cd "$PACKDIR" && find . -type f -name '*.pack' \
+			| sed -e 's/^\.\///' -e 's/\.pack$//'`
+		do
+			if [ -e "$PACKDIR/$e.keep" ]; then
+				: keep
+			else
+				args="$args --unpacked=$e.pack"
+				existing="$existing $e"
+			fi
+		done
+	fi
+	[ -z "$args" ] && args='--unpacked --incremental'
 	;;
 esac
 
@@ -86,17 +94,16 @@ fi
 
 if test "$remove_redundant" = t
 then
-	# We know $existing are all redundant only when
-	# all-into-one is used.
-	if test "$all_into_one" != '' && test "$existing" != ''
+	# We know $existing are all redundant.
+	if [ -n "$existing" ]
 	then
 		sync
 		( cd "$PACKDIR" &&
 		  for e in $existing
 		  do
 			case "$e" in
-			./pack-$name.pack | ./pack-$name.idx) ;;
-			*)	rm -f $e ;;
+			pack-$name) ;;
+			*)	rm -f "$e.pack" "$e.idx" "$e.keep" ;;
 			esac
 		  done
 		)

receive-pack.c

 #include "cache.h"
+#include "pack.h"
 #include "refs.h"
 #include "pkt-line.h"
 #include "run-command.h"
+#include "exec_cmd.h"
 #include "commit.h"
 #include "object.h"
+#include <sys/wait.h>
 
 static const char receive_pack_usage[] = "git-receive-pack <git-dir>";
 
-static const char *unpacker[] = { "unpack-objects", NULL };
-
 static int deny_non_fast_forwards = 0;
+static int unpack_limit = 5000;
 static int report_status;
 
 static char capabilities[] = "report-status";
@@ -25,6 +27,12 @@ static int receive_pack_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (strcmp(var, "receive.unpacklimit") == 0)
+	{
+		unpack_limit = git_config_int(var, value);
+		return 0;
+	}
+
 	return 0;
 }
 
@@ -227,29 +235,127 @@ static void read_head_info(void)
 	}
 }
 
-static const char *unpack(int *error_code)
+static const char *parse_pack_header(struct pack_header *hdr)
 {
-	int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD);
+	char *c = (char*)hdr;
+	ssize_t remaining = sizeof(struct pack_header);
+	do {
+		ssize_t r = xread(0, c, remaining);
+		if (r <= 0)
+			return "eof before pack header was fully read";
+		remaining -= r;
+		c += r;
+	} while (remaining > 0);
+	if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
+		return "protocol error (pack signature mismatch detected)";
+	if (!pack_version_ok(hdr->hdr_version))
+		return "protocol error (pack version unsupported)";
+	return NULL;
+}
 
-	*error_code = 0;
-	switch (code) {
-	case 0:
-		return NULL;
-	case -ERR_RUN_COMMAND_FORK:
-		return "unpack fork failed";
-	case -ERR_RUN_COMMAND_EXEC:
-		return "unpack execute failed";
-	case -ERR_RUN_COMMAND_WAITPID:
-		return "waitpid failed";
-	case -ERR_RUN_COMMAND_WAITPID_WRONG_PID:
-		return "waitpid is confused";
-	case -ERR_RUN_COMMAND_WAITPID_SIGNAL:
-		return "unpacker died of signal";
-	case -ERR_RUN_COMMAND_WAITPID_NOEXIT:
-		return "unpacker died strangely";
-	default:
-		*error_code = -code;
-		return "unpacker exited with error code";
+static const char *pack_lockfile;
+
+static const char *unpack(void)
+{
+	struct pack_header hdr;
+	const char *hdr_err;
+	char hdr_arg[38];
+
+	hdr_err = parse_pack_header(&hdr);
+	if (hdr_err)
+		return hdr_err;
+	snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
+			ntohl(hdr.hdr_version), ntohl(hdr.hdr_entries));
+
+	if (ntohl(hdr.hdr_entries) < unpack_limit) {
+		int code;
+		const char *unpacker[3];
+		unpacker[0] = "unpack-objects";
+		unpacker[1] = hdr_arg;
+		unpacker[2] = NULL;
+		code = run_command_v_opt(1, unpacker, RUN_GIT_CMD);
+		switch (code) {
+		case 0:
+			return NULL;
+		case -ERR_RUN_COMMAND_FORK:
+			return "unpack fork failed";
+		case -ERR_RUN_COMMAND_EXEC:
+			return "unpack execute failed";
+		case -ERR_RUN_COMMAND_WAITPID:
+			return "waitpid failed";
+		case -ERR_RUN_COMMAND_WAITPID_WRONG_PID:
+			return "waitpid is confused";
+		case -ERR_RUN_COMMAND_WAITPID_SIGNAL:
+			return "unpacker died of signal";
+		case -ERR_RUN_COMMAND_WAITPID_NOEXIT:
+			return "unpacker died strangely";
+		default:
+			return "unpacker exited with error code";
+		}
+	} else {
+		const char *keeper[6];
+		int fd[2], s, len, status;
+		pid_t pid;
+		char keep_arg[256];
+		char packname[46];
+
+		s = sprintf(keep_arg, "--keep=receive-pack %i on ", getpid());
+		if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
+			strcpy(keep_arg + s, "localhost");
+
+		keeper[0] = "index-pack";
+		keeper[1] = "--stdin";
+		keeper[2] = "--fix-thin";
+		keeper[3] = hdr_arg;
+		keeper[4] = keep_arg;
+		keeper[5] = NULL;
+
+		if (pipe(fd) < 0)
+			return "index-pack pipe failed";
+		pid = fork();
+		if (pid < 0)
+			return "index-pack fork failed";
+		if (!pid) {
+			dup2(fd[1], 1);
+			close(fd[1]);
+			close(fd[0]);
+			execv_git_cmd(keeper);
+			die("execv of index-pack failed");
+		}
+		close(fd[1]);
+
+		/*
+		 * The first thing we expects from index-pack's output
+		 * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
+		 * %40s is the newly created pack SHA1 name.  In the "keep"
+		 * case, we need it to remove the corresponding .keep file
+		 * later on.  If we don't get that then tough luck with it.
+		 */
+		for (len = 0;
+		     len < 46 && (s = xread(fd[0], packname+len, 46-len)) > 0;
+		     len += s);
+		close(fd[0]);
+		if (len == 46 && packname[45] == '\n' &&
+		    memcmp(packname, "keep\t", 5) == 0) {
+			char path[PATH_MAX];
+			packname[45] = 0;
+			snprintf(path, sizeof(path), "%s/pack/pack-%s.keep",
+				 get_object_directory(), packname + 5);
+			pack_lockfile = xstrdup(path);
+		}
+
+		/* Then wrap our index-pack process. */
+		while (waitpid(pid, &status, 0) < 0)
+			if (errno != EINTR)
+				return "waitpid failed";
+		if (WIFEXITED(status)) {
+			int code = WEXITSTATUS(status);
+			if (code)
+				return "index-pack exited with error code";
+			reprepare_packed_git();
+			return NULL;
+		}
+		return "index-pack abnormal exit";
 	}
 }
 
@@ -302,10 +408,11 @@ int main(int argc, char **argv)
 
 	read_head_info();
 	if (commands) {
-		int code;
-		const char *unpack_status = unpack(&code);
+		const char *unpack_status = unpack();
 		if (!unpack_status)
 			execute_commands();
+		if (pack_lockfile)
+			unlink(pack_lockfile);
 		if (report_status)
 			report(unpack_status);
 	}

sha1_file.c

@@ -663,7 +663,7 @@ void prepare_packed_git(void)
 	prepare_packed_git_run_once = 1;
 }
 
-static void reprepare_packed_git(void)
+void reprepare_packed_git(void)
 {
 	prepare_packed_git_run_once = 0;
 	prepare_packed_git();
@@ -1203,6 +1203,24 @@ unsigned long find_pack_entry_one(const unsigned char *sha1,
 	return 0;
 }
 
+static int matches_pack_name(struct packed_git *p, const char *ig)
+{
+	const char *last_c, *c;
+
+	if (!strcmp(p->pack_name, ig))
+		return 0;
+
+	for (c = p->pack_name, last_c = c; *c;)
+		if (*c == '/')
+			last_c = ++c;
+		else
+			++c;
+	if (!strcmp(last_c, ig))
+		return 0;
+
+	return 1;
+}
+
 static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
 {
 	struct packed_git *p;
@@ -1214,7 +1232,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, cons
 		if (ignore_packed) {
 			const char **ig;
 			for (ig = ignore_packed; *ig; ig++)
-				if (!strcmp(p->pack_name, *ig))
+				if (!matches_pack_name(p, *ig))
 					break;
 			if (*ig)
 				continue;