builtin-pack-objects.c 42.6 KB
Newer Older
1
#include "builtin.h"
2 3
#include "cache.h"
#include "object.h"
4 5 6 7
#include "blob.h"
#include "commit.h"
#include "tag.h"
#include "tree.h"
8
#include "delta.h"
9
#include "pack.h"
10
#include "csum-file.h"
J
Junio C Hamano 已提交
11
#include "tree-walk.h"
12 13 14
#include "diff.h"
#include "revision.h"
#include "list-objects.h"
15

16 17 18 19
static const char pack_usage[] = "\
git-pack-objects [{ -q | --progress | --all-progress }] \n\
	[--local] [--incremental] [--window=N] [--depth=N] \n\
	[--no-reuse-delta] [--delta-base-offset] [--non-empty] \n\
20
	[--revs [--unpacked | --all]*] [--reflog] [--stdout | base-name] \n\
21
	[<ref-list | <object-list]";
22 23 24

struct object_entry {
	unsigned char sha1[20];
25
	unsigned long size;	/* uncompressed size */
26 27 28
	unsigned long offset;	/* offset into the final pack file;
				 * nonzero if already written.
				 */
29
	unsigned int depth;	/* delta depth */
30
	unsigned int delta_limit;	/* base adjustment for in-pack delta */
31
	unsigned int hash;	/* name hint hash */
32
	enum object_type type;
33
	enum object_type in_pack_type;	/* could be delta */
34
	unsigned long delta_size;	/* delta data size (uncompressed) */
35
#define in_pack_header_size delta_size	/* only when reusing pack data */
36 37 38
	struct object_entry *delta;	/* delta base object */
	struct packed_git *in_pack; 	/* already in pack */
	unsigned int in_pack_offset;
39
	struct object_entry *delta_child; /* deltified objects who bases me */
40 41 42
	struct object_entry *delta_sibling; /* other deltified objects who
					     * uses the same base as me
					     */
43 44 45 46
	int preferred_base;	/* we do not pack this, but is encouraged to
				 * be used as the base objectto delta huge
				 * objects against.
				 */
47 48
};

49
/*
50
 * Objects we are going to pack are collected in objects array (dynamically
51 52 53 54 55 56 57 58 59 60 61
 * expanded).  nr_objects & nr_alloc controls this array.  They are stored
 * in the order we see -- typically rev-list --objects order that gives us
 * nice "minimum seek" order.
 *
 * sorted-by-sha ans sorted-by-type are arrays of pointers that point at
 * elements in the objects array.  The former is used to build the pack
 * index (lists object names in the ascending order to help offset lookup),
 * and the latter is used to group similar things together by try_delta()
 * heuristics.
 */

62
static unsigned char object_list_sha1[20];
63 64 65 66
static int non_empty;
static int no_reuse_delta;
static int local;
static int incremental;
67 68
static int allow_ofs_delta;

69
static struct object_entry **sorted_by_sha, **sorted_by_type;
70 71
static struct object_entry *objects;
static int nr_objects, nr_alloc, nr_result;
72
static const char *base_name;
73
static unsigned char pack_file_sha1[20];
J
Junio C Hamano 已提交
74
static int progress = 1;
75
static volatile sig_atomic_t progress_update;
76
static int window = 10;
77
static int pack_to_stdout;
78
static int num_preferred_base;
79

80 81 82 83 84 85
/*
 * The object names in objects array are hashed with this hashtable,
 * to help looking up the entry by object name.  Binary search from
 * sorted_by_sha is also possible but this was easier to code and faster.
 * This hashtable is built after all the objects are seen.
 */
86 87
static int *object_ix;
static int object_ix_hashsz;
88 89 90 91 92

/*
 * Pack index for existing packs give us easy access to the offsets into
 * corresponding pack file where each object's data starts, but the entries
 * do not store the size of the compressed representation (uncompressed
93 94 95 96 97 98 99 100 101
 * size is easily available by examining the pack entry header).  It is
 * also rather expensive to find the sha1 for an object given its offset.
 *
 * We build a hashtable of existing packs (pack_revindex), and keep reverse
 * index here -- pack index file is sorted by object name mapping to offset;
 * this pack_revindex[].revindex array is a list of offset/index_nr pairs
 * ordered by offset, so if you know the offset of an object, next offset
 * is where its packed representation ends and the index_nr can be used to
 * get the object sha1 from the main index.
102
 */
103 104 105 106
struct revindex_entry {
	unsigned int offset;
	unsigned int nr;
};
107 108
struct pack_revindex {
	struct packed_git *p;
109 110 111
	struct revindex_entry *revindex;
};
static struct  pack_revindex *pack_revindex;
112
static int pack_revindex_hashsz;
113 114 115 116

/*
 * stats
 */
117 118 119 120
static int written;
static int written_delta;
static int reused;
static int reused_delta;
121 122 123

static int pack_revindex_ix(struct packed_git *p)
{
L
Luck, Tony 已提交
124
	unsigned long ui = (unsigned long)p;
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
	int i;

	ui = ui ^ (ui >> 16); /* defeat structure alignment */
	i = (int)(ui % pack_revindex_hashsz);
	while (pack_revindex[i].p) {
		if (pack_revindex[i].p == p)
			return i;
		if (++i == pack_revindex_hashsz)
			i = 0;
	}
	return -1 - i;
}

static void prepare_pack_ix(void)
{
	int num;
	struct packed_git *p;
	for (num = 0, p = packed_git; p; p = p->next)
		num++;
	if (!num)
		return;
	pack_revindex_hashsz = num * 11;
	pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
	for (p = packed_git; p; p = p->next) {
		num = pack_revindex_ix(p);
		num = - 1 - num;
		pack_revindex[num].p = p;
	}
	/* revindex elements are lazily initialized */
}

static int cmp_offset(const void *a_, const void *b_)
{
158 159 160
	const struct revindex_entry *a = a_;
	const struct revindex_entry *b = b_;
	return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
161 162 163 164 165 166 167 168 169 170 171 172
}

/*
 * Ordered list of offsets of objects in the pack.
 */
static void prepare_pack_revindex(struct pack_revindex *rix)
{
	struct packed_git *p = rix->p;
	int num_ent = num_packed_objects(p);
	int i;
	void *index = p->index_base + 256;

173
	rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
174
	for (i = 0; i < num_ent; i++) {
175
		unsigned int hl = *((unsigned int *)((char *) index + 24*i));
176 177
		rix->revindex[i].offset = ntohl(hl);
		rix->revindex[i].nr = i;
178 179 180 181
	}
	/* This knows the pack format -- the 20-byte trailer
	 * follows immediately after the last object data.
	 */
182 183 184
	rix->revindex[num_ent].offset = p->pack_size - 20;
	rix->revindex[num_ent].nr = -1;
	qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
185 186
}

187 188
static struct revindex_entry * find_packed_object(struct packed_git *p,
						  unsigned int ofs)
189 190 191 192
{
	int num;
	int lo, hi;
	struct pack_revindex *rix;
193
	struct revindex_entry *revindex;
194 195 196 197 198 199 200 201 202 203 204
	num = pack_revindex_ix(p);
	if (num < 0)
		die("internal error: pack revindex uninitialized");
	rix = &pack_revindex[num];
	if (!rix->revindex)
		prepare_pack_revindex(rix);
	revindex = rix->revindex;
	lo = 0;
	hi = num_packed_objects(p) + 1;
	do {
		int mi = (lo + hi) / 2;
205 206
		if (revindex[mi].offset == ofs) {
			return revindex + mi;
207
		}
208
		else if (ofs < revindex[mi].offset)
209 210 211 212 213 214 215
			hi = mi;
		else
			lo = mi + 1;
	} while (lo < hi);
	die("internal error: pack revindex corrupt");
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229
static unsigned long find_packed_object_size(struct packed_git *p,
					     unsigned long ofs)
{
	struct revindex_entry *entry = find_packed_object(p, ofs);
	return entry[1].offset - ofs;
}

static unsigned char *find_packed_object_name(struct packed_git *p,
					      unsigned long ofs)
{
	struct revindex_entry *entry = find_packed_object(p, ofs);
	return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
}

230 231 232 233 234 235 236 237 238
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
{
	unsigned long othersize, delta_size;
	char type[10];
	void *otherbuf = read_sha1_file(entry->delta->sha1, type, &othersize);
	void *delta_buf;

	if (!otherbuf)
		die("unable to read %s", sha1_to_hex(entry->delta->sha1));
239
        delta_buf = diff_delta(otherbuf, othersize,
240
			       buf, size, &delta_size, 0);
241
        if (!delta_buf || delta_size != entry->delta_size)
242 243 244 245 246 247
        	die("delta size changed");
        free(buf);
        free(otherbuf);
	return delta_buf;
}

248 249 250 251 252 253 254 255 256
/*
 * The per-object header is a pretty dense thing, which is
 *  - first byte: low four bits are "size", then three bits of "type",
 *    and the high bit is "size continues".
 *  - each byte afterwards: low seven bits are size continuation,
 *    with the high bit being "size continues"
 */
static int encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
{
257
	int n = 1;
258 259
	unsigned char c;

260
	if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
261 262
		die("bad type %d", type);

263 264 265
	c = (type << 4) | (size & 15);
	size >>= 4;
	while (size) {
266
		*hdr++ = c | 0x80;
267 268 269
		c = size & 0x7f;
		size >>= 7;
		n++;
270 271 272 273 274
	}
	*hdr = c;
	return n;
}

275 276 277 278
/*
 * we are going to reuse the existing object data as is.  make
 * sure it is not corrupt.
 */
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
static int check_pack_inflate(struct packed_git *p,
		struct pack_window **w_curs,
		unsigned long offset,
		unsigned long len,
		unsigned long expect)
{
	z_stream stream;
	unsigned char fakebuf[4096], *in;
	int st;

	memset(&stream, 0, sizeof(stream));
	inflateInit(&stream);
	do {
		in = use_pack(p, w_curs, offset, &stream.avail_in);
		stream.next_in = in;
		stream.next_out = fakebuf;
		stream.avail_out = sizeof(fakebuf);
		st = inflate(&stream, Z_FINISH);
		offset += stream.next_in - in;
	} while (st == Z_OK || st == Z_BUF_ERROR);
	inflateEnd(&stream);
	return (st == Z_STREAM_END &&
		stream.total_out == expect &&
		stream.total_in == len) ? 0 : -1;
}

static void copy_pack_data(struct sha1file *f,
		struct packed_git *p,
		struct pack_window **w_curs,
		unsigned long offset,
		unsigned long len)
{
	unsigned char *in;
	unsigned int avail;

	while (len) {
		in = use_pack(p, w_curs, offset, &avail);
		if (avail > len)
			avail = len;
		sha1write(f, in, avail);
		offset += avail;
		len -= avail;
	}
}

static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)
325
{
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
	z_stream stream;
	unsigned char fakebuf[4096];
	int st;

	memset(&stream, 0, sizeof(stream));
	stream.next_in = data;
	stream.avail_in = len;
	stream.next_out = fakebuf;
	stream.avail_out = sizeof(fakebuf);
	inflateInit(&stream);

	while (1) {
		st = inflate(&stream, Z_FINISH);
		if (st == Z_STREAM_END || st == Z_OK) {
			st = (stream.total_out == expect &&
			      stream.total_in == len) ? 0 : -1;
			break;
		}
		if (st != Z_BUF_ERROR) {
			st = -1;
			break;
		}
		stream.next_out = fakebuf;
		stream.avail_out = sizeof(fakebuf);
	}
	inflateEnd(&stream);
	return st;
353 354 355 356 357 358 359
}

static int revalidate_loose_object(struct object_entry *entry,
				   unsigned char *map,
				   unsigned long mapsize)
{
	/* we already know this is a loose object with new type header. */
360 361
	enum object_type type;
	unsigned long size, used;
362 363 364 365

	if (pack_to_stdout)
		return 0;

366 367 368 369 370
	used = unpack_object_header_gently(map, mapsize, &type, &size);
	if (!used)
		return -1;
	map += used;
	mapsize -= used;
371
	return check_loose_inflate(map, mapsize, size);
372 373
}

374 375
static unsigned long write_object(struct sha1file *f,
				  struct object_entry *entry)
376 377 378
{
	unsigned long size;
	char type[10];
379
	void *buf;
380
	unsigned char header[10];
381
	unsigned hdrlen, datalen;
382
	enum object_type obj_type;
383
	int to_reuse = 0;
384

385
	obj_type = entry->type;
386 387
	if (! entry->in_pack)
		to_reuse = 0;	/* can't reuse what we don't have */
388
	else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
389 390 391 392 393 394 395 396 397 398
		to_reuse = 1;	/* check_object() decided it for us */
	else if (obj_type != entry->in_pack_type)
		to_reuse = 0;	/* pack has delta which is unusable */
	else if (entry->delta)
		to_reuse = 0;	/* we want to pack afresh */
	else
		to_reuse = 1;	/* we have it in-pack undeltified,
				 * and we do not need to deltify it.
				 */

399 400 401 402 403 404
	if (!entry->in_pack && !entry->delta) {
		unsigned char *map;
		unsigned long mapsize;
		map = map_sha1_file(entry->sha1, &mapsize);
		if (map && !legacy_loose_object(map)) {
			/* We can copy straight into the pack file */
405 406 407
			if (revalidate_loose_object(entry, map, mapsize))
				die("corrupt loose object %s",
				    sha1_to_hex(entry->sha1));
408 409 410 411 412 413 414 415 416 417
			sha1write(f, map, mapsize);
			munmap(map, mapsize);
			written++;
			reused++;
			return mapsize;
		}
		if (map)
			munmap(map, mapsize);
	}

418
	if (!to_reuse) {
419 420 421 422 423 424 425 426 427
		buf = read_sha1_file(entry->sha1, type, &size);
		if (!buf)
			die("unable to read %s", sha1_to_hex(entry->sha1));
		if (size != entry->size)
			die("object %s size inconsistency (%lu vs %lu)",
			    sha1_to_hex(entry->sha1), size, entry->size);
		if (entry->delta) {
			buf = delta_against(buf, size, entry);
			size = entry->delta_size;
428 429
			obj_type = (allow_ofs_delta && entry->delta->offset) ?
				OBJ_OFS_DELTA : OBJ_REF_DELTA;
430 431 432
		}
		/*
		 * The object header is a byte of 'type' followed by zero or
433
		 * more bytes of length.
434 435 436 437
		 */
		hdrlen = encode_header(obj_type, size, header);
		sha1write(f, header, hdrlen);

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
		if (obj_type == OBJ_OFS_DELTA) {
			/*
			 * Deltas with relative base contain an additional
			 * encoding of the relative offset for the delta
			 * base from this object's position in the pack.
			 */
			unsigned long ofs = entry->offset - entry->delta->offset;
			unsigned pos = sizeof(header) - 1;
			header[pos] = ofs & 127;
			while (ofs >>= 7)
				header[--pos] = 128 | (--ofs & 127);
			sha1write(f, header + pos, sizeof(header) - pos);
			hdrlen += sizeof(header) - pos;
		} else if (obj_type == OBJ_REF_DELTA) {
			/*
			 * Deltas with a base reference contain
			 * an additional 20 bytes for the base sha1.
			 */
			sha1write(f, entry->delta->sha1, 20);
457 458 459 460
			hdrlen += 20;
		}
		datalen = sha1write_compressed(f, buf, size);
		free(buf);
461
	}
462 463
	else {
		struct packed_git *p = entry->in_pack;
464
		struct pack_window *w_curs = NULL;
465
		unsigned long offset;
466

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
		if (entry->delta) {
			obj_type = (allow_ofs_delta && entry->delta->offset) ?
				OBJ_OFS_DELTA : OBJ_REF_DELTA;
			reused_delta++;
		}
		hdrlen = encode_header(obj_type, entry->size, header);
		sha1write(f, header, hdrlen);
		if (obj_type == OBJ_OFS_DELTA) {
			unsigned long ofs = entry->offset - entry->delta->offset;
			unsigned pos = sizeof(header) - 1;
			header[pos] = ofs & 127;
			while (ofs >>= 7)
				header[--pos] = 128 | (--ofs & 127);
			sha1write(f, header + pos, sizeof(header) - pos);
			hdrlen += sizeof(header) - pos;
		} else if (obj_type == OBJ_REF_DELTA) {
			sha1write(f, entry->delta->sha1, 20);
			hdrlen += 20;
		}
486

487
		offset = entry->in_pack_offset + entry->in_pack_header_size;
488 489
		datalen = find_packed_object_size(p, entry->in_pack_offset)
				- entry->in_pack_header_size;
490 491
		if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
				offset, datalen, entry->size))
492
			die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
493
		copy_pack_data(f, p, &w_curs, offset, datalen);
494
		unuse_pack(&w_curs);
495
		reused++;
496
	}
497
	if (entry->delta)
498
		written_delta++;
499
	written++;
500 501 502
	return hdrlen + datalen;
}

503 504 505 506
static unsigned long write_one(struct sha1file *f,
			       struct object_entry *e,
			       unsigned long offset)
{
507
	if (e->offset || e->preferred_base)
508 509 510 511
		/* offset starts from header size and cannot be zero
		 * if it is written already.
		 */
		return offset;
512
	/* if we are deltified, write out its base object first. */
513 514
	if (e->delta)
		offset = write_one(f, e->delta, offset);
515 516
	e->offset = offset;
	return offset + write_object(f, e);
517 518
}

519 520 521
static void write_pack_file(void)
{
	int i;
522
	struct sha1file *f;
523 524
	unsigned long offset;
	struct pack_header hdr;
525
	unsigned last_percent = 999;
N
Nicolas Pitre 已提交
526
	int do_progress = progress;
527

N
Nicolas Pitre 已提交
528
	if (!base_name) {
529
		f = sha1fd(1, "<stdout>");
N
Nicolas Pitre 已提交
530 531 532
		do_progress >>= 1;
	}
	else
533 534 535
		f = sha1create("%s-%s.%s", base_name,
			       sha1_to_hex(object_list_sha1), "pack");
	if (do_progress)
536
		fprintf(stderr, "Writing %d objects.\n", nr_result);
537

538
	hdr.hdr_signature = htonl(PACK_SIGNATURE);
539
	hdr.hdr_version = htonl(PACK_VERSION);
540
	hdr.hdr_entries = htonl(nr_result);
541 542
	sha1write(f, &hdr, sizeof(hdr));
	offset = sizeof(hdr);
543 544
	if (!nr_result)
		goto done;
545
	for (i = 0; i < nr_objects; i++) {
546
		offset = write_one(f, objects + i, offset);
547
		if (do_progress) {
548
			unsigned percent = written * 100 / nr_result;
549 550
			if (progress_update || percent != last_percent) {
				fprintf(stderr, "%4u%% (%u/%u) done\r",
551
					percent, written, nr_result);
552 553 554
				progress_update = 0;
				last_percent = percent;
			}
555 556
		}
	}
557 558
	if (do_progress)
		fputc('\n', stderr);
559
 done:
560 561
	if (written != nr_result)
		die("wrote %d objects while expecting %d", written, nr_result);
562
	sha1close(f, pack_file_sha1, 1);
563 564 565 566 567
}

static void write_index_file(void)
{
	int i;
568 569
	struct sha1file *f = sha1create("%s-%s.%s", base_name,
					sha1_to_hex(object_list_sha1), "idx");
570
	struct object_entry **list = sorted_by_sha;
571
	struct object_entry **last = list + nr_result;
572
	uint32_t array[256];
573 574 575 576

	/*
	 * Write the first-level table (the list is sorted,
	 * but we use a 256-entry lookup to be able to avoid
577
	 * having to do eight extra binary search iterations).
578 579 580 581 582 583 584 585 586 587 588 589
	 */
	for (i = 0; i < 256; i++) {
		struct object_entry **next = list;
		while (next < last) {
			struct object_entry *entry = *next;
			if (entry->sha1[0] != i)
				break;
			next++;
		}
		array[i] = htonl(next - sorted_by_sha);
		list = next;
	}
590
	sha1write(f, array, 256 * 4);
591 592 593 594 595

	/*
	 * Write the actual SHA1 entries..
	 */
	list = sorted_by_sha;
596
	for (i = 0; i < nr_result; i++) {
597
		struct object_entry *entry = *list++;
598
		uint32_t offset = htonl(entry->offset);
599 600
		sha1write(f, &offset, 4);
		sha1write(f, entry->sha1, 20);
601
	}
602 603
	sha1write(f, pack_file_sha1, 20);
	sha1close(f, NULL, 1);
604 605
}

606 607 608 609 610 611 612
static int locate_object_entry_hash(const unsigned char *sha1)
{
	int i;
	unsigned int ui;
	memcpy(&ui, sha1, sizeof(unsigned int));
	i = ui % object_ix_hashsz;
	while (0 < object_ix[i]) {
613
		if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
			return i;
		if (++i == object_ix_hashsz)
			i = 0;
	}
	return -1 - i;
}

static struct object_entry *locate_object_entry(const unsigned char *sha1)
{
	int i;

	if (!object_ix_hashsz)
		return NULL;

	i = locate_object_entry_hash(sha1);
	if (0 <= i)
		return &objects[object_ix[i]-1];
	return NULL;
}

static void rehash_objects(void)
635
{
636 637 638 639 640 641 642
	int i;
	struct object_entry *oe;

	object_ix_hashsz = nr_objects * 3;
	if (object_ix_hashsz < 1024)
		object_ix_hashsz = 1024;
	object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
643
	memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
644 645 646 647 648 649 650 651 652
	for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
		int ix = locate_object_entry_hash(oe->sha1);
		if (0 <= ix)
			continue;
		ix = -1 - ix;
		object_ix[ix] = i + 1;
	}
}

653
static unsigned name_hash(const char *name)
654
{
655 656 657
	unsigned char c;
	unsigned hash = 0;

658
	/*
659 660 661
	 * This effectively just creates a sortable number from the
	 * last sixteen non-whitespace characters. Last characters
	 * count "most", so things that end in ".c" sort together.
662
	 */
663 664 665 666 667
	while ((c = *name++) != 0) {
		if (isspace(c))
			continue;
		hash = (hash >> 2) + (c << 24);
	}
668 669 670 671
	return hash;
}

static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude)
672 673 674
{
	unsigned int idx = nr_objects;
	struct object_entry *entry;
675
	struct packed_git *p;
676 677
	unsigned int found_offset = 0;
	struct packed_git *found_pack = NULL;
678
	int ix, status = 0;
679

680
	if (!exclude) {
L
Linus Torvalds 已提交
681
		for (p = packed_git; p; p = p->next) {
N
Nicolas Pitre 已提交
682 683
			unsigned long offset = find_pack_entry_one(sha1, p);
			if (offset) {
L
Linus Torvalds 已提交
684 685 686 687
				if (incremental)
					return 0;
				if (local && !p->pack_local)
					return 0;
688
				if (!found_pack) {
N
Nicolas Pitre 已提交
689 690
					found_offset = offset;
					found_pack = p;
691
				}
L
Linus Torvalds 已提交
692 693 694
			}
		}
	}
695 696
	if ((entry = locate_object_entry(sha1)) != NULL)
		goto already_added;
697

698 699 700 701 702 703
	if (idx >= nr_alloc) {
		unsigned int needed = (idx + 1024) * 3 / 2;
		objects = xrealloc(objects, needed * sizeof(*entry));
		nr_alloc = needed;
	}
	entry = objects + idx;
704
	nr_objects = idx + 1;
705
	memset(entry, 0, sizeof(*entry));
706
	hashcpy(entry->sha1, sha1);
707
	entry->hash = hash;
708 709 710 711 712 713 714 715

	if (object_ix_hashsz * 3 <= nr_objects * 4)
		rehash_objects();
	else {
		ix = locate_object_entry_hash(entry->sha1);
		if (0 <= ix)
			die("internal error in object hashing.");
		object_ix[-1 - ix] = idx + 1;
716
	}
717
	status = 1;
718 719

 already_added:
720 721 722 723
	if (progress_update) {
		fprintf(stderr, "Counting objects...%d\r", nr_objects);
		progress_update = 0;
	}
724 725 726 727 728 729 730
	if (exclude)
		entry->preferred_base = 1;
	else {
		if (found_pack) {
			entry->in_pack = found_pack;
			entry->in_pack_offset = found_offset;
		}
731
	}
732
	return status;
733 734
}

735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
struct pbase_tree_cache {
	unsigned char sha1[20];
	int ref;
	int temporary;
	void *tree_data;
	unsigned long tree_size;
};

static struct pbase_tree_cache *(pbase_tree_cache[256]);
static int pbase_tree_cache_ix(const unsigned char *sha1)
{
	return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
}
static int pbase_tree_cache_ix_incr(int ix)
{
	return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
}

static struct pbase_tree {
	struct pbase_tree *next;
	/* This is a phony "cache" entry; we are not
	 * going to evict it nor find it through _get()
	 * mechanism -- this is for the toplevel node that
	 * would almost always change with any commit.
	 */
	struct pbase_tree_cache pcache;
} *pbase_tree;

static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
{
	struct pbase_tree_cache *ent, *nent;
	void *data;
	unsigned long size;
	char type[20];
	int neigh;
	int my_ix = pbase_tree_cache_ix(sha1);
	int available_ix = -1;

	/* pbase-tree-cache acts as a limited hashtable.
	 * your object will be found at your index or within a few
	 * slots after that slot if it is cached.
	 */
	for (neigh = 0; neigh < 8; neigh++) {
		ent = pbase_tree_cache[my_ix];
779
		if (ent && !hashcmp(ent->sha1, sha1)) {
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
			ent->ref++;
			return ent;
		}
		else if (((available_ix < 0) && (!ent || !ent->ref)) ||
			 ((0 <= available_ix) &&
			  (!ent && pbase_tree_cache[available_ix])))
			available_ix = my_ix;
		if (!ent)
			break;
		my_ix = pbase_tree_cache_ix_incr(my_ix);
	}

	/* Did not find one.  Either we got a bogus request or
	 * we need to read and perhaps cache.
	 */
	data = read_sha1_file(sha1, type, &size);
	if (!data)
		return NULL;
	if (strcmp(type, tree_type)) {
		free(data);
		return NULL;
	}

	/* We need to either cache or return a throwaway copy */

	if (available_ix < 0)
		ent = NULL;
	else {
		ent = pbase_tree_cache[available_ix];
		my_ix = available_ix;
	}

	if (!ent) {
		nent = xmalloc(sizeof(*nent));
		nent->temporary = (available_ix < 0);
	}
	else {
		/* evict and reuse */
		free(ent->tree_data);
		nent = ent;
	}
821
	hashcpy(nent->sha1, sha1);
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
	nent->tree_data = data;
	nent->tree_size = size;
	nent->ref = 1;
	if (!nent->temporary)
		pbase_tree_cache[my_ix] = nent;
	return nent;
}

static void pbase_tree_put(struct pbase_tree_cache *cache)
{
	if (!cache->temporary) {
		cache->ref--;
		return;
	}
	free(cache->tree_data);
	free(cache);
}

static int name_cmp_len(const char *name)
{
	int i;
	for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
		;
	return i;
}

static void add_pbase_object(struct tree_desc *tree,
			     const char *name,
850 851
			     int cmplen,
			     const char *fullname)
852
{
853 854 855
	struct name_entry entry;

	while (tree_entry(tree,&entry)) {
856 857 858
		unsigned long size;
		char type[20];

859 860 861 862
		if (entry.pathlen != cmplen ||
		    memcmp(entry.path, name, cmplen) ||
		    !has_sha1_file(entry.sha1) ||
		    sha1_object_info(entry.sha1, type, &size))
863
			continue;
864
		if (name[cmplen] != '/') {
865
			unsigned hash = name_hash(fullname);
866
			add_object_entry(entry.sha1, hash, 1);
867 868
			return;
		}
869
		if (!strcmp(type, tree_type)) {
870
			struct tree_desc sub;
871 872 873 874
			struct pbase_tree_cache *tree;
			const char *down = name+cmplen+1;
			int downlen = name_cmp_len(down);

875
			tree = pbase_tree_get(entry.sha1);
876 877 878 879 880
			if (!tree)
				return;
			sub.buf = tree->tree_data;
			sub.size = tree->tree_size;

881
			add_pbase_object(&sub, down, downlen, fullname);
882 883 884 885
			pbase_tree_put(tree);
		}
	}
}
886

887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
static unsigned *done_pbase_paths;
static int done_pbase_paths_num;
static int done_pbase_paths_alloc;
static int done_pbase_path_pos(unsigned hash)
{
	int lo = 0;
	int hi = done_pbase_paths_num;
	while (lo < hi) {
		int mi = (hi + lo) / 2;
		if (done_pbase_paths[mi] == hash)
			return mi;
		if (done_pbase_paths[mi] < hash)
			hi = mi;
		else
			lo = mi + 1;
	}
	return -lo-1;
}

static int check_pbase_path(unsigned hash)
{
	int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
	if (0 <= pos)
		return 1;
	pos = -pos - 1;
	if (done_pbase_paths_alloc <= done_pbase_paths_num) {
		done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
		done_pbase_paths = xrealloc(done_pbase_paths,
					    done_pbase_paths_alloc *
					    sizeof(unsigned));
	}
	done_pbase_paths_num++;
	if (pos < done_pbase_paths_num)
		memmove(done_pbase_paths + pos + 1,
			done_pbase_paths + pos,
			(done_pbase_paths_num - pos - 1) * sizeof(unsigned));
	done_pbase_paths[pos] = hash;
	return 0;
}

927
static void add_preferred_base_object(const char *name, unsigned hash)
928 929 930 931 932 933 934 935 936
{
	struct pbase_tree *it;
	int cmplen = name_cmp_len(name);

	if (check_pbase_path(hash))
		return;

	for (it = pbase_tree; it; it = it->next) {
		if (cmplen == 0) {
937
			hash = name_hash("");
938 939 940 941 942 943
			add_object_entry(it->pcache.sha1, hash, 1);
		}
		else {
			struct tree_desc tree;
			tree.buf = it->pcache.tree_data;
			tree.size = it->pcache.tree_size;
944
			add_pbase_object(&tree, name, cmplen, name);
945
		}
946 947 948
	}
}

949
static void add_preferred_base(unsigned char *sha1)
950
{
951 952 953 954
	struct pbase_tree *it;
	void *data;
	unsigned long size;
	unsigned char tree_sha1[20];
955

956 957 958
	if (window <= num_preferred_base++)
		return;

959 960
	data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
	if (!data)
961
		return;
962 963

	for (it = pbase_tree; it; it = it->next) {
964
		if (!hashcmp(it->pcache.sha1, tree_sha1)) {
965 966 967 968 969 970 971 972 973
			free(data);
			return;
		}
	}

	it = xcalloc(1, sizeof(*it));
	it->next = pbase_tree;
	pbase_tree = it;

974
	hashcpy(it->pcache.sha1, tree_sha1);
975 976
	it->pcache.tree_data = data;
	it->pcache.tree_size = size;
977 978
}

979 980
static void check_object(struct object_entry *entry)
{
981 982
	char type[20];

983
	if (entry->in_pack && !entry->preferred_base) {
984
		struct packed_git *p = entry->in_pack;
985
		struct pack_window *w_curs = NULL;
986 987 988 989 990
		unsigned long left = p->pack_size - entry->in_pack_offset;
		unsigned long size, used;
		unsigned char *buf;
		struct object_entry *base_entry = NULL;

991
		buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL);
992 993 994 995

		/* We want in_pack_type even if we do not reuse delta.
		 * There is no point not reusing non-delta representations.
		 */
996 997
		used = unpack_object_header_gently(buf, left,
						   &entry->in_pack_type, &size);
998

999 1000 1001 1002
		/* Check if it is delta, and the base is also an object
		 * we are going to pack.  If so we will reuse the existing
		 * delta.
		 */
1003 1004 1005
		if (!no_reuse_delta) {
			unsigned char c, *base_name;
			unsigned long ofs;
1006
			unsigned long used_0;
1007 1008 1009
			/* there is at least 20 bytes left in the pack */
			switch (entry->in_pack_type) {
			case OBJ_REF_DELTA:
1010 1011
				base_name = use_pack(p, &w_curs,
					entry->in_pack_offset + used, NULL);
1012 1013 1014
				used += 20;
				break;
			case OBJ_OFS_DELTA:
1015 1016
				buf = use_pack(p, &w_curs,
					entry->in_pack_offset + used, NULL);
1017 1018
				used_0 = 0;
				c = buf[used_0++];
1019 1020 1021 1022 1023 1024
				ofs = c & 127;
				while (c & 128) {
					ofs += 1;
					if (!ofs || ofs & ~(~0UL >> 7))
						die("delta base offset overflow in pack for %s",
						    sha1_to_hex(entry->sha1));
1025
					c = buf[used_0++];
1026 1027 1028 1029 1030 1031 1032
					ofs = (ofs << 7) + (c & 127);
				}
				if (ofs >= entry->in_pack_offset)
					die("delta base offset out of bound for %s",
					    sha1_to_hex(entry->sha1));
				ofs = entry->in_pack_offset - ofs;
				base_name = find_packed_object_name(p, ofs);
1033
				used += used_0;
1034 1035 1036 1037 1038 1039 1040
				break;
			default:
				base_name = NULL;
			}
			if (base_name)
				base_entry = locate_object_entry(base_name);
		}
1041
		unuse_pack(&w_curs);
1042 1043
		entry->in_pack_header_size = used;

1044
		if (base_entry) {
1045 1046 1047 1048 1049

			/* Depth value does not matter - find_deltas()
			 * will never consider reused delta as the
			 * base object to deltify other objects
			 * against, in order to avoid circular deltas.
1050
			 */
1051 1052

			/* uncompressed size of the delta data */
1053
			entry->size = size;
1054
			entry->delta = base_entry;
1055
			entry->type = entry->in_pack_type;
1056

1057 1058
			entry->delta_sibling = base_entry->delta_child;
			base_entry->delta_child = entry;
1059

1060 1061 1062
			return;
		}
		/* Otherwise we would do the usual */
1063
	}
1064 1065

	if (sha1_object_info(entry->sha1, type, &entry->size))
1066 1067
		die("unable to get type of object %s",
		    sha1_to_hex(entry->sha1));
1068

1069
	if (!strcmp(type, commit_type)) {
1070
		entry->type = OBJ_COMMIT;
1071
	} else if (!strcmp(type, tree_type)) {
1072
		entry->type = OBJ_TREE;
1073
	} else if (!strcmp(type, blob_type)) {
1074
		entry->type = OBJ_BLOB;
1075
	} else if (!strcmp(type, tag_type)) {
1076 1077 1078 1079 1080 1081
		entry->type = OBJ_TAG;
	} else
		die("unable to pack object %s of type %s",
		    sha1_to_hex(entry->sha1), type);
}

1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
{
	struct object_entry *child = me->delta_child;
	unsigned int m = n;
	while (child) {
		unsigned int c = check_delta_limit(child, n + 1);
		if (m < c)
			m = c;
		child = child->delta_sibling;
	}
	return m;
}

1095 1096 1097
static void get_object_details(void)
{
	int i;
1098
	struct object_entry *entry;
1099

1100
	prepare_pack_ix();
1101 1102
	for (i = 0, entry = objects; i < nr_objects; i++, entry++)
		check_object(entry);
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119

	if (nr_objects == nr_result) {
		/*
		 * Depth of objects that depend on the entry -- this
		 * is subtracted from depth-max to break too deep
		 * delta chain because of delta data reusing.
		 * However, we loosen this restriction when we know we
		 * are creating a thin pack -- it will have to be
		 * expanded on the other end anyway, so do not
		 * artificially cut the delta chain and let it go as
		 * deep as it wants.
		 */
		for (i = 0, entry = objects; i < nr_objects; i++, entry++)
			if (!entry->delta && entry->delta_child)
				entry->delta_limit =
					check_delta_limit(entry, 1);
	}
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
}

typedef int (*entry_sort_t)(const struct object_entry *, const struct object_entry *);

static entry_sort_t current_sort;

static int sort_comparator(const void *_a, const void *_b)
{
	struct object_entry *a = *(struct object_entry **)_a;
	struct object_entry *b = *(struct object_entry **)_b;
	return current_sort(a,b);
}

static struct object_entry **create_sorted_list(entry_sort_t sort)
{
	struct object_entry **list = xmalloc(nr_objects * sizeof(struct object_entry *));
	int i;

	for (i = 0; i < nr_objects; i++)
		list[i] = objects + i;
	current_sort = sort;
	qsort(list, nr_objects, sizeof(struct object_entry *), sort_comparator);
	return list;
}

static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
{
1147
	return hashcmp(a->sha1, b->sha1);
1148 1149
}

T
Timo Hirvonen 已提交
1150
static struct object_entry **create_final_object_list(void)
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
{
	struct object_entry **list;
	int i, j;

	for (i = nr_result = 0; i < nr_objects; i++)
		if (!objects[i].preferred_base)
			nr_result++;
	list = xmalloc(nr_result * sizeof(struct object_entry *));
	for (i = j = 0; i < nr_objects; i++) {
		if (!objects[i].preferred_base)
			list[j++] = objects + i;
	}
	current_sort = sha1_sort;
	qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator);
	return list;
}

1168 1169 1170 1171 1172 1173
static int type_size_sort(const struct object_entry *a, const struct object_entry *b)
{
	if (a->type < b->type)
		return -1;
	if (a->type > b->type)
		return 1;
1174 1175 1176 1177
	if (a->hash < b->hash)
		return -1;
	if (a->hash > b->hash)
		return 1;
1178 1179 1180 1181
	if (a->preferred_base < b->preferred_base)
		return -1;
	if (a->preferred_base > b->preferred_base)
		return 1;
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
	if (a->size < b->size)
		return -1;
	if (a->size > b->size)
		return 1;
	return a < b ? -1 : (a > b);
}

struct unpacked {
	struct object_entry *entry;
	void *data;
1192
	struct delta_index *index;
1193 1194 1195
};

/*
1196 1197 1198 1199 1200 1201
 * We search for deltas _backwards_ in a list sorted by type and
 * by size, so that we see progressively smaller and smaller files.
 * That's because we prefer deltas to be from the bigger file
 * to the smaller - deletes are potentially cheaper, but perhaps
 * more importantly, the bigger file is likely the more recent
 * one.
1202
 */
1203
static int try_delta(struct unpacked *trg, struct unpacked *src,
1204
		     unsigned max_depth)
1205
{
1206 1207
	struct object_entry *trg_entry = trg->entry;
	struct object_entry *src_entry = src->entry;
1208 1209
	unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
	char type[10];
1210 1211 1212
	void *delta_buf;

	/* Don't bother doing diffs between different types */
1213
	if (trg_entry->type != src_entry->type)
1214 1215
		return -1;

1216 1217 1218
	/* We do not compute delta to *create* objects we are not
	 * going to pack.
	 */
1219
	if (trg_entry->preferred_base)
1220
		return -1;
1221

1222 1223
	/*
	 * We do not bother to try a delta that we discarded
1224
	 * on an earlier try, but only when reusing delta data.
1225
	 */
1226
	if (!no_reuse_delta && trg_entry->in_pack &&
1227 1228 1229
	    trg_entry->in_pack == src_entry->in_pack &&
	    trg_entry->in_pack_type != OBJ_REF_DELTA &&
	    trg_entry->in_pack_type != OBJ_OFS_DELTA)
1230 1231
		return 0;

1232 1233
	/*
	 * If the current object is at pack edge, take the depth the
1234 1235
	 * objects that depend on the current object into account --
	 * otherwise they would become too deep.
1236
	 */
1237 1238
	if (trg_entry->delta_child) {
		if (max_depth <= trg_entry->delta_limit)
1239
			return 0;
1240
		max_depth -= trg_entry->delta_limit;
1241
	}
1242
	if (src_entry->depth >= max_depth)
1243
		return 0;
1244

1245
	/* Now some size filtering heuristics. */
1246 1247
	trg_size = trg_entry->size;
	max_size = trg_size/2 - 20;
1248 1249 1250
	max_size = max_size * (max_depth - src_entry->depth) / max_depth;
	if (max_size == 0)
		return 0;
1251
	if (trg_entry->delta && trg_entry->delta_size <= max_size)
1252 1253
		max_size = trg_entry->delta_size-1;
	src_size = src_entry->size;
1254
	sizediff = src_size < trg_size ? trg_size - src_size : 0;
1255
	if (sizediff >= max_size)
1256
		return 0;
1257

1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
	/* Load data if not already done */
	if (!trg->data) {
		trg->data = read_sha1_file(trg_entry->sha1, type, &sz);
		if (sz != trg_size)
			die("object %s inconsistent object length (%lu vs %lu)",
			    sha1_to_hex(trg_entry->sha1), sz, trg_size);
	}
	if (!src->data) {
		src->data = read_sha1_file(src_entry->sha1, type, &sz);
		if (sz != src_size)
			die("object %s inconsistent object length (%lu vs %lu)",
			    sha1_to_hex(src_entry->sha1), sz, src_size);
	}
	if (!src->index) {
		src->index = create_delta_index(src->data, src_size);
		if (!src->index)
			die("out of memory");
	}

	delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
1278
	if (!delta_buf)
1279
		return 0;
1280 1281 1282 1283

	trg_entry->delta = src_entry;
	trg_entry->delta_size = delta_size;
	trg_entry->depth = src_entry->depth + 1;
1284
	free(delta_buf);
1285
	return 1;
1286 1287
}

1288 1289 1290 1291 1292
static void progress_interval(int signum)
{
	progress_update = 1;
}

1293
static void find_deltas(struct object_entry **list, int window, int depth)
1294
{
1295
	int i, idx;
1296 1297
	unsigned int array_size = window * sizeof(struct unpacked);
	struct unpacked *array = xmalloc(array_size);
1298 1299
	unsigned processed = 0;
	unsigned last_percent = 999;
1300 1301

	memset(array, 0, array_size);
1302 1303
	i = nr_objects;
	idx = 0;
1304
	if (progress)
1305
		fprintf(stderr, "Deltifying %d objects.\n", nr_result);
1306

1307
	while (--i >= 0) {
1308 1309 1310 1311
		struct object_entry *entry = list[i];
		struct unpacked *n = array + idx;
		int j;

1312 1313 1314
		if (!entry->preferred_base)
			processed++;

1315
		if (progress) {
1316
			unsigned percent = processed * 100 / nr_result;
1317 1318
			if (percent != last_percent || progress_update) {
				fprintf(stderr, "%4u%% (%u/%u) done\r",
1319
					percent, processed, nr_result);
1320 1321 1322
				progress_update = 0;
				last_percent = percent;
			}
1323
		}
1324 1325 1326

		if (entry->delta)
			/* This happens if we decided to reuse existing
1327
			 * delta from a pack.  "!no_reuse_delta &&" is implied.
1328 1329 1330
			 */
			continue;

1331 1332
		if (entry->size < 50)
			continue;
1333 1334
		free_delta_index(n->index);
		n->index = NULL;
1335
		free(n->data);
1336
		n->data = NULL;
1337
		n->entry = entry;
1338

L
Linus Torvalds 已提交
1339 1340 1341
		j = window;
		while (--j > 0) {
			unsigned int other_idx = idx + j;
1342
			struct unpacked *m;
L
Linus Torvalds 已提交
1343 1344
			if (other_idx >= window)
				other_idx -= window;
1345 1346 1347
			m = array + other_idx;
			if (!m->entry)
				break;
1348
			if (try_delta(n, m, depth) < 0)
1349 1350
				break;
		}
1351 1352 1353 1354 1355 1356
		/* if we made n a delta, and if n is already at max
		 * depth, leaving it in the window is pointless.  we
		 * should evict it first.
		 */
		if (entry->delta && depth <= entry->depth)
			continue;
1357

1358 1359 1360
		idx++;
		if (idx >= window)
			idx = 0;
1361
	}
1362

1363 1364 1365
	if (progress)
		fputc('\n', stderr);

1366
	for (i = 0; i < window; ++i) {
1367
		free_delta_index(array[i].index);
1368
		free(array[i].data);
1369
	}
1370
	free(array);
1371 1372
}

1373 1374
static void prepare_pack(int window, int depth)
{
1375
	get_object_details();
1376 1377 1378 1379 1380
	sorted_by_type = create_sorted_list(type_size_sort);
	if (window && depth)
		find_deltas(sorted_by_type, window+1, depth);
}

1381
static int reuse_cached_pack(unsigned char *sha1)
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
{
	static const char cache[] = "pack-cache/pack-%s.%s";
	char *cached_pack, *cached_idx;
	int ifd, ofd, ifd_ix = -1;

	cached_pack = git_path(cache, sha1_to_hex(sha1), "pack");
	ifd = open(cached_pack, O_RDONLY);
	if (ifd < 0)
		return 0;

	if (!pack_to_stdout) {
		cached_idx = git_path(cache, sha1_to_hex(sha1), "idx");
		ifd_ix = open(cached_idx, O_RDONLY);
		if (ifd_ix < 0) {
			close(ifd);
			return 0;
		}
	}

1401 1402 1403
	if (progress)
		fprintf(stderr, "Reusing %d objects pack %s\n", nr_objects,
			sha1_to_hex(sha1));
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434

	if (pack_to_stdout) {
		if (copy_fd(ifd, 1))
			exit(1);
		close(ifd);
	}
	else {
		char name[PATH_MAX];
		snprintf(name, sizeof(name),
			 "%s-%s.%s", base_name, sha1_to_hex(sha1), "pack");
		ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666);
		if (ofd < 0)
			die("unable to open %s (%s)", name, strerror(errno));
		if (copy_fd(ifd, ofd))
			exit(1);
		close(ifd);

		snprintf(name, sizeof(name),
			 "%s-%s.%s", base_name, sha1_to_hex(sha1), "idx");
		ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666);
		if (ofd < 0)
			die("unable to open %s (%s)", name, strerror(errno));
		if (copy_fd(ifd_ix, ofd))
			exit(1);
		close(ifd_ix);
		puts(sha1_to_hex(sha1));
	}

	return 1;
}

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
static void setup_progress_signal(void)
{
	struct sigaction sa;
	struct itimerval v;

	memset(&sa, 0, sizeof(sa));
	sa.sa_handler = progress_interval;
	sigemptyset(&sa.sa_mask);
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);

	v.it_interval.tv_sec = 1;
	v.it_interval.tv_usec = 0;
	v.it_value = v.it_interval;
	setitimer(ITIMER_REAL, &v, NULL);
}

1452 1453 1454 1455 1456 1457 1458 1459 1460
static int git_pack_config(const char *k, const char *v)
{
	if(!strcmp(k, "pack.window")) {
		window = git_config_int(k, v);
		return 0;
	}
	return git_default_config(k, v);
}

1461
static void read_object_list_from_stdin(void)
1462
{
1463 1464 1465
	char line[40 + 1 + PATH_MAX + 2];
	unsigned char sha1[20];
	unsigned hash;
1466

1467 1468 1469 1470 1471 1472
	for (;;) {
		if (!fgets(line, sizeof(line), stdin)) {
			if (feof(stdin))
				break;
			if (!ferror(stdin))
				die("fgets returned NULL, not EOF, not error!");
1473 1474 1475 1476
			if (errno != EINTR)
				die("fgets: %s", strerror(errno));
			clearerr(stdin);
			continue;
1477
		}
1478 1479 1480
		if (line[0] == '-') {
			if (get_sha1_hex(line+1, sha1))
				die("expected edge sha1, got garbage:\n %s",
1481
				    line);
1482
			add_preferred_base(sha1);
1483
			continue;
1484
		}
1485
		if (get_sha1_hex(line, sha1))
1486
			die("expected sha1, got garbage:\n %s", line);
1487

1488
		hash = name_hash(line+41);
1489 1490
		add_preferred_base_object(line+41, hash);
		add_object_entry(sha1, hash, 0);
1491
	}
1492 1493 1494 1495 1496
}

static void show_commit(struct commit *commit)
{
	unsigned hash = name_hash("");
1497
	add_preferred_base_object("", hash);
1498 1499 1500 1501 1502 1503
	add_object_entry(commit->object.sha1, hash, 0);
}

static void show_object(struct object_array_entry *p)
{
	unsigned hash = name_hash(p->name);
1504
	add_preferred_base_object(p->name, hash);
1505 1506 1507
	add_object_entry(p->item->sha1, hash, 0);
}

1508 1509 1510 1511 1512 1513
static void show_edge(struct commit *commit)
{
	add_preferred_base(commit->object.sha1);
}

static void get_object_list(int ac, const char **av)
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
{
	struct rev_info revs;
	char line[1000];
	int flags = 0;

	init_revisions(&revs, NULL);
	save_commit_buffer = 0;
	track_object_refs = 0;
	setup_revisions(ac, av, &revs, NULL);

	while (fgets(line, sizeof(line), stdin) != NULL) {
		int len = strlen(line);
		if (line[len - 1] == '\n')
			line[--len] = 0;
		if (!len)
			break;
		if (*line == '-') {
			if (!strcmp(line, "--not")) {
				flags ^= UNINTERESTING;
				continue;
			}
			die("not a rev '%s'", line);
		}
		if (handle_revision_arg(line, &revs, flags, 1))
			die("bad revision '%s'", line);
	}

	prepare_revision_walk(&revs);
1542
	mark_edges_uninteresting(revs.commits, &revs, show_edge);
1543 1544 1545 1546 1547 1548 1549 1550 1551
	traverse_commit_list(&revs, show_commit, show_object);
}

int cmd_pack_objects(int argc, const char **argv, const char *prefix)
{
	SHA_CTX ctx;
	int depth = 10;
	struct object_entry **list;
	int use_internal_rev_list = 0;
1552
	int thin = 0;
1553
	int i;
1554 1555
	const char **rp_av;
	int rp_ac_alloc = 64;
1556 1557
	int rp_ac;

1558 1559
	rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));

1560 1561 1562
	rp_av[0] = "pack-objects";
	rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
	rp_ac = 2;
1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584

	git_config(git_pack_config);

	progress = isatty(2);
	for (i = 1; i < argc; i++) {
		const char *arg = argv[i];

		if (*arg != '-')
			break;

		if (!strcmp("--non-empty", arg)) {
			non_empty = 1;
			continue;
		}
		if (!strcmp("--local", arg)) {
			local = 1;
			continue;
		}
		if (!strcmp("--incremental", arg)) {
			incremental = 1;
			continue;
		}
1585
		if (!prefixcmp(arg, "--window=")) {
1586 1587 1588 1589 1590 1591
			char *end;
			window = strtoul(arg+9, &end, 0);
			if (!arg[9] || *end)
				usage(pack_usage);
			continue;
		}
1592
		if (!prefixcmp(arg, "--depth=")) {
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
			char *end;
			depth = strtoul(arg+8, &end, 0);
			if (!arg[8] || *end)
				usage(pack_usage);
			continue;
		}
		if (!strcmp("--progress", arg)) {
			progress = 1;
			continue;
		}
1603 1604 1605 1606
		if (!strcmp("--all-progress", arg)) {
			progress = 2;
			continue;
		}
1607 1608 1609 1610 1611 1612 1613 1614
		if (!strcmp("-q", arg)) {
			progress = 0;
			continue;
		}
		if (!strcmp("--no-reuse-delta", arg)) {
			no_reuse_delta = 1;
			continue;
		}
1615
		if (!strcmp("--delta-base-offset", arg)) {
1616
			allow_ofs_delta = 1;
1617 1618
			continue;
		}
1619 1620 1621 1622 1623 1624 1625 1626
		if (!strcmp("--stdout", arg)) {
			pack_to_stdout = 1;
			continue;
		}
		if (!strcmp("--revs", arg)) {
			use_internal_rev_list = 1;
			continue;
		}
1627
		if (!strcmp("--unpacked", arg) ||
1628
		    !prefixcmp(arg, "--unpacked=") ||
1629
		    !strcmp("--reflog", arg) ||
1630 1631
		    !strcmp("--all", arg)) {
			use_internal_rev_list = 1;
1632 1633 1634 1635 1636
			if (rp_ac >= rp_ac_alloc - 1) {
				rp_ac_alloc = alloc_nr(rp_ac_alloc);
				rp_av = xrealloc(rp_av,
						 rp_ac_alloc * sizeof(*rp_av));
			}
1637
			rp_av[rp_ac++] = arg;
1638 1639
			continue;
		}
1640 1641 1642 1643
		if (!strcmp("--thin", arg)) {
			use_internal_rev_list = 1;
			thin = 1;
			rp_av[1] = "--objects-edge";
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667
			continue;
		}
		usage(pack_usage);
	}

	/* Traditionally "pack-objects [options] base extra" failed;
	 * we would however want to take refs parameter that would
	 * have been given to upstream rev-list ourselves, which means
	 * we somehow want to say what the base name is.  So the
	 * syntax would be:
	 *
	 * pack-objects [options] base <refs...>
	 *
	 * in other words, we would treat the first non-option as the
	 * base_name and send everything else to the internal revision
	 * walker.
	 */

	if (!pack_to_stdout)
		base_name = argv[i++];

	if (pack_to_stdout != !base_name)
		usage(pack_usage);

1668 1669
	if (!pack_to_stdout && thin)
		die("--thin cannot be used to build an indexable pack.");
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679

	prepare_packed_git();

	if (progress) {
		fprintf(stderr, "Generating pack...\n");
		setup_progress_signal();
	}

	if (!use_internal_rev_list)
		read_object_list_from_stdin();
1680 1681 1682 1683
	else {
		rp_av[rp_ac] = NULL;
		get_object_list(rp_ac, rp_av);
	}
1684

1685 1686
	if (progress)
		fprintf(stderr, "Done counting %d objects.\n", nr_objects);
1687 1688
	sorted_by_sha = create_final_object_list();
	if (non_empty && !nr_result)
1689
		return 0;
1690

J
Junio C Hamano 已提交
1691 1692
	SHA1_Init(&ctx);
	list = sorted_by_sha;
1693
	for (i = 0; i < nr_result; i++) {
J
Junio C Hamano 已提交
1694 1695 1696 1697
		struct object_entry *entry = *list++;
		SHA1_Update(&ctx, entry->sha1, 20);
	}
	SHA1_Final(object_list_sha1, &ctx);
1698 1699
	if (progress && (nr_objects != nr_result))
		fprintf(stderr, "Result has %d objects.\n", nr_result);
J
Junio C Hamano 已提交
1700

1701
	if (reuse_cached_pack(object_list_sha1))
1702 1703
		;
	else {
1704 1705
		if (nr_result)
			prepare_pack(window, depth);
N
Nicolas Pitre 已提交
1706
		if (progress == 1 && pack_to_stdout) {
1707 1708 1709 1710 1711 1712 1713
			/* the other end usually displays progress itself */
			struct itimerval v = {{0,},};
			setitimer(ITIMER_REAL, &v, NULL);
			signal(SIGALRM, SIG_IGN );
			progress_update = 0;
		}
		write_pack_file();
1714 1715 1716 1717
		if (!pack_to_stdout) {
			write_index_file();
			puts(sha1_to_hex(object_list_sha1));
		}
1718
	}
1719
	if (progress)
1720 1721
		fprintf(stderr, "Total %d (delta %d), reused %d (delta %d)\n",
			written, written_delta, reused, reused_delta);
1722 1723
	return 0;
}