builtin-pack-objects.c 42.4 KB
Newer Older
1
#include "builtin.h"
2 3
#include "cache.h"
#include "object.h"
4 5 6 7
#include "blob.h"
#include "commit.h"
#include "tag.h"
#include "tree.h"
8
#include "delta.h"
9
#include "pack.h"
10
#include "csum-file.h"
J
Junio C Hamano 已提交
11
#include "tree-walk.h"
12 13 14
#include "diff.h"
#include "revision.h"
#include "list-objects.h"
15

16 17 18 19
static const char pack_usage[] = "\
git-pack-objects [{ -q | --progress | --all-progress }] \n\
	[--local] [--incremental] [--window=N] [--depth=N] \n\
	[--no-reuse-delta] [--delta-base-offset] [--non-empty] \n\
20
	[--revs [--unpacked | --all]*] [--reflog] [--stdout | base-name] \n\
21
	[<ref-list | <object-list]";
22 23 24

struct object_entry {
	unsigned char sha1[20];
25
	unsigned long size;	/* uncompressed size */
26 27 28
	unsigned long offset;	/* offset into the final pack file;
				 * nonzero if already written.
				 */
29
	unsigned int depth;	/* delta depth */
30
	unsigned int delta_limit;	/* base adjustment for in-pack delta */
31
	unsigned int hash;	/* name hint hash */
32
	enum object_type type;
33
	enum object_type in_pack_type;	/* could be delta */
34
	unsigned long delta_size;	/* delta data size (uncompressed) */
35
#define in_pack_header_size delta_size	/* only when reusing pack data */
36 37 38
	struct object_entry *delta;	/* delta base object */
	struct packed_git *in_pack; 	/* already in pack */
	unsigned int in_pack_offset;
39
	struct object_entry *delta_child; /* deltified objects who bases me */
40 41 42
	struct object_entry *delta_sibling; /* other deltified objects who
					     * uses the same base as me
					     */
43 44 45 46
	int preferred_base;	/* we do not pack this, but is encouraged to
				 * be used as the base objectto delta huge
				 * objects against.
				 */
47 48
};

49
/*
50
 * Objects we are going to pack are collected in objects array (dynamically
51 52 53 54 55 56 57 58 59 60 61
 * expanded).  nr_objects & nr_alloc controls this array.  They are stored
 * in the order we see -- typically rev-list --objects order that gives us
 * nice "minimum seek" order.
 *
 * sorted-by-sha ans sorted-by-type are arrays of pointers that point at
 * elements in the objects array.  The former is used to build the pack
 * index (lists object names in the ascending order to help offset lookup),
 * and the latter is used to group similar things together by try_delta()
 * heuristics.
 */

62
static unsigned char object_list_sha1[20];
63 64 65 66
static int non_empty;
static int no_reuse_delta;
static int local;
static int incremental;
67 68
static int allow_ofs_delta;

69
static struct object_entry **sorted_by_sha, **sorted_by_type;
70 71
static struct object_entry *objects;
static int nr_objects, nr_alloc, nr_result;
72
static const char *base_name;
73
static unsigned char pack_file_sha1[20];
J
Junio C Hamano 已提交
74
static int progress = 1;
75
static volatile sig_atomic_t progress_update;
76
static int window = 10;
77
static int pack_to_stdout;
78
static int num_preferred_base;
79

80 81 82 83 84 85
/*
 * The object names in objects array are hashed with this hashtable,
 * to help looking up the entry by object name.  Binary search from
 * sorted_by_sha is also possible but this was easier to code and faster.
 * This hashtable is built after all the objects are seen.
 */
86 87
static int *object_ix;
static int object_ix_hashsz;
88 89 90 91 92

/*
 * Pack index for existing packs give us easy access to the offsets into
 * corresponding pack file where each object's data starts, but the entries
 * do not store the size of the compressed representation (uncompressed
93 94 95 96 97 98 99 100 101
 * size is easily available by examining the pack entry header).  It is
 * also rather expensive to find the sha1 for an object given its offset.
 *
 * We build a hashtable of existing packs (pack_revindex), and keep reverse
 * index here -- pack index file is sorted by object name mapping to offset;
 * this pack_revindex[].revindex array is a list of offset/index_nr pairs
 * ordered by offset, so if you know the offset of an object, next offset
 * is where its packed representation ends and the index_nr can be used to
 * get the object sha1 from the main index.
102
 */
103 104 105 106
struct revindex_entry {
	unsigned int offset;
	unsigned int nr;
};
107 108
struct pack_revindex {
	struct packed_git *p;
109 110 111
	struct revindex_entry *revindex;
};
static struct  pack_revindex *pack_revindex;
112
static int pack_revindex_hashsz;
113 114 115 116

/*
 * stats
 */
117 118 119 120
static int written;
static int written_delta;
static int reused;
static int reused_delta;
121 122 123

static int pack_revindex_ix(struct packed_git *p)
{
L
Luck, Tony 已提交
124
	unsigned long ui = (unsigned long)p;
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
	int i;

	ui = ui ^ (ui >> 16); /* defeat structure alignment */
	i = (int)(ui % pack_revindex_hashsz);
	while (pack_revindex[i].p) {
		if (pack_revindex[i].p == p)
			return i;
		if (++i == pack_revindex_hashsz)
			i = 0;
	}
	return -1 - i;
}

static void prepare_pack_ix(void)
{
	int num;
	struct packed_git *p;
	for (num = 0, p = packed_git; p; p = p->next)
		num++;
	if (!num)
		return;
	pack_revindex_hashsz = num * 11;
	pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
	for (p = packed_git; p; p = p->next) {
		num = pack_revindex_ix(p);
		num = - 1 - num;
		pack_revindex[num].p = p;
	}
	/* revindex elements are lazily initialized */
}

static int cmp_offset(const void *a_, const void *b_)
{
158 159 160
	const struct revindex_entry *a = a_;
	const struct revindex_entry *b = b_;
	return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
161 162 163 164 165 166 167 168 169 170 171 172
}

/*
 * Ordered list of offsets of objects in the pack.
 */
static void prepare_pack_revindex(struct pack_revindex *rix)
{
	struct packed_git *p = rix->p;
	int num_ent = num_packed_objects(p);
	int i;
	void *index = p->index_base + 256;

173
	rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
174
	for (i = 0; i < num_ent; i++) {
175
		unsigned int hl = *((unsigned int *)((char *) index + 24*i));
176 177
		rix->revindex[i].offset = ntohl(hl);
		rix->revindex[i].nr = i;
178 179 180 181
	}
	/* This knows the pack format -- the 20-byte trailer
	 * follows immediately after the last object data.
	 */
182 183 184
	rix->revindex[num_ent].offset = p->pack_size - 20;
	rix->revindex[num_ent].nr = -1;
	qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
185 186
}

187 188
static struct revindex_entry * find_packed_object(struct packed_git *p,
						  unsigned int ofs)
189 190 191 192
{
	int num;
	int lo, hi;
	struct pack_revindex *rix;
193
	struct revindex_entry *revindex;
194 195 196 197 198 199 200 201 202 203 204
	num = pack_revindex_ix(p);
	if (num < 0)
		die("internal error: pack revindex uninitialized");
	rix = &pack_revindex[num];
	if (!rix->revindex)
		prepare_pack_revindex(rix);
	revindex = rix->revindex;
	lo = 0;
	hi = num_packed_objects(p) + 1;
	do {
		int mi = (lo + hi) / 2;
205 206
		if (revindex[mi].offset == ofs) {
			return revindex + mi;
207
		}
208
		else if (ofs < revindex[mi].offset)
209 210 211 212 213 214 215
			hi = mi;
		else
			lo = mi + 1;
	} while (lo < hi);
	die("internal error: pack revindex corrupt");
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229
static unsigned long find_packed_object_size(struct packed_git *p,
					     unsigned long ofs)
{
	struct revindex_entry *entry = find_packed_object(p, ofs);
	return entry[1].offset - ofs;
}

static unsigned char *find_packed_object_name(struct packed_git *p,
					      unsigned long ofs)
{
	struct revindex_entry *entry = find_packed_object(p, ofs);
	return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
}

230 231 232 233 234 235 236 237 238
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
{
	unsigned long othersize, delta_size;
	char type[10];
	void *otherbuf = read_sha1_file(entry->delta->sha1, type, &othersize);
	void *delta_buf;

	if (!otherbuf)
		die("unable to read %s", sha1_to_hex(entry->delta->sha1));
239
        delta_buf = diff_delta(otherbuf, othersize,
240
			       buf, size, &delta_size, 0);
241
        if (!delta_buf || delta_size != entry->delta_size)
242 243 244 245 246 247
        	die("delta size changed");
        free(buf);
        free(otherbuf);
	return delta_buf;
}

248 249 250 251 252 253 254 255 256
/*
 * The per-object header is a pretty dense thing, which is
 *  - first byte: low four bits are "size", then three bits of "type",
 *    and the high bit is "size continues".
 *  - each byte afterwards: low seven bits are size continuation,
 *    with the high bit being "size continues"
 */
static int encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
{
257
	int n = 1;
258 259
	unsigned char c;

260
	if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
261 262
		die("bad type %d", type);

263 264 265
	c = (type << 4) | (size & 15);
	size >>= 4;
	while (size) {
266
		*hdr++ = c | 0x80;
267 268 269
		c = size & 0x7f;
		size >>= 7;
		n++;
270 271 272 273 274
	}
	*hdr = c;
	return n;
}

275 276 277 278
/*
 * we are going to reuse the existing object data as is.  make
 * sure it is not corrupt.
 */
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
static int check_pack_inflate(struct packed_git *p,
		struct pack_window **w_curs,
		unsigned long offset,
		unsigned long len,
		unsigned long expect)
{
	z_stream stream;
	unsigned char fakebuf[4096], *in;
	int st;

	memset(&stream, 0, sizeof(stream));
	inflateInit(&stream);
	do {
		in = use_pack(p, w_curs, offset, &stream.avail_in);
		stream.next_in = in;
		stream.next_out = fakebuf;
		stream.avail_out = sizeof(fakebuf);
		st = inflate(&stream, Z_FINISH);
		offset += stream.next_in - in;
	} while (st == Z_OK || st == Z_BUF_ERROR);
	inflateEnd(&stream);
	return (st == Z_STREAM_END &&
		stream.total_out == expect &&
		stream.total_in == len) ? 0 : -1;
}

static void copy_pack_data(struct sha1file *f,
		struct packed_git *p,
		struct pack_window **w_curs,
		unsigned long offset,
		unsigned long len)
{
	unsigned char *in;
	unsigned int avail;

	while (len) {
		in = use_pack(p, w_curs, offset, &avail);
		if (avail > len)
			avail = len;
		sha1write(f, in, avail);
		offset += avail;
		len -= avail;
	}
}

static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)
325
{
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
	z_stream stream;
	unsigned char fakebuf[4096];
	int st;

	memset(&stream, 0, sizeof(stream));
	stream.next_in = data;
	stream.avail_in = len;
	stream.next_out = fakebuf;
	stream.avail_out = sizeof(fakebuf);
	inflateInit(&stream);

	while (1) {
		st = inflate(&stream, Z_FINISH);
		if (st == Z_STREAM_END || st == Z_OK) {
			st = (stream.total_out == expect &&
			      stream.total_in == len) ? 0 : -1;
			break;
		}
		if (st != Z_BUF_ERROR) {
			st = -1;
			break;
		}
		stream.next_out = fakebuf;
		stream.avail_out = sizeof(fakebuf);
	}
	inflateEnd(&stream);
	return st;
353 354 355 356 357 358 359
}

static int revalidate_loose_object(struct object_entry *entry,
				   unsigned char *map,
				   unsigned long mapsize)
{
	/* we already know this is a loose object with new type header. */
360 361
	enum object_type type;
	unsigned long size, used;
362 363 364 365

	if (pack_to_stdout)
		return 0;

366 367 368 369 370
	used = unpack_object_header_gently(map, mapsize, &type, &size);
	if (!used)
		return -1;
	map += used;
	mapsize -= used;
371
	return check_loose_inflate(map, mapsize, size);
372 373
}

374 375
static unsigned long write_object(struct sha1file *f,
				  struct object_entry *entry)
376 377 378
{
	unsigned long size;
	char type[10];
379
	void *buf;
380
	unsigned char header[10];
381
	unsigned hdrlen, datalen;
382
	enum object_type obj_type;
383
	int to_reuse = 0;
384

385
	obj_type = entry->type;
386 387
	if (! entry->in_pack)
		to_reuse = 0;	/* can't reuse what we don't have */
388
	else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
389 390 391 392 393 394 395 396 397 398
		to_reuse = 1;	/* check_object() decided it for us */
	else if (obj_type != entry->in_pack_type)
		to_reuse = 0;	/* pack has delta which is unusable */
	else if (entry->delta)
		to_reuse = 0;	/* we want to pack afresh */
	else
		to_reuse = 1;	/* we have it in-pack undeltified,
				 * and we do not need to deltify it.
				 */

399 400 401 402 403 404
	if (!entry->in_pack && !entry->delta) {
		unsigned char *map;
		unsigned long mapsize;
		map = map_sha1_file(entry->sha1, &mapsize);
		if (map && !legacy_loose_object(map)) {
			/* We can copy straight into the pack file */
405 406 407
			if (revalidate_loose_object(entry, map, mapsize))
				die("corrupt loose object %s",
				    sha1_to_hex(entry->sha1));
408 409 410 411 412 413 414 415 416 417
			sha1write(f, map, mapsize);
			munmap(map, mapsize);
			written++;
			reused++;
			return mapsize;
		}
		if (map)
			munmap(map, mapsize);
	}

418
	if (!to_reuse) {
419 420 421 422 423 424 425 426 427
		buf = read_sha1_file(entry->sha1, type, &size);
		if (!buf)
			die("unable to read %s", sha1_to_hex(entry->sha1));
		if (size != entry->size)
			die("object %s size inconsistency (%lu vs %lu)",
			    sha1_to_hex(entry->sha1), size, entry->size);
		if (entry->delta) {
			buf = delta_against(buf, size, entry);
			size = entry->delta_size;
428 429
			obj_type = (allow_ofs_delta && entry->delta->offset) ?
				OBJ_OFS_DELTA : OBJ_REF_DELTA;
430 431 432
		}
		/*
		 * The object header is a byte of 'type' followed by zero or
433
		 * more bytes of length.
434 435 436 437
		 */
		hdrlen = encode_header(obj_type, size, header);
		sha1write(f, header, hdrlen);

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
		if (obj_type == OBJ_OFS_DELTA) {
			/*
			 * Deltas with relative base contain an additional
			 * encoding of the relative offset for the delta
			 * base from this object's position in the pack.
			 */
			unsigned long ofs = entry->offset - entry->delta->offset;
			unsigned pos = sizeof(header) - 1;
			header[pos] = ofs & 127;
			while (ofs >>= 7)
				header[--pos] = 128 | (--ofs & 127);
			sha1write(f, header + pos, sizeof(header) - pos);
			hdrlen += sizeof(header) - pos;
		} else if (obj_type == OBJ_REF_DELTA) {
			/*
			 * Deltas with a base reference contain
			 * an additional 20 bytes for the base sha1.
			 */
			sha1write(f, entry->delta->sha1, 20);
457 458 459 460
			hdrlen += 20;
		}
		datalen = sha1write_compressed(f, buf, size);
		free(buf);
461
	}
462 463
	else {
		struct packed_git *p = entry->in_pack;
464
		struct pack_window *w_curs = NULL;
465
		unsigned long offset;
466

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
		if (entry->delta) {
			obj_type = (allow_ofs_delta && entry->delta->offset) ?
				OBJ_OFS_DELTA : OBJ_REF_DELTA;
			reused_delta++;
		}
		hdrlen = encode_header(obj_type, entry->size, header);
		sha1write(f, header, hdrlen);
		if (obj_type == OBJ_OFS_DELTA) {
			unsigned long ofs = entry->offset - entry->delta->offset;
			unsigned pos = sizeof(header) - 1;
			header[pos] = ofs & 127;
			while (ofs >>= 7)
				header[--pos] = 128 | (--ofs & 127);
			sha1write(f, header + pos, sizeof(header) - pos);
			hdrlen += sizeof(header) - pos;
		} else if (obj_type == OBJ_REF_DELTA) {
			sha1write(f, entry->delta->sha1, 20);
			hdrlen += 20;
		}
486

487
		offset = entry->in_pack_offset + entry->in_pack_header_size;
488 489
		datalen = find_packed_object_size(p, entry->in_pack_offset)
				- entry->in_pack_header_size;
490 491
		if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
				offset, datalen, entry->size))
492
			die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
493
		copy_pack_data(f, p, &w_curs, offset, datalen);
494
		unuse_pack(&w_curs);
495
		reused++;
496
	}
497
	if (entry->delta)
498
		written_delta++;
499
	written++;
500 501 502
	return hdrlen + datalen;
}

503 504 505 506
static unsigned long write_one(struct sha1file *f,
			       struct object_entry *e,
			       unsigned long offset)
{
507
	if (e->offset || e->preferred_base)
508 509 510 511
		/* offset starts from header size and cannot be zero
		 * if it is written already.
		 */
		return offset;
512
	/* if we are deltified, write out its base object first. */
513 514
	if (e->delta)
		offset = write_one(f, e->delta, offset);
515 516
	e->offset = offset;
	return offset + write_object(f, e);
517 518
}

519 520 521
static void write_pack_file(void)
{
	int i;
522
	struct sha1file *f;
523 524
	unsigned long offset;
	struct pack_header hdr;
525
	unsigned last_percent = 999;
N
Nicolas Pitre 已提交
526
	int do_progress = progress;
527

N
Nicolas Pitre 已提交
528
	if (!base_name) {
529
		f = sha1fd(1, "<stdout>");
N
Nicolas Pitre 已提交
530 531 532
		do_progress >>= 1;
	}
	else
533 534 535
		f = sha1create("%s-%s.%s", base_name,
			       sha1_to_hex(object_list_sha1), "pack");
	if (do_progress)
536
		fprintf(stderr, "Writing %d objects.\n", nr_result);
537

538
	hdr.hdr_signature = htonl(PACK_SIGNATURE);
539
	hdr.hdr_version = htonl(PACK_VERSION);
540
	hdr.hdr_entries = htonl(nr_result);
541 542
	sha1write(f, &hdr, sizeof(hdr));
	offset = sizeof(hdr);
543 544
	if (!nr_result)
		goto done;
545
	for (i = 0; i < nr_objects; i++) {
546
		offset = write_one(f, objects + i, offset);
547
		if (do_progress) {
548
			unsigned percent = written * 100 / nr_result;
549 550
			if (progress_update || percent != last_percent) {
				fprintf(stderr, "%4u%% (%u/%u) done\r",
551
					percent, written, nr_result);
552 553 554
				progress_update = 0;
				last_percent = percent;
			}
555 556
		}
	}
557 558
	if (do_progress)
		fputc('\n', stderr);
559
 done:
560 561
	if (written != nr_result)
		die("wrote %d objects while expecting %d", written, nr_result);
562
	sha1close(f, pack_file_sha1, 1);
563 564 565 566 567
}

static void write_index_file(void)
{
	int i;
568 569
	struct sha1file *f = sha1create("%s-%s.%s", base_name,
					sha1_to_hex(object_list_sha1), "idx");
570
	struct object_entry **list = sorted_by_sha;
571
	struct object_entry **last = list + nr_result;
572 573 574 575 576
	unsigned int array[256];

	/*
	 * Write the first-level table (the list is sorted,
	 * but we use a 256-entry lookup to be able to avoid
577
	 * having to do eight extra binary search iterations).
578 579 580 581 582 583 584 585 586 587 588 589
	 */
	for (i = 0; i < 256; i++) {
		struct object_entry **next = list;
		while (next < last) {
			struct object_entry *entry = *next;
			if (entry->sha1[0] != i)
				break;
			next++;
		}
		array[i] = htonl(next - sorted_by_sha);
		list = next;
	}
590
	sha1write(f, array, 256 * sizeof(int));
591 592 593 594 595

	/*
	 * Write the actual SHA1 entries..
	 */
	list = sorted_by_sha;
596
	for (i = 0; i < nr_result; i++) {
597 598
		struct object_entry *entry = *list++;
		unsigned int offset = htonl(entry->offset);
599 600
		sha1write(f, &offset, 4);
		sha1write(f, entry->sha1, 20);
601
	}
602 603
	sha1write(f, pack_file_sha1, 20);
	sha1close(f, NULL, 1);
604 605
}

606 607 608 609 610 611 612
static int locate_object_entry_hash(const unsigned char *sha1)
{
	int i;
	unsigned int ui;
	memcpy(&ui, sha1, sizeof(unsigned int));
	i = ui % object_ix_hashsz;
	while (0 < object_ix[i]) {
613
		if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
			return i;
		if (++i == object_ix_hashsz)
			i = 0;
	}
	return -1 - i;
}

static struct object_entry *locate_object_entry(const unsigned char *sha1)
{
	int i;

	if (!object_ix_hashsz)
		return NULL;

	i = locate_object_entry_hash(sha1);
	if (0 <= i)
		return &objects[object_ix[i]-1];
	return NULL;
}

static void rehash_objects(void)
635
{
636 637 638 639 640 641 642
	int i;
	struct object_entry *oe;

	object_ix_hashsz = nr_objects * 3;
	if (object_ix_hashsz < 1024)
		object_ix_hashsz = 1024;
	object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
643
	memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
644 645 646 647 648 649 650 651 652
	for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
		int ix = locate_object_entry_hash(oe->sha1);
		if (0 <= ix)
			continue;
		ix = -1 - ix;
		object_ix[ix] = i + 1;
	}
}

653
static unsigned name_hash(const char *name)
654
{
655 656 657
	unsigned char c;
	unsigned hash = 0;

658
	/*
659 660 661
	 * This effectively just creates a sortable number from the
	 * last sixteen non-whitespace characters. Last characters
	 * count "most", so things that end in ".c" sort together.
662
	 */
663 664 665 666 667
	while ((c = *name++) != 0) {
		if (isspace(c))
			continue;
		hash = (hash >> 2) + (c << 24);
	}
668 669 670 671
	return hash;
}

static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude)
672 673 674
{
	unsigned int idx = nr_objects;
	struct object_entry *entry;
675
	struct packed_git *p;
676 677
	unsigned int found_offset = 0;
	struct packed_git *found_pack = NULL;
678
	int ix, status = 0;
679

680
	if (!exclude) {
L
Linus Torvalds 已提交
681
		for (p = packed_git; p; p = p->next) {
N
Nicolas Pitre 已提交
682 683
			unsigned long offset = find_pack_entry_one(sha1, p);
			if (offset) {
L
Linus Torvalds 已提交
684 685 686 687
				if (incremental)
					return 0;
				if (local && !p->pack_local)
					return 0;
688
				if (!found_pack) {
N
Nicolas Pitre 已提交
689 690
					found_offset = offset;
					found_pack = p;
691
				}
L
Linus Torvalds 已提交
692 693 694
			}
		}
	}
695 696
	if ((entry = locate_object_entry(sha1)) != NULL)
		goto already_added;
697

698 699 700 701 702 703
	if (idx >= nr_alloc) {
		unsigned int needed = (idx + 1024) * 3 / 2;
		objects = xrealloc(objects, needed * sizeof(*entry));
		nr_alloc = needed;
	}
	entry = objects + idx;
704
	nr_objects = idx + 1;
705
	memset(entry, 0, sizeof(*entry));
706
	hashcpy(entry->sha1, sha1);
707
	entry->hash = hash;
708 709 710 711 712 713 714 715

	if (object_ix_hashsz * 3 <= nr_objects * 4)
		rehash_objects();
	else {
		ix = locate_object_entry_hash(entry->sha1);
		if (0 <= ix)
			die("internal error in object hashing.");
		object_ix[-1 - ix] = idx + 1;
716
	}
717
	status = 1;
718 719

 already_added:
720 721 722 723
	if (progress_update) {
		fprintf(stderr, "Counting objects...%d\r", nr_objects);
		progress_update = 0;
	}
724 725 726 727 728 729 730
	if (exclude)
		entry->preferred_base = 1;
	else {
		if (found_pack) {
			entry->in_pack = found_pack;
			entry->in_pack_offset = found_offset;
		}
731
	}
732
	return status;
733 734
}

735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
struct pbase_tree_cache {
	unsigned char sha1[20];
	int ref;
	int temporary;
	void *tree_data;
	unsigned long tree_size;
};

static struct pbase_tree_cache *(pbase_tree_cache[256]);
static int pbase_tree_cache_ix(const unsigned char *sha1)
{
	return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
}
static int pbase_tree_cache_ix_incr(int ix)
{
	return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
}

static struct pbase_tree {
	struct pbase_tree *next;
	/* This is a phony "cache" entry; we are not
	 * going to evict it nor find it through _get()
	 * mechanism -- this is for the toplevel node that
	 * would almost always change with any commit.
	 */
	struct pbase_tree_cache pcache;
} *pbase_tree;

static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
{
	struct pbase_tree_cache *ent, *nent;
	void *data;
	unsigned long size;
	char type[20];
	int neigh;
	int my_ix = pbase_tree_cache_ix(sha1);
	int available_ix = -1;

	/* pbase-tree-cache acts as a limited hashtable.
	 * your object will be found at your index or within a few
	 * slots after that slot if it is cached.
	 */
	for (neigh = 0; neigh < 8; neigh++) {
		ent = pbase_tree_cache[my_ix];
779
		if (ent && !hashcmp(ent->sha1, sha1)) {
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
			ent->ref++;
			return ent;
		}
		else if (((available_ix < 0) && (!ent || !ent->ref)) ||
			 ((0 <= available_ix) &&
			  (!ent && pbase_tree_cache[available_ix])))
			available_ix = my_ix;
		if (!ent)
			break;
		my_ix = pbase_tree_cache_ix_incr(my_ix);
	}

	/* Did not find one.  Either we got a bogus request or
	 * we need to read and perhaps cache.
	 */
	data = read_sha1_file(sha1, type, &size);
	if (!data)
		return NULL;
	if (strcmp(type, tree_type)) {
		free(data);
		return NULL;
	}

	/* We need to either cache or return a throwaway copy */

	if (available_ix < 0)
		ent = NULL;
	else {
		ent = pbase_tree_cache[available_ix];
		my_ix = available_ix;
	}

	if (!ent) {
		nent = xmalloc(sizeof(*nent));
		nent->temporary = (available_ix < 0);
	}
	else {
		/* evict and reuse */
		free(ent->tree_data);
		nent = ent;
	}
821
	hashcpy(nent->sha1, sha1);
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
	nent->tree_data = data;
	nent->tree_size = size;
	nent->ref = 1;
	if (!nent->temporary)
		pbase_tree_cache[my_ix] = nent;
	return nent;
}

static void pbase_tree_put(struct pbase_tree_cache *cache)
{
	if (!cache->temporary) {
		cache->ref--;
		return;
	}
	free(cache->tree_data);
	free(cache);
}

static int name_cmp_len(const char *name)
{
	int i;
	for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
		;
	return i;
}

static void add_pbase_object(struct tree_desc *tree,
			     const char *name,
850 851
			     int cmplen,
			     const char *fullname)
852
{
853 854 855
	struct name_entry entry;

	while (tree_entry(tree,&entry)) {
856 857 858
		unsigned long size;
		char type[20];

859 860 861 862
		if (entry.pathlen != cmplen ||
		    memcmp(entry.path, name, cmplen) ||
		    !has_sha1_file(entry.sha1) ||
		    sha1_object_info(entry.sha1, type, &size))
863
			continue;
864
		if (name[cmplen] != '/') {
865
			unsigned hash = name_hash(fullname);
866
			add_object_entry(entry.sha1, hash, 1);
867 868
			return;
		}
869
		if (!strcmp(type, tree_type)) {
870
			struct tree_desc sub;
871 872 873 874
			struct pbase_tree_cache *tree;
			const char *down = name+cmplen+1;
			int downlen = name_cmp_len(down);

875
			tree = pbase_tree_get(entry.sha1);
876 877 878 879 880
			if (!tree)
				return;
			sub.buf = tree->tree_data;
			sub.size = tree->tree_size;

881
			add_pbase_object(&sub, down, downlen, fullname);
882 883 884 885
			pbase_tree_put(tree);
		}
	}
}
886

887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
static unsigned *done_pbase_paths;
static int done_pbase_paths_num;
static int done_pbase_paths_alloc;
static int done_pbase_path_pos(unsigned hash)
{
	int lo = 0;
	int hi = done_pbase_paths_num;
	while (lo < hi) {
		int mi = (hi + lo) / 2;
		if (done_pbase_paths[mi] == hash)
			return mi;
		if (done_pbase_paths[mi] < hash)
			hi = mi;
		else
			lo = mi + 1;
	}
	return -lo-1;
}

static int check_pbase_path(unsigned hash)
{
	int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
	if (0 <= pos)
		return 1;
	pos = -pos - 1;
	if (done_pbase_paths_alloc <= done_pbase_paths_num) {
		done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
		done_pbase_paths = xrealloc(done_pbase_paths,
					    done_pbase_paths_alloc *
					    sizeof(unsigned));
	}
	done_pbase_paths_num++;
	if (pos < done_pbase_paths_num)
		memmove(done_pbase_paths + pos + 1,
			done_pbase_paths + pos,
			(done_pbase_paths_num - pos - 1) * sizeof(unsigned));
	done_pbase_paths[pos] = hash;
	return 0;
}

927
static void add_preferred_base_object(const char *name, unsigned hash)
928 929 930 931 932 933 934 935 936
{
	struct pbase_tree *it;
	int cmplen = name_cmp_len(name);

	if (check_pbase_path(hash))
		return;

	for (it = pbase_tree; it; it = it->next) {
		if (cmplen == 0) {
937
			hash = name_hash("");
938 939 940 941 942 943
			add_object_entry(it->pcache.sha1, hash, 1);
		}
		else {
			struct tree_desc tree;
			tree.buf = it->pcache.tree_data;
			tree.size = it->pcache.tree_size;
944
			add_pbase_object(&tree, name, cmplen, name);
945
		}
946 947 948
	}
}

949
static void add_preferred_base(unsigned char *sha1)
950
{
951 952 953 954
	struct pbase_tree *it;
	void *data;
	unsigned long size;
	unsigned char tree_sha1[20];
955

956 957 958
	if (window <= num_preferred_base++)
		return;

959 960
	data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
	if (!data)
961
		return;
962 963

	for (it = pbase_tree; it; it = it->next) {
964
		if (!hashcmp(it->pcache.sha1, tree_sha1)) {
965 966 967 968 969 970 971 972 973
			free(data);
			return;
		}
	}

	it = xcalloc(1, sizeof(*it));
	it->next = pbase_tree;
	pbase_tree = it;

974
	hashcpy(it->pcache.sha1, tree_sha1);
975 976
	it->pcache.tree_data = data;
	it->pcache.tree_size = size;
977 978
}

979 980
static void check_object(struct object_entry *entry)
{
981 982
	char type[20];

983
	if (entry->in_pack && !entry->preferred_base) {
984
		struct packed_git *p = entry->in_pack;
985
		struct pack_window *w_curs = NULL;
986 987 988 989 990
		unsigned long left = p->pack_size - entry->in_pack_offset;
		unsigned long size, used;
		unsigned char *buf;
		struct object_entry *base_entry = NULL;

991
		buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL);
992 993 994 995

		/* We want in_pack_type even if we do not reuse delta.
		 * There is no point not reusing non-delta representations.
		 */
996 997 998 999
		used = unpack_object_header_gently(buf, left,
						   &entry->in_pack_type, &size);
		if (!used || left - used <= 20)
			die("corrupt pack for %s", sha1_to_hex(entry->sha1));
1000

1001 1002 1003 1004
		/* Check if it is delta, and the base is also an object
		 * we are going to pack.  If so we will reuse the existing
		 * delta.
		 */
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
		if (!no_reuse_delta) {
			unsigned char c, *base_name;
			unsigned long ofs;
			/* there is at least 20 bytes left in the pack */
			switch (entry->in_pack_type) {
			case OBJ_REF_DELTA:
				base_name = buf + used;
				used += 20;
				break;
			case OBJ_OFS_DELTA:
				c = buf[used++];
				ofs = c & 127;
				while (c & 128) {
					ofs += 1;
					if (!ofs || ofs & ~(~0UL >> 7))
						die("delta base offset overflow in pack for %s",
						    sha1_to_hex(entry->sha1));
					c = buf[used++];
					ofs = (ofs << 7) + (c & 127);
				}
				if (ofs >= entry->in_pack_offset)
					die("delta base offset out of bound for %s",
					    sha1_to_hex(entry->sha1));
				ofs = entry->in_pack_offset - ofs;
				base_name = find_packed_object_name(p, ofs);
				break;
			default:
				base_name = NULL;
			}
			if (base_name)
				base_entry = locate_object_entry(base_name);
		}
1037
		unuse_pack(&w_curs);
1038 1039
		entry->in_pack_header_size = used;

1040
		if (base_entry) {
1041 1042 1043 1044 1045

			/* Depth value does not matter - find_deltas()
			 * will never consider reused delta as the
			 * base object to deltify other objects
			 * against, in order to avoid circular deltas.
1046
			 */
1047 1048

			/* uncompressed size of the delta data */
1049
			entry->size = size;
1050
			entry->delta = base_entry;
1051
			entry->type = entry->in_pack_type;
1052

1053 1054
			entry->delta_sibling = base_entry->delta_child;
			base_entry->delta_child = entry;
1055

1056 1057 1058
			return;
		}
		/* Otherwise we would do the usual */
1059
	}
1060 1061

	if (sha1_object_info(entry->sha1, type, &entry->size))
1062 1063
		die("unable to get type of object %s",
		    sha1_to_hex(entry->sha1));
1064

1065
	if (!strcmp(type, commit_type)) {
1066
		entry->type = OBJ_COMMIT;
1067
	} else if (!strcmp(type, tree_type)) {
1068
		entry->type = OBJ_TREE;
1069
	} else if (!strcmp(type, blob_type)) {
1070
		entry->type = OBJ_BLOB;
1071
	} else if (!strcmp(type, tag_type)) {
1072 1073 1074 1075 1076 1077
		entry->type = OBJ_TAG;
	} else
		die("unable to pack object %s of type %s",
		    sha1_to_hex(entry->sha1), type);
}

1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
{
	struct object_entry *child = me->delta_child;
	unsigned int m = n;
	while (child) {
		unsigned int c = check_delta_limit(child, n + 1);
		if (m < c)
			m = c;
		child = child->delta_sibling;
	}
	return m;
}

1091 1092 1093
static void get_object_details(void)
{
	int i;
1094
	struct object_entry *entry;
1095

1096
	prepare_pack_ix();
1097 1098
	for (i = 0, entry = objects; i < nr_objects; i++, entry++)
		check_object(entry);
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115

	if (nr_objects == nr_result) {
		/*
		 * Depth of objects that depend on the entry -- this
		 * is subtracted from depth-max to break too deep
		 * delta chain because of delta data reusing.
		 * However, we loosen this restriction when we know we
		 * are creating a thin pack -- it will have to be
		 * expanded on the other end anyway, so do not
		 * artificially cut the delta chain and let it go as
		 * deep as it wants.
		 */
		for (i = 0, entry = objects; i < nr_objects; i++, entry++)
			if (!entry->delta && entry->delta_child)
				entry->delta_limit =
					check_delta_limit(entry, 1);
	}
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
}

typedef int (*entry_sort_t)(const struct object_entry *, const struct object_entry *);

static entry_sort_t current_sort;

static int sort_comparator(const void *_a, const void *_b)
{
	struct object_entry *a = *(struct object_entry **)_a;
	struct object_entry *b = *(struct object_entry **)_b;
	return current_sort(a,b);
}

static struct object_entry **create_sorted_list(entry_sort_t sort)
{
	struct object_entry **list = xmalloc(nr_objects * sizeof(struct object_entry *));
	int i;

	for (i = 0; i < nr_objects; i++)
		list[i] = objects + i;
	current_sort = sort;
	qsort(list, nr_objects, sizeof(struct object_entry *), sort_comparator);
	return list;
}

static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
{
1143
	return hashcmp(a->sha1, b->sha1);
1144 1145
}

T
Timo Hirvonen 已提交
1146
static struct object_entry **create_final_object_list(void)
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
{
	struct object_entry **list;
	int i, j;

	for (i = nr_result = 0; i < nr_objects; i++)
		if (!objects[i].preferred_base)
			nr_result++;
	list = xmalloc(nr_result * sizeof(struct object_entry *));
	for (i = j = 0; i < nr_objects; i++) {
		if (!objects[i].preferred_base)
			list[j++] = objects + i;
	}
	current_sort = sha1_sort;
	qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator);
	return list;
}

1164 1165 1166 1167 1168 1169
static int type_size_sort(const struct object_entry *a, const struct object_entry *b)
{
	if (a->type < b->type)
		return -1;
	if (a->type > b->type)
		return 1;
1170 1171 1172 1173
	if (a->hash < b->hash)
		return -1;
	if (a->hash > b->hash)
		return 1;
1174 1175 1176 1177
	if (a->preferred_base < b->preferred_base)
		return -1;
	if (a->preferred_base > b->preferred_base)
		return 1;
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
	if (a->size < b->size)
		return -1;
	if (a->size > b->size)
		return 1;
	return a < b ? -1 : (a > b);
}

struct unpacked {
	struct object_entry *entry;
	void *data;
1188
	struct delta_index *index;
1189 1190 1191
};

/*
1192 1193 1194 1195 1196 1197
 * We search for deltas _backwards_ in a list sorted by type and
 * by size, so that we see progressively smaller and smaller files.
 * That's because we prefer deltas to be from the bigger file
 * to the smaller - deletes are potentially cheaper, but perhaps
 * more importantly, the bigger file is likely the more recent
 * one.
1198
 */
1199
static int try_delta(struct unpacked *trg, struct unpacked *src,
1200
		     unsigned max_depth)
1201
{
1202 1203
	struct object_entry *trg_entry = trg->entry;
	struct object_entry *src_entry = src->entry;
1204 1205
	unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
	char type[10];
1206 1207 1208
	void *delta_buf;

	/* Don't bother doing diffs between different types */
1209
	if (trg_entry->type != src_entry->type)
1210 1211
		return -1;

1212 1213 1214
	/* We do not compute delta to *create* objects we are not
	 * going to pack.
	 */
1215
	if (trg_entry->preferred_base)
1216
		return -1;
1217

1218 1219
	/*
	 * We do not bother to try a delta that we discarded
1220
	 * on an earlier try, but only when reusing delta data.
1221
	 */
1222
	if (!no_reuse_delta && trg_entry->in_pack &&
1223 1224 1225
	    trg_entry->in_pack == src_entry->in_pack &&
	    trg_entry->in_pack_type != OBJ_REF_DELTA &&
	    trg_entry->in_pack_type != OBJ_OFS_DELTA)
1226 1227
		return 0;

1228 1229
	/*
	 * If the current object is at pack edge, take the depth the
1230 1231
	 * objects that depend on the current object into account --
	 * otherwise they would become too deep.
1232
	 */
1233 1234
	if (trg_entry->delta_child) {
		if (max_depth <= trg_entry->delta_limit)
1235
			return 0;
1236
		max_depth -= trg_entry->delta_limit;
1237
	}
1238
	if (src_entry->depth >= max_depth)
1239
		return 0;
1240

1241
	/* Now some size filtering heuristics. */
1242 1243
	trg_size = trg_entry->size;
	max_size = trg_size/2 - 20;
1244 1245 1246
	max_size = max_size * (max_depth - src_entry->depth) / max_depth;
	if (max_size == 0)
		return 0;
1247
	if (trg_entry->delta && trg_entry->delta_size <= max_size)
1248 1249
		max_size = trg_entry->delta_size-1;
	src_size = src_entry->size;
1250
	sizediff = src_size < trg_size ? trg_size - src_size : 0;
1251
	if (sizediff >= max_size)
1252
		return 0;
1253

1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
	/* Load data if not already done */
	if (!trg->data) {
		trg->data = read_sha1_file(trg_entry->sha1, type, &sz);
		if (sz != trg_size)
			die("object %s inconsistent object length (%lu vs %lu)",
			    sha1_to_hex(trg_entry->sha1), sz, trg_size);
	}
	if (!src->data) {
		src->data = read_sha1_file(src_entry->sha1, type, &sz);
		if (sz != src_size)
			die("object %s inconsistent object length (%lu vs %lu)",
			    sha1_to_hex(src_entry->sha1), sz, src_size);
	}
	if (!src->index) {
		src->index = create_delta_index(src->data, src_size);
		if (!src->index)
			die("out of memory");
	}

	delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
1274
	if (!delta_buf)
1275
		return 0;
1276 1277 1278 1279

	trg_entry->delta = src_entry;
	trg_entry->delta_size = delta_size;
	trg_entry->depth = src_entry->depth + 1;
1280
	free(delta_buf);
1281
	return 1;
1282 1283
}

1284 1285 1286 1287 1288
static void progress_interval(int signum)
{
	progress_update = 1;
}

1289
static void find_deltas(struct object_entry **list, int window, int depth)
1290
{
1291
	int i, idx;
1292 1293
	unsigned int array_size = window * sizeof(struct unpacked);
	struct unpacked *array = xmalloc(array_size);
1294 1295
	unsigned processed = 0;
	unsigned last_percent = 999;
1296 1297

	memset(array, 0, array_size);
1298 1299
	i = nr_objects;
	idx = 0;
1300
	if (progress)
1301
		fprintf(stderr, "Deltifying %d objects.\n", nr_result);
1302

1303
	while (--i >= 0) {
1304 1305 1306 1307
		struct object_entry *entry = list[i];
		struct unpacked *n = array + idx;
		int j;

1308 1309 1310
		if (!entry->preferred_base)
			processed++;

1311
		if (progress) {
1312
			unsigned percent = processed * 100 / nr_result;
1313 1314
			if (percent != last_percent || progress_update) {
				fprintf(stderr, "%4u%% (%u/%u) done\r",
1315
					percent, processed, nr_result);
1316 1317 1318
				progress_update = 0;
				last_percent = percent;
			}
1319
		}
1320 1321 1322

		if (entry->delta)
			/* This happens if we decided to reuse existing
1323
			 * delta from a pack.  "!no_reuse_delta &&" is implied.
1324 1325 1326
			 */
			continue;

1327 1328
		if (entry->size < 50)
			continue;
1329 1330
		free_delta_index(n->index);
		n->index = NULL;
1331
		free(n->data);
1332
		n->data = NULL;
1333
		n->entry = entry;
1334

L
Linus Torvalds 已提交
1335 1336 1337
		j = window;
		while (--j > 0) {
			unsigned int other_idx = idx + j;
1338
			struct unpacked *m;
L
Linus Torvalds 已提交
1339 1340
			if (other_idx >= window)
				other_idx -= window;
1341 1342 1343
			m = array + other_idx;
			if (!m->entry)
				break;
1344
			if (try_delta(n, m, depth) < 0)
1345 1346
				break;
		}
1347 1348 1349 1350 1351 1352
		/* if we made n a delta, and if n is already at max
		 * depth, leaving it in the window is pointless.  we
		 * should evict it first.
		 */
		if (entry->delta && depth <= entry->depth)
			continue;
1353

1354 1355 1356
		idx++;
		if (idx >= window)
			idx = 0;
1357
	}
1358

1359 1360 1361
	if (progress)
		fputc('\n', stderr);

1362
	for (i = 0; i < window; ++i) {
1363
		free_delta_index(array[i].index);
1364
		free(array[i].data);
1365
	}
1366
	free(array);
1367 1368
}

1369 1370
static void prepare_pack(int window, int depth)
{
1371
	get_object_details();
1372 1373 1374 1375 1376
	sorted_by_type = create_sorted_list(type_size_sort);
	if (window && depth)
		find_deltas(sorted_by_type, window+1, depth);
}

1377
static int reuse_cached_pack(unsigned char *sha1)
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
{
	static const char cache[] = "pack-cache/pack-%s.%s";
	char *cached_pack, *cached_idx;
	int ifd, ofd, ifd_ix = -1;

	cached_pack = git_path(cache, sha1_to_hex(sha1), "pack");
	ifd = open(cached_pack, O_RDONLY);
	if (ifd < 0)
		return 0;

	if (!pack_to_stdout) {
		cached_idx = git_path(cache, sha1_to_hex(sha1), "idx");
		ifd_ix = open(cached_idx, O_RDONLY);
		if (ifd_ix < 0) {
			close(ifd);
			return 0;
		}
	}

1397 1398 1399
	if (progress)
		fprintf(stderr, "Reusing %d objects pack %s\n", nr_objects,
			sha1_to_hex(sha1));
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430

	if (pack_to_stdout) {
		if (copy_fd(ifd, 1))
			exit(1);
		close(ifd);
	}
	else {
		char name[PATH_MAX];
		snprintf(name, sizeof(name),
			 "%s-%s.%s", base_name, sha1_to_hex(sha1), "pack");
		ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666);
		if (ofd < 0)
			die("unable to open %s (%s)", name, strerror(errno));
		if (copy_fd(ifd, ofd))
			exit(1);
		close(ifd);

		snprintf(name, sizeof(name),
			 "%s-%s.%s", base_name, sha1_to_hex(sha1), "idx");
		ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666);
		if (ofd < 0)
			die("unable to open %s (%s)", name, strerror(errno));
		if (copy_fd(ifd_ix, ofd))
			exit(1);
		close(ifd_ix);
		puts(sha1_to_hex(sha1));
	}

	return 1;
}

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
static void setup_progress_signal(void)
{
	struct sigaction sa;
	struct itimerval v;

	memset(&sa, 0, sizeof(sa));
	sa.sa_handler = progress_interval;
	sigemptyset(&sa.sa_mask);
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);

	v.it_interval.tv_sec = 1;
	v.it_interval.tv_usec = 0;
	v.it_value = v.it_interval;
	setitimer(ITIMER_REAL, &v, NULL);
}

1448 1449 1450 1451 1452 1453 1454 1455 1456
static int git_pack_config(const char *k, const char *v)
{
	if(!strcmp(k, "pack.window")) {
		window = git_config_int(k, v);
		return 0;
	}
	return git_default_config(k, v);
}

1457
static void read_object_list_from_stdin(void)
1458
{
1459 1460 1461
	char line[40 + 1 + PATH_MAX + 2];
	unsigned char sha1[20];
	unsigned hash;
1462

1463 1464 1465 1466 1467 1468
	for (;;) {
		if (!fgets(line, sizeof(line), stdin)) {
			if (feof(stdin))
				break;
			if (!ferror(stdin))
				die("fgets returned NULL, not EOF, not error!");
1469 1470 1471 1472
			if (errno != EINTR)
				die("fgets: %s", strerror(errno));
			clearerr(stdin);
			continue;
1473
		}
1474 1475 1476
		if (line[0] == '-') {
			if (get_sha1_hex(line+1, sha1))
				die("expected edge sha1, got garbage:\n %s",
1477
				    line);
1478
			add_preferred_base(sha1);
1479
			continue;
1480
		}
1481
		if (get_sha1_hex(line, sha1))
1482
			die("expected sha1, got garbage:\n %s", line);
1483

1484
		hash = name_hash(line+41);
1485 1486
		add_preferred_base_object(line+41, hash);
		add_object_entry(sha1, hash, 0);
1487
	}
1488 1489 1490 1491 1492
}

static void show_commit(struct commit *commit)
{
	unsigned hash = name_hash("");
1493
	add_preferred_base_object("", hash);
1494 1495 1496 1497 1498 1499
	add_object_entry(commit->object.sha1, hash, 0);
}

static void show_object(struct object_array_entry *p)
{
	unsigned hash = name_hash(p->name);
1500
	add_preferred_base_object(p->name, hash);
1501 1502 1503
	add_object_entry(p->item->sha1, hash, 0);
}

1504 1505 1506 1507 1508 1509
static void show_edge(struct commit *commit)
{
	add_preferred_base(commit->object.sha1);
}

static void get_object_list(int ac, const char **av)
1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
{
	struct rev_info revs;
	char line[1000];
	int flags = 0;

	init_revisions(&revs, NULL);
	save_commit_buffer = 0;
	track_object_refs = 0;
	setup_revisions(ac, av, &revs, NULL);

	while (fgets(line, sizeof(line), stdin) != NULL) {
		int len = strlen(line);
		if (line[len - 1] == '\n')
			line[--len] = 0;
		if (!len)
			break;
		if (*line == '-') {
			if (!strcmp(line, "--not")) {
				flags ^= UNINTERESTING;
				continue;
			}
			die("not a rev '%s'", line);
		}
		if (handle_revision_arg(line, &revs, flags, 1))
			die("bad revision '%s'", line);
	}

	prepare_revision_walk(&revs);
1538
	mark_edges_uninteresting(revs.commits, &revs, show_edge);
1539 1540 1541 1542 1543 1544 1545 1546 1547
	traverse_commit_list(&revs, show_commit, show_object);
}

int cmd_pack_objects(int argc, const char **argv, const char *prefix)
{
	SHA_CTX ctx;
	int depth = 10;
	struct object_entry **list;
	int use_internal_rev_list = 0;
1548
	int thin = 0;
1549
	int i;
1550 1551 1552 1553 1554 1555
	const char *rp_av[64];
	int rp_ac;

	rp_av[0] = "pack-objects";
	rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
	rp_ac = 2;
1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595

	git_config(git_pack_config);

	progress = isatty(2);
	for (i = 1; i < argc; i++) {
		const char *arg = argv[i];

		if (*arg != '-')
			break;

		if (!strcmp("--non-empty", arg)) {
			non_empty = 1;
			continue;
		}
		if (!strcmp("--local", arg)) {
			local = 1;
			continue;
		}
		if (!strcmp("--incremental", arg)) {
			incremental = 1;
			continue;
		}
		if (!strncmp("--window=", arg, 9)) {
			char *end;
			window = strtoul(arg+9, &end, 0);
			if (!arg[9] || *end)
				usage(pack_usage);
			continue;
		}
		if (!strncmp("--depth=", arg, 8)) {
			char *end;
			depth = strtoul(arg+8, &end, 0);
			if (!arg[8] || *end)
				usage(pack_usage);
			continue;
		}
		if (!strcmp("--progress", arg)) {
			progress = 1;
			continue;
		}
1596 1597 1598 1599
		if (!strcmp("--all-progress", arg)) {
			progress = 2;
			continue;
		}
1600 1601 1602 1603 1604 1605 1606 1607
		if (!strcmp("-q", arg)) {
			progress = 0;
			continue;
		}
		if (!strcmp("--no-reuse-delta", arg)) {
			no_reuse_delta = 1;
			continue;
		}
1608
		if (!strcmp("--delta-base-offset", arg)) {
1609
			allow_ofs_delta = 1;
1610 1611
			continue;
		}
1612 1613 1614 1615 1616 1617 1618 1619
		if (!strcmp("--stdout", arg)) {
			pack_to_stdout = 1;
			continue;
		}
		if (!strcmp("--revs", arg)) {
			use_internal_rev_list = 1;
			continue;
		}
1620 1621
		if (!strcmp("--unpacked", arg) ||
		    !strncmp("--unpacked=", arg, 11) ||
1622
		    !strcmp("--reflog", arg) ||
1623 1624 1625 1626 1627
		    !strcmp("--all", arg)) {
			use_internal_rev_list = 1;
			if (ARRAY_SIZE(rp_av) - 1 <= rp_ac)
				die("too many internal rev-list options");
			rp_av[rp_ac++] = arg;
1628 1629
			continue;
		}
1630 1631 1632 1633
		if (!strcmp("--thin", arg)) {
			use_internal_rev_list = 1;
			thin = 1;
			rp_av[1] = "--objects-edge";
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
			continue;
		}
		usage(pack_usage);
	}

	/* Traditionally "pack-objects [options] base extra" failed;
	 * we would however want to take refs parameter that would
	 * have been given to upstream rev-list ourselves, which means
	 * we somehow want to say what the base name is.  So the
	 * syntax would be:
	 *
	 * pack-objects [options] base <refs...>
	 *
	 * in other words, we would treat the first non-option as the
	 * base_name and send everything else to the internal revision
	 * walker.
	 */

	if (!pack_to_stdout)
		base_name = argv[i++];

	if (pack_to_stdout != !base_name)
		usage(pack_usage);

1658 1659
	if (!pack_to_stdout && thin)
		die("--thin cannot be used to build an indexable pack.");
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669

	prepare_packed_git();

	if (progress) {
		fprintf(stderr, "Generating pack...\n");
		setup_progress_signal();
	}

	if (!use_internal_rev_list)
		read_object_list_from_stdin();
1670 1671 1672 1673
	else {
		rp_av[rp_ac] = NULL;
		get_object_list(rp_ac, rp_av);
	}
1674

1675 1676
	if (progress)
		fprintf(stderr, "Done counting %d objects.\n", nr_objects);
1677 1678
	sorted_by_sha = create_final_object_list();
	if (non_empty && !nr_result)
1679
		return 0;
1680

J
Junio C Hamano 已提交
1681 1682
	SHA1_Init(&ctx);
	list = sorted_by_sha;
1683
	for (i = 0; i < nr_result; i++) {
J
Junio C Hamano 已提交
1684 1685 1686 1687
		struct object_entry *entry = *list++;
		SHA1_Update(&ctx, entry->sha1, 20);
	}
	SHA1_Final(object_list_sha1, &ctx);
1688 1689
	if (progress && (nr_objects != nr_result))
		fprintf(stderr, "Result has %d objects.\n", nr_result);
J
Junio C Hamano 已提交
1690

1691
	if (reuse_cached_pack(object_list_sha1))
1692 1693
		;
	else {
1694 1695
		if (nr_result)
			prepare_pack(window, depth);
N
Nicolas Pitre 已提交
1696
		if (progress == 1 && pack_to_stdout) {
1697 1698 1699 1700 1701 1702 1703
			/* the other end usually displays progress itself */
			struct itimerval v = {{0,},};
			setitimer(ITIMER_REAL, &v, NULL);
			signal(SIGALRM, SIG_IGN );
			progress_update = 0;
		}
		write_pack_file();
1704 1705 1706 1707
		if (!pack_to_stdout) {
			write_index_file();
			puts(sha1_to_hex(object_list_sha1));
		}
1708
	}
1709
	if (progress)
1710 1711
		fprintf(stderr, "Total %d (delta %d), reused %d (delta %d)\n",
			written, written_delta, reused, reused_delta);
1712 1713
	return 0;
}