diff --git a/archive-zip.c b/archive-zip.c index 8fbac2741aad958272f05cb9db02b5915a195ffb..72d55a58ac60019f9f6175d4c86af47348630762 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -90,7 +90,7 @@ static void copy_le32(unsigned char *dest, unsigned int n) static void *zlib_deflate(void *data, unsigned long size, int compression_level, unsigned long *compressed_size) { - z_stream stream; + git_zstream stream; unsigned long maxsize; void *buffer; int result; diff --git a/builtin/apply.c b/builtin/apply.c index 530d4bb7e76da8e402f78ab7f981a080d25d6e78..f2edc52818e817e83717f992a6bbd66dad4ef24d 100644 --- a/builtin/apply.c +++ b/builtin/apply.c @@ -1634,7 +1634,7 @@ static inline int metadata_changes(struct patch *patch) static char *inflate_it(const void *data, unsigned long size, unsigned long inflated_size) { - z_stream stream; + git_zstream stream; void *out; int st; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index dbfb5da52a9f9bf097c6336ab5bdc55eccf9caae..f65cb373bfee4fb1fcb40d79480eb104e71d5cad 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -265,7 +265,7 @@ static void unlink_base_data(struct base_data *c) static void *unpack_entry_data(unsigned long offset, unsigned long size) { int status; - z_stream stream; + git_zstream stream; void *buf = xmalloc(size); memset(&stream, 0, sizeof(stream)); @@ -355,7 +355,7 @@ static void *get_data_from_pack(struct object_entry *obj) off_t from = obj[0].idx.offset + obj[0].hdr_size; unsigned long len = obj[1].idx.offset - from; unsigned char *data, *inbuf; - z_stream stream; + git_zstream stream; int status; data = xmalloc(obj->size); @@ -666,7 +666,7 @@ static void parse_pack_objects(unsigned char *sha1) static int write_compressed(struct sha1file *f, void *in, unsigned int size) { - z_stream stream; + git_zstream stream; int status; unsigned char outbuf[4096]; diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 8981dd60669b14ed25449b6afde9a1526bb813c0..c6e2d8766b0ec15fcfe9dc0a60ee81db6750b527 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -126,7 +126,7 @@ static void *get_delta(struct object_entry *entry) static unsigned long do_compress(void **pptr, unsigned long size) { - z_stream stream; + git_zstream stream; void *in, *out; unsigned long maxsize; @@ -160,7 +160,7 @@ static int check_pack_inflate(struct packed_git *p, off_t len, unsigned long expect) { - z_stream stream; + git_zstream stream; unsigned char fakebuf[4096], *in; int st; @@ -187,12 +187,12 @@ static void copy_pack_data(struct sha1file *f, off_t len) { unsigned char *in; - unsigned int avail; + unsigned long avail; while (len) { in = use_pack(p, w_curs, offset, &avail); if (avail > len) - avail = (unsigned int)len; + avail = (unsigned long)len; sha1write(f, in, avail); offset += avail; len -= avail; @@ -994,7 +994,7 @@ static void check_object(struct object_entry *entry) const unsigned char *base_ref = NULL; struct object_entry *base_entry; unsigned long used, used_0; - unsigned int avail; + unsigned long avail; off_t ofs; unsigned char *buf, c; diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index f63973c9143c4109a3d1e40df9ec66a32498fe00..14e04e6795bd514fb1fec506073d8a9a71668cfa 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -90,7 +90,7 @@ static void use(int bytes) static void *get_data(unsigned long size) { - z_stream stream; + git_zstream stream; void *buf = xmalloc(size); memset(&stream, 0, sizeof(stream)); diff --git a/cache.h b/cache.h index 85ac5ec17c9cf4a8dfd725e1f871be8ec4fef867..1784bab7d5ade3c8efa70980c906c2c77085ade6 100644 --- a/cache.h +++ b/cache.h @@ -16,18 +16,27 @@ #endif #include - -void git_inflate_init(z_streamp strm); -void git_inflate_init_gzip_only(z_streamp strm); -void git_inflate_end(z_streamp strm); -int git_inflate(z_streamp strm, int flush); - -void git_deflate_init(z_streamp strm, int level); -void git_deflate_init_gzip(z_streamp strm, int level); -void git_deflate_end(z_streamp strm); -int git_deflate_end_gently(z_streamp strm); -int git_deflate(z_streamp strm, int flush); -unsigned long git_deflate_bound(z_streamp, unsigned long); +typedef struct git_zstream { + z_stream z; + unsigned long avail_in; + unsigned long avail_out; + unsigned long total_in; + unsigned long total_out; + unsigned char *next_in; + unsigned char *next_out; +} git_zstream; + +void git_inflate_init(git_zstream *); +void git_inflate_init_gzip_only(git_zstream *); +void git_inflate_end(git_zstream *); +int git_inflate(git_zstream *, int flush); + +void git_deflate_init(git_zstream *, int level); +void git_deflate_init_gzip(git_zstream *, int level); +void git_deflate_end(git_zstream *); +int git_deflate_end_gently(git_zstream *); +int git_deflate(git_zstream *, int flush); +unsigned long git_deflate_bound(git_zstream *, unsigned long); #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) @@ -991,7 +1000,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1, extern void pack_report(void); extern int open_pack_index(struct packed_git *); extern void close_pack_index(struct packed_git *); -extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *); +extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); diff --git a/diff.c b/diff.c index bac9a4bc22f96fd284a89c8c33a183672da3361b..431873d43ff35d69326053567f91745a904af93a 100644 --- a/diff.c +++ b/diff.c @@ -1729,7 +1729,7 @@ static unsigned char *deflate_it(char *data, { int bound; unsigned char *deflated; - z_stream stream; + git_zstream stream; memset(&stream, 0, sizeof(stream)); git_deflate_init(&stream, zlib_compression_level); diff --git a/fast-import.c b/fast-import.c index e4116a4ace927f2d6fb323a654130863ed047dff..1d5e3336a51a4bf19c0ab700565826b438c43266 100644 --- a/fast-import.c +++ b/fast-import.c @@ -1017,7 +1017,7 @@ static int store_object( unsigned char sha1[20]; unsigned long hdrlen, deltalen; git_SHA_CTX c; - z_stream s; + git_zstream s; hdrlen = sprintf((char *)hdr,"%s %lu", typename(type), (unsigned long)dat->len) + 1; @@ -1163,7 +1163,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) off_t offset; git_SHA_CTX c; git_SHA_CTX pack_file_ctx; - z_stream s; + git_zstream s; int status = Z_OK; /* Determine if we should auto-checkpoint. */ diff --git a/http-backend.c b/http-backend.c index ab5015d9d90cdbc5ac6126090e7b5a1cbf0bd626..59ad7da605f711af6970d6832db32efd62b6ea97 100644 --- a/http-backend.c +++ b/http-backend.c @@ -271,7 +271,7 @@ static struct rpc_service *select_service(const char *name) static void inflate_request(const char *prog_name, int out) { - z_stream stream; + git_zstream stream; unsigned char in_buf[8192]; unsigned char out_buf[8192]; unsigned long cnt = 0; diff --git a/http-push.c b/http-push.c index ecd67cf40a2780db4d73f9a28e2b961e6752cfeb..1e8a83093758a27b0312f8e0a9e570a138f1177b 100644 --- a/http-push.c +++ b/http-push.c @@ -352,7 +352,7 @@ static void start_put(struct transfer_request *request) unsigned long len; int hdrlen; ssize_t size; - z_stream stream; + git_zstream stream; unpacked = read_sha1_file(request->obj->sha1, &type, &len); hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; diff --git a/http.h b/http.h index e9ed3c2e8272e43208e75459ec29fd8b6548ae4d..a39304a7ab81a228da73d85fadfd6d7c1103715d 100644 --- a/http.h +++ b/http.h @@ -172,7 +172,7 @@ struct http_object_request { unsigned char sha1[20]; unsigned char real_sha1[20]; git_SHA_CTX c; - z_stream stream; + git_zstream stream; int zret; int rename; struct active_request_slot *slot; diff --git a/pack-check.c b/pack-check.c index a1a521648deee8e7e82c4e2cb7fe8fe03605dc06..0c19b6e5a5677bd14989175abddc119381fac4ef 100644 --- a/pack-check.c +++ b/pack-check.c @@ -26,7 +26,7 @@ int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, uint32_t data_crc = crc32(0, NULL, 0); do { - unsigned int avail; + unsigned long avail; void *data = use_pack(p, w_curs, offset, &avail); if (avail > len) avail = len; @@ -61,7 +61,7 @@ static int verify_packfile(struct packed_git *p, git_SHA1_Init(&ctx); do { - unsigned int remaining; + unsigned long remaining; unsigned char *in = use_pack(p, w_curs, offset, &remaining); offset += remaining; if (!pack_sig_ofs) diff --git a/remote-curl.c b/remote-curl.c index 13d8ceede345893b76fa2f15597d90304a50cf8e..bc48a36ef55603def36fa61144c68698a1e0fc15 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -471,7 +471,7 @@ static int post_rpc(struct rpc_state *rpc) * the transfer time. */ size_t size; - z_stream stream; + git_zstream stream; int ret; memset(&stream, 0, sizeof(stream)); diff --git a/sha1_file.c b/sha1_file.c index 0eefb6194e177fc36a70da1a90e416389edc7708..94d431907cc6f974277ab912411d2a9de2cb98bf 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -833,7 +833,7 @@ static int in_window(struct pack_window *win, off_t offset) unsigned char *use_pack(struct packed_git *p, struct pack_window **w_cursor, off_t offset, - unsigned int *left) + unsigned long *left) { struct pack_window *win = *w_cursor; @@ -1244,7 +1244,7 @@ unsigned long unpack_object_header_buffer(const unsigned char *buf, return used; } -static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) +static int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { unsigned long size, used; static const char valid_loose_object_type[8] = { @@ -1291,7 +1291,7 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon return 0; } -static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1) +static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; unsigned char *buf = xmallocz(size); @@ -1390,7 +1390,7 @@ static int parse_sha1_header(const char *hdr, unsigned long *sizep) static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) { int ret; - z_stream stream; + git_zstream stream; char hdr[8192]; ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); @@ -1406,7 +1406,7 @@ unsigned long get_size_from_delta(struct packed_git *p, { const unsigned char *data; unsigned char delta_head[20], *in; - z_stream stream; + git_zstream stream; int st; memset(&stream, 0, sizeof(stream)); @@ -1528,7 +1528,7 @@ static int unpack_object_header(struct packed_git *p, unsigned long *sizep) { unsigned char *base; - unsigned int left; + unsigned long left; unsigned long used; enum object_type type; @@ -1641,7 +1641,7 @@ static void *unpack_compressed_entry(struct packed_git *p, unsigned long size) { int st; - z_stream stream; + git_zstream stream; unsigned char *buffer, *in; buffer = xmallocz(size); @@ -2074,7 +2074,7 @@ static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *size int status; unsigned long mapsize, size; void *map; - z_stream stream; + git_zstream stream; char hdr[32]; map = map_sha1_file(sha1, &mapsize); @@ -2425,7 +2425,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, { int fd, ret; unsigned char compressed[4096]; - z_stream stream; + git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; char *filename; diff --git a/zlib.c b/zlib.c index 8f19d2fe388b2a75699d873618da05a15dd4e2a0..fe537e3ff69880f2d5c69575c2c2575bac503323 100644 --- a/zlib.c +++ b/zlib.c @@ -22,45 +22,90 @@ static const char *zerr_to_string(int status) } } -void git_inflate_init(z_streamp strm) +/* + * avail_in and avail_out in zlib are counted in uInt, which typically + * limits the size of the buffer we can use to 4GB when interacting + * with zlib in a single call to inflate/deflate. + */ +#define ZLIB_BUF_MAX ((uInt)-1) +static inline uInt zlib_buf_cap(unsigned long len) +{ + if (ZLIB_BUF_MAX < len) + die("working buffer for zlib too large"); + return len; +} + +static void zlib_pre_call(git_zstream *s) +{ + s->z.next_in = s->next_in; + s->z.next_out = s->next_out; + s->z.total_in = s->total_in; + s->z.total_out = s->total_out; + s->z.avail_in = zlib_buf_cap(s->avail_in); + s->z.avail_out = zlib_buf_cap(s->avail_out); +} + +static void zlib_post_call(git_zstream *s) +{ + s->next_in = s->z.next_in; + s->next_out = s->z.next_out; + s->total_in = s->z.total_in; + s->total_out = s->z.total_out; + s->avail_in = s->z.avail_in; + s->avail_out = s->z.avail_out; +} + +void git_inflate_init(git_zstream *strm) { - int status = inflateInit(strm); + int status; + zlib_pre_call(strm); + status = inflateInit(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; die("inflateInit: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_inflate_init_gzip_only(z_streamp strm) +void git_inflate_init_gzip_only(git_zstream *strm) { /* * Use default 15 bits, +16 is to accept only gzip and to * yield Z_DATA_ERROR when fed zlib format. */ const int windowBits = 15 + 16; - int status = inflateInit2(strm, windowBits); + int status; + zlib_pre_call(strm); + status = inflateInit2(&strm->z, windowBits); + zlib_post_call(strm); if (status == Z_OK) return; die("inflateInit2: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_inflate_end(z_streamp strm) +void git_inflate_end(git_zstream *strm) { - int status = inflateEnd(strm); + int status; + zlib_pre_call(strm); + status = inflateEnd(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; error("inflateEnd: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -int git_inflate(z_streamp strm, int flush) +int git_inflate(git_zstream *strm, int flush) { - int status = inflate(strm, flush); + int status; + zlib_pre_call(strm); + status = inflate(&strm->z, flush); + zlib_post_call(strm); switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: @@ -74,7 +119,7 @@ int git_inflate(z_streamp strm, int flush) break; } error("inflate: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); return status; } @@ -82,56 +127,74 @@ int git_inflate(z_streamp strm, int flush) #define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) #endif -unsigned long git_deflate_bound(z_streamp strm, unsigned long size) +unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) { - return deflateBound(strm, size); + return deflateBound(&strm->z, size); } -void git_deflate_init(z_streamp strm, int level) +void git_deflate_init(git_zstream *strm, int level) { - int status = deflateInit(strm, level); + int status; + zlib_pre_call(strm); + status = deflateInit(&strm->z, level); + zlib_post_call(strm); if (status == Z_OK) return; die("deflateInit: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_deflate_init_gzip(z_streamp strm, int level) +void git_deflate_init_gzip(git_zstream *strm, int level) { /* * Use default 15 bits, +16 is to generate gzip header/trailer * instead of the zlib wrapper. */ const int windowBits = 15 + 16; - int status = deflateInit2(strm, level, + int status; + + zlib_pre_call(strm); + status = deflateInit2(&strm->z, level, Z_DEFLATED, windowBits, 8, Z_DEFAULT_STRATEGY); + zlib_post_call(strm); if (status == Z_OK) return; die("deflateInit2: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -void git_deflate_end(z_streamp strm) +void git_deflate_end(git_zstream *strm) { - int status = deflateEnd(strm); + int status; + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); if (status == Z_OK) return; error("deflateEnd: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); } -int git_deflate_end_gently(z_streamp strm) +int git_deflate_end_gently(git_zstream *strm) { - return deflateEnd(strm); + int status; + + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); + return status; } -int git_deflate(z_streamp strm, int flush) +int git_deflate(git_zstream *strm, int flush) { - int status = deflate(strm, flush); + int status; + zlib_pre_call(strm); + status = deflate(&strm->z, flush); + zlib_post_call(strm); switch (status) { /* Z_BUF_ERROR: normal, needs more space in the output buffer */ case Z_BUF_ERROR: @@ -145,6 +208,6 @@ int git_deflate(z_streamp strm, int flush) break; } error("deflate: %s (%s)", zerr_to_string(status), - strm->msg ? strm->msg : "no message"); + strm->z.msg ? strm->z.msg : "no message"); return status; }