From 4d703a1a9016cd0a08994ddf7fc2f4739f223112 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:25 -0500 Subject: Replace unpack_entry_gently with unpack_entry. The unpack_entry_gently function currently has only two callers: the delta base resolution in sha1_file.c and the main loop of pack-check.c. Both of these must change to using unpack_entry directly when we implement sliding window mmap logic, so I'm doing it earlier to help break down the change set. This may cause a slight performance decrease for delta base resolution as well as for pack-check.c's verify_packfile(), as the pack use counter will be incremented and decremented for every object that is unpacked. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 2 +- pack-check.c | 2 +- sha1_file.c | 33 ++++++++++++--------------------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cache.h b/cache.h index 29dd290c9..e0a26e9d0 100644 --- a/cache.h +++ b/cache.h @@ -394,7 +394,7 @@ extern struct packed_git *add_packed_git(char *, int, int); extern int num_packed_objects(const struct packed_git *p); extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*); extern unsigned long find_pack_entry_one(const unsigned char *, struct packed_git *); -extern void *unpack_entry_gently(struct packed_git *, unsigned long, char *, unsigned long *); +extern void *unpack_entry(struct packed_git *, unsigned long, char *, unsigned long *); extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); extern void packed_object_info_detail(struct packed_git *, unsigned long, char *, unsigned long *, unsigned long *, unsigned int *, unsigned char *); diff --git a/pack-check.c b/pack-check.c index c0caaee09..491bad2ae 100644 --- a/pack-check.c +++ b/pack-check.c @@ -51,7 +51,7 @@ static int verify_packfile(struct packed_git *p) offset = find_pack_entry_one(sha1, p); if (!offset) die("internal error pack-check find-pack-entry-one"); - data = unpack_entry_gently(p, offset, type, &size); + data = unpack_entry(p, offset, type, &size); if (!data) { err = error("cannot unpack %s from %s", sha1_to_hex(sha1), p->pack_name); diff --git a/sha1_file.c b/sha1_file.c index 1c4df5b73..4824a5d4d 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1110,7 +1110,7 @@ static void *unpack_delta_entry(struct packed_git *p, unsigned long result_size, base_size, base_offset; offset = get_delta_base(p, offset, kind, obj_offset, &base_offset); - base = unpack_entry_gently(p, base_offset, type, &base_size); + base = unpack_entry(p, base_offset, type, &base_size); if (!base) die("failed to read delta base object at %lu from %s", base_offset, p->pack_name); @@ -1127,43 +1127,34 @@ static void *unpack_delta_entry(struct packed_git *p, return result; } -static void *unpack_entry(struct pack_entry *entry, +void *unpack_entry(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep) { - struct packed_git *p = entry->p; + unsigned long size, obj_offset = offset; + enum object_type kind; void *retval; if (use_packed_git(p)) die("cannot map packed file"); - retval = unpack_entry_gently(p, entry->offset, type, sizep); - unuse_packed_git(p); - if (!retval) - die("corrupted pack file %s", p->pack_name); - return retval; -} - -/* The caller is responsible for use_packed_git()/unuse_packed_git() pair */ -void *unpack_entry_gently(struct packed_git *p, unsigned long offset, - char *type, unsigned long *sizep) -{ - unsigned long size, obj_offset = offset; - enum object_type kind; - offset = unpack_object_header(p, offset, &kind, &size); switch (kind) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - return unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep); + retval = unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep); + break; case OBJ_COMMIT: case OBJ_TREE: case OBJ_BLOB: case OBJ_TAG: strcpy(type, type_names[kind]); *sizep = size; - return unpack_compressed_entry(p, offset, size); + retval = unpack_compressed_entry(p, offset, size); + break; default: - return NULL; + die("unknown object type %i in %s", kind, p->pack_name); } + unuse_packed_git(p); + return retval; } int num_packed_objects(const struct packed_git *p) @@ -1312,7 +1303,7 @@ static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned lo error("cannot read sha1_file for %s", sha1_to_hex(sha1)); return NULL; } - return unpack_entry(&e, type, size); + return unpack_entry(e.p, e.offset, type, size); } void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size) -- cgit v1.2.1 From 77ccc5bbd1bd403abd5f552be7210073bea856a6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:35 -0500 Subject: Introduce new config option for mmap limit. Rather than hardcoding the maximum number of bytes which can be mmapped from pack files we should make this value configurable, allowing the end user to increase or decrease this limit on a per-repository basis depending on the size of the repository and the capabilities of their operating system. In general users should not need to manually tune such a low-level setting within the core code, but being able to artifically limit the number of bytes which we can mmap at once from pack files will make it easier to craft test cases for the new mmap sliding window implementation. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 9 +++++++++ cache.h | 1 + config.c | 5 +++++ environment.c | 1 + sha1_file.c | 3 +-- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 178e0e1e2..28fe6942c 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -118,6 +118,15 @@ core.legacyheaders:: database directly (where the "http://" and "rsync://" protocols count as direct access). +core.packedGitLimit:: + Maximum number of bytes to map simultaneously into memory + from pack files. If Git needs to access more than this many + bytes at once to complete an operation it will unmap existing + regions to reclaim virtual address space within the process. + Default is 256 MiB, which should be reasonable for all + users/operating systems, except on largest Git projects. + You probably do not need to adjust this value. + alias.*:: Command aliases for the gitlink:git[1] command wrapper - e.g. after defining "alias.last = cat-file commit HEAD", the invocation diff --git a/cache.h b/cache.h index e0a26e9d0..816239bea 100644 --- a/cache.h +++ b/cache.h @@ -196,6 +196,7 @@ extern int warn_ambiguous_refs; extern int shared_repository; extern const char *apply_default_whitespace; extern int zlib_compression_level; +extern size_t packed_git_limit; #define GIT_REPO_VERSION 0 extern int repository_format_version; diff --git a/config.c b/config.c index fcccf7e2a..0c21286cb 100644 --- a/config.c +++ b/config.c @@ -298,6 +298,11 @@ int git_default_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.packedgitlimit")) { + packed_git_limit = git_config_int(var, value); + return 0; + } + if (!strcmp(var, "user.name")) { strlcpy(git_default_name, value, sizeof(git_default_name)); return 0; diff --git a/environment.c b/environment.c index a1502c4e8..a3ddae68a 100644 --- a/environment.c +++ b/environment.c @@ -23,6 +23,7 @@ char *git_log_output_encoding; int shared_repository = PERM_UMASK; const char *apply_default_whitespace; int zlib_compression_level = Z_DEFAULT_COMPRESSION; +size_t packed_git_limit = 256 * 1024 * 1024; int pager_in_use; int pager_use_color = 1; diff --git a/sha1_file.c b/sha1_file.c index 4824a5d4d..4183f595e 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -397,7 +397,6 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) return NULL; } -#define PACK_MAX_SZ (1<<26) static int pack_used_ctr; static unsigned long pack_mapped; struct packed_git *packed_git; @@ -490,7 +489,7 @@ int use_packed_git(struct packed_git *p) struct pack_header *hdr; pack_mapped += p->pack_size; - while (PACK_MAX_SZ < pack_mapped && unuse_one_packed_git()) + while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ fd = open(p->pack_name, O_RDONLY); if (fd < 0) -- cgit v1.2.1 From c41ee586dc95b757cdff4deae10a30a691ba758b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:44 -0500 Subject: Refactor packed_git to prepare for sliding mmap windows. The idea behind the sliding mmap window pack reader implementation is to have multiple mmap regions active against the same pack file, thereby allowing the process to mmap in only the active/hot sections of the pack and reduce overall virtual address space usage. To implement this we need to refactor the mmap related data (pack_base, pack_use_cnt) out of struct packed_git and move them into a new struct pack_window. We are refactoring the code to support a single struct pack_window per packfile, thereby emulating the prior behavior of mmap'ing the entire pack file. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 4 ++-- cache.h | 13 ++++++++++--- pack-check.c | 6 +++--- sha1_file.c | 51 +++++++++++++++++++++++++------------------------- 4 files changed, 40 insertions(+), 34 deletions(-) diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 9e15beb3b..4a00a1206 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -438,7 +438,7 @@ static unsigned long write_object(struct sha1file *f, } use_packed_git(p); - buf = (char *) p->pack_base + buf = p->windows->base + entry->in_pack_offset + entry->in_pack_header_size; datalen = find_packed_object_size(p, entry->in_pack_offset) @@ -943,7 +943,7 @@ static void check_object(struct object_entry *entry) struct object_entry *base_entry = NULL; use_packed_git(p); - buf = p->pack_base; + buf = p->windows->base; buf += entry->in_pack_offset; /* We want in_pack_type even if we do not reuse delta. diff --git a/cache.h b/cache.h index 816239bea..ae7bceca5 100644 --- a/cache.h +++ b/cache.h @@ -336,14 +336,21 @@ extern struct alternate_object_database { } *alt_odb_list; extern void prepare_alt_odb(void); +struct pack_window { + struct pack_window *next; + unsigned char *base; + off_t offset; + size_t len; + unsigned int last_used; + unsigned int inuse_cnt; +}; + extern struct packed_git { struct packed_git *next; unsigned long index_size; unsigned long pack_size; + struct pack_window *windows; unsigned int *index_base; - void *pack_base; - unsigned int pack_last_used; - unsigned int pack_use_cnt; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ diff --git a/pack-check.c b/pack-check.c index 491bad2ae..761cc852e 100644 --- a/pack-check.c +++ b/pack-check.c @@ -13,7 +13,8 @@ static int verify_packfile(struct packed_git *p) int nr_objects, err, i; /* Header consistency check */ - hdr = p->pack_base; + pack_base = p->windows->base; + hdr = (struct pack_header*)pack_base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) return error("Packfile %s signature mismatch", p->pack_name); if (!pack_version_ok(hdr->hdr_version)) @@ -26,7 +27,6 @@ static int verify_packfile(struct packed_git *p) num_packed_objects(p)); SHA1_Init(&ctx); - pack_base = p->pack_base; SHA1_Update(&ctx, pack_base, pack_size - 20); SHA1_Final(sha1, &ctx); if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20)) @@ -78,7 +78,7 @@ static void show_pack_info(struct packed_git *p) int nr_objects, i; unsigned int chain_histogram[MAX_CHAIN]; - hdr = p->pack_base; + hdr = (struct pack_header*)p->windows->base; nr_objects = ntohl(hdr->hdr_entries); memset(chain_histogram, 0, sizeof(chain_histogram)); diff --git a/sha1_file.c b/sha1_file.c index 4183f595e..a9f374e60 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -455,21 +455,23 @@ static int unuse_one_packed_git(void) struct packed_git *p, *lru = NULL; for (p = packed_git; p; p = p->next) { - if (p->pack_use_cnt || !p->pack_base) + if (!p->windows || p->windows->inuse_cnt) continue; - if (!lru || p->pack_last_used < lru->pack_last_used) + if (!lru || p->windows->last_used < lru->windows->last_used) lru = p; } if (!lru) return 0; - munmap(lru->pack_base, lru->pack_size); - lru->pack_base = NULL; + munmap(lru->windows->base, lru->windows->len); + free(lru->windows); + lru->windows = NULL; return 1; } void unuse_packed_git(struct packed_git *p) { - p->pack_use_cnt--; + if (p->windows) + p->windows->inuse_cnt--; } int use_packed_git(struct packed_git *p) @@ -482,10 +484,10 @@ int use_packed_git(struct packed_git *p) die("packfile %s not a regular file", p->pack_name); p->pack_size = st.st_size; } - if (!p->pack_base) { + if (!p->windows) { int fd; struct stat st; - void *map; + struct pack_window *win; struct pack_header *hdr; pack_mapped += p->pack_size; @@ -500,16 +502,18 @@ int use_packed_git(struct packed_git *p) } if (st.st_size != p->pack_size) die("packfile %s size mismatch.", p->pack_name); - map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); + win = xcalloc(1, sizeof(*win)); + win->len = p->pack_size; + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (map == MAP_FAILED) + if (win->base == MAP_FAILED) die("packfile %s cannot be mapped.", p->pack_name); - p->pack_base = map; + p->windows = win; /* Check if we understand this pack file. If we don't we're * likely too old to handle it. */ - hdr = map; + hdr = (struct pack_header*)win->base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) die("packfile %s isn't actually a pack.", p->pack_name); if (!pack_version_ok(hdr->hdr_version)) @@ -522,13 +526,13 @@ int use_packed_git(struct packed_git *p) */ if (hashcmp((unsigned char *)(p->index_base) + p->index_size - 40, - (unsigned char *)p->pack_base + + p->windows->base + p->pack_size - 20)) { die("packfile %s does not match index.", p->pack_name); } } - p->pack_last_used = pack_used_ctr++; - p->pack_use_cnt++; + p->windows->last_used = pack_used_ctr++; + p->windows->inuse_cnt++; return 0; } @@ -558,9 +562,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local) p->pack_size = st.st_size; p->index_base = idx_map; p->next = NULL; - p->pack_base = NULL; - p->pack_last_used = 0; - p->pack_use_cnt = 0; + p->windows = NULL; p->pack_local = local; if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1)) hashcpy(p->sha1, sha1); @@ -591,9 +593,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa p->pack_size = 0; p->index_base = idx_map; p->next = NULL; - p->pack_base = NULL; - p->pack_last_used = 0; - p->pack_use_cnt = 0; + p->windows = NULL; hashcpy(p->sha1, sha1); return p; } @@ -882,7 +882,7 @@ static unsigned long get_delta_base(struct packed_git *p, unsigned long delta_obj_offset, unsigned long *base_obj_offset) { - unsigned char *base_info = (unsigned char *) p->pack_base + offset; + unsigned char *base_info = p->windows->base + offset; unsigned long base_offset; /* there must be at least 20 bytes left regardless of delta type */ @@ -949,7 +949,7 @@ static int packed_delta_info(struct packed_git *p, memset(&stream, 0, sizeof(stream)); - stream.next_in = (unsigned char *) p->pack_base + offset; + stream.next_in = p->windows->base + offset; stream.avail_in = p->pack_size - offset; stream.next_out = delta_head; stream.avail_out = sizeof(delta_head); @@ -984,8 +984,7 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of if (p->pack_size <= offset) die("object offset outside of pack file"); - used = unpack_object_header_gently((unsigned char *)p->pack_base + - offset, + used = unpack_object_header_gently(p->windows->base + offset, p->pack_size - offset, type, sizep); if (!used) die("object offset outside of pack file"); @@ -1031,7 +1030,7 @@ void packed_object_info_detail(struct packed_git *p, if (p->pack_size <= offset + 20) die("pack file %s records an incomplete delta base", p->pack_name); - next_sha1 = (unsigned char *) p->pack_base + offset; + next_sha1 = p->windows->base + offset; if (*delta_chain_length == 0) hashcpy(base_sha1, next_sha1); offset = find_pack_entry_one(next_sha1, p); @@ -1081,7 +1080,7 @@ static void *unpack_compressed_entry(struct packed_git *p, buffer = xmalloc(size + 1); buffer[size] = 0; memset(&stream, 0, sizeof(stream)); - stream.next_in = (unsigned char*)p->pack_base + offset; + stream.next_in = p->windows->base + offset; stream.avail_in = p->pack_size - offset; stream.next_out = buffer; stream.avail_out = size; -- cgit v1.2.1 From 2dc3a234094afc61c367be3e6018722cd9d92ea9 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:47 -0500 Subject: Use off_t for index and pack file lengths. Since the index_size and pack_size members of struct packed_git are the lengths of those corresponding files we should use the off_t size of the operating system to store these file lengths, rather than an unsigned long. This would help in the future should we ever resurrect Junio's 64 bit index implementation. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache.h b/cache.h index ae7bceca5..cc872440e 100644 --- a/cache.h +++ b/cache.h @@ -347,10 +347,10 @@ struct pack_window { extern struct packed_git { struct packed_git *next; - unsigned long index_size; - unsigned long pack_size; struct pack_window *windows; unsigned int *index_base; + off_t index_size; + off_t pack_size; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ -- cgit v1.2.1 From 75025ccdb79b5d6b290c630f8777c9f7bb9d257c Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:55 -0500 Subject: Create read_or_die utility routine. Like write_or_die read_or_die reads the entire length requested or it kills the current process with a die call. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + write_or_die.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cache.h b/cache.h index cc872440e..e6b716495 100644 --- a/cache.h +++ b/cache.h @@ -428,6 +428,7 @@ extern char *git_commit_encoding; extern char *git_log_output_encoding; extern int copy_fd(int ifd, int ofd); +extern void read_or_die(int fd, void *buf, size_t count); extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); diff --git a/write_or_die.c b/write_or_die.c index bfe4eeb64..8cf648602 100644 --- a/write_or_die.c +++ b/write_or_die.c @@ -1,5 +1,21 @@ #include "cache.h" +void read_or_die(int fd, void *buf, size_t count) +{ + char *p = buf; + ssize_t loaded; + + while (count > 0) { + loaded = xread(fd, p, count); + if (loaded == 0) + die("unexpected end of file"); + else if (loaded < 0) + die("read error (%s)", strerror(errno)); + count -= loaded; + p += loaded; + } +} + void write_or_die(int fd, const void *buf, size_t count) { const char *p = buf; -- cgit v1.2.1 From 9bc879c1ced505089e2a1e420d32599bb15b35b5 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:01 -0500 Subject: Refactor how we open pack files to prepare for multiple windows. To efficiently support mmaping of multiple regions of the same pack file we want to keep the pack's file descriptor open while we are actively working with that pack. So we are now keeping that file descriptor in packed_git.pack_fd and closing it only after we unmap the last window. This is going to increase the number of file descriptors that are in use at once, however that will be bounded by the total number of pack files present and therefore should not be very high. It is a small tradeoff which we may need to revisit after some testing can be done on various repositories and systems. For code clarity we also want to seperate out the implementation of how we open a pack file from the implementation which locates a suitable window (or makes a new one) from the given pack file. Since this is a rather large delta I'm taking advantage of doing it now, in a fairly isolated change. When we open a pack file we need to examine the header and trailer without having a mmap in place, as we may only need to mmap the middle section of this particular pack. Consequently the verification code has been refactored to make use of the new read_or_die function. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + sha1_file.c | 86 +++++++++++++++++++++++++++++++++---------------------------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/cache.h b/cache.h index e6b716495..f8a89280f 100644 --- a/cache.h +++ b/cache.h @@ -351,6 +351,7 @@ extern struct packed_git { unsigned int *index_base; off_t index_size; off_t pack_size; + int pack_fd; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ diff --git a/sha1_file.c b/sha1_file.c index a9f374e60..79d2d9dac 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -465,6 +465,8 @@ static int unuse_one_packed_git(void) munmap(lru->windows->base, lru->windows->len); free(lru->windows); lru->windows = NULL; + close(p->pack_fd); + p->pack_fd = -1; return 1; } @@ -474,62 +476,64 @@ void unuse_packed_git(struct packed_git *p) p->windows->inuse_cnt--; } -int use_packed_git(struct packed_git *p) +static void open_packed_git(struct packed_git *p) { + struct stat st; + struct pack_header hdr; + unsigned char sha1[20]; + unsigned char *idx_sha1; + + p->pack_fd = open(p->pack_name, O_RDONLY); + if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) + die("packfile %s cannot be opened", p->pack_name); + + /* If we created the struct before we had the pack we lack size. */ if (!p->pack_size) { - struct stat st; - /* We created the struct before we had the pack */ - stat(p->pack_name, &st); if (!S_ISREG(st.st_mode)) die("packfile %s not a regular file", p->pack_name); p->pack_size = st.st_size; - } + } else if (p->pack_size != st.st_size) + die("packfile %s size changed", p->pack_name); + + /* Verify we recognize this pack file format. */ + read_or_die(p->pack_fd, &hdr, sizeof(hdr)); + if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) + die("file %s is not a GIT packfile", p->pack_name); + if (!pack_version_ok(hdr.hdr_version)) + die("packfile %s is version %u and not supported" + " (try upgrading GIT to a newer version)", + p->pack_name, ntohl(hdr.hdr_version)); + + /* Verify the pack matches its index. */ + if (num_packed_objects(p) != ntohl(hdr.hdr_entries)) + die("packfile %s claims to have %u objects" + " while index size indicates %u objects", + p->pack_name, ntohl(hdr.hdr_entries), + num_packed_objects(p)); + if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1) + die("end of packfile %s is unavailable", p->pack_name); + read_or_die(p->pack_fd, sha1, sizeof(sha1)); + idx_sha1 = ((unsigned char *)p->index_base) + p->index_size - 40; + if (hashcmp(sha1, idx_sha1)) + die("packfile %s does not match index", p->pack_name); +} + +int use_packed_git(struct packed_git *p) +{ + if (p->pack_fd == -1) + open_packed_git(p); if (!p->windows) { - int fd; - struct stat st; struct pack_window *win; - struct pack_header *hdr; pack_mapped += p->pack_size; while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ - fd = open(p->pack_name, O_RDONLY); - if (fd < 0) - die("packfile %s cannot be opened", p->pack_name); - if (fstat(fd, &st)) { - close(fd); - die("packfile %s cannot be opened", p->pack_name); - } - if (st.st_size != p->pack_size) - die("packfile %s size mismatch.", p->pack_name); win = xcalloc(1, sizeof(*win)); win->len = p->pack_size; - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0); if (win->base == MAP_FAILED) die("packfile %s cannot be mapped.", p->pack_name); p->windows = win; - - /* Check if we understand this pack file. If we don't we're - * likely too old to handle it. - */ - hdr = (struct pack_header*)win->base; - if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) - die("packfile %s isn't actually a pack.", p->pack_name); - if (!pack_version_ok(hdr->hdr_version)) - die("packfile %s is version %i and not supported" - " (try upgrading GIT to a newer version)", - p->pack_name, ntohl(hdr->hdr_version)); - - /* Check if the pack file matches with the index file. - * this is cheap. - */ - if (hashcmp((unsigned char *)(p->index_base) + - p->index_size - 40, - p->windows->base + - p->pack_size - 20)) { - die("packfile %s does not match index.", p->pack_name); - } } p->windows->last_used = pack_used_ctr++; p->windows->inuse_cnt++; @@ -563,6 +567,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local) p->index_base = idx_map; p->next = NULL; p->windows = NULL; + p->pack_fd = -1; p->pack_local = local; if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1)) hashcpy(p->sha1, sha1); @@ -594,6 +599,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa p->index_base = idx_map; p->next = NULL; p->windows = NULL; + p->pack_fd = -1; hashcpy(p->sha1, sha1); return p; } -- cgit v1.2.1 From 03e79c88aa34ce188eb4fb7509e6d127c79c507d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:08 -0500 Subject: Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 16 ++++--- cache.h | 4 +- pack-check.c | 22 +++++----- sha1_file.c | 110 ++++++++++++++++++++++++++++--------------------- 4 files changed, 85 insertions(+), 67 deletions(-) diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 4a00a1206..6d7ae7f1a 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -416,6 +416,7 @@ static unsigned long write_object(struct sha1file *f, } else { struct packed_git *p = entry->in_pack; + struct pack_window *w_curs = NULL; if (entry->delta) { obj_type = (allow_ofs_delta && entry->delta->offset) ? @@ -437,16 +438,14 @@ static unsigned long write_object(struct sha1file *f, hdrlen += 20; } - use_packed_git(p); - buf = p->windows->base - + entry->in_pack_offset - + entry->in_pack_header_size; + buf = use_pack(p, &w_curs, entry->in_pack_offset + + entry->in_pack_header_size, NULL); datalen = find_packed_object_size(p, entry->in_pack_offset) - entry->in_pack_header_size; if (!pack_to_stdout && check_inflate(buf, datalen, entry->size)) die("corrupt delta in pack %s", sha1_to_hex(entry->sha1)); sha1write(f, buf, datalen); - unuse_packed_git(p); + unuse_pack(&w_curs); reused++; } if (entry->delta) @@ -937,14 +936,13 @@ static void check_object(struct object_entry *entry) if (entry->in_pack && !entry->preferred_base) { struct packed_git *p = entry->in_pack; + struct pack_window *w_curs = NULL; unsigned long left = p->pack_size - entry->in_pack_offset; unsigned long size, used; unsigned char *buf; struct object_entry *base_entry = NULL; - use_packed_git(p); - buf = p->windows->base; - buf += entry->in_pack_offset; + buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL); /* We want in_pack_type even if we do not reuse delta. * There is no point not reusing non-delta representations. @@ -990,7 +988,7 @@ static void check_object(struct object_entry *entry) if (base_name) base_entry = locate_object_entry(base_name); } - unuse_packed_git(p); + unuse_pack(&w_curs); entry->in_pack_header_size = used; if (base_entry) { diff --git a/cache.h b/cache.h index f8a89280f..c927f29d3 100644 --- a/cache.h +++ b/cache.h @@ -397,8 +397,8 @@ extern void install_packed_git(struct packed_git *pack); extern struct packed_git *find_sha1_pack(const unsigned char *sha1, struct packed_git *packs); -extern int use_packed_git(struct packed_git *); -extern void unuse_packed_git(struct packed_git *); +extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *); +extern void unuse_pack(struct pack_window **); extern struct packed_git *add_packed_git(char *, int, int); extern int num_packed_objects(const struct packed_git *p); extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*); diff --git a/pack-check.c b/pack-check.c index 761cc852e..972916f40 100644 --- a/pack-check.c +++ b/pack-check.c @@ -1,7 +1,8 @@ #include "cache.h" #include "pack.h" -static int verify_packfile(struct packed_git *p) +static int verify_packfile(struct packed_git *p, + struct pack_window **w_curs) { unsigned long index_size = p->index_size; void *index_base = p->index_base; @@ -13,7 +14,7 @@ static int verify_packfile(struct packed_git *p) int nr_objects, err, i; /* Header consistency check */ - pack_base = p->windows->base; + pack_base = use_pack(p, w_curs, 0, NULL); hdr = (struct pack_header*)pack_base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) return error("Packfile %s signature mismatch", p->pack_name); @@ -72,13 +73,14 @@ static int verify_packfile(struct packed_git *p) #define MAX_CHAIN 40 -static void show_pack_info(struct packed_git *p) +static void show_pack_info(struct packed_git *p, + struct pack_window **w_curs) { struct pack_header *hdr; int nr_objects, i; unsigned int chain_histogram[MAX_CHAIN]; - hdr = (struct pack_header*)p->windows->base; + hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL); nr_objects = ntohl(hdr->hdr_entries); memset(chain_histogram, 0, sizeof(chain_histogram)); @@ -142,18 +144,18 @@ int verify_pack(struct packed_git *p, int verbose) if (!ret) { /* Verify pack file */ - use_packed_git(p); - ret = verify_packfile(p); - unuse_packed_git(p); + struct pack_window *w_curs = NULL; + ret = verify_packfile(p, &w_curs); + unuse_pack(&w_curs); } if (verbose) { if (ret) printf("%s: bad\n", p->pack_name); else { - use_packed_git(p); - show_pack_info(p); - unuse_packed_git(p); + struct pack_window *w_curs = NULL; + show_pack_info(p, &w_curs); + unuse_pack(&w_curs); printf("%s: ok\n", p->pack_name); } } diff --git a/sha1_file.c b/sha1_file.c index 79d2d9dac..4d80527ba 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -470,10 +470,13 @@ static int unuse_one_packed_git(void) return 1; } -void unuse_packed_git(struct packed_git *p) +void unuse_pack(struct pack_window **w_cursor) { - if (p->windows) - p->windows->inuse_cnt--; + struct pack_window *w = *w_cursor; + if (w) { + w->inuse_cnt--; + *w_cursor = NULL; + } } static void open_packed_git(struct packed_git *p) @@ -518,13 +521,16 @@ static void open_packed_git(struct packed_git *p) die("packfile %s does not match index", p->pack_name); } -int use_packed_git(struct packed_git *p) +unsigned char* use_pack(struct packed_git *p, + struct pack_window **w_cursor, + unsigned long offset, + unsigned int *left) { + struct pack_window *win = p->windows; + if (p->pack_fd == -1) open_packed_git(p); - if (!p->windows) { - struct pack_window *win; - + if (!win) { pack_mapped += p->pack_size; while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ @@ -535,9 +541,14 @@ int use_packed_git(struct packed_git *p) die("packfile %s cannot be mapped.", p->pack_name); p->windows = win; } - p->windows->last_used = pack_used_ctr++; - p->windows->inuse_cnt++; - return 0; + if (win != *w_cursor) { + win->last_used = pack_used_ctr++; + win->inuse_cnt++; + *w_cursor = win; + } + if (left) + *left = win->len - offset; + return win->base + offset; } struct packed_git *add_packed_git(char *path, int path_len, int local) @@ -883,12 +894,13 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l } static unsigned long get_delta_base(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, enum object_type kind, unsigned long delta_obj_offset, unsigned long *base_obj_offset) { - unsigned char *base_info = p->windows->base + offset; + unsigned char *base_info = use_pack(p, w_curs, offset, NULL); unsigned long base_offset; /* there must be at least 20 bytes left regardless of delta type */ @@ -928,6 +940,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep); static int packed_delta_info(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, enum object_type kind, unsigned long obj_offset, @@ -936,7 +949,8 @@ static int packed_delta_info(struct packed_git *p, { unsigned long base_offset; - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset); + offset = get_delta_base(p, w_curs, offset, kind, + obj_offset, &base_offset); /* We choose to only get the type of the base object and * ignore potentially corrupt pack file that expects the delta @@ -955,8 +969,7 @@ static int packed_delta_info(struct packed_git *p, memset(&stream, 0, sizeof(stream)); - stream.next_in = p->windows->base + offset; - stream.avail_in = p->pack_size - offset; + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = delta_head; stream.avail_out = sizeof(delta_head); @@ -982,16 +995,18 @@ static int packed_delta_info(struct packed_git *p, return 0; } -static unsigned long unpack_object_header(struct packed_git *p, unsigned long offset, - enum object_type *type, unsigned long *sizep) +static unsigned long unpack_object_header(struct packed_git *p, + struct pack_window **w_curs, + unsigned long offset, + enum object_type *type, + unsigned long *sizep) { + unsigned char *base; + unsigned int left; unsigned long used; - if (p->pack_size <= offset) - die("object offset outside of pack file"); - - used = unpack_object_header_gently(p->windows->base + offset, - p->pack_size - offset, type, sizep); + base = use_pack(p, w_curs, offset, &left); + used = unpack_object_header_gently(base, left, type, sizep); if (!used) die("object offset outside of pack file"); @@ -1006,13 +1021,14 @@ void packed_object_info_detail(struct packed_git *p, unsigned int *delta_chain_length, unsigned char *base_sha1) { + struct pack_window *w_curs = NULL; unsigned long obj_offset, val; unsigned char *next_sha1; enum object_type kind; *delta_chain_length = 0; obj_offset = offset; - offset = unpack_object_header(p, offset, &kind, size); + offset = unpack_object_header(p, &w_curs, offset, &kind, size); for (;;) { switch (kind) { @@ -1025,25 +1041,24 @@ void packed_object_info_detail(struct packed_git *p, case OBJ_TAG: strcpy(type, type_names[kind]); *store_size = 0; /* notyet */ + unuse_pack(&w_curs); return; case OBJ_OFS_DELTA: - get_delta_base(p, offset, kind, obj_offset, &offset); + get_delta_base(p, &w_curs, offset, kind, + obj_offset, &offset); if (*delta_chain_length == 0) { /* TODO: find base_sha1 as pointed by offset */ } break; case OBJ_REF_DELTA: - if (p->pack_size <= offset + 20) - die("pack file %s records an incomplete delta base", - p->pack_name); - next_sha1 = p->windows->base + offset; + next_sha1 = use_pack(p, &w_curs, offset, NULL); if (*delta_chain_length == 0) hashcpy(base_sha1, next_sha1); offset = find_pack_entry_one(next_sha1, p); break; } obj_offset = offset; - offset = unpack_object_header(p, offset, &kind, &val); + offset = unpack_object_header(p, &w_curs, offset, &kind, &val); (*delta_chain_length)++; } } @@ -1051,20 +1066,26 @@ void packed_object_info_detail(struct packed_git *p, static int packed_object_info(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep) { + struct pack_window *w_curs = NULL; unsigned long size, obj_offset = offset; enum object_type kind; + int r; - offset = unpack_object_header(p, offset, &kind, &size); + offset = unpack_object_header(p, &w_curs, offset, &kind, &size); switch (kind) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - return packed_delta_info(p, offset, kind, obj_offset, type, sizep); + r = packed_delta_info(p, &w_curs, offset, kind, + obj_offset, type, sizep); + unuse_pack(&w_curs); + return r; case OBJ_COMMIT: case OBJ_TREE: case OBJ_BLOB: case OBJ_TAG: strcpy(type, type_names[kind]); + unuse_pack(&w_curs); break; default: die("pack %s contains unknown object type %d", @@ -1076,6 +1097,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset, } static void *unpack_compressed_entry(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, unsigned long size) { @@ -1086,8 +1108,7 @@ static void *unpack_compressed_entry(struct packed_git *p, buffer = xmalloc(size + 1); buffer[size] = 0; memset(&stream, 0, sizeof(stream)); - stream.next_in = p->windows->base + offset; - stream.avail_in = p->pack_size - offset; + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = buffer; stream.avail_out = size; @@ -1103,6 +1124,7 @@ static void *unpack_compressed_entry(struct packed_git *p, } static void *unpack_delta_entry(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, unsigned long delta_size, enum object_type kind, @@ -1113,13 +1135,14 @@ static void *unpack_delta_entry(struct packed_git *p, void *delta_data, *result, *base; unsigned long result_size, base_size, base_offset; - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset); + offset = get_delta_base(p, w_curs, offset, kind, + obj_offset, &base_offset); base = unpack_entry(p, base_offset, type, &base_size); if (!base) die("failed to read delta base object at %lu from %s", base_offset, p->pack_name); - delta_data = unpack_compressed_entry(p, offset, delta_size); + delta_data = unpack_compressed_entry(p, w_curs, offset, delta_size); result = patch_delta(base, base_size, delta_data, delta_size, &result_size); @@ -1134,17 +1157,17 @@ static void *unpack_delta_entry(struct packed_git *p, void *unpack_entry(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep) { + struct pack_window *w_curs = NULL; unsigned long size, obj_offset = offset; enum object_type kind; void *retval; - if (use_packed_git(p)) - die("cannot map packed file"); - offset = unpack_object_header(p, offset, &kind, &size); + offset = unpack_object_header(p, &w_curs, offset, &kind, &size); switch (kind) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - retval = unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep); + retval = unpack_delta_entry(p, &w_curs, offset, size, + kind, obj_offset, type, sizep); break; case OBJ_COMMIT: case OBJ_TREE: @@ -1152,12 +1175,12 @@ void *unpack_entry(struct packed_git *p, unsigned long offset, case OBJ_TAG: strcpy(type, type_names[kind]); *sizep = size; - retval = unpack_compressed_entry(p, offset, size); + retval = unpack_compressed_entry(p, &w_curs, offset, size); break; default: die("unknown object type %i in %s", kind, p->pack_name); } - unuse_packed_git(p); + unuse_pack(&w_curs); return retval; } @@ -1284,7 +1307,6 @@ static int sha1_loose_object_info(const unsigned char *sha1, char *type, unsigne int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep) { - int status; struct pack_entry e; if (!find_pack_entry(sha1, &e, NULL)) { @@ -1292,11 +1314,7 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep if (!find_pack_entry(sha1, &e, NULL)) return sha1_loose_object_info(sha1, type, sizep); } - if (use_packed_git(e.p)) - die("cannot map packed file"); - status = packed_object_info(e.p, e.offset, type, sizep); - unuse_packed_git(e.p); - return status; + return packed_object_info(e.p, e.offset, type, sizep); } static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned long *size) -- cgit v1.2.1 From 079afb18fed078af01bd9ab02e2ebbe5d31893b1 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:13 -0500 Subject: Loop over pack_windows when inflating/accessing data. When multiple mmaps start getting used for all pack file access it is not possible to get all data associated with a specific object in one contiguous memory region. This limitation prevents simply passing a single address and length to SHA1_Update or to inflate. Instead we need to loop until we have processed all data of interest. As we loop over the data we are always interested in reusing the same window 'cursor', as the prior window will no longer be of any use to us. This allows the use_pack() call to automatically decrement the use count of the prior window before setting up access for us to the next window. Within each loop we need to make use of the available length output parameter of use_pack() to tell us how many bytes are available in the current memory region, as we cannot tell otherwise. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 58 ++++++++++++++++++++++++++++++++++++++++++++------ pack-check.c | 46 +++++++++++++++++---------------------- sha1_file.c | 22 +++++++++++++------ 3 files changed, 87 insertions(+), 39 deletions(-) diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 6d7ae7f1a..afb926a34 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -276,7 +276,52 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha * we are going to reuse the existing object data as is. make * sure it is not corrupt. */ -static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect) +static int check_pack_inflate(struct packed_git *p, + struct pack_window **w_curs, + unsigned long offset, + unsigned long len, + unsigned long expect) +{ + z_stream stream; + unsigned char fakebuf[4096], *in; + int st; + + memset(&stream, 0, sizeof(stream)); + inflateInit(&stream); + do { + in = use_pack(p, w_curs, offset, &stream.avail_in); + stream.next_in = in; + stream.next_out = fakebuf; + stream.avail_out = sizeof(fakebuf); + st = inflate(&stream, Z_FINISH); + offset += stream.next_in - in; + } while (st == Z_OK || st == Z_BUF_ERROR); + inflateEnd(&stream); + return (st == Z_STREAM_END && + stream.total_out == expect && + stream.total_in == len) ? 0 : -1; +} + +static void copy_pack_data(struct sha1file *f, + struct packed_git *p, + struct pack_window **w_curs, + unsigned long offset, + unsigned long len) +{ + unsigned char *in; + unsigned int avail; + + while (len) { + in = use_pack(p, w_curs, offset, &avail); + if (avail > len) + avail = len; + sha1write(f, in, avail); + offset += avail; + len -= avail; + } +} + +static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect) { z_stream stream; unsigned char fakebuf[4096]; @@ -323,7 +368,7 @@ static int revalidate_loose_object(struct object_entry *entry, return -1; map += used; mapsize -= used; - return check_inflate(map, mapsize, size); + return check_loose_inflate(map, mapsize, size); } static unsigned long write_object(struct sha1file *f, @@ -417,6 +462,7 @@ static unsigned long write_object(struct sha1file *f, else { struct packed_git *p = entry->in_pack; struct pack_window *w_curs = NULL; + unsigned long offset; if (entry->delta) { obj_type = (allow_ofs_delta && entry->delta->offset) ? @@ -438,13 +484,13 @@ static unsigned long write_object(struct sha1file *f, hdrlen += 20; } - buf = use_pack(p, &w_curs, entry->in_pack_offset - + entry->in_pack_header_size, NULL); + offset = entry->in_pack_offset + entry->in_pack_header_size; datalen = find_packed_object_size(p, entry->in_pack_offset) - entry->in_pack_header_size; - if (!pack_to_stdout && check_inflate(buf, datalen, entry->size)) + if (!pack_to_stdout && check_pack_inflate(p, &w_curs, + offset, datalen, entry->size)) die("corrupt delta in pack %s", sha1_to_hex(entry->sha1)); - sha1write(f, buf, datalen); + copy_pack_data(f, p, &w_curs, offset, datalen); unuse_pack(&w_curs); reused++; } diff --git a/pack-check.c b/pack-check.c index 972916f40..08a9fd8dc 100644 --- a/pack-check.c +++ b/pack-check.c @@ -8,39 +8,38 @@ static int verify_packfile(struct packed_git *p, void *index_base = p->index_base; SHA_CTX ctx; unsigned char sha1[20]; - unsigned long pack_size = p->pack_size; - void *pack_base; - struct pack_header *hdr; + unsigned long offset = 0, pack_sig = p->pack_size - 20; int nr_objects, err, i; - /* Header consistency check */ - pack_base = use_pack(p, w_curs, 0, NULL); - hdr = (struct pack_header*)pack_base; - if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) - return error("Packfile %s signature mismatch", p->pack_name); - if (!pack_version_ok(hdr->hdr_version)) - return error("Packfile version %d unsupported", - ntohl(hdr->hdr_version)); - nr_objects = ntohl(hdr->hdr_entries); - if (num_packed_objects(p) != nr_objects) - return error("Packfile claims to have %d objects, " - "while idx size expects %d", nr_objects, - num_packed_objects(p)); + /* Note that the pack header checks are actually performed by + * use_pack when it first opens the pack file. If anything + * goes wrong during those checks then the call will die out + * immediately. + */ SHA1_Init(&ctx); - SHA1_Update(&ctx, pack_base, pack_size - 20); + while (offset < pack_sig) { + unsigned int remaining; + unsigned char *in = use_pack(p, w_curs, offset, &remaining); + offset += remaining; + if (offset > pack_sig) + remaining -= offset - pack_sig; + SHA1_Update(&ctx, in, remaining); + } SHA1_Final(sha1, &ctx); - if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20)) + if (hashcmp(sha1, use_pack(p, w_curs, pack_sig, NULL))) return error("Packfile %s SHA1 mismatch with itself", p->pack_name); if (hashcmp(sha1, (unsigned char *)index_base + index_size - 40)) return error("Packfile %s SHA1 mismatch with idx", p->pack_name); + unuse_pack(w_curs); /* Make sure everything reachable from idx is valid. Since we * have verified that nr_objects matches between idx and pack, * we do not do scan-streaming check on the pack file. */ + nr_objects = num_packed_objects(p); for (i = err = 0; i < nr_objects; i++) { unsigned char sha1[20]; void *data; @@ -73,15 +72,12 @@ static int verify_packfile(struct packed_git *p, #define MAX_CHAIN 40 -static void show_pack_info(struct packed_git *p, - struct pack_window **w_curs) +static void show_pack_info(struct packed_git *p) { - struct pack_header *hdr; int nr_objects, i; unsigned int chain_histogram[MAX_CHAIN]; - hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL); - nr_objects = ntohl(hdr->hdr_entries); + nr_objects = num_packed_objects(p); memset(chain_histogram, 0, sizeof(chain_histogram)); for (i = 0; i < nr_objects; i++) { @@ -153,9 +149,7 @@ int verify_pack(struct packed_git *p, int verbose) if (ret) printf("%s: bad\n", p->pack_name); else { - struct pack_window *w_curs = NULL; - show_pack_info(p, &w_curs); - unuse_pack(&w_curs); + show_pack_info(p); printf("%s: ok\n", p->pack_name); } } diff --git a/sha1_file.c b/sha1_file.c index 4d80527ba..c30452251 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -962,19 +962,23 @@ static int packed_delta_info(struct packed_git *p, if (sizep) { const unsigned char *data; - unsigned char delta_head[20]; + unsigned char delta_head[20], *in; unsigned long result_size; z_stream stream; int st; memset(&stream, 0, sizeof(stream)); - - stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = delta_head; stream.avail_out = sizeof(delta_head); inflateInit(&stream); - st = inflate(&stream, Z_FINISH); + do { + in = use_pack(p, w_curs, offset, &stream.avail_in); + stream.next_in = in; + st = inflate(&stream, Z_FINISH); + offset += stream.next_in - in; + } while ((st == Z_OK || st == Z_BUF_ERROR) + && stream.total_out < sizeof(delta_head)); inflateEnd(&stream); if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) @@ -1103,17 +1107,21 @@ static void *unpack_compressed_entry(struct packed_git *p, { int st; z_stream stream; - unsigned char *buffer; + unsigned char *buffer, *in; buffer = xmalloc(size + 1); buffer[size] = 0; memset(&stream, 0, sizeof(stream)); - stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = buffer; stream.avail_out = size; inflateInit(&stream); - st = inflate(&stream, Z_FINISH); + do { + in = use_pack(p, w_curs, offset, &stream.avail_in); + stream.next_in = in; + st = inflate(&stream, Z_FINISH); + offset += stream.next_in - in; + } while (st == Z_OK || st == Z_BUF_ERROR); inflateEnd(&stream); if ((st != Z_STREAM_END) || stream.total_out != size) { free(buffer); -- cgit v1.2.1 From 8d8a4ea5530ef9c738341887a7dcece4abd7dcbe Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:18 -0500 Subject: Document why header parsing won't exceed a window. When we parse the object header or the delta base reference we don't bother to loop over use_pack() calls. The reason we don't need to bother with calling use_pack for each byte accessed is that use_pack will always promise us at least 20 bytes (really the hash size) after the offset. This promise from use_pack simplifies a lot of code in the header parsing logic, as well as helps out the zlib library by ensuring there's always some data for it to consume during an inflate call. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- sha1_file.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sha1_file.c b/sha1_file.c index c30452251..346696934 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -903,10 +903,12 @@ static unsigned long get_delta_base(struct packed_git *p, unsigned char *base_info = use_pack(p, w_curs, offset, NULL); unsigned long base_offset; - /* there must be at least 20 bytes left regardless of delta type */ - if (p->pack_size <= offset + 20) - die("truncated pack file"); - + /* use_pack() assured us we have [base_info, base_info + 20) + * as a range that we can look at without walking off the + * end of the mapped window. Its actually the hash size + * that is assured. An OFS_DELTA longer than the hash size + * is stupid, as then a REF_DELTA would be smaller to store. + */ if (kind == OBJ_OFS_DELTA) { unsigned used = 0; unsigned char c = base_info[used++]; @@ -1009,6 +1011,12 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned int left; unsigned long used; + /* use_pack() assures us we have [base, base + 20) available + * as a range that we can look at at. (Its actually the hash + * size that is assurred.) With our object header encoding + * the maximum deflated object size is 2^137, which is just + * insane, so we know won't exceed what we have been given. + */ base = use_pack(p, w_curs, offset, &left); used = unpack_object_header_gently(base, left, type, sizep); if (!used) -- cgit v1.2.1 From 54044bf825d311751e30552248be1e0cac99a5a3 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:23 -0500 Subject: Unmap individual windows rather than entire files. To support multiple windows per packfile we need to unmap only one window at a time from that packfile, leaving any other windows in place and available for reference. We treat all windows from all packfiles equally; the least recently used, not-in-use window across all packfiles will always be closed first. If we have unmapped all windows in a packfile then we can also close the packfile's file descriptor as its possible we won't need to map any window from that file in the near future. This decision about when to close the pack file descriptor may need to be revisited in the future after additional testing on several different platforms can be performed. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- sha1_file.c | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/sha1_file.c b/sha1_file.c index 346696934..8e14a5a88 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -450,24 +450,39 @@ static int check_packed_git_idx(const char *path, unsigned long *idx_size_, return 0; } -static int unuse_one_packed_git(void) +static int unuse_one_window(void) { - struct packed_git *p, *lru = NULL; + struct packed_git *p, *lru_p = NULL; + struct pack_window *w, *w_l, *lru_w = NULL, *lru_l = NULL; for (p = packed_git; p; p = p->next) { - if (!p->windows || p->windows->inuse_cnt) - continue; - if (!lru || p->windows->last_used < lru->windows->last_used) - lru = p; + for (w_l = NULL, w = p->windows; w; w = w->next) { + if (!w->inuse_cnt) { + if (!lru_w || w->last_used < lru_w->last_used) { + lru_p = p; + lru_w = w; + lru_l = w_l; + } + } + w_l = w; + } } - if (!lru) - return 0; - munmap(lru->windows->base, lru->windows->len); - free(lru->windows); - lru->windows = NULL; - close(p->pack_fd); - p->pack_fd = -1; - return 1; + if (lru_p) { + munmap(lru_w->base, lru_w->len); + pack_mapped -= lru_w->len; + if (lru_l) + lru_l->next = lru_w->next; + else { + lru_p->windows = lru_w->next; + if (!lru_p->windows) { + close(lru_p->pack_fd); + lru_p->pack_fd = -1; + } + } + free(lru_w); + return 1; + } + return 0; } void unuse_pack(struct pack_window **w_cursor) @@ -532,7 +547,7 @@ unsigned char* use_pack(struct packed_git *p, open_packed_git(p); if (!win) { pack_mapped += p->pack_size; - while (packed_git_limit < pack_mapped && unuse_one_packed_git()) + while (packed_git_limit < pack_mapped && unuse_one_window()) ; /* nothing */ win = xcalloc(1, sizeof(*win)); win->len = p->pack_size; -- cgit v1.2.1 From 60bb8b1453e2f93d13e7bb44e8e46c085d2dd752 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:28 -0500 Subject: Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 11 ++++++++ cache.h | 1 + config.c | 10 ++++++++ environment.c | 1 + sha1_file.c | 66 ++++++++++++++++++++++++++++++++++++++---------- 5 files changed, 76 insertions(+), 13 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 28fe6942c..d71653dc6 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -118,6 +118,17 @@ core.legacyheaders:: database directly (where the "http://" and "rsync://" protocols count as direct access). +core.packedGitWindowSize:: + Number of bytes of a pack file to map into memory in a + single mapping operation. Larger window sizes may allow + your system to process a smaller number of large pack files + more quickly. Smaller window sizes will negatively affect + performance due to increased calls to the opreating system's + memory manager, but may improve performance when accessing + a large number of large pack files. Default is 32 MiB, + which should be reasonable for all users/operating systems. + You probably do not need to adjust this value. + core.packedGitLimit:: Maximum number of bytes to map simultaneously into memory from pack files. If Git needs to access more than this many diff --git a/cache.h b/cache.h index c927f29d3..c7d7e4dcc 100644 --- a/cache.h +++ b/cache.h @@ -196,6 +196,7 @@ extern int warn_ambiguous_refs; extern int shared_repository; extern const char *apply_default_whitespace; extern int zlib_compression_level; +extern size_t packed_git_window_size; extern size_t packed_git_limit; #define GIT_REPO_VERSION 0 diff --git a/config.c b/config.c index 0c21286cb..21f166e60 100644 --- a/config.c +++ b/config.c @@ -298,6 +298,16 @@ int git_default_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.packedgitwindowsize")) { + int pgsz = getpagesize(); + packed_git_window_size = git_config_int(var, value); + packed_git_window_size /= pgsz; + if (!packed_git_window_size) + packed_git_window_size = 1; + packed_git_window_size *= pgsz; + return 0; + } + if (!strcmp(var, "core.packedgitlimit")) { packed_git_limit = git_config_int(var, value); return 0; diff --git a/environment.c b/environment.c index a3ddae68a..e559fd69c 100644 --- a/environment.c +++ b/environment.c @@ -23,6 +23,7 @@ char *git_log_output_encoding; int shared_repository = PERM_UMASK; const char *apply_default_whitespace; int zlib_compression_level = Z_DEFAULT_COMPRESSION; +size_t packed_git_window_size = 32 * 1024 * 1024; size_t packed_git_limit = 256 * 1024 * 1024; int pager_in_use; int pager_use_color = 1; diff --git a/sha1_file.c b/sha1_file.c index 8e14a5a88..548535c84 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -397,8 +397,9 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) return NULL; } -static int pack_used_ctr; -static unsigned long pack_mapped; +static unsigned int pack_used_ctr; +static size_t pack_mapped; +static size_t page_size; struct packed_git *packed_git; static int check_packed_git_idx(const char *path, unsigned long *idx_size_, @@ -536,31 +537,70 @@ static void open_packed_git(struct packed_git *p) die("packfile %s does not match index", p->pack_name); } +static int in_window(struct pack_window *win, unsigned long offset) +{ + /* We must promise at least 20 bytes (one hash) after the + * offset is available from this window, otherwise the offset + * is not actually in this window and a different window (which + * has that one hash excess) must be used. This is to support + * the object header and delta base parsing routines below. + */ + off_t win_off = win->offset; + return win_off <= offset + && (offset + 20) <= (win_off + win->len); +} + unsigned char* use_pack(struct packed_git *p, struct pack_window **w_cursor, unsigned long offset, unsigned int *left) { - struct pack_window *win = p->windows; + struct pack_window *win = *w_cursor; if (p->pack_fd == -1) open_packed_git(p); - if (!win) { - pack_mapped += p->pack_size; - while (packed_git_limit < pack_mapped && unuse_one_window()) - ; /* nothing */ - win = xcalloc(1, sizeof(*win)); - win->len = p->pack_size; - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0); - if (win->base == MAP_FAILED) - die("packfile %s cannot be mapped.", p->pack_name); - p->windows = win; + + /* Since packfiles end in a hash of their content and its + * pointless to ask for an offset into the middle of that + * hash, and the in_window function above wouldn't match + * don't allow an offset too close to the end of the file. + */ + if (offset > (p->pack_size - 20)) + die("offset beyond end of packfile (truncated pack?)"); + + if (!win || !in_window(win, offset)) { + if (win) + win->inuse_cnt--; + for (win = p->windows; win; win = win->next) { + if (in_window(win, offset)) + break; + } + if (!win) { + if (!page_size) + page_size = getpagesize(); + win = xcalloc(1, sizeof(*win)); + win->offset = (offset / page_size) * page_size; + win->len = p->pack_size - win->offset; + if (win->len > packed_git_window_size) + win->len = packed_git_window_size; + pack_mapped += win->len; + while (packed_git_limit < pack_mapped && unuse_one_window()) + ; /* nothing */ + win->base = mmap(NULL, win->len, + PROT_READ, MAP_PRIVATE, + p->pack_fd, win->offset); + if (win->base == MAP_FAILED) + die("packfile %s cannot be mapped.", p->pack_name); + win->next = p->windows; + p->windows = win; + } } if (win != *w_cursor) { win->last_used = pack_used_ctr++; win->inuse_cnt++; *w_cursor = win; } + offset -= win->offset; if (left) *left = win->len - offset; return win->base + offset; -- cgit v1.2.1 From bac2614de3c162de2328f21f2272ffbc30122230 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:33 -0500 Subject: Load core configuration in git-verify-pack. Now that our pack access code's behavior may be altered by the setting of core.packedGitWindowSize or core.packedGitLimit we need to make sure these values are set as configured in the repository's configuration file rather than to their defaults. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-verify-pack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/builtin-verify-pack.c b/builtin-verify-pack.c index 7d39d9bcd..4e31c273f 100644 --- a/builtin-verify-pack.c +++ b/builtin-verify-pack.c @@ -55,6 +55,7 @@ int cmd_verify_pack(int argc, const char **argv, const char *prefix) int no_more_options = 0; int nothing_done = 1; + git_config(git_default_config); while (1 < argc) { if (!no_more_options && argv[1][0] == '-') { if (!strcmp("-v", argv[1])) -- cgit v1.2.1 From 40be82723ca35d4e95daccc3ccf6456f6876430e Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:38 -0500 Subject: Ensure core.packedGitWindowSize cannot be less than 2 pages. We cannot allow a window to be smaller than 2 system pages. This limitation is necessary to support the feature of use_pack() where we always supply at least 20 bytes after the offset to help the object header and delta base parsing routines. If packedGitWindowSize were allowed to be as small as 1 system page then we would be completely unable to access an object header which spanned over a page as we would never be able to arrange a mapping such that the header was contiguous in virtual memory. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.c b/config.c index 21f166e60..3b8da569f 100644 --- a/config.c +++ b/config.c @@ -302,8 +302,8 @@ int git_default_config(const char *var, const char *value) int pgsz = getpagesize(); packed_git_window_size = git_config_int(var, value); packed_git_window_size /= pgsz; - if (!packed_git_window_size) - packed_git_window_size = 1; + if (packed_git_window_size < 2) + packed_git_window_size = 2; packed_git_window_size *= pgsz; return 0; } -- cgit v1.2.1 From 73b4e4be7128f75346ce6053d6da1672f243bc74 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:41 -0500 Subject: Improve error message when packfile mmap fails. If we are unable to mmap the a region of the packfile with the mmap() system call there may be a good reason why, such as a closed file descriptor or out of address space. Reporting the system level error message can help to debug such problems. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- sha1_file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sha1_file.c b/sha1_file.c index 548535c84..63123cc47 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -590,7 +590,9 @@ unsigned char* use_pack(struct packed_git *p, PROT_READ, MAP_PRIVATE, p->pack_fd, win->offset); if (win->base == MAP_FAILED) - die("packfile %s cannot be mapped.", p->pack_name); + die("packfile %s cannot be mapped: %s", + p->pack_name, + strerror(errno)); win->next = p->windows; p->windows = win; } -- cgit v1.2.1 From 11daf39b74321d02d574479def29939d67c319ad Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:44 -0500 Subject: Support unmapping windows on 'temporary' packfiles. If a command opens a packfile for only temporary access and does not install the struct packed_git* into the global packed_git list then we are unable to unmap any inactive windows within that packed_git, causing the overall process to exceed core.packedGitLimit. We cannot force the callers to install their temporary packfile into the packed_git chain as doing so would allow that (possibly corrupt but currently being verified) temporary packfile to become part of the local ODB, which may allow it to be considered for object resolution when it may not actually be a valid packfile. So to support unmapping the windows of these temporary packfiles we also scan the windows of the struct packed_git which was supplied to use_pack(). Since commands only work with one temporary packfile at a time scanning the one supplied to use_pack() and all packs installed into packed_git should cover everything available in memory. We also have to be careful to not close the file descriptor of the packed_git which was handed to use_pack() when all of that packfile's windows have been unmapped, as we are already past the open call that would open the packfile and need the file descriptor to be ready for mmap() after unuse_one_window returns. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- sha1_file.c | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/sha1_file.c b/sha1_file.c index 63123cc47..01a2f8779 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -451,23 +451,34 @@ static int check_packed_git_idx(const char *path, unsigned long *idx_size_, return 0; } -static int unuse_one_window(void) -{ - struct packed_git *p, *lru_p = NULL; - struct pack_window *w, *w_l, *lru_w = NULL, *lru_l = NULL; - - for (p = packed_git; p; p = p->next) { - for (w_l = NULL, w = p->windows; w; w = w->next) { - if (!w->inuse_cnt) { - if (!lru_w || w->last_used < lru_w->last_used) { - lru_p = p; - lru_w = w; - lru_l = w_l; - } +static void scan_windows(struct packed_git *p, + struct packed_git **lru_p, + struct pack_window **lru_w, + struct pack_window **lru_l) +{ + struct pack_window *w, *w_l; + + for (w_l = NULL, w = p->windows; w; w = w->next) { + if (!w->inuse_cnt) { + if (!*lru_w || w->last_used < (*lru_w)->last_used) { + *lru_p = p; + *lru_w = w; + *lru_l = w_l; } - w_l = w; } + w_l = w; } +} + +static int unuse_one_window(struct packed_git *current) +{ + struct packed_git *p, *lru_p = NULL; + struct pack_window *lru_w = NULL, *lru_l = NULL; + + if (current) + scan_windows(current, &lru_p, &lru_w, &lru_l); + for (p = packed_git; p; p = p->next) + scan_windows(p, &lru_p, &lru_w, &lru_l); if (lru_p) { munmap(lru_w->base, lru_w->len); pack_mapped -= lru_w->len; @@ -475,7 +486,7 @@ static int unuse_one_window(void) lru_l->next = lru_w->next; else { lru_p->windows = lru_w->next; - if (!lru_p->windows) { + if (!lru_p->windows && lru_p != current) { close(lru_p->pack_fd); lru_p->pack_fd = -1; } @@ -584,7 +595,8 @@ unsigned char* use_pack(struct packed_git *p, if (win->len > packed_git_window_size) win->len = packed_git_window_size; pack_mapped += win->len; - while (packed_git_limit < pack_mapped && unuse_one_window()) + while (packed_git_limit < pack_mapped + && unuse_one_window(p)) ; /* nothing */ win->base = mmap(NULL, win->len, PROT_READ, MAP_PRIVATE, -- cgit v1.2.1 From a53128b60162d7a84adca4206540df5b8e3d9dc8 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:47 -0500 Subject: Create pack_report() as a debugging aid. Much like the alloc_report() function can be useful to report on object allocation statistics while debugging the new pack_report() function can be useful to report on the behavior of the mmap window code used for packfile access. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + sha1_file.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/cache.h b/cache.h index c7d7e4dcc..a5fc23235 100644 --- a/cache.h +++ b/cache.h @@ -398,6 +398,7 @@ extern void install_packed_git(struct packed_git *pack); extern struct packed_git *find_sha1_pack(const unsigned char *sha1, struct packed_git *packs); +extern void pack_report(); extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *); extern void unuse_pack(struct pack_window **); extern struct packed_git *add_packed_git(char *, int, int); diff --git a/sha1_file.c b/sha1_file.c index 01a2f8779..8de8ce0a7 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -398,10 +398,34 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) } static unsigned int pack_used_ctr; +static unsigned int pack_mmap_calls; +static unsigned int peak_pack_open_windows; +static unsigned int pack_open_windows; +static size_t peak_pack_mapped; static size_t pack_mapped; static size_t page_size; struct packed_git *packed_git; +void pack_report() +{ + fprintf(stderr, + "pack_report: getpagesize() = %10lu\n" + "pack_report: core.packedGitWindowSize = %10lu\n" + "pack_report: core.packedGitLimit = %10lu\n", + page_size, + packed_git_window_size, + packed_git_limit); + fprintf(stderr, + "pack_report: pack_used_ctr = %10u\n" + "pack_report: pack_mmap_calls = %10u\n" + "pack_report: pack_open_windows = %10u / %10u\n" + "pack_report: pack_mapped = %10lu / %10lu\n", + pack_used_ctr, + pack_mmap_calls, + pack_open_windows, peak_pack_open_windows, + pack_mapped, peak_pack_mapped); +} + static int check_packed_git_idx(const char *path, unsigned long *idx_size_, void **idx_map_) { @@ -492,6 +516,7 @@ static int unuse_one_window(struct packed_git *current) } } free(lru_w); + pack_open_windows--; return 1; } return 0; @@ -605,6 +630,12 @@ unsigned char* use_pack(struct packed_git *p, die("packfile %s cannot be mapped: %s", p->pack_name, strerror(errno)); + pack_mmap_calls++; + pack_open_windows++; + if (pack_mapped > peak_pack_mapped) + peak_pack_mapped = pack_mapped; + if (pack_open_windows > peak_pack_open_windows) + peak_pack_open_windows = pack_open_windows; win->next = p->windows; p->windows = win; } -- cgit v1.2.1 From 2dee8af67688bb24575dd6b07106bc41d9997923 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:51 -0500 Subject: Test suite for sliding window mmap implementation. This is a basic set of tests for the sliding window mmap. We mostly focus on the verify-pack and pack-objects implementations (including delta reuse) as these commands appear to cover the bulk of the affected portions of sha1_file.c. The test cases don't verify the virtual memory size used, as this can differ from system to system. Instead it just verifies that we can run with very low values for core.packedGitLimit and core.packedGitWindowSize. Adding pack_report() to the end of both builtin-verify-pack.c and builtin-pack-objects.c and manually inspecting the statistics output can help to verify that the total virtual memory size attributed to pack mmap usage is what one might expect on the current system. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- t/t5301-sliding-window.sh | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 t/t5301-sliding-window.sh diff --git a/t/t5301-sliding-window.sh b/t/t5301-sliding-window.sh new file mode 100755 index 000000000..5a7232a57 --- /dev/null +++ b/t/t5301-sliding-window.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Copyright (c) 2006 Shawn Pearce +# + +test_description='mmap sliding window tests' +. ./test-lib.sh + +test_expect_success \ + 'setup' \ + 'rm -f .git/index* + for i in a b c + do + echo $i >$i && + dd if=/dev/urandom bs=32k count=1 >>$i && + git-update-index --add $i || return 1 + done && + echo d >d && cat c >>d && git-update-index --add d && + tree=`git-write-tree` && + commit1=`git-commit-tree $tree Date: Sun, 24 Dec 2006 00:46:13 -0500 Subject: Default core.packdGitWindowSize to 1 MiB if NO_MMAP. If the compiler has asked us to disable use of mmap() on their platform then we are forced to use git_mmap and its emulation via pread. In this case large (e.g. 32 MiB) windows for pack access are simply too big as a command will wind up reading a lot more data than it will ever need, significantly reducing response time. To prevent a high latency when NO_MMAP has been selected we now use a default of 1 MiB for core.packedGitWindowSize. Credit goes to Linus and Junio for recommending this more reasonable setting. [jc: upcased the name of the symbolic constant, and made another hardcoded constant into a symbolic constant while at it. ] Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- environment.c | 4 ++-- git-compat-util.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/environment.c b/environment.c index e559fd69c..09976c7bf 100644 --- a/environment.c +++ b/environment.c @@ -23,8 +23,8 @@ char *git_log_output_encoding; int shared_repository = PERM_UMASK; const char *apply_default_whitespace; int zlib_compression_level = Z_DEFAULT_COMPRESSION; -size_t packed_git_window_size = 32 * 1024 * 1024; -size_t packed_git_limit = 256 * 1024 * 1024; +size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE; +size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT; int pager_in_use; int pager_use_color = 1; diff --git a/git-compat-util.h b/git-compat-util.h index 5d9eb2615..4764087d8 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -92,12 +92,17 @@ extern void set_warn_routine(void (*routine)(const char *warn, va_list params)); extern void *git_mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); extern int git_munmap(void *start, size_t length); +#define DEFAULT_PACKED_GIT_WINDOW_SIZE (1 * 1024 * 1024) + #else /* NO_MMAP */ #include +#define DEFAULT_PACKED_GIT_WINDOW_SIZE (32 * 1024 * 1024) #endif /* NO_MMAP */ +#define DEFAULT_PACKED_GIT_LIMIT (256 * 1024 * 1024) + #ifdef NO_SETENV #define setenv gitsetenv extern int gitsetenv(const char *, const char *, int); -- cgit v1.2.1 From 97bfeb34df1aa8a1cf232278624a5a5c924ee380 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sun, 24 Dec 2006 00:47:19 -0500 Subject: Release pack windows before reporting out of memory. If we are about to fail because this process has run out of memory we should first try to automatically control our appetite for address space by releasing enough least-recently-used pack windows to gain back enough memory such that we might actually be able to meet the current allocation request. This should help users who have fairly large repositories but are working on systems with relatively small virtual address space. Many times we see reports on the mailing list of these users running out of memory during various Git operations. Dynamically decreasing the amount of pack memory used when the demand for heap memory is increasing is an intelligent solution to this problem. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- git-compat-util.h | 40 ++++++++++++++++++++++++++++++++-------- sha1_file.c | 7 +++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/git-compat-util.h b/git-compat-util.h index 4764087d8..0f856747e 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -123,11 +123,17 @@ extern char *gitstrcasestr(const char *haystack, const char *needle); extern size_t gitstrlcpy(char *, const char *, size_t); #endif +extern void release_pack_memory(size_t); + static inline char* xstrdup(const char *str) { char *ret = strdup(str); - if (!ret) - die("Out of memory, strdup failed"); + if (!ret) { + release_pack_memory(strlen(str) + 1); + ret = strdup(str); + if (!ret) + die("Out of memory, strdup failed"); + } return ret; } @@ -136,8 +142,14 @@ static inline void *xmalloc(size_t size) void *ret = malloc(size); if (!ret && !size) ret = malloc(1); - if (!ret) - die("Out of memory, malloc failed"); + if (!ret) { + release_pack_memory(size); + ret = malloc(size); + if (!ret && !size) + ret = malloc(1); + if (!ret) + die("Out of memory, malloc failed"); + } #ifdef XMALLOC_POISON memset(ret, 0xA5, size); #endif @@ -149,8 +161,14 @@ static inline void *xrealloc(void *ptr, size_t size) void *ret = realloc(ptr, size); if (!ret && !size) ret = realloc(ptr, 1); - if (!ret) - die("Out of memory, realloc failed"); + if (!ret) { + release_pack_memory(size); + ret = realloc(ptr, size); + if (!ret && !size) + ret = realloc(ptr, 1); + if (!ret) + die("Out of memory, realloc failed"); + } return ret; } @@ -159,8 +177,14 @@ static inline void *xcalloc(size_t nmemb, size_t size) void *ret = calloc(nmemb, size); if (!ret && (!nmemb || !size)) ret = calloc(1, 1); - if (!ret) - die("Out of memory, calloc failed"); + if (!ret) { + release_pack_memory(nmemb * size); + ret = calloc(nmemb, size); + if (!ret && (!nmemb || !size)) + ret = calloc(1, 1); + if (!ret) + die("Out of memory, calloc failed"); + } return ret; } diff --git a/sha1_file.c b/sha1_file.c index 8de8ce0a7..fb1032b0f 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -522,6 +522,13 @@ static int unuse_one_window(struct packed_git *current) return 0; } +void release_pack_memory(size_t need) +{ + size_t cur = pack_mapped; + while (need >= (cur - pack_mapped) && unuse_one_window(NULL)) + ; /* nothing */ +} + void unuse_pack(struct pack_window **w_cursor) { struct pack_window *w = *w_cursor; -- cgit v1.2.1 From c4712e4553f13d87d655325a57538f299402d457 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sun, 24 Dec 2006 00:47:23 -0500 Subject: Replace mmap with xmmap, better handling MAP_FAILED. In some cases we did not even bother to check the return value of mmap() and just assume it worked. This is bad, because if we are out of virtual address space the kernel returned MAP_FAILED and we would attempt to dereference that address, segfaulting without any real error output to the user. We are replacing all calls to mmap() with xmmap() and moving all MAP_FAILED checking into that single location. If a mmap call fails we try to release enough least-recently-used pack windows to possibly succeed, then retry the mmap() attempt. If we cannot mmap even after releasing pack memory then we die() as none of our callers have any reasonable recovery strategy for a failed mmap. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- config.c | 2 +- diff.c | 4 +--- git-compat-util.h | 13 +++++++++++++ read-cache.c | 2 +- refs.c | 2 +- sha1_file.c | 18 +++++------------- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/config.c b/config.c index 3b8da569f..2e0d5a868 100644 --- a/config.c +++ b/config.c @@ -704,7 +704,7 @@ int git_config_set_multivar(const char* key, const char* value, } fstat(in_fd, &st); - contents = mmap(NULL, st.st_size, PROT_READ, + contents = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in_fd, 0); close(in_fd); diff --git a/diff.c b/diff.c index f14288bb8..244292a70 100644 --- a/diff.c +++ b/diff.c @@ -1341,10 +1341,8 @@ int diff_populate_filespec(struct diff_filespec *s, int size_only) fd = open(s->path, O_RDONLY); if (fd < 0) goto err_empty; - s->data = mmap(NULL, s->size, PROT_READ, MAP_PRIVATE, fd, 0); + s->data = xmmap(NULL, s->size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (s->data == MAP_FAILED) - goto err_empty; s->should_munmap = 1; } else { diff --git a/git-compat-util.h b/git-compat-util.h index 0f856747e..f243b86d3 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -188,6 +188,19 @@ static inline void *xcalloc(size_t nmemb, size_t size) return ret; } +static inline void *xmmap(void *start, size_t length, + int prot, int flags, int fd, off_t offset) +{ + void *ret = mmap(start, length, prot, flags, fd, offset); + if (ret == MAP_FAILED) { + release_pack_memory(length); + ret = mmap(start, length, prot, flags, fd, offset); + if (ret == MAP_FAILED) + die("Out of memory? mmap failed: %s", strerror(errno)); + } + return ret; +} + static inline ssize_t xread(int fd, void *buf, size_t len) { ssize_t nr; diff --git a/read-cache.c b/read-cache.c index b8d83ccd9..ca3efbbf0 100644 --- a/read-cache.c +++ b/read-cache.c @@ -798,7 +798,7 @@ int read_cache_from(const char *path) cache_mmap_size = st.st_size; errno = EINVAL; if (cache_mmap_size >= sizeof(struct cache_header) + 20) - cache_mmap = mmap(NULL, cache_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + cache_mmap = xmmap(NULL, cache_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); } close(fd); if (cache_mmap == MAP_FAILED) diff --git a/refs.c b/refs.c index e88ed8b2d..121774cb3 100644 --- a/refs.c +++ b/refs.c @@ -1026,7 +1026,7 @@ int read_ref_at(const char *ref, unsigned long at_time, int cnt, unsigned char * fstat(logfd, &st); if (!st.st_size) die("Log %s is empty.", logfile); - logdata = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, logfd, 0); + logdata = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, logfd, 0); close(logfd); lastrec = NULL; diff --git a/sha1_file.c b/sha1_file.c index fb1032b0f..84037fe98 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -355,10 +355,8 @@ static void read_info_alternates(const char * relative_base, int depth) close(fd); return; } - map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + map = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (map == MAP_FAILED) - return; link_alt_odb_entries(map, map + st.st_size, '\n', relative_base, depth); @@ -442,10 +440,8 @@ static int check_packed_git_idx(const char *path, unsigned long *idx_size_, return -1; } idx_size = st.st_size; - idx_map = mmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0); + idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (idx_map == MAP_FAILED) - return -1; index = idx_map; *idx_map_ = idx_map; @@ -630,7 +626,7 @@ unsigned char* use_pack(struct packed_git *p, while (packed_git_limit < pack_mapped && unuse_one_window(p)) ; /* nothing */ - win->base = mmap(NULL, win->len, + win->base = xmmap(NULL, win->len, PROT_READ, MAP_PRIVATE, p->pack_fd, win->offset); if (win->base == MAP_FAILED) @@ -828,10 +824,8 @@ void *map_sha1_file(const unsigned char *sha1, unsigned long *size) */ sha1_file_open_flag = 0; } - map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + map = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (map == MAP_FAILED) - return NULL; *size = st.st_size; return map; } @@ -1987,10 +1981,8 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, con buf = ""; if (size) - buf = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); + buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (buf == MAP_FAILED) - return -1; if (!type) type = blob_type; -- cgit v1.2.1 From 5fe5c8300da67a07bd430a33c474ef1498cb1d4b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 25 Dec 2006 23:40:58 -0500 Subject: Cleanup read_cache_from error handling. When I converted the mmap() call to xmmap() I failed to cleanup the way this routine handles errors and left some crufty code behind. This is a small cleanup, suggested by Johannes. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- read-cache.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/read-cache.c b/read-cache.c index ca3efbbf0..29cf9abe6 100644 --- a/read-cache.c +++ b/read-cache.c @@ -793,16 +793,16 @@ int read_cache_from(const char *path) die("index file open failed (%s)", strerror(errno)); } - cache_mmap = MAP_FAILED; if (!fstat(fd, &st)) { cache_mmap_size = st.st_size; errno = EINVAL; if (cache_mmap_size >= sizeof(struct cache_header) + 20) cache_mmap = xmmap(NULL, cache_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - } + else + die("index file smaller than expected"); + } else + die("cannot stat the open index (%s)", strerror(errno)); close(fd); - if (cache_mmap == MAP_FAILED) - die("index file mmap failed (%s)", strerror(errno)); hdr = cache_mmap; if (verify_hdr(hdr, cache_mmap_size) < 0) -- cgit v1.2.1 From f5b1b5a07e2d715c5ee7c098e95bd3dbc8ee745d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 27 Dec 2006 02:46:23 -0500 Subject: Fix random segfaults in pack-objects. Junio noticed that 'non-trivial' pushes were failing if executed using the sliding window mmap changes. This was somewhat difficult to track down as the failure was appearing randomly. It turns out this was a failure caused by the delta base reference (either ref or offset format) spanning over the end of a mmap window. The error in pack-objects was we were not recalling use_pack after the object header was unpacked, and therefore we did not get the promise of at least 20 bytes in the buffer for the delta base parsing. This would case later memcmp() calls to walk into unassigned address space at the end of the window. The reason Junio and I had hard time tracking this down in current Git repositories is we were both probably packing with offset deltas, which minimized the odds of the delta base reference spanning over the end of the mmap window. Stepping back and repacking with version 1.3.3 (which only supported reference deltas) increased the likelyhood of seeing the bug. The correct technique (as used in sha1_file.c) is to invoke use_pack() after unpack_object_header_gently to ensure we have enough data available for the delta base decoding. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index afb926a34..e9a1804fa 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -995,8 +995,6 @@ static void check_object(struct object_entry *entry) */ used = unpack_object_header_gently(buf, left, &entry->in_pack_type, &size); - if (!used || left - used <= 20) - die("corrupt pack for %s", sha1_to_hex(entry->sha1)); /* Check if it is delta, and the base is also an object * we are going to pack. If so we will reuse the existing @@ -1008,10 +1006,13 @@ static void check_object(struct object_entry *entry) /* there is at least 20 bytes left in the pack */ switch (entry->in_pack_type) { case OBJ_REF_DELTA: - base_name = buf + used; + base_name = use_pack(p, &w_curs, + entry->in_pack_offset + used, NULL); used += 20; break; case OBJ_OFS_DELTA: + buf = use_pack(p, &w_curs, + entry->in_pack_offset + used, NULL); c = buf[used++]; ofs = c & 127; while (c & 128) { -- cgit v1.2.1 From 9c18df19073bad62e16d6b6b8e1939fd15424612 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 28 Dec 2006 22:29:06 -0800 Subject: pack-objects: fix use of use_pack(). The code calls use_pack() to make that the variably encoded offset fits in the mmap'ed window, but it forgot that the operation gives the pointer to the beginning of the asked region. Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index e9a1804fa..42dd8c87a 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -1003,6 +1003,7 @@ static void check_object(struct object_entry *entry) if (!no_reuse_delta) { unsigned char c, *base_name; unsigned long ofs; + unsigned long used_0; /* there is at least 20 bytes left in the pack */ switch (entry->in_pack_type) { case OBJ_REF_DELTA: @@ -1013,14 +1014,15 @@ static void check_object(struct object_entry *entry) case OBJ_OFS_DELTA: buf = use_pack(p, &w_curs, entry->in_pack_offset + used, NULL); - c = buf[used++]; + used_0 = 0; + c = buf[used_0++]; ofs = c & 127; while (c & 128) { ofs += 1; if (!ofs || ofs & ~(~0UL >> 7)) die("delta base offset overflow in pack for %s", sha1_to_hex(entry->sha1)); - c = buf[used++]; + c = buf[used_0++]; ofs = (ofs << 7) + (c & 127); } if (ofs >= entry->in_pack_offset) @@ -1028,6 +1030,7 @@ static void check_object(struct object_entry *entry) sha1_to_hex(entry->sha1)); ofs = entry->in_pack_offset - ofs; base_name = find_packed_object_name(p, ofs); + used += used_0; break; default: base_name = NULL; -- cgit v1.2.1 From 2c039da804ee0542ff41d2f22a444d04a2d37856 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 29 Dec 2006 00:30:01 -0800 Subject: mmap: set FD_CLOEXEC for file descriptors we keep open for mmap() I do not have any proof that this matters to any existing problems I am seeing, but I do not think of any reason not to do this. Signed-off-by: Junio C Hamano --- sha1_file.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sha1_file.c b/sha1_file.c index 84037fe98..d9622d95e 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -540,6 +540,7 @@ static void open_packed_git(struct packed_git *p) struct pack_header hdr; unsigned char sha1[20]; unsigned char *idx_sha1; + long fd_flag; p->pack_fd = open(p->pack_name, O_RDONLY); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) @@ -553,6 +554,16 @@ static void open_packed_git(struct packed_git *p) } else if (p->pack_size != st.st_size) die("packfile %s size changed", p->pack_name); + /* We leave these file descriptors open with sliding mmap; + * there is no point keeping them open across exec(), though. + */ + fd_flag = fcntl(p->pack_fd, F_GETFD, 0); + if (fd_flag < 0) + die("cannot determine file descriptor flags"); + fd_flag |= FD_CLOEXEC; + if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) + die("cannot set FD_CLOEXEC"); + /* Verify we recognize this pack file format. */ read_or_die(p->pack_fd, &hdr, sizeof(hdr)); if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) -- cgit v1.2.1 From eb92242f19e58de9c930220caf6bf1b83df54160 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 30 Dec 2006 22:13:43 -0500 Subject: Update packedGit config option documentation. Corrected minor typos and documented the new k/m/g suffix for core.packedGitWindowSize and core.packedGitLimit. [jc: with a minor markup fix.] Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 744484b98..6c8382901 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -123,11 +123,13 @@ core.packedGitWindowSize:: single mapping operation. Larger window sizes may allow your system to process a smaller number of large pack files more quickly. Smaller window sizes will negatively affect - performance due to increased calls to the opreating system's + performance due to increased calls to the operating system's memory manager, but may improve performance when accessing a large number of large pack files. Default is 32 MiB, which should be reasonable for all users/operating systems. You probably do not need to adjust this value. ++ +Common unit suffixes of 'k', 'm', or 'g' are supported. core.packedGitLimit:: Maximum number of bytes to map simultaneously into memory @@ -135,8 +137,10 @@ core.packedGitLimit:: bytes at once to complete an operation it will unmap existing regions to reclaim virtual address space within the process. Default is 256 MiB, which should be reasonable for all - users/operating systems, except on largest Git projects. + users/operating systems, except on the largest projects. You probably do not need to adjust this value. ++ +Common unit suffixes of 'k', 'm', or 'g' are supported. alias.*:: Command aliases for the gitlink:git[1] command wrapper - e.g. -- cgit v1.2.1 From e54eef9e1f8adb055c7bbe0e4f50a3716f611dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20R=C3=BChle?= Date: Tue, 2 Jan 2007 20:26:20 +0100 Subject: Clarify syntax and role of git-add in status output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This uses the actual (simplified) synopsis line from the git-add man page and advertises its incremental nature. Signed-off-by: JΓΌrgen RΓΌhle Signed-off-by: Junio C Hamano --- wt-status.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wt-status.c b/wt-status.c index db427384f..34be91b77 100644 --- a/wt-status.c +++ b/wt-status.c @@ -15,7 +15,7 @@ static char wt_status_colors[][COLOR_MAXLEN] = { "\033[31m", /* WT_STATUS_CHANGED: red */ "\033[31m", /* WT_STATUS_UNTRACKED: red */ }; -static const char* use_add_msg = "use \"git add file1 file2\" to include for commit"; +static const char* use_add_msg = "use \"git add ...\" to incrementally add content to commit"; static int parse_status_slot(const char *var, int offset) { -- cgit v1.2.1 From 6e458bf63f48fb7d15cb70ad7c7b7b71915d94a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20R=C3=BChle?= Date: Tue, 2 Jan 2007 20:26:22 +0100 Subject: Improve "nothing to commit" part of status output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously git-status in a clean working directory would advice the user to use git add. This isn't very helpful when there is nothing to add in the working directory, therefore note a clean working directory while displaying the other sections and print the appropriate message for each case. Signed-off-by: JΓΌrgen RΓΌhle Signed-off-by: Junio C Hamano --- wt-status.c | 24 ++++++++++++++++++------ wt-status.h | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/wt-status.c b/wt-status.c index 34be91b77..ca4690e86 100644 --- a/wt-status.c +++ b/wt-status.c @@ -51,6 +51,8 @@ void wt_status_prepare(struct wt_status *s) s->verbose = 0; s->commitable = 0; s->untracked = 0; + + s->workdir_clean = 1; } static void wt_status_print_header(const char *main, const char *sub) @@ -162,9 +164,12 @@ static void wt_status_print_changed_cb(struct diff_queue_struct *q, struct diff_options *options, void *data) { + struct wt_status *s = data; int i; - if (q->nr) + if (q->nr) { + s->workdir_clean = 0; wt_status_print_header("Changed but not added", use_add_msg); + } for (i = 0; i < q->nr; i++) wt_status_print_filepair(WT_STATUS_CHANGED, q->queue[i]); if (q->nr) @@ -215,7 +220,7 @@ static void wt_status_print_changed(struct wt_status *s) run_diff_files(&rev, 0); } -static void wt_status_print_untracked(const struct wt_status *s) +static void wt_status_print_untracked(struct wt_status *s) { struct dir_struct dir; const char *x; @@ -250,6 +255,7 @@ static void wt_status_print_untracked(const struct wt_status *s) continue; } if (!shown_header) { + s->workdir_clean = 0; wt_status_print_header("Untracked files", use_add_msg); shown_header = 1; } @@ -291,10 +297,16 @@ void wt_status_print(struct wt_status *s) if (s->verbose && !s->is_initial) wt_status_print_verbose(s); - if (!s->commitable) - printf("%s (%s)\n", - s->amend ? "# No changes" : "nothing to commit", - use_add_msg); + if (!s->commitable) { + if (s->amend) + printf("# No changes\n"); + else if (s->workdir_clean) + printf(s->is_initial + ? "nothing to commit\n" + : "nothing to commit (working directory matches HEAD)\n"); + else + printf("no changes added to commit (use \"git add\" and/or \"git commit [-a|-i|-o]\")\n"); + } } int git_status_config(const char *k, const char *v) diff --git a/wt-status.h b/wt-status.h index 0a5a5b7ba..892a86c76 100644 --- a/wt-status.h +++ b/wt-status.h @@ -16,6 +16,7 @@ struct wt_status { int verbose; int amend; int untracked; + int workdir_clean; }; int git_status_config(const char *var, const char *value); -- cgit v1.2.1 From 98bf8a47c296f51ea9722fef4bb81dbfb70cd4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20R=C3=BChle?= Date: Tue, 2 Jan 2007 20:26:23 +0100 Subject: Support --amend on initial commit in status output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We check the existence of the parent commit to determine whether the status is requested for an initial commit. Since the parent commit depends on the presence of the --amend switch do initial commit detection after command line arguments have been handled. Signed-off-by: JΓΌrgen RΓΌhle Signed-off-by: Junio C Hamano --- wt-status.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wt-status.c b/wt-status.c index ca4690e86..9b777a569 100644 --- a/wt-status.c +++ b/wt-status.c @@ -41,8 +41,6 @@ void wt_status_prepare(struct wt_status *s) unsigned char sha1[20]; const char *head; - s->is_initial = get_sha1("HEAD", sha1) ? 1 : 0; - head = resolve_ref("HEAD", sha1, 0, NULL); s->branch = head ? xstrdup(head) : NULL; @@ -277,6 +275,9 @@ static void wt_status_print_verbose(struct wt_status *s) void wt_status_print(struct wt_status *s) { + unsigned char sha1[20]; + s->is_initial = get_sha1(s->reference, sha1) ? 1 : 0; + if (s->branch) color_printf_ln(color(WT_STATUS_HEADER), "# On branch %s", s->branch); -- cgit v1.2.1 From 3c1eb9cb2d57769f770ddc36fd6b49706608ebb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20R=C3=BChle?= Date: Tue, 2 Jan 2007 20:26:21 +0100 Subject: Improve cached content header of status output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This tries to be more to the point while also including a pointer on how to unstage changes from the index. Since this header is printed in two different code paths and the name of the reference commit is needed for the unstage part, provide a new printing function. Signed-off-by: JΓΌrgen RΓΌhle Signed-off-by: Junio C Hamano --- wt-status.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/wt-status.c b/wt-status.c index 9b777a569..8aac52644 100644 --- a/wt-status.c +++ b/wt-status.c @@ -53,6 +53,18 @@ void wt_status_prepare(struct wt_status *s) s->workdir_clean = 1; } +static void wt_status_print_cached_header(const char *reference) +{ + const char *c = color(WT_STATUS_HEADER); + color_printf_ln(c, "# Cached changes to be committed:"); + if (reference) { + color_printf_ln(c, "# (use \"git reset %s ...\" and \"git rm --cached ...\" to unstage)", reference); + } else { + color_printf_ln(c, "# (use \"git rm --cached ...\" to unstage)"); + } + color_printf_ln(c, "#"); +} + static void wt_status_print_header(const char *main, const char *sub) { const char *c = color(WT_STATUS_HEADER); @@ -147,8 +159,7 @@ static void wt_status_print_updated_cb(struct diff_queue_struct *q, if (q->queue[i]->status == 'U') continue; if (!shown_header) { - wt_status_print_header("Added but not yet committed", - "will commit"); + wt_status_print_cached_header(s->reference); s->commitable = 1; shown_header = 1; } @@ -182,8 +193,7 @@ void wt_status_print_initial(struct wt_status *s) read_cache(); if (active_nr) { s->commitable = 1; - wt_status_print_header("Added but not yet committed", - "will commit"); + wt_status_print_cached_header(NULL); } for (i = 0; i < active_nr; i++) { color_printf(color(WT_STATUS_HEADER), "#\t"); -- cgit v1.2.1 From e194cd1e0e08611462eb9c5a731a7a3d797f9252 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 3 Jan 2007 12:13:04 -0800 Subject: git-remote It might be handy to have a single command that helps you manage your configuration that relates to downloading from remote repositories. This currently does only about 20% of what I want it to do. $ git remote shows the list of 'remotes' you have defined somewhere, and $ git remote origin shows the details about the named remote (in this case "origin"). How the branches are tracked, if you have a tracking branch that is stale, etc. $ git add another git://git.kernel.org/pub/... defines the default remote.another.url and remote.another.fetch entries just like a clone does; you can say "git fetch another" afterwards. For it to be useful, I think it should be enhanced to: - check overlaps of tracking branches and warn; - offer to remove stale tracking branches in one go; - offer ways to remove or rename remote; - offer ways to update an existing remote, perhaps have an interactive mode; Other enhancements might be also possible, but I do not think of anything that is absolutely necessary other than the above right now. Signed-off-by: Junio C Hamano --- Makefile | 2 +- git-remote.perl | 277 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+), 1 deletion(-) create mode 100755 git-remote.perl diff --git a/Makefile b/Makefile index 180e1e0b9..5c712b94c 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ SCRIPT_SH = \ SCRIPT_PERL = \ git-add--interactive.perl \ git-archimport.perl git-cvsimport.perl git-relink.perl \ - git-cvsserver.perl \ + git-cvsserver.perl git-remote.perl \ git-svnimport.perl git-cvsexportcommit.perl \ git-send-email.perl git-svn.perl diff --git a/git-remote.perl b/git-remote.perl new file mode 100755 index 000000000..059c141b6 --- /dev/null +++ b/git-remote.perl @@ -0,0 +1,277 @@ +#!/usr/bin/perl -w + +use Git; +my $git = Git->repository(); + +sub add_remote_config { + my ($hash, $name, $what, $value) = @_; + if ($what eq 'url') { + if (exists $hash->{$name}{'URL'}) { + print STDERR "Warning: more than one remote.$name.url\n"; + } + $hash->{$name}{'URL'} = $value; + } + elsif ($what eq 'fetch') { + $hash->{$name}{'FETCH'} ||= []; + push @{$hash->{$name}{'FETCH'}}, $value; + } + if (!exists $hash->{$name}{'SOURCE'}) { + $hash->{$name}{'SOURCE'} = 'config'; + } +} + +sub add_remote_remotes { + my ($hash, $file, $name) = @_; + + if (exists $hash->{$name}) { + $hash->{$name}{'WARNING'} = 'ignored due to config'; + return; + } + + my $fh; + if (!open($fh, '<', $file)) { + print STDERR "Warning: cannot open $file\n"; + return; + } + my $it = { 'SOURCE' => 'remotes' }; + $hash->{$name} = $it; + while (<$fh>) { + chomp; + if (/^URL:\s*(.*)$/) { + # Having more than one is Ok -- it is used for push. + if (! exists $it->{'URL'}) { + $it->{'URL'} = $1; + } + } + elsif (/^Push:\s*(.*)$/) { + ; # later + } + elsif (/^Pull:\s*(.*)$/) { + $it->{'FETCH'} ||= []; + push @{$it->{'FETCH'}}, $1; + } + elsif (/^\#/) { + ; # ignore + } + else { + print STDERR "Warning: funny line in $file: $_\n"; + } + } + close($fh); +} + +sub list_remote { + my ($git) = @_; + my %seen = (); + my @remotes = eval { + $git->command(qw(repo-config --get-regexp), '^remote\.'); + }; + for (@remotes) { + if (/^remote\.([^.]*)\.(\S*)\s+(.*)$/) { + add_remote_config(\%seen, $1, $2, $3); + } + } + + my $dir = $git->repo_path() . "/remotes"; + if (opendir(my $dh, $dir)) { + local $_; + while ($_ = readdir($dh)) { + chomp; + next if (! -f "$dir/$_" || ! -r _); + add_remote_remotes(\%seen, "$dir/$_", $_); + } + } + + return \%seen; +} + +sub add_branch_config { + my ($hash, $name, $what, $value) = @_; + if ($what eq 'remote') { + if (exists $hash->{$name}{'REMOTE'}) { + print STDERR "Warning: more than one branch.$name.remote\n"; + } + $hash->{$name}{'REMOTE'} = $value; + } + elsif ($what eq 'merge') { + $hash->{$name}{'MERGE'} ||= []; + push @{$hash->{$name}{'MERGE'}}, $value; + } +} + +sub list_branch { + my ($git) = @_; + my %seen = (); + my @branches = eval { + $git->command(qw(repo-config --get-regexp), '^branch\.'); + }; + for (@branches) { + if (/^branch\.([^.]*)\.(\S*)\s+(.*)$/) { + add_branch_config(\%seen, $1, $2, $3); + } + } + + return \%seen; +} + +my $remote = list_remote($git); +my $branch = list_branch($git); + +sub update_ls_remote { + my ($harder, $info) = @_; + + return if (($harder == 0) || + (($harder == 1) && exists $info->{'LS_REMOTE'})); + + my @ref = map { + s|^[0-9a-f]{40}\s+refs/heads/||; + $_; + } $git->command(qw(ls-remote --heads), $info->{'URL'}); + $info->{'LS_REMOTE'} = \@ref; +} + +sub show_wildcard_mapping { + my ($forced, $ours, $ls) = @_; + my %refs; + for (@$ls) { + $refs{$_} = 01; # bit #0 to say "they have" + } + for ($git->command('for-each-ref', "refs/remotes/$ours")) { + chomp; + next unless (s|^[0-9a-f]{40}\s[a-z]+\srefs/remotes/$ours/||); + next if ($_ eq 'HEAD'); + $refs{$_} ||= 0; + $refs{$_} |= 02; # bit #1 to say "we have" + } + my (@new, @stale, @tracked); + for (sort keys %refs) { + my $have = $refs{$_}; + if ($have == 1) { + push @new, $_; + } + elsif ($have == 2) { + push @stale, $_; + } + elsif ($have == 3) { + push @tracked, $_; + } + } + if (@new) { + print " New remote branches (next fetch will store in remotes/$ours)\n"; + print " @new\n"; + } + if (@stale) { + print " Stale tracking branches in remotes/$ours (you'd better remove them)\n"; + print " @stale\n"; + } + if (@tracked) { + print " Tracked remote branches\n"; + print " @tracked\n"; + } +} + +sub show_mapping { + my ($name, $info) = @_; + my $fetch = $info->{'FETCH'}; + my $ls = $info->{'LS_REMOTE'}; + my (@stale, @tracked); + + for (@$fetch) { + next unless (/(\+)?([^:]+):(.*)/); + my ($forced, $theirs, $ours) = ($1, $2, $3); + if ($theirs eq 'refs/heads/*' && + $ours =~ /^refs\/remotes\/(.*)\/\*$/) { + # wildcard mapping + show_wildcard_mapping($forced, $1, $ls); + } + elsif ($theirs =~ /\*/ || $ours =~ /\*/) { + print STDERR "Warning: unrecognized mapping in remotes.$name.fetch: $_\n"; + } + elsif ($theirs =~ s|^refs/heads/||) { + if (!grep { $_ eq $theirs } @$ls) { + push @stale, $theirs; + } + elsif ($ours ne '') { + push @tracked, $theirs; + } + } + } + if (@stale) { + print " Stale tracking branches in remotes/$name (you'd better remove them)\n"; + print " @stale\n"; + } + if (@tracked) { + print " Tracked remote branches\n"; + print " @tracked\n"; + } +} + +sub show_remote { + my ($name, $ls_remote) = @_; + if (!exists $remote->{$name}) { + print STDERR "No such remote $name\n"; + return; + } + my $info = $remote->{$name}; + update_ls_remote($ls_remote, $info); + + print "* remote $name\n"; + print " URL: $info->{'URL'}\n"; + for my $branchname (sort keys %$branch) { + next if ($branch->{$branchname}{'REMOTE'} ne $name); + my @merged = map { + s|^refs/heads/||; + $_; + } split(' ',"@{$branch->{$branchname}{'MERGE'}}"); + next unless (@merged); + print " Remote branch(es) merged with 'git pull' while on branch $branchname\n"; + print " @merged\n"; + } + if ($info->{'LS_REMOTE'}) { + show_mapping($name, $info); + } +} + +sub add_remote { + my ($name, $url) = @_; + if (exists $remote->{$name}) { + print STDERR "remote $name already exists.\n"; + exit(1); + } + $git->command('repo-config', "remote.$name.url", $url); + $git->command('repo-config', "remote.$name.fetch", + "+refs/heads/*:refs/remotes/$name/*"); +} + +if (!@ARGV) { + for (sort keys %$remote) { + print "$_\n"; + } +} +elsif ($ARGV[0] eq 'show') { + my $ls_remote = 1; + my $i; + for ($i = 1; $i < @ARGV; $i++) { + if ($ARGV[$i] eq '-n') { + $ls_remote = 0; + } + else { + last; + } + } + if ($i >= @ARGV) { + print STDERR "Usage: git remote show \n"; + exit(1); + } + for (; $i < @ARGV; $i++) { + show_remote($ARGV[$i], $ls_remote); + } +} +elsif ($ARGV[0] eq 'add') { + if (@ARGV != 3) { + print STDERR "Usage: git remote add \n"; + exit(1); + } + add_remote($ARGV[1], $ARGV[2]); +} + -- cgit v1.2.1 From 22bac0ea528fd419cb833cab5de79a36fad91524 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 4 Jan 2007 22:28:08 -0500 Subject: Increase packedGit{Limit,WindowSize} on 64 bit systems. If we have a 64 bit address space we can easily afford to commit a larger amount of virtual address space to pack file access. So on these platforms we should increase the default settings of core.packedGit{Limit,WindowSize} to something that will better handle very large projects. Thanks to Andy Whitcroft for pointing out that we can safely increase these defaults on such systems. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 16 ++++++++++------ git-compat-util.h | 10 ++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 6c8382901..b24d9dff6 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -125,9 +125,12 @@ core.packedGitWindowSize:: more quickly. Smaller window sizes will negatively affect performance due to increased calls to the operating system's memory manager, but may improve performance when accessing - a large number of large pack files. Default is 32 MiB, - which should be reasonable for all users/operating systems. - You probably do not need to adjust this value. + a large number of large pack files. ++ +Default is 1 MiB if NO_MMAP was set at compile time, otherwise 32 +MiB on 32 bit platforms and 1 GiB on 64 bit platforms. This should +be reasonable for all users/operating systems. You probably do +not need to adjust this value. + Common unit suffixes of 'k', 'm', or 'g' are supported. @@ -136,9 +139,10 @@ core.packedGitLimit:: from pack files. If Git needs to access more than this many bytes at once to complete an operation it will unmap existing regions to reclaim virtual address space within the process. - Default is 256 MiB, which should be reasonable for all - users/operating systems, except on the largest projects. - You probably do not need to adjust this value. ++ +Default is 256 MiB on 32 bit platforms and 8 GiB on 64 bit platforms. +This should be reasonable for all users/operating systems, except on +the largest projects. You probably do not need to adjust this value. + Common unit suffixes of 'k', 'm', or 'g' are supported. diff --git a/git-compat-util.h b/git-compat-util.h index f243b86d3..55456da37 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -97,11 +97,17 @@ extern int git_munmap(void *start, size_t length); #else /* NO_MMAP */ #include -#define DEFAULT_PACKED_GIT_WINDOW_SIZE (32 * 1024 * 1024) +#define DEFAULT_PACKED_GIT_WINDOW_SIZE \ + (sizeof(void*) >= 8 \ + ? 1 * 1024 * 1024 * 1024 \ + : 32 * 1024 * 1024) #endif /* NO_MMAP */ -#define DEFAULT_PACKED_GIT_LIMIT (256 * 1024 * 1024) +#define DEFAULT_PACKED_GIT_LIMIT \ + (sizeof(void*) >= 8 \ + ? 8 * 1024 * 1024 * 1024 \ + : 256 * 1024 * 1024) #ifdef NO_SETENV #define setenv gitsetenv -- cgit v1.2.1 From 24304816141d16aacdc63612797faa1426222ef7 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 6 Jan 2007 02:16:10 -0800 Subject: builtin-prune: make file-scope static struct to an argument. I want to make the first part of 'git prune' that marks the reachable objects callable as a library, so this starts the first step toward the goal by making the callchain to pass rev_info structure as an argument. No functionality change should be in this step. Signed-off-by: Junio C Hamano --- builtin-prune.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/builtin-prune.c b/builtin-prune.c index b469c43bc..95228645d 100644 --- a/builtin-prune.c +++ b/builtin-prune.c @@ -12,7 +12,6 @@ static const char prune_usage[] = "git-prune [-n]"; static int show_only; -static struct rev_info revs; static int prune_object(char *path, const char *filename, const unsigned char *sha1) { @@ -184,45 +183,48 @@ static void walk_commit_list(struct rev_info *revs) static int add_one_reflog_ent(unsigned char *osha1, unsigned char *nsha1, char *datail, void *cb_data) { struct object *object; + struct rev_info *revs = (struct rev_info *)cb_data; object = parse_object(osha1); if (object) - add_pending_object(&revs, object, ""); + add_pending_object(revs, object, ""); object = parse_object(nsha1); if (object) - add_pending_object(&revs, object, ""); + add_pending_object(revs, object, ""); return 0; } static int add_one_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data) { struct object *object = parse_object(sha1); + struct rev_info *revs = (struct rev_info *)cb_data; + if (!object) die("bad object ref: %s:%s", path, sha1_to_hex(sha1)); - add_pending_object(&revs, object, ""); + add_pending_object(revs, object, ""); - for_each_reflog_ent(path, add_one_reflog_ent, NULL); + for_each_reflog_ent(path, add_one_reflog_ent, cb_data); return 0; } -static void add_one_tree(const unsigned char *sha1) +static void add_one_tree(const unsigned char *sha1, struct rev_info *revs) { struct tree *tree = lookup_tree(sha1); - add_pending_object(&revs, &tree->object, ""); + add_pending_object(revs, &tree->object, ""); } -static void add_cache_tree(struct cache_tree *it) +static void add_cache_tree(struct cache_tree *it, struct rev_info *revs) { int i; if (it->entry_count >= 0) - add_one_tree(it->sha1); + add_one_tree(it->sha1, revs); for (i = 0; i < it->subtree_nr; i++) - add_cache_tree(it->down[i]->cache_tree); + add_cache_tree(it->down[i]->cache_tree, revs); } -static void add_cache_refs(void) +static void add_cache_refs(struct rev_info *revs) { int i; @@ -237,12 +239,13 @@ static void add_cache_refs(void) */ } if (active_cache_tree) - add_cache_tree(active_cache_tree); + add_cache_tree(active_cache_tree, revs); } int cmd_prune(int argc, const char **argv, const char *prefix) { int i; + struct rev_info revs; for (i = 1; i < argc; i++) { const char *arg = argv[i]; @@ -264,11 +267,11 @@ int cmd_prune(int argc, const char **argv, const char *prefix) revs.blob_objects = 1; revs.tree_objects = 1; - /* Add all external refs */ - for_each_ref(add_one_ref, NULL); + /* Add all external refs, along with its reflog info */ + for_each_ref(add_one_ref, &revs); /* Add all refs from the index file */ - add_cache_refs(); + add_cache_refs(&revs); /* * Set up the revision walk - this will move all commits -- cgit v1.2.1 From ca4f293fb492efdd2b984b992796b075c30e230d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 6 Jan 2007 02:16:14 -0800 Subject: builtin-prune: separate ref walking from reflog walking. This is necessary for the next step, because the reason I am making the connectivity walker into a library is because I want to use it for cleaning up stale reflog entries. Signed-off-by: Junio C Hamano --- builtin-prune.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/builtin-prune.c b/builtin-prune.c index 95228645d..cd079b419 100644 --- a/builtin-prune.c +++ b/builtin-prune.c @@ -203,8 +203,12 @@ static int add_one_ref(const char *path, const unsigned char *sha1, int flag, vo die("bad object ref: %s:%s", path, sha1_to_hex(sha1)); add_pending_object(revs, object, ""); - for_each_reflog_ent(path, add_one_reflog_ent, cb_data); + return 0; +} +static int add_one_reflog(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + for_each_reflog_ent(path, add_one_reflog_ent, cb_data); return 0; } @@ -267,12 +271,15 @@ int cmd_prune(int argc, const char **argv, const char *prefix) revs.blob_objects = 1; revs.tree_objects = 1; - /* Add all external refs, along with its reflog info */ + /* Add all external refs */ for_each_ref(add_one_ref, &revs); /* Add all refs from the index file */ add_cache_refs(&revs); + /* Add all reflog info from refs */ + for_each_ref(add_one_reflog, &revs); + /* * Set up the revision walk - this will move all commits * from the pending list to the commit walking list. -- cgit v1.2.1 From 94421474e068c2f0a7bef3d658216a0f1e75b906 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 6 Jan 2007 02:16:17 -0800 Subject: Move traversal of reachable objects into a separate library. This moves major part of builtin-prune into a separate file, reachable.c. It is used to mark the objects that are reachable from refs, and optionally from reflogs. The patch looks very large, but if you look at it with diff -C, which this message is formatted in, most of them are copied lines and there are very little additions. Signed-off-by: Junio C Hamano --- Makefile | 1 + builtin-prune.c | 195 +----------------------------------------------------- reachable.c | 199 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ reachable.h | 6 ++ 4 files changed, 208 insertions(+), 193 deletions(-) create mode 100644 reachable.c create mode 100644 reachable.h diff --git a/Makefile b/Makefile index 180e1e0b9..d8bfb6b87 100644 --- a/Makefile +++ b/Makefile @@ -251,6 +251,7 @@ LIB_OBJS = \ interpolate.o \ lockfile.o \ object.o pack-check.o patch-delta.o path.o pkt-line.o sideband.o \ + reachable.o \ quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \ server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \ tag.o tree.o usage.o config.o environment.o ctype.o copy.o \ diff --git a/builtin-prune.c b/builtin-prune.c index cd079b419..6f0ba0d04 100644 --- a/builtin-prune.c +++ b/builtin-prune.c @@ -1,14 +1,9 @@ #include "cache.h" -#include "refs.h" -#include "tag.h" #include "commit.h" -#include "tree.h" -#include "blob.h" -#include "tree-walk.h" #include "diff.h" #include "revision.h" #include "builtin.h" -#include "cache-tree.h" +#include "reachable.h" static const char prune_usage[] = "git-prune [-n]"; static int show_only; @@ -84,168 +79,6 @@ static void prune_object_dir(const char *path) } } -static void process_blob(struct blob *blob, - struct object_array *p, - struct name_path *path, - const char *name) -{ - struct object *obj = &blob->object; - - if (obj->flags & SEEN) - return; - obj->flags |= SEEN; - /* Nothing to do, really .. The blob lookup was the important part */ -} - -static void process_tree(struct tree *tree, - struct object_array *p, - struct name_path *path, - const char *name) -{ - struct object *obj = &tree->object; - struct tree_desc desc; - struct name_entry entry; - struct name_path me; - - if (obj->flags & SEEN) - return; - obj->flags |= SEEN; - if (parse_tree(tree) < 0) - die("bad tree object %s", sha1_to_hex(obj->sha1)); - name = xstrdup(name); - add_object(obj, p, path, name); - me.up = path; - me.elem = name; - me.elem_len = strlen(name); - - desc.buf = tree->buffer; - desc.size = tree->size; - - while (tree_entry(&desc, &entry)) { - if (S_ISDIR(entry.mode)) - process_tree(lookup_tree(entry.sha1), p, &me, entry.path); - else - process_blob(lookup_blob(entry.sha1), p, &me, entry.path); - } - free(tree->buffer); - tree->buffer = NULL; -} - -static void process_tag(struct tag *tag, struct object_array *p, const char *name) -{ - struct object *obj = &tag->object; - struct name_path me; - - if (obj->flags & SEEN) - return; - obj->flags |= SEEN; - - me.up = NULL; - me.elem = "tag:/"; - me.elem_len = 5; - - if (parse_tag(tag) < 0) - die("bad tag object %s", sha1_to_hex(obj->sha1)); - add_object(tag->tagged, p, NULL, name); -} - -static void walk_commit_list(struct rev_info *revs) -{ - int i; - struct commit *commit; - struct object_array objects = { 0, 0, NULL }; - - /* Walk all commits, process their trees */ - while ((commit = get_revision(revs)) != NULL) - process_tree(commit->tree, &objects, NULL, ""); - - /* Then walk all the pending objects, recursively processing them too */ - for (i = 0; i < revs->pending.nr; i++) { - struct object_array_entry *pending = revs->pending.objects + i; - struct object *obj = pending->item; - const char *name = pending->name; - if (obj->type == OBJ_TAG) { - process_tag((struct tag *) obj, &objects, name); - continue; - } - if (obj->type == OBJ_TREE) { - process_tree((struct tree *)obj, &objects, NULL, name); - continue; - } - if (obj->type == OBJ_BLOB) { - process_blob((struct blob *)obj, &objects, NULL, name); - continue; - } - die("unknown pending object %s (%s)", sha1_to_hex(obj->sha1), name); - } -} - -static int add_one_reflog_ent(unsigned char *osha1, unsigned char *nsha1, char *datail, void *cb_data) -{ - struct object *object; - struct rev_info *revs = (struct rev_info *)cb_data; - - object = parse_object(osha1); - if (object) - add_pending_object(revs, object, ""); - object = parse_object(nsha1); - if (object) - add_pending_object(revs, object, ""); - return 0; -} - -static int add_one_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data) -{ - struct object *object = parse_object(sha1); - struct rev_info *revs = (struct rev_info *)cb_data; - - if (!object) - die("bad object ref: %s:%s", path, sha1_to_hex(sha1)); - add_pending_object(revs, object, ""); - - return 0; -} - -static int add_one_reflog(const char *path, const unsigned char *sha1, int flag, void *cb_data) -{ - for_each_reflog_ent(path, add_one_reflog_ent, cb_data); - return 0; -} - -static void add_one_tree(const unsigned char *sha1, struct rev_info *revs) -{ - struct tree *tree = lookup_tree(sha1); - add_pending_object(revs, &tree->object, ""); -} - -static void add_cache_tree(struct cache_tree *it, struct rev_info *revs) -{ - int i; - - if (it->entry_count >= 0) - add_one_tree(it->sha1, revs); - for (i = 0; i < it->subtree_nr; i++) - add_cache_tree(it->down[i]->cache_tree, revs); -} - -static void add_cache_refs(struct rev_info *revs) -{ - int i; - - read_cache(); - for (i = 0; i < active_nr; i++) { - lookup_blob(active_cache[i]->sha1); - /* - * We could add the blobs to the pending list, but quite - * frankly, we don't care. Once we've looked them up, and - * added them as objects, we've really done everything - * there is to do for a blob - */ - } - if (active_cache_tree) - add_cache_tree(active_cache_tree, revs); -} - int cmd_prune(int argc, const char **argv, const char *prefix) { int i; @@ -261,32 +94,8 @@ int cmd_prune(int argc, const char **argv, const char *prefix) } save_commit_buffer = 0; - - /* - * Set up revision parsing, and mark us as being interested - * in all object types, not just commits. - */ init_revisions(&revs, prefix); - revs.tag_objects = 1; - revs.blob_objects = 1; - revs.tree_objects = 1; - - /* Add all external refs */ - for_each_ref(add_one_ref, &revs); - - /* Add all refs from the index file */ - add_cache_refs(&revs); - - /* Add all reflog info from refs */ - for_each_ref(add_one_reflog, &revs); - - /* - * Set up the revision walk - this will move all commits - * from the pending list to the commit walking list. - */ - prepare_revision_walk(&revs); - - walk_commit_list(&revs); + mark_reachable_objects(&revs, 1); prune_object_dir(get_object_directory()); diff --git a/reachable.c b/reachable.c new file mode 100644 index 000000000..4dfee1dbe --- /dev/null +++ b/reachable.c @@ -0,0 +1,199 @@ +#include "cache.h" +#include "refs.h" +#include "tag.h" +#include "commit.h" +#include "blob.h" +#include "diff.h" +#include "revision.h" +#include "reachable.h" +#include "cache-tree.h" + +static void process_blob(struct blob *blob, + struct object_array *p, + struct name_path *path, + const char *name) +{ + struct object *obj = &blob->object; + + if (obj->flags & SEEN) + return; + obj->flags |= SEEN; + /* Nothing to do, really .. The blob lookup was the important part */ +} + +static void process_tree(struct tree *tree, + struct object_array *p, + struct name_path *path, + const char *name) +{ + struct object *obj = &tree->object; + struct tree_desc desc; + struct name_entry entry; + struct name_path me; + + if (obj->flags & SEEN) + return; + obj->flags |= SEEN; + if (parse_tree(tree) < 0) + die("bad tree object %s", sha1_to_hex(obj->sha1)); + name = xstrdup(name); + add_object(obj, p, path, name); + me.up = path; + me.elem = name; + me.elem_len = strlen(name); + + desc.buf = tree->buffer; + desc.size = tree->size; + + while (tree_entry(&desc, &entry)) { + if (S_ISDIR(entry.mode)) + process_tree(lookup_tree(entry.sha1), p, &me, entry.path); + else + process_blob(lookup_blob(entry.sha1), p, &me, entry.path); + } + free(tree->buffer); + tree->buffer = NULL; +} + +static void process_tag(struct tag *tag, struct object_array *p, const char *name) +{ + struct object *obj = &tag->object; + struct name_path me; + + if (obj->flags & SEEN) + return; + obj->flags |= SEEN; + + me.up = NULL; + me.elem = "tag:/"; + me.elem_len = 5; + + if (parse_tag(tag) < 0) + die("bad tag object %s", sha1_to_hex(obj->sha1)); + add_object(tag->tagged, p, NULL, name); +} + +static void walk_commit_list(struct rev_info *revs) +{ + int i; + struct commit *commit; + struct object_array objects = { 0, 0, NULL }; + + /* Walk all commits, process their trees */ + while ((commit = get_revision(revs)) != NULL) + process_tree(commit->tree, &objects, NULL, ""); + + /* Then walk all the pending objects, recursively processing them too */ + for (i = 0; i < revs->pending.nr; i++) { + struct object_array_entry *pending = revs->pending.objects + i; + struct object *obj = pending->item; + const char *name = pending->name; + if (obj->type == OBJ_TAG) { + process_tag((struct tag *) obj, &objects, name); + continue; + } + if (obj->type == OBJ_TREE) { + process_tree((struct tree *)obj, &objects, NULL, name); + continue; + } + if (obj->type == OBJ_BLOB) { + process_blob((struct blob *)obj, &objects, NULL, name); + continue; + } + die("unknown pending object %s (%s)", sha1_to_hex(obj->sha1), name); + } +} + +static int add_one_reflog_ent(unsigned char *osha1, unsigned char *nsha1, char *datail, void *cb_data) +{ + struct object *object; + struct rev_info *revs = (struct rev_info *)cb_data; + + object = parse_object(osha1); + if (object) + add_pending_object(revs, object, ""); + object = parse_object(nsha1); + if (object) + add_pending_object(revs, object, ""); + return 0; +} + +static int add_one_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + struct object *object = parse_object(sha1); + struct rev_info *revs = (struct rev_info *)cb_data; + + if (!object) + die("bad object ref: %s:%s", path, sha1_to_hex(sha1)); + add_pending_object(revs, object, ""); + + return 0; +} + +static int add_one_reflog(const char *path, const unsigned char *sha1, int flag, void *cb_data) +{ + for_each_reflog_ent(path, add_one_reflog_ent, cb_data); + return 0; +} + +static void add_one_tree(const unsigned char *sha1, struct rev_info *revs) +{ + struct tree *tree = lookup_tree(sha1); + add_pending_object(revs, &tree->object, ""); +} + +static void add_cache_tree(struct cache_tree *it, struct rev_info *revs) +{ + int i; + + if (it->entry_count >= 0) + add_one_tree(it->sha1, revs); + for (i = 0; i < it->subtree_nr; i++) + add_cache_tree(it->down[i]->cache_tree, revs); +} + +static void add_cache_refs(struct rev_info *revs) +{ + int i; + + read_cache(); + for (i = 0; i < active_nr; i++) { + lookup_blob(active_cache[i]->sha1); + /* + * We could add the blobs to the pending list, but quite + * frankly, we don't care. Once we've looked them up, and + * added them as objects, we've really done everything + * there is to do for a blob + */ + } + if (active_cache_tree) + add_cache_tree(active_cache_tree, revs); +} + +void mark_reachable_objects(struct rev_info *revs, int mark_reflog) +{ + /* + * Set up revision parsing, and mark us as being interested + * in all object types, not just commits. + */ + revs->tag_objects = 1; + revs->blob_objects = 1; + revs->tree_objects = 1; + + /* Add all refs from the index file */ + add_cache_refs(revs); + + /* Add all external refs */ + for_each_ref(add_one_ref, revs); + + /* Add all reflog info from refs */ + if (mark_reflog) + for_each_ref(add_one_reflog, revs); + + /* + * Set up the revision walk - this will move all commits + * from the pending list to the commit walking list. + */ + prepare_revision_walk(revs); + walk_commit_list(revs); +} diff --git a/reachable.h b/reachable.h new file mode 100644 index 000000000..40751810b --- /dev/null +++ b/reachable.h @@ -0,0 +1,6 @@ +#ifndef REACHEABLE_H +#define REACHEABLE_H + +extern void mark_reachable_objects(struct rev_info *revs, int mark_reflog); + +#endif -- cgit v1.2.1 From 1389d9ddaa68a4cbf5018d88f971b9bbb7aaa3c9 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 6 Jan 2007 02:16:19 -0800 Subject: reflog expire --fix-stale The logic in an earlier round to detect reflog entries that point at a broken commit was not sufficient. Just like we do not trust presense of a commit during pack transfer (we trust only our refs), we should not trust a commit's presense, even if the tree of that commit is complete. A repository that had reflog enabled on some of the refs that was rewound and then run git-repack or git-prune from older versions of git can have reflog entries that point at a commit that still exist but lack commits (or trees and blobs needed for that commit) between it and some commit that is reachable from one of the refs. This revamps the logic -- the definition of "broken commit" becomes: a commit that is not reachable from any of the refs and there is a missing object among the commit, tree, or blob objects reachable from it that is not reachable from any of the refs. Entries in the reflog that refer to such a commit are expired. Since this computation involves traversing all the reachable objects, i.e. it has the same cost as 'git prune', it is enabled only when a new option --fix-stale. Fortunately, once this is run, we should not have to ever worry about missing objects, because the current prune and pack-objects know about reflogs and protect objects referred by them. Unfortunately, this will be absolutely necessary to help people migrate to the newer prune and repack. Signed-off-by: Junio C Hamano --- builtin-reflog.c | 159 +++++++++++++++++++++++++++++++++++++++++------- t/t1410-reflog.sh | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 314 insertions(+), 21 deletions(-) create mode 100755 t/t1410-reflog.sh diff --git a/builtin-reflog.c b/builtin-reflog.c index d3f2f50d2..1da7da091 100644 --- a/builtin-reflog.c +++ b/builtin-reflog.c @@ -4,16 +4,34 @@ #include "refs.h" #include "dir.h" #include "tree-walk.h" +#include "diff.h" +#include "revision.h" +#include "reachable.h" + +/* + * reflog expire + */ + +static const char reflog_expire_usage[] = +"git-reflog expire [--verbose] [--dry-run] [--fix-stale] [--expire=