diff options
| -rw-r--r-- | Documentation/git-index-pack.txt | 3 | ||||
| -rw-r--r-- | builtin/clone.c | 15 | ||||
| -rw-r--r-- | builtin/index-pack.c | 38 | ||||
| -rw-r--r-- | commit.h | 2 | ||||
| -rw-r--r-- | connected.c | 34 | ||||
| -rw-r--r-- | connected.h | 5 | ||||
| -rw-r--r-- | fetch-pack.c | 84 | ||||
| -rw-r--r-- | fetch-pack.h | 4 | ||||
| -rw-r--r-- | git.c | 7 | ||||
| -rw-r--r-- | shallow.c | 42 | ||||
| -rwxr-xr-x | t/t5500-fetch-pack.sh | 7 | ||||
| -rw-r--r-- | transport.c | 4 | ||||
| -rw-r--r-- | transport.h | 2 |
13 files changed, 192 insertions, 55 deletions
diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index bde8eec30..7a4e05552 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -74,6 +74,9 @@ OPTIONS --strict:: Die, if the pack contains broken objects or links. +--check-self-contained-and-connected:: + Die if the pack contains broken links. For internal use only. + --threads=<n>:: Specifies the number of threads to spawn when resolving deltas. This requires that index-pack be compiled with diff --git a/builtin/clone.c b/builtin/clone.c index b6ffc6b4f..66bff5700 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -541,12 +541,18 @@ static void update_remote_refs(const struct ref *refs, const struct ref *mapped_refs, const struct ref *remote_head_points_at, const char *branch_top, - const char *msg) + const char *msg, + struct transport *transport) { const struct ref *rm = mapped_refs; - if (check_everything_connected(iterate_ref_map, 0, &rm)) + if (0 <= option_verbosity) + printf(_("Checking connectivity... ")); + if (check_everything_connected_with_transport(iterate_ref_map, + 0, &rm, transport)) die(_("remote did not send all necessary objects")); + if (0 <= option_verbosity) + printf(_("done\n")); if (refs) { write_remote_refs(mapped_refs); @@ -890,6 +896,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_upload_pack) transport_set_option(transport, TRANS_OPT_UPLOADPACK, option_upload_pack); + + if (transport->smart_options && !option_depth) + transport->smart_options->check_self_contained_and_connected = 1; } refs = transport_get_remote_refs(transport); @@ -951,7 +960,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) transport_fetch_refs(transport, mapped_refs); update_remote_refs(refs, mapped_refs, remote_head_points_at, - branch_top.buf, reflog_msg.buf); + branch_top.buf, reflog_msg.buf, transport); update_head(our_head_points_at, remote_head, reflog_msg.buf); diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 79dfe4732..9c1cfac44 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -77,8 +77,10 @@ static int nr_threads; static int from_stdin; static int strict; +static int do_fsck_object; static int verbose; static int show_stat; +static int check_self_contained_and_connected; static struct progress *progress; @@ -187,13 +189,13 @@ static int mark_link(struct object *obj, int type, void *data) /* The content of each linked object must have been checked or it must be already present in the object database */ -static void check_object(struct object *obj) +static unsigned check_object(struct object *obj) { if (!obj) - return; + return 0; if (!(obj->flags & FLAG_LINK)) - return; + return 0; if (!(obj->flags & FLAG_CHECKED)) { unsigned long size; @@ -201,17 +203,20 @@ static void check_object(struct object *obj) if (type != obj->type || type <= 0) die(_("object of unexpected type")); obj->flags |= FLAG_CHECKED; - return; + return 1; } + + return 0; } -static void check_objects(void) +static unsigned check_objects(void) { - unsigned i, max; + unsigned i, max, foreign_nr = 0; max = get_max_object_index(); for (i = 0; i < max; i++) - check_object(get_indexed_object(i)); + foreign_nr += check_object(get_indexed_object(i)); + return foreign_nr; } @@ -747,8 +752,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, int eaten; void *buf = (void *) data; - if (!buf) - buf = new_data = get_data_from_pack(obj_entry); + assert(data && "data can only be NULL for large _blobs_"); /* * we do not need to free the memory here, as the @@ -757,7 +761,8 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, obj = parse_object_buffer(sha1, type, size, buf, &eaten); if (!obj) die(_("invalid %s"), typename(type)); - if (fsck_object(obj, 1, fsck_error_function)) + if (do_fsck_object && + fsck_object(obj, 1, fsck_error_function)) die(_("Error in object")); if (fsck_walk(obj, mark_link, NULL)) die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1)); @@ -1491,6 +1496,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) struct pack_idx_entry **idx_objects; struct pack_idx_option opts; unsigned char pack_sha1[20]; + unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ if (argc == 2 && !strcmp(argv[1], "-h")) usage(index_pack_usage); @@ -1512,6 +1518,10 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) fix_thin_pack = 1; } else if (!strcmp(arg, "--strict")) { strict = 1; + do_fsck_object = 1; + } else if (!strcmp(arg, "--check-self-contained-and-connected")) { + strict = 1; + check_self_contained_and_connected = 1; } else if (!strcmp(arg, "--verify")) { verify = 1; } else if (!strcmp(arg, "--verify-stat")) { @@ -1625,7 +1635,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) conclude_pack(fix_thin_pack, curr_pack, pack_sha1); free(deltas); if (strict) - check_objects(); + foreign_nr = check_objects(); if (show_stat) show_pack_info(stat_only); @@ -1651,5 +1661,11 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) if (index_name == NULL) free((void *) curr_index); + /* + * Let the caller know this pack is not self contained + */ + if (check_self_contained_and_connected && foreign_nr) + return 1; + return 0; } @@ -176,6 +176,8 @@ extern int for_each_commit_graft(each_commit_graft_fn, void *); extern int is_repository_shallow(void); extern struct commit_list *get_shallow_commits(struct object_array *heads, int depth, int shallow_flag, int not_shallow_flag); +extern void check_shallow_file_for_update(void); +extern void set_alternate_shallow_file(const char *path); int is_descendant_of(struct commit *, struct commit_list *); int in_merge_bases(struct commit *, struct commit *); diff --git a/connected.c b/connected.c index 1e89c1cd1..fae8d64c1 100644 --- a/connected.c +++ b/connected.c @@ -2,7 +2,12 @@ #include "run-command.h" #include "sigchain.h" #include "connected.h" +#include "transport.h" +int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data) +{ + return check_everything_connected_with_transport(fn, quiet, cb_data, NULL); +} /* * If we feed all the commits we want to verify to this command * @@ -14,7 +19,10 @@ * * Returns 0 if everything is connected, non-zero otherwise. */ -int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data) +int check_everything_connected_with_transport(sha1_iterate_fn fn, + int quiet, + void *cb_data, + struct transport *transport) { struct child_process rev_list; const char *argv[] = {"rev-list", "--objects", @@ -22,10 +30,23 @@ int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data) char commit[41]; unsigned char sha1[20]; int err = 0; + struct packed_git *new_pack = NULL; if (fn(cb_data, sha1)) return err; + if (transport && transport->smart_options && + transport->smart_options->self_contained_and_connected && + transport->pack_lockfile && + !suffixcmp(transport->pack_lockfile, ".keep")) { + struct strbuf idx_file = STRBUF_INIT; + strbuf_addstr(&idx_file, transport->pack_lockfile); + strbuf_setlen(&idx_file, idx_file.len - 5); /* ".keep" */ + strbuf_addstr(&idx_file, ".idx"); + new_pack = add_packed_git(idx_file.buf, idx_file.len, 1); + strbuf_release(&idx_file); + } + if (quiet) argv[5] = "--quiet"; @@ -42,6 +63,17 @@ int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data) commit[40] = '\n'; do { + /* + * If index-pack already checked that: + * - there are no dangling pointers in the new pack + * - the pack is self contained + * Then if the updated ref is in the new pack, then we + * are sure the ref is good and not sending it to + * rev-list for verification. + */ + if (new_pack && find_pack_entry_one(sha1, new_pack)) + continue; + memcpy(commit, sha1_to_hex(sha1), 40); if (write_in_full(rev_list.in, commit, 41) < 0) { if (errno != EPIPE && errno != EINVAL) diff --git a/connected.h b/connected.h index 7e4585a6c..0b060b742 100644 --- a/connected.h +++ b/connected.h @@ -1,6 +1,8 @@ #ifndef CONNECTED_H #define CONNECTED_H +struct transport; + /* * Take callback data, and return next object name in the buffer. * When called after returning the name for the last object, return -1 @@ -16,5 +18,8 @@ typedef int (*sha1_iterate_fn)(void *, unsigned char [20]); * Return 0 if Ok, non zero otherwise (i.e. some missing objects) */ extern int check_everything_connected(sha1_iterate_fn, int quiet, void *cb_data); +extern int check_everything_connected_with_transport(sha1_iterate_fn, int quiet, + void *cb_data, + struct transport *transport); #endif /* CONNECTED_H */ diff --git a/fetch-pack.c b/fetch-pack.c index f156dd4fa..abe5ffbba 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -20,6 +20,8 @@ static int no_done; static int fetch_fsck_objects = -1; static int transfer_fsck_objects = -1; static int agent_supported; +static struct lock_file shallow_lock; +static const char *alternate_shallow_file; #define COMPLETE (1U << 0) #define COMMON (1U << 1) @@ -683,12 +685,13 @@ static int get_pack(struct fetch_pack_args *args, int xd[2], char **pack_lockfile) { struct async demux; - const char *argv[20]; + const char *argv[22]; char keep_arg[256]; char hdr_arg[256]; const char **av; int do_keep = args->keep_pack; struct child_process cmd; + int ret; memset(&demux, 0, sizeof(demux)); if (use_sideband) { @@ -724,6 +727,11 @@ static int get_pack(struct fetch_pack_args *args, do_keep = 1; } + if (alternate_shallow_file) { + *av++ = "--shallow-file"; + *av++ = alternate_shallow_file; + } + if (do_keep) { if (pack_lockfile) cmd.out = -1; @@ -740,11 +748,14 @@ static int get_pack(struct fetch_pack_args *args, strcpy(keep_arg + s, "localhost"); *av++ = keep_arg; } + if (args->check_self_contained_and_connected) + *av++ = "--check-self-contained-and-connected"; } else { *av++ = "unpack-objects"; if (args->quiet || args->no_progress) *av++ = "-q"; + args->check_self_contained_and_connected = 0; } if (*hdr_arg) *av++ = hdr_arg; @@ -765,7 +776,12 @@ static int get_pack(struct fetch_pack_args *args, close(cmd.out); } - if (finish_command(&cmd)) + ret = finish_command(&cmd); + if (!ret || (args->check_self_contained_and_connected && ret == 1)) + args->self_contained_and_connected = + args->check_self_contained_and_connected && + ret == 0; + else die("%s failed", argv[0]); if (use_sideband && finish_async(&demux)) die("error in sideband demultiplexer"); @@ -779,6 +795,27 @@ static int cmp_ref_by_name(const void *a_, const void *b_) return strcmp(a->name, b->name); } +static void setup_alternate_shallow(void) +{ + struct strbuf sb = STRBUF_INIT; + int fd; + + check_shallow_file_for_update(); + fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"), + LOCK_DIE_ON_ERROR); + if (write_shallow_commits(&sb, 0)) { + if (write_in_full(fd, sb.buf, sb.len) != sb.len) + die_errno("failed to write to %s", shallow_lock.filename); + alternate_shallow_file = shallow_lock.filename; + } else + /* + * is_repository_shallow() sees empty string as "no + * shallow file". + */ + alternate_shallow_file = ""; + strbuf_release(&sb); +} + static struct ref *do_fetch_pack(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, @@ -858,6 +895,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->stateless_rpc) packet_flush(fd[1]); + if (args->depth > 0) + setup_alternate_shallow(); if (get_pack(args, fd, pack_lockfile)) die("git fetch-pack: fetch failed."); @@ -936,15 +975,9 @@ struct ref *fetch_pack(struct fetch_pack_args *args, struct ref **sought, int nr_sought, char **pack_lockfile) { - struct stat st; struct ref *ref_cpy; fetch_pack_setup(); - if (args->depth > 0) { - if (stat(git_path("shallow"), &st)) - st.st_mtime = 0; - } - if (nr_sought) nr_sought = remove_duplicates_in_refs(sought, nr_sought); @@ -954,35 +987,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args, } ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - if (args->depth > 0) { - static struct lock_file lock; - struct cache_time mtime; - struct strbuf sb = STRBUF_INIT; - char *shallow = git_path("shallow"); - int fd; - - mtime.sec = st.st_mtime; - mtime.nsec = ST_MTIME_NSEC(st); - if (stat(shallow, &st)) { - if (mtime.sec) - die("shallow file was removed during fetch"); - } else if (st.st_mtime != mtime.sec -#ifdef USE_NSEC - || ST_MTIME_NSEC(st) != mtime.nsec -#endif - ) - die("shallow file was changed during fetch"); - - fd = hold_lock_file_for_update(&lock, shallow, - LOCK_DIE_ON_ERROR); - if (!write_shallow_commits(&sb, 0) - || write_in_full(fd, sb.buf, sb.len) != sb.len) { - unlink_or_warn(shallow); - rollback_lock_file(&lock); - } else { - commit_lock_file(&lock); - } - strbuf_release(&sb); + if (alternate_shallow_file) { + if (*alternate_shallow_file == '\0') { /* --unshallow */ + unlink_or_warn(git_path("shallow")); + rollback_lock_file(&shallow_lock); + } else + commit_lock_file(&shallow_lock); } reprepare_packed_git(); diff --git a/fetch-pack.h b/fetch-pack.h index dc5266c97..40f08bab2 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -16,7 +16,9 @@ struct fetch_pack_args { verbose:1, no_progress:1, include_tag:1, - stateless_rpc:1; + stateless_rpc:1, + check_self_contained_and_connected:1, + self_contained_and_connected:1; }; /* @@ -4,6 +4,7 @@ #include "help.h" #include "quote.h" #include "run-command.h" +#include "commit.h" const char git_usage_string[] = "git [--version] [--help] [-c name=value]\n" @@ -146,6 +147,12 @@ static int handle_options(const char ***argv, int *argc, int *envchanged) setenv(GIT_LITERAL_PATHSPECS_ENVIRONMENT, "0", 1); if (envchanged) *envchanged = 1; + } else if (!strcmp(cmd, "--shallow-file")) { + (*argv)++; + (*argc)--; + set_alternate_shallow_file((*argv)[0]); + if (envchanged) + *envchanged = 1; } else { fprintf(stderr, "Unknown option: %s\n", cmd); usage(git_usage_string); @@ -3,6 +3,16 @@ #include "tag.h" static int is_shallow = -1; +static struct stat shallow_stat; +static char *alternate_shallow_file; + +void set_alternate_shallow_file(const char *path) +{ + if (is_shallow != -1) + die("BUG: is_repository_shallow must not be called before set_alternate_shallow_file"); + free(alternate_shallow_file); + alternate_shallow_file = path ? xstrdup(path) : NULL; +} int register_shallow(const unsigned char *sha1) { @@ -21,12 +31,21 @@ int is_repository_shallow(void) { FILE *fp; char buf[1024]; + const char *path = alternate_shallow_file; if (is_shallow >= 0) return is_shallow; - fp = fopen(git_path("shallow"), "r"); - if (!fp) { + if (!path) + path = git_path("shallow"); + /* + * fetch-pack sets '--shallow-file ""' as an indicator that no + * shallow file should be used. We could just open it and it + * will likely fail. But let's do an explicit check instead. + */ + if (!*path || + stat(path, &shallow_stat) || + (fp = fopen(path, "r")) == NULL) { is_shallow = 0; return is_shallow; } @@ -108,3 +127,22 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, return result; } + +void check_shallow_file_for_update(void) +{ + struct stat st; + + if (!is_shallow) + return; + else if (is_shallow == -1) + die("BUG: shallow must be initialized by now"); + + if (stat(git_path("shallow"), &st)) + die("shallow file was removed during fetch"); + else if (st.st_mtime != shallow_stat.st_mtime +#ifdef USE_NSEC + || ST_MTIME_NSEC(st) != ST_MTIME_NSEC(shallow_stat) +#endif + ) + die("shallow file was changed during fetch"); +} diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index 6133d9ed1..fd2598e60 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -135,6 +135,13 @@ test_expect_success 'clone shallow depth 1' ' test "`git --git-dir=shallow0/.git rev-list --count HEAD`" = 1 ' +test_expect_success 'clone shallow depth 1 with fsck' ' + git config --global fetch.fsckobjects true && + git clone --no-single-branch --depth 1 "file://$(pwd)/." shallow0fsck && + test "`git --git-dir=shallow0fsck/.git rev-list --count HEAD`" = 1 && + git config --global --unset fetch.fsckobjects +' + test_expect_success 'clone shallow' ' git clone --no-single-branch --depth 2 "file://$(pwd)/." shallow ' diff --git a/transport.c b/transport.c index ba5d8afb1..359a671c8 100644 --- a/transport.c +++ b/transport.c @@ -534,6 +534,8 @@ static int fetch_refs_via_pack(struct transport *transport, args.quiet = (transport->verbose < 0); args.no_progress = !transport->progress; args.depth = data->options.depth; + args.check_self_contained_and_connected = + data->options.check_self_contained_and_connected; if (!data->got_remote_heads) { connect_setup(transport, 0, 0); @@ -551,6 +553,8 @@ static int fetch_refs_via_pack(struct transport *transport, refs = NULL; data->conn = NULL; data->got_remote_heads = 0; + data->options.self_contained_and_connected = + args.self_contained_and_connected; free_refs(refs_tmp); diff --git a/transport.h b/transport.h index fcb1d25d9..4edebc535 100644 --- a/transport.h +++ b/transport.h @@ -8,6 +8,8 @@ struct git_transport_options { unsigned thin : 1; unsigned keep : 1; unsigned followtags : 1; + unsigned check_self_contained_and_connected : 1; + unsigned self_contained_and_connected : 1; int depth; const char *uploadpack; const char *receivepack; |