From 29e4d3635709778bcc808dbad0477efad82f8d7e Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 20 Dec 2005 00:02:15 -0800 Subject: Racy GIT This fixes the longstanding "Racy GIT" problem, which was pretty much there from the beginning of time, but was first demonstrated by Pasky in this message on October 24, 2005: http://marc.theaimsgroup.com/?l=git&m=113014629716878 If you run the following sequence of commands: echo frotz >infocom git update-index --add infocom echo xyzzy >infocom so that the second update to file "infocom" does not change st_mtime, what is recorded as the stat information for the cache entry "infocom" exactly matches what is on the filesystem (owner, group, inum, mtime, ctime, mode, length). After this sequence, we incorrectly think "infocom" file still has string "frotz" in it, and get really confused. E.g. git-diff-files would say there is no change, git-update-index --refresh would not even look at the filesystem to correct the situation. Some ways of working around this issue were already suggested by Linus in the same thread on the same day, including waiting until the next second before returning from update-index if a cache entry written out has the current timestamp, but that means we can make at most one commit per second, and given that the e-mail patch workflow used by Linus needs to process at least 5 commits per second, it is not an acceptable solution. Linus notes that git-apply is primarily used to update the index while processing e-mailed patches, which is true, and git-apply's up-to-date check is fooled by the same problem but luckily in the other direction, so it is not really a big issue, but still it is disturbing. The function ce_match_stat() is called to bypass the comparison against filesystem data when the stat data recorded in the cache entry matches what stat() returns from the filesystem. This patch tackles the problem by changing it to actually go to the filesystem data for cache entries that have the same mtime as the index file itself. This works as long as the index file and working tree files are on the filesystems that share the same monotonic clock. Files on network mounted filesystems sometimes get skewed timestamps compared to "date" output, but as long as working tree files' timestamps are skewed the same way as the index file's, this approach still works. The only problematic files are the ones that have the same timestamp as the index file's, because two file updates that sandwitch the index file update must happen within the same second to trigger the problem. Signed-off-by: Junio C Hamano --- read-cache.c | 140 ++++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 54 deletions(-) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 693273620..780601f07 100644 --- a/read-cache.c +++ b/read-cache.c @@ -6,6 +6,7 @@ #include "cache.h" struct cache_entry **active_cache = NULL; +static time_t index_file_timestamp; unsigned int active_nr = 0, active_alloc = 0, active_cache_changed = 0; /* @@ -28,6 +29,64 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st) ce->ce_size = htonl(st->st_size); } +static int ce_compare_data(struct cache_entry *ce, struct stat *st) +{ + int match = -1; + int fd = open(ce->name, O_RDONLY); + + if (fd >= 0) { + unsigned char sha1[20]; + if (!index_fd(sha1, fd, st, 0, NULL)) + match = memcmp(sha1, ce->sha1, 20); + close(fd); + } + return match; +} + +static int ce_compare_link(struct cache_entry *ce, unsigned long expected_size) +{ + int match = -1; + char *target; + void *buffer; + unsigned long size; + char type[10]; + int len; + + target = xmalloc(expected_size); + len = readlink(ce->name, target, expected_size); + if (len != expected_size) { + free(target); + return -1; + } + buffer = read_sha1_file(ce->sha1, type, &size); + if (!buffer) { + free(target); + return -1; + } + if (size == expected_size) + match = memcmp(buffer, target, size); + free(buffer); + free(target); + return match; +} + +static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st) +{ + switch (st->st_mode & S_IFMT) { + case S_IFREG: + if (ce_compare_data(ce, st)) + return DATA_CHANGED; + break; + case S_IFLNK: + if (ce_compare_link(ce, st->st_size)) + return DATA_CHANGED; + break; + default: + return TYPE_CHANGED; + } + return 0; +} + int ce_match_stat(struct cache_entry *ce, struct stat *st) { unsigned int changed = 0; @@ -83,57 +142,37 @@ int ce_match_stat(struct cache_entry *ce, struct stat *st) if (ce->ce_size != htonl(st->st_size)) changed |= DATA_CHANGED; - return changed; -} - -static int ce_compare_data(struct cache_entry *ce, struct stat *st) -{ - int match = -1; - int fd = open(ce->name, O_RDONLY); - if (fd >= 0) { - unsigned char sha1[20]; - if (!index_fd(sha1, fd, st, 0, NULL)) - match = memcmp(sha1, ce->sha1, 20); - close(fd); - } - return match; -} - -static int ce_compare_link(struct cache_entry *ce, unsigned long expected_size) -{ - int match = -1; - char *target; - void *buffer; - unsigned long size; - char type[10]; - int len; + /* + * Within 1 second of this sequence: + * echo xyzzy >file && git-update-index --add file + * running this command: + * echo frotz >file + * would give a falsely clean cache entry. The mtime and + * length match the cache, and other stat fields do not change. + * + * We could detect this at update-index time (the cache entry + * being registered/updated records the same time as "now") + * and delay the return from git-update-index, but that would + * effectively mean we can make at most one commit per second, + * which is not acceptable. Instead, we check cache entries + * whose mtime are the same as the index file timestamp more + * careful than others. + */ + if (!changed && + index_file_timestamp && + index_file_timestamp <= ntohl(ce->ce_mtime.sec)) + changed |= ce_modified_check_fs(ce, st); - target = xmalloc(expected_size); - len = readlink(ce->name, target, expected_size); - if (len != expected_size) { - free(target); - return -1; - } - buffer = read_sha1_file(ce->sha1, type, &size); - if (!buffer) { - free(target); - return -1; - } - if (size == expected_size) - match = memcmp(buffer, target, size); - free(buffer); - free(target); - return match; + return changed; } int ce_modified(struct cache_entry *ce, struct stat *st) { - int changed; + int changed, changed_fs; changed = ce_match_stat(ce, st); if (!changed) return 0; - /* * If the mode or type has changed, there's no point in trying * to refresh the entry - it's not going to match @@ -148,18 +187,9 @@ int ce_modified(struct cache_entry *ce, struct stat *st) if ((changed & DATA_CHANGED) && ce->ce_size != htonl(0)) return changed; - switch (st->st_mode & S_IFMT) { - case S_IFREG: - if (ce_compare_data(ce, st)) - return changed | DATA_CHANGED; - break; - case S_IFLNK: - if (ce_compare_link(ce, st->st_size)) - return changed | DATA_CHANGED; - break; - default: - return changed | TYPE_CHANGED; - } + changed_fs = ce_modified_check_fs(ce, st); + if (changed_fs) + return changed | changed_fs; return 0; } @@ -471,6 +501,7 @@ int read_cache(void) return active_nr; errno = ENOENT; + index_file_timestamp = 0; fd = open(get_index_file(), O_RDONLY); if (fd < 0) { if (errno == ENOENT) @@ -504,6 +535,7 @@ int read_cache(void) offset = offset + ce_size(ce); active_cache[i] = ce; } + index_file_timestamp = st.st_mtime; return active_nr; unmap: -- cgit v1.2.1 From 407c8eb0d09d4b84bbfda9e04895a35c8fd6fef6 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 20 Dec 2005 12:12:18 -0800 Subject: Racy GIT (part #2) The previous round caught the most trivial case well, but broke down once index file is updated again. Smudge problematic entries (they should be very few if any under normal interactive workflow) before writing a new index file out. Signed-off-by: Junio C Hamano --- read-cache.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 780601f07..afdc2b075 100644 --- a/read-cache.c +++ b/read-cache.c @@ -87,7 +87,7 @@ static int ce_modified_check_fs(struct cache_entry *ce, struct stat *st) return 0; } -int ce_match_stat(struct cache_entry *ce, struct stat *st) +static int ce_match_stat_basic(struct cache_entry *ce, struct stat *st) { unsigned int changed = 0; @@ -143,6 +143,13 @@ int ce_match_stat(struct cache_entry *ce, struct stat *st) if (ce->ce_size != htonl(st->st_size)) changed |= DATA_CHANGED; + return changed; +} + +int ce_match_stat(struct cache_entry *ce, struct stat *st) +{ + unsigned int changed = ce_match_stat_basic(ce, st); + /* * Within 1 second of this sequence: * echo xyzzy >file && git-update-index --add file @@ -594,6 +601,26 @@ static int ce_flush(SHA_CTX *context, int fd) return 0; } +static void ce_smudge_racily_clean_entry(struct cache_entry *ce) +{ + /* + * The only thing we care about in this function is to smudge the + * falsely clean entry due to touch-update-touch race, so we leave + * everything else as they are. We are called for entries whose + * ce_mtime match the index file mtime. + */ + struct stat st; + + if (lstat(ce->name, &st) < 0) + return; + if (ce_match_stat_basic(ce, &st)) + return; + if (ce_modified_check_fs(ce, &st)) { + /* This is "racily clean"; smudge it */ + ce->ce_size = htonl(0); + } +} + int write_cache(int newfd, struct cache_entry **cache, int entries) { SHA_CTX c; @@ -616,6 +643,9 @@ int write_cache(int newfd, struct cache_entry **cache, int entries) struct cache_entry *ce = cache[i]; if (!ce->ce_mode) continue; + if (index_file_timestamp && + index_file_timestamp <= ntohl(ce->ce_mtime.sec)) + ce_smudge_racily_clean_entry(ce); if (ce_write(&c, newfd, ce, ce_size(ce)) < 0) return -1; } -- cgit v1.2.1 From 4b3511b0f8422fd2c5b1b37c9655ae2ce904bca5 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 20 Dec 2005 14:18:47 -0800 Subject: ce_smudge_racily_clean_entry: explain why it works. This is a tricky code and warrants extra commenting. I wasted 30 minutes trying to break it until I realized why it works. Signed-off-by: Junio C Hamano --- read-cache.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index afdc2b075..c5474d497 100644 --- a/read-cache.c +++ b/read-cache.c @@ -616,7 +616,31 @@ static void ce_smudge_racily_clean_entry(struct cache_entry *ce) if (ce_match_stat_basic(ce, &st)) return; if (ce_modified_check_fs(ce, &st)) { - /* This is "racily clean"; smudge it */ + /* This is "racily clean"; smudge it. Note that this + * is a tricky code. At first glance, it may appear + * that it can break with this sequence: + * + * $ echo xyzzy >frotz + * $ git-update-index --add frotz + * $ : >frotz + * $ sleep 3 + * $ echo filfre >nitfol + * $ git-update-index --add nitfol + * + * but it does not. Whe the second update-index runs, + * it notices that the entry "frotz" has the same timestamp + * as index, and if we were to smudge it by resetting its + * size to zero here, then the object name recorded + * in index is the 6-byte file but the cached stat information + * becomes zero --- which would then match what we would + * obtain from the filesystem next time we stat("frotz"). + * + * However, the second update-index, before calling + * this function, notices that the cached size is 6 + * bytes and what is on the filesystem is an empty + * file, and never calls us, so the cached size information + * for "frotz" stays 6 which does not match the filesystem. + */ ce->ce_size = htonl(0); } } -- cgit v1.2.1