Merge branch 'jk/pack-idx-corruption-safety'

The code to read the pack data using the offsets stored in the pack
idx file has been made more carefully check the validity of the
data in the idx.

* jk/pack-idx-corruption-safety:
  sha1_file.c: mark strings for translation
  use_pack: handle signed off_t overflow
  nth_packed_object_offset: bounds-check extended offset
  t5313: test bounds-checks of corrupted/malicious pack/idx files

1 files changed, 179 insertions, 0 deletions

diff --git a/t/t5313-pack-bounds-checks.sh b/t/t5313-pack-bounds-checks.sh
new file mode 100755
index 000000000..a8a587abc
--- /dev/null
+++ b/t/t5313-pack-bounds-checks.sh
@@ -0,0 +1,179 @@
+#!/bin/sh
+
+test_description='bounds-checking of access to mmapped on-disk file formats'
+. ./test-lib.sh
+
+clear_base () {
+	test_when_finished 'restore_base' &&
+	rm -f $base
+}
+
+restore_base () {
+	cp base-backup/* .git/objects/pack/
+}
+
+do_pack () {
+	pack_objects=$1; shift
+	sha1=$(
+		for i in $pack_objects
+		do
+			echo $i
+		done | git pack-objects "$@" .git/objects/pack/pack
+	) &&
+	pack=.git/objects/pack/pack-$sha1.pack &&
+	idx=.git/objects/pack/pack-$sha1.idx &&
+	chmod +w $pack $idx &&
+	test_when_finished 'rm -f "$pack" "$idx"'
+}
+
+munge () {
+	printf "$3" | dd of="$1" bs=1 conv=notrunc seek=$2
+}
+
+# Offset in a v2 .idx to its initial and extended offset tables. For an index
+# with "nr" objects, this is:
+#
+#   magic(4) + version(4) + fan-out(4*256) + sha1s(20*nr) + crc(4*nr),
+#
+# for the initial, and another ofs(4*nr) past that for the extended.
+#
+ofs_table () {
+	echo $((4 + 4 + 4*256 + 20*$1 + 4*$1))
+}
+extended_table () {
+	echo $(($(ofs_table "$1") + 4*$1))
+}
+
+test_expect_success 'set up base packfile and variables' '
+	# the hash of this content starts with ff, which
+	# makes some later computations much simpler
+	echo 74 >file &&
+	git add file &&
+	git commit -m base &&
+	git repack -ad &&
+	base=$(echo .git/objects/pack/*) &&
+	chmod +w $base &&
+	mkdir base-backup &&
+	cp $base base-backup/ &&
+	object=$(git rev-parse HEAD:file)
+'
+
+test_expect_success 'pack/index object count mismatch' '
+	do_pack $object &&
+	munge $pack 8 "\377\0\0\0" &&
+	clear_base &&
+
+	# We enumerate the objects from the completely-fine
+	# .idx, but notice later that the .pack is bogus
+	# and fail to show any data.
+	echo "$object missing" >expect &&
+	git cat-file --batch-all-objects --batch-check >actual &&
+	test_cmp expect actual &&
+
+	# ...and here fail to load the object (without segfaulting),
+	# but fallback to a good copy if available.
+	test_must_fail git cat-file blob $object &&
+	restore_base &&
+	git cat-file blob $object >actual &&
+	test_cmp file actual &&
+
+	# ...and make sure that index-pack --verify, which has its
+	# own reading routines, does not segfault.
+	test_must_fail git index-pack --verify $pack
+'
+
+test_expect_success 'matched bogus object count' '
+	do_pack $object &&
+	munge $pack 8 "\377\0\0\0" &&
+	munge $idx $((255 * 4)) "\377\0\0\0" &&
+	clear_base &&
+
+	# Unlike above, we should notice early that the .idx is totally
+	# bogus, and not even enumerate its contents.
+	>expect &&
+	git cat-file --batch-all-objects --batch-check >actual &&
+	test_cmp expect actual &&
+
+	# But as before, we can do the same object-access checks.
+	test_must_fail git cat-file blob $object &&
+	restore_base &&
+	git cat-file blob $object >actual &&
+	test_cmp file actual &&
+
+	test_must_fail git index-pack --verify $pack
+'
+
+# Note that we cannot check the fallback case for these
+# further .idx tests, as we notice the problem in functions
+# whose interface doesn't allow an error return (like use_pack()),
+# and thus we just die().
+#
+# There's also no point in doing enumeration tests, as
+# we are munging offsets here, which are about looking up
+# specific objects.
+
+test_expect_success 'bogus object offset (v1)' '
+	do_pack $object --index-version=1 &&
+	munge $idx $((4 * 256)) "\377\0\0\0" &&
+	clear_base &&
+	test_must_fail git cat-file blob $object &&
+	test_must_fail git index-pack --verify $pack
+'
+
+test_expect_success 'bogus object offset (v2, no msb)' '
+	do_pack $object --index-version=2 &&
+	munge $idx $(ofs_table 1) "\0\377\0\0" &&
+	clear_base &&
+	test_must_fail git cat-file blob $object &&
+	test_must_fail git index-pack --verify $pack
+'
+
+test_expect_success 'bogus offset into v2 extended table' '
+	do_pack $object --index-version=2 &&
+	munge $idx $(ofs_table 1) "\377\0\0\0" &&
+	clear_base &&
+	test_must_fail git cat-file blob $object &&
+	test_must_fail git index-pack --verify $pack
+'
+
+test_expect_success 'bogus offset inside v2 extended table' '
+	# We need two objects here, so we can plausibly require
+	# an extended table (if the first object were larger than 2^31).
+	do_pack "$object $(git rev-parse HEAD)" --index-version=2 &&
+
+	# We have to make extra room for the table, so we cannot
+	# just munge in place as usual.
+	{
+		dd if=$idx bs=1 count=$(($(ofs_table 2) + 4)) &&
+		printf "\200\0\0\0" &&
+		printf "\377\0\0\0\0\0\0\0" &&
+		dd if=$idx bs=1 skip=$(extended_table 2)
+	} >tmp &&
+	mv tmp "$idx" &&
+	clear_base &&
+	test_must_fail git cat-file blob $object &&
+	test_must_fail git index-pack --verify $pack
+'
+
+test_expect_success 'bogus OFS_DELTA in packfile' '
+	# Generate a pack with a delta in it.
+	base=$(test-genrandom foo 3000 | git hash-object --stdin -w) &&
+	delta=$(test-genrandom foo 2000 | git hash-object --stdin -w) &&
+	do_pack "$base $delta" --delta-base-offset &&
+	rm -f .git/objects/??/* &&
+
+	# Double check that we have the delta we expect.
+	echo $base >expect &&
+	echo $delta | git cat-file --batch-check="%(deltabase)" >actual &&
+	test_cmp expect actual &&
+
+	# Now corrupt it. We assume the varint size for the delta is small
+	# enough to fit in the first byte (which it should be, since it
+	# is a pure deletion from the base), and that original ofs_delta
+	# takes 2 bytes (which it should, as it should be ~3000).
+	ofs=$(git show-index <$idx | grep $delta | cut -d" " -f1) &&
+	munge $pack $(($ofs + 1)) "\177\377" &&
+	test_must_fail git cat-file blob $delta >/dev/null
+'
+
+test_done