Merge branch 'rt/zlib-smaller-window'

* rt/zlib-smaller-window:
  test: consolidate definition of $LF
  Tolerate zlib deflation with window size < 32Kb

23 files changed, 99 insertions, 13 deletions

diff --git a/sha1_file.c b/sha1_file.c
index d5616dca0..44444ae8f 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1217,14 +1217,34 @@ static int experimental_loose_object(unsigned char *map)
 	unsigned int word;
 
 	/*
-	 * Is it a zlib-compressed buffer? If so, the first byte
-	 * must be 0x78 (15-bit window size, deflated), and the
-	 * first 16-bit word is evenly divisible by 31. If so,
-	 * we are looking at the official format, not the experimental
-	 * one.
+	 * We must determine if the buffer contains the standard
+	 * zlib-deflated stream or the experimental format based
+	 * on the in-pack object format. Compare the header byte
+	 * for each format:
+	 *
+	 * RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7
+	 * Experimental pack-based : Stttssss : ttt = 1,2,3,4
+	 *
+	 * If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be
+	 * in standard loose-object format, UNLESS it is a Git-pack
+	 * format object *exactly* 8 bytes in size when inflated.
+	 *
+	 * However, RFC1950 also specifies that the 1st 16-bit word
+	 * must be divisible by 31 - this checksum tells us our buffer
+	 * is in the standard format, giving a false positive only if
+	 * the 1st word of the Git-pack format object happens to be
+	 * divisible by 31, ie:
+	 *      ((byte0 * 256) + byte1) % 31 = 0
+	 *   =>        0ttt10000www1000 % 31 = 0
+	 *
+	 * As it happens, this case can only arise for www=3 & ttt=1
+	 * - ie, a Commit object, which would have to be 8 bytes in
+	 * size. As no Commit can be that small, we find that the
+	 * combination of these two criteria (bitmask & checksum)
+	 * can always correctly determine the buffer format.
 	 */
 	word = (map[0] << 8) + map[1];
-	if (map[0] == 0x78 && !(word % 31))
+	if ((map[0] & 0x8F) == 0x08 && !(word % 31))
 		return 0;
 	else
 		return 1;
diff --git a/t/t1013-loose-object-format.sh b/t/t1013-loose-object-format.sh
new file mode 100755
index 000000000..0a9cedd37
--- /dev/null
+++ b/t/t1013-loose-object-format.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# Copyright (c) 2011 Roberto Tyley
+#
+
+test_description='Correctly identify and parse loose object headers
+
+There are two file formats for loose objects - the original standard
+format, and the experimental format introduced with Git v1.4.3, later
+deprecated with v1.5.3. Although Git no longer writes the
+experimental format, objects in both formats must be read, with the
+format for a given file being determined by the header.
+
+Detecting file format based on header is not entirely trivial, not
+least because the first byte of a zlib-deflated stream will vary
+depending on how much memory was allocated for the deflation window
+buffer when the object was written out (for example 4KB on Android,
+rather that 32KB on a normal PC).
+
+The loose objects used as test vectors have been generated with the
+following Git versions:
+
+standard format: Git v1.7.4.1
+experimental format: Git v1.4.3 (legacyheaders=false)
+standard format, deflated with 4KB window size: Agit/JGit on Android
+'
+
+. ./test-lib.sh
+
+assert_blob_equals() {
+	printf "%s" "$2" >expected &&
+	git cat-file -p "$1" >actual &&
+	test_cmp expected actual
+}
+
+test_expect_success setup '
+	cp -R "$TEST_DIRECTORY/t1013/objects" .git/
+	git --version
+'
+
+test_expect_success 'read standard-format loose objects' '
+	git cat-file tag 8d4e360d6c70fbd72411991c02a09c442cf7a9fa &&
+	git cat-file commit 6baee0540ea990d9761a3eb9ab183003a71c3696 &&
+	git ls-tree 7a37b887a73791d12d26c0d3e39568a8fb0fa6e8 &&
+	assert_blob_equals "257cc5642cb1a054f08cc83f2d943e56fd3ebe99" "foo$LF"
+'
+
+test_expect_success 'read experimental-format loose objects' '
+	git cat-file tag 76e7fa9941f4d5f97f64fea65a2cba436bc79cbb &&
+	git cat-file commit 7875c6237d3fcdd0ac2f0decc7d3fa6a50b66c09 &&
+	git ls-tree 95b1625de3ba8b2214d1e0d0591138aea733f64f &&
+	assert_blob_equals "2e65efe2a145dda7ee51d1741299f848e5bf752e" "a" &&
+	assert_blob_equals "9ae9e86b7bd6cb1472d9373702d8249973da0832" "ab" &&
+	assert_blob_equals "85df50785d62d3b05ab03d9cbf7e4a0b49449730" "abcd" &&
+	assert_blob_equals "1656f9233d999f61ef23ef390b9c71d75399f435" "abcdefgh" &&
+	assert_blob_equals "1e72a6b2c4a577ab0338860fa9fe87f761fc9bbd" "abcdefghi" &&
+	assert_blob_equals "70e6a83d8dcb26fc8bc0cf702e2ddeb6adca18fd" "abcdefghijklmnop" &&
+	assert_blob_equals "bd15045f6ce8ff75747562173640456a394412c8" "abcdefghijklmnopqrstuvwx"
+'
+
+test_expect_success 'read standard-format objects deflated with smaller window buffer' '
+	git cat-file tag f816d5255855ac160652ee5253b06cd8ee14165a &&
+	git cat-file tag 149cedb5c46929d18e0f118e9fa31927487af3b6
+'
+
+test_done
diff --git a/t/t1013/objects/14/9cedb5c46929d18e0f118e9fa31927487af3b6 b/t/t1013/objects/14/9cedb5c46929d18e0f118e9fa31927487af3b6
new file mode 100644
index 000000000..472fd1458
--- /dev/null
+++ b/t/t1013/objects/14/9cedb5c46929d18e0f118e9fa31927487af3b6
diff --git a/t/t1013/objects/16/56f9233d999f61ef23ef390b9c71d75399f435 b/t/t1013/objects/16/56f9233d999f61ef23ef390b9c71d75399f435
new file mode 100644
index 000000000..c379d74ae
--- /dev/null
+++ b/t/t1013/objects/16/56f9233d999f61ef23ef390b9c71d75399f435
diff --git a/t/t1013/objects/1e/72a6b2c4a577ab0338860fa9fe87f761fc9bbd b/t/t1013/objects/1e/72a6b2c4a577ab0338860fa9fe87f761fc9bbd
new file mode 100644
index 000000000..93706305b
--- /dev/null
+++ b/t/t1013/objects/1e/72a6b2c4a577ab0338860fa9fe87f761fc9bbd
diff --git a/t/t1013/objects/25/7cc5642cb1a054f08cc83f2d943e56fd3ebe99 b/t/t1013/objects/25/7cc5642cb1a054f08cc83f2d943e56fd3ebe99
new file mode 100644
index 000000000..bdcf704c9
--- /dev/null
+++ b/t/t1013/objects/25/7cc5642cb1a054f08cc83f2d943e56fd3ebe99
diff --git a/t/t1013/objects/2e/65efe2a145dda7ee51d1741299f848e5bf752e b/t/t1013/objects/2e/65efe2a145dda7ee51d1741299f848e5bf752e
new file mode 100644
index 000000000..ad62c43e4
--- /dev/null
+++ b/t/t1013/objects/2e/65efe2a145dda7ee51d1741299f848e5bf752e
diff --git a/t/t1013/objects/6b/aee0540ea990d9761a3eb9ab183003a71c3696 b/t/t1013/objects/6b/aee0540ea990d9761a3eb9ab183003a71c3696
new file mode 100644
index 000000000..3d2f0337d
--- /dev/null
+++ b/t/t1013/objects/6b/aee0540ea990d9761a3eb9ab183003a71c3696
diff --git a/t/t1013/objects/70/e6a83d8dcb26fc8bc0cf702e2ddeb6adca18fd b/t/t1013/objects/70/e6a83d8dcb26fc8bc0cf702e2ddeb6adca18fd
new file mode 100644
index 000000000..b3f71a6ee
--- /dev/null
+++ b/t/t1013/objects/70/e6a83d8dcb26fc8bc0cf702e2ddeb6adca18fd
diff --git a/t/t1013/objects/76/e7fa9941f4d5f97f64fea65a2cba436bc79cbb b/t/t1013/objects/76/e7fa9941f4d5f97f64fea65a2cba436bc79cbb
new file mode 100644
index 000000000..af4e9a7b0
--- /dev/null
+++ b/t/t1013/objects/76/e7fa9941f4d5f97f64fea65a2cba436bc79cbb
@@ -0,0 +1,2 @@
+�x�%�A�0@�}O1{cSZ(��ν��th���Z��ޠ��?
�m�6d�i��9��G�h�ب�ZR'Q���R������p���qL9��=g���sI�oop���eϫ_1����$��*Si��NwpP�RB�����
+��[(�d-���L9�
\ No newline at end of file
diff --git a/t/t1013/objects/78/75c6237d3fcdd0ac2f0decc7d3fa6a50b66c09 b/t/t1013/objects/78/75c6237d3fcdd0ac2f0decc7d3fa6a50b66c09
new file mode 100644
index 000000000..3dd28be5c
--- /dev/null
+++ b/t/t1013/objects/78/75c6237d3fcdd0ac2f0decc7d3fa6a50b66c09
diff --git a/t/t1013/objects/7a/37b887a73791d12d26c0d3e39568a8fb0fa6e8 b/t/t1013/objects/7a/37b887a73791d12d26c0d3e39568a8fb0fa6e8
new file mode 100644
index 000000000..2b97b264c
--- /dev/null
+++ b/t/t1013/objects/7a/37b887a73791d12d26c0d3e39568a8fb0fa6e8
diff --git a/t/t1013/objects/85/df50785d62d3b05ab03d9cbf7e4a0b49449730 b/t/t1013/objects/85/df50785d62d3b05ab03d9cbf7e4a0b49449730
new file mode 100644
index 000000000..6dff74687
--- /dev/null
+++ b/t/t1013/objects/85/df50785d62d3b05ab03d9cbf7e4a0b49449730
diff --git a/t/t1013/objects/8d/4e360d6c70fbd72411991c02a09c442cf7a9fa b/t/t1013/objects/8d/4e360d6c70fbd72411991c02a09c442cf7a9fa
new file mode 100644
index 000000000..cb41e92d0
--- /dev/null
+++ b/t/t1013/objects/8d/4e360d6c70fbd72411991c02a09c442cf7a9fa
diff --git a/t/t1013/objects/95/b1625de3ba8b2214d1e0d0591138aea733f64f b/t/t1013/objects/95/b1625de3ba8b2214d1e0d0591138aea733f64f
new file mode 100644
index 000000000..7ac46b4f7
--- /dev/null
+++ b/t/t1013/objects/95/b1625de3ba8b2214d1e0d0591138aea733f64f
diff --git a/t/t1013/objects/9a/e9e86b7bd6cb1472d9373702d8249973da0832 b/t/t1013/objects/9a/e9e86b7bd6cb1472d9373702d8249973da0832
new file mode 100644
index 000000000..9d8316d4e
--- /dev/null
+++ b/t/t1013/objects/9a/e9e86b7bd6cb1472d9373702d8249973da0832
diff --git a/t/t1013/objects/bd/15045f6ce8ff75747562173640456a394412c8 b/t/t1013/objects/bd/15045f6ce8ff75747562173640456a394412c8
new file mode 100644
index 000000000..eebf23956
--- /dev/null
+++ b/t/t1013/objects/bd/15045f6ce8ff75747562173640456a394412c8
diff --git a/t/t1013/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 b/t/t1013/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
new file mode 100644
index 000000000..134cf1937
--- /dev/null
+++ b/t/t1013/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/t/t1013/objects/f8/16d5255855ac160652ee5253b06cd8ee14165a b/t/t1013/objects/f8/16d5255855ac160652ee5253b06cd8ee14165a
new file mode 100644
index 000000000..26b75aec5
--- /dev/null
+++ b/t/t1013/objects/f8/16d5255855ac160652ee5253b06cd8ee14165a
@@ -0,0 +1 @@
+H����0�a�{��
I�e&��*��
G��^�����D�҆wU�җ�S�4������ ,f�[��VAۺ��x���6[wtG�Lu�?������@�"g�{�+by�%M
\ No newline at end of file
diff --git a/t/t1020-subdirectory.sh b/t/t1020-subdirectory.sh
index 865b8ed26..3b1b98599 100755
--- a/t/t1020-subdirectory.sh
+++ b/t/t1020-subdirectory.sh
@@ -17,8 +17,6 @@ test_expect_success setup '
 	cp one original.one &&
 	cp dir/two original.two
 '
-LF='
-'
 
 test_expect_success 'update-index and ls-files' '
 	git update-index --add one &&
diff --git a/t/t3902-quoted.sh b/t/t3902-quoted.sh
index da82b655b..534ee08a4 100755
--- a/t/t3902-quoted.sh
+++ b/t/t3902-quoted.sh
@@ -10,8 +10,6 @@ test_description='quoted output'
 FN='濱野'
 GN='純'
 HT='	'
-LF='
-'
 DQ='"'
 
 echo foo 2>/dev/null > "Name and an${HT}HT"
diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh
index 4048d106d..395adfc8a 100755
--- a/t/t7800-difftool.sh
+++ b/t/t7800-difftool.sh
@@ -10,9 +10,6 @@ Testing basic diff tool invocation
 
 . ./test-lib.sh
 
-LF='
-'
-
 remove_config_vars()
 {
 	# Unset all config variables used by git-difftool
diff --git a/t/test-lib.sh b/t/test-lib.sh
index 57c3d5329..e27422217 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -92,6 +92,10 @@ _x40="$_x05$_x05$_x05$_x05$_x05$_x05$_x05$_x05"
 # Zero SHA-1
 _z40=0000000000000000000000000000000000000000
 
+# Line feed
+LF='
+'
+
 # Each test should start with something like this, after copyright notices:
 #
 # test_description='Description of this test...