diff: fix 2 whitespace issues

When whitespace or whitespace change was ignored, the function
xdl_recmatch() returned memcmp() style differences, which is wrong,
since it should return 0 on non-match.

Also, there were three horrible off-by-one bugs, even leading to wrong
hashes in the whitespace special handling.

The issue was noticed by Ray Lehtiniemi.

For good measure, this commit adds a test.

Signed-off-by: Johannes Schindelin <Johannes.Schindelin@gmx.de>
Signed-off-by: Junio C Hamano <junkio@cox.net>

1 files changed, 12 insertions, 17 deletions

diff --git a/xdiff/xutils.c b/xdiff/xutils.c
index f7bdd395a..9e4bb47ee 100644
--- a/xdiff/xutils.c
+++ b/xdiff/xutils.c
@@ -191,36 +191,30 @@ int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags)
 	int i1, i2;
 
 	if (flags & XDF_IGNORE_WHITESPACE) {
-		for (i1 = i2 = 0; i1 < s1 && i2 < s2; i1++, i2++) {
+		for (i1 = i2 = 0; i1 < s1 && i2 < s2; ) {
 			if (isspace(l1[i1]))
 				while (isspace(l1[i1]) && i1 < s1)
 					i1++;
-			else if (isspace(l2[i2]))
+			if (isspace(l2[i2]))
 				while (isspace(l2[i2]) && i2 < s2)
 					i2++;
-			else if (l1[i1] != l2[i2])
-				return l2[i2] - l1[i1];
+			if (i1 < s1 && i2 < s2 && l1[i1++] != l2[i2++])
+				return 0;
 		}
-		if (i1 >= s1)
-			return 1;
-		else if (i2 >= s2)
-			return -1;
+		return (i1 >= s1 && i2 >= s2);
 	} else if (flags & XDF_IGNORE_WHITESPACE_CHANGE) {
-		for (i1 = i2 = 0; i1 < s1 && i2 < s2; i1++, i2++) {
+		for (i1 = i2 = 0; i1 < s1 && i2 < s2; ) {
 			if (isspace(l1[i1])) {
 				if (!isspace(l2[i2]))
-					return -1;
+					return 0;
 				while (isspace(l1[i1]) && i1 < s1)
 					i1++;
 				while (isspace(l2[i2]) && i2 < s2)
 					i2++;
-			} else if (l1[i1] != l2[i2])
-				return l2[i2] - l1[i1];
+			} else if (l1[i1++] != l2[i2++])
+				return 0;
 		}
-		if (i1 >= s1)
-			return 1;
-		else if (i2 >= s2)
-			return -1;
+		return (i1 >= s1 && i2 >= s2);
 	} else
 		return s1 == s2 && !memcmp(l1, l2, s1);
 
@@ -233,7 +227,8 @@ unsigned long xdl_hash_record(char const **data, char const *top, long flags) {
 
 	for (; ptr < top && *ptr != '\n'; ptr++) {
 		if (isspace(*ptr) && (flags & XDF_WHITESPACE_FLAGS)) {
-			while (ptr < top && isspace(*ptr) && ptr[1] != '\n')
+			while (ptr + 1 < top && isspace(ptr[1])
+					&& ptr[1] != '\n')
 				ptr++;
 			if (flags & XDF_IGNORE_WHITESPACE_CHANGE) {
 				ha += (ha << 5);