Sane use of test_expect_failure

Originally, test_expect_failure was designed to be the opposite
of test_expect_success, but this was a bad decision.  Most tests
run a series of commands that leads to the single command that
needs to be tested, like this:

    test_expect_{success,failure} 'test title' '
	setup1 &&
        setup2 &&
        setup3 &&
        what is to be tested
    '

And expecting a failure exit from the whole sequence misses the
point of writing tests.  Your setup$N that are supposed to
succeed may have failed without even reaching what you are
trying to test.  The only valid use of test_expect_failure is to
check a trivial single command that is expected to fail, which
is a minority in tests of Porcelain-ish commands.

This large-ish patch rewrites all uses of test_expect_failure to
use test_expect_success and rewrites the condition of what is
tested, like this:

    test_expect_success 'test title' '
	setup1 &&
        setup2 &&
        setup3 &&
        ! this command should fail
    '

test_expect_failure is redefined to serve as a reminder that
that test *should* succeed but due to a known breakage in git it
currently does not pass.  So if git-foo command should create a
file 'bar' but you discovered a bug that it doesn't, you can
write a test like this:

    test_expect_failure 'git-foo should create bar' '
        rm -f bar &&
        git foo &&
        test -f bar
    '

This construct acts similar to test_expect_success, but instead
of reporting "ok/FAIL" like test_expect_success does, the
outcome is reported as "FIXED/still broken".

Signed-off-by: Junio C Hamano <gitster@pobox.com>

1 files changed, 89 insertions, 72 deletions

diff --git a/t/t1000-read-tree-m-3way.sh b/t/t1000-read-tree-m-3way.sh
index 37add1b50..6c065bfa2 100755
--- a/t/t1000-read-tree-m-3way.sh
+++ b/t/t1000-read-tree-m-3way.sh
@@ -210,12 +210,12 @@ DF (file) when tree B require DF to be a directory by having DF/DF
 
 END_OF_CASE_TABLE
 
-test_expect_failure \
-    '1 - must not have an entry not in A.' \
-    "rm -f .git/index XX &&
+test_expect_success '1 - must not have an entry not in A.' "
+     rm -f .git/index XX &&
      echo XX >XX &&
      git update-index --add XX &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '2 - must match B in !O && !A && B case.' \
@@ -248,13 +248,14 @@ test_expect_success \
      echo extra >>AN &&
      git read-tree -m $tree_O $tree_A $tree_B"
 
-test_expect_failure \
-    '3 (fail) - must match A in !O && A && !B case.' \
-    "rm -f .git/index AN &&
+test_expect_success \
+    '3 (fail) - must match A in !O && A && !B case.' "
+     rm -f .git/index AN &&
      cp .orig-A/AN AN &&
      echo extra >>AN &&
      git update-index --add AN &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '4 - must match and be up-to-date in !O && A && B && A!=B case.' \
@@ -264,21 +265,23 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '4 (fail) - must match and be up-to-date in !O && A && B && A!=B case.' \
-    "rm -f .git/index AA &&
+test_expect_success \
+    '4 (fail) - must match and be up-to-date in !O && A && B && A!=B case.' "
+     rm -f .git/index AA &&
      cp .orig-A/AA AA &&
      git update-index --add AA &&
      echo extra >>AA &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '4 (fail) - must match and be up-to-date in !O && A && B && A!=B case.' \
-    "rm -f .git/index AA &&
+test_expect_success \
+    '4 (fail) - must match and be up-to-date in !O && A && B && A!=B case.' "
+     rm -f .git/index AA &&
      cp .orig-A/AA AA &&
      echo extra >>AA &&
      git update-index --add AA &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '5 - must match in !O && A && B && A==B case.' \
@@ -297,34 +300,38 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '5 (fail) - must match A in !O && A && B && A==B case.' \
-    "rm -f .git/index LL &&
+test_expect_success \
+    '5 (fail) - must match A in !O && A && B && A==B case.' "
+     rm -f .git/index LL &&
      cp .orig-A/LL LL &&
      echo extra >>LL &&
      git update-index --add LL &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '6 - must not exist in O && !A && !B case' \
-    "rm -f .git/index DD &&
+test_expect_success \
+    '6 - must not exist in O && !A && !B case' "
+     rm -f .git/index DD &&
      echo DD >DD
      git update-index --add DD &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '7 - must not exist in O && !A && B && O!=B case' \
-    "rm -f .git/index DM &&
+test_expect_success \
+    '7 - must not exist in O && !A && B && O!=B case' "
+     rm -f .git/index DM &&
      cp .orig-B/DM DM &&
      git update-index --add DM &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '8 - must not exist in O && !A && B && O==B case' \
-    "rm -f .git/index DN &&
+test_expect_success \
+    '8 - must not exist in O && !A && B && O==B case' "
+     rm -f .git/index DN &&
      cp .orig-B/DN DN &&
      git update-index --add DN &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '9 - must match and be up-to-date in O && A && !B && O!=A case' \
@@ -334,21 +341,23 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '9 (fail) - must match and be up-to-date in O && A && !B && O!=A case' \
-    "rm -f .git/index MD &&
+test_expect_success \
+    '9 (fail) - must match and be up-to-date in O && A && !B && O!=A case' "
+     rm -f .git/index MD &&
      cp .orig-A/MD MD &&
      git update-index --add MD &&
      echo extra >>MD &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '9 (fail) - must match and be up-to-date in O && A && !B && O!=A case' \
-    "rm -f .git/index MD &&
+test_expect_success \
+    '9 (fail) - must match and be up-to-date in O && A && !B && O!=A case' "
+     rm -f .git/index MD &&
      cp .orig-A/MD MD &&
      echo extra >>MD &&
      git update-index --add MD &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '10 - must match and be up-to-date in O && A && !B && O==A case' \
@@ -358,21 +367,23 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '10 (fail) - must match and be up-to-date in O && A && !B && O==A case' \
-    "rm -f .git/index ND &&
+test_expect_success \
+    '10 (fail) - must match and be up-to-date in O && A && !B && O==A case' "
+     rm -f .git/index ND &&
      cp .orig-A/ND ND &&
      git update-index --add ND &&
      echo extra >>ND &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '10 (fail) - must match and be up-to-date in O && A && !B && O==A case' \
-    "rm -f .git/index ND &&
+test_expect_success \
+    '10 (fail) - must match and be up-to-date in O && A && !B && O==A case' "
+     rm -f .git/index ND &&
      cp .orig-A/ND ND &&
      echo extra >>ND &&
      git update-index --add ND &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '11 - must match and be up-to-date in O && A && B && O!=A && O!=B && A!=B case' \
@@ -382,21 +393,23 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '11 (fail) - must match and be up-to-date in O && A && B && O!=A && O!=B && A!=B case' \
-    "rm -f .git/index MM &&
+test_expect_success \
+    '11 (fail) - must match and be up-to-date in O && A && B && O!=A && O!=B && A!=B case' "
+     rm -f .git/index MM &&
      cp .orig-A/MM MM &&
      git update-index --add MM &&
      echo extra >>MM &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '11 (fail) - must match and be up-to-date in O && A && B && O!=A && O!=B && A!=B case' \
-    "rm -f .git/index MM &&
+test_expect_success \
+    '11 (fail) - must match and be up-to-date in O && A && B && O!=A && O!=B && A!=B case' "
+     rm -f .git/index MM &&
      cp .orig-A/MM MM &&
      echo extra >>MM &&
      git update-index --add MM &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '12 - must match A in O && A && B && O!=A && A==B case' \
@@ -415,13 +428,14 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '12 (fail) - must match A in O && A && B && O!=A && A==B case' \
-    "rm -f .git/index SS &&
+test_expect_success \
+    '12 (fail) - must match A in O && A && B && O!=A && A==B case' "
+     rm -f .git/index SS &&
      cp .orig-A/SS SS &&
      echo extra >>SS &&
      git update-index --add SS &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '13 - must match A in O && A && B && O!=A && O==B case' \
@@ -457,21 +471,23 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '14 (fail) - must match and be up-to-date in O && A && B && O==A && O!=B case' \
-    "rm -f .git/index NM &&
+test_expect_success \
+    '14 (fail) - must match and be up-to-date in O && A && B && O==A && O!=B case' "
+     rm -f .git/index NM &&
      cp .orig-A/NM NM &&
      git update-index --add NM &&
      echo extra >>NM &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
-test_expect_failure \
-    '14 (fail) - must match and be up-to-date in O && A && B && O==A && O!=B case' \
-    "rm -f .git/index NM &&
+test_expect_success \
+    '14 (fail) - must match and be up-to-date in O && A && B && O==A && O!=B case' "
+     rm -f .git/index NM &&
      cp .orig-A/NM NM &&
      echo extra >>NM &&
      git update-index --add NM &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 test_expect_success \
     '15 - must match A in O && A && B && O==A && O==B case' \
@@ -490,13 +506,14 @@ test_expect_success \
      git read-tree -m $tree_O $tree_A $tree_B &&
      check_result"
 
-test_expect_failure \
-    '15 (fail) - must match A in O && A && B && O==A && O==B case' \
-    "rm -f .git/index NN &&
+test_expect_success \
+    '15 (fail) - must match A in O && A && B && O==A && O==B case' "
+     rm -f .git/index NN &&
      cp .orig-A/NN NN &&
      echo extra >>NN &&
      git update-index --add NN &&
-     git read-tree -m $tree_O $tree_A $tree_B"
+     ! git read-tree -m $tree_O $tree_A $tree_B
+"
 
 # #16
 test_expect_success \