From 495ea6cd41216f16dfd5051e456fd86082d0593d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SZEDER=20G=C3=A1bor?= Date: Sun, 31 Dec 2017 11:12:03 +0100 Subject: travis-ci: print the "tip of branch is exactly at tag" message in color MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To make this info message stand out from the regular build job trace output. Signed-off-by: SZEDER Gábor Reviewed-by: Lars Schneider Signed-off-by: Junio C Hamano --- ci/lib-travisci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index 348fe3c3c..9d379db8a 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -16,7 +16,7 @@ skip_branch_tip_with_tag () { if TAG=$(git describe --exact-match "$TRAVIS_BRANCH" 2>/dev/null) && test "$TAG" != "$TRAVIS_BRANCH" then - echo "Tip of $TRAVIS_BRANCH is exactly at $TAG" + echo "$(tput setaf 2)Tip of $TRAVIS_BRANCH is exactly at $TAG$(tput sgr0)" exit 0 fi } -- cgit v1.2.1 From b4a2fdc9bdc87fc8bd62e8bf7890482c590e558c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SZEDER=20G=C3=A1bor?= Date: Sun, 31 Dec 2017 11:12:04 +0100 Subject: travis-ci: create the cache directory early in the build process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems that Travis CI creates the cache directory for us anyway, even when a previous cache doesn't exist for the current build job. Alas, this behavior is not explicitly documented, therefore we don't rely on it and create the cache directory ourselves in those build jobs that read/write cached data (currently only the prove state). In the following commit we'll start to cache additional data in every build job, and will access the cache much earlier in the build process. Therefore move creating the cache directory to 'ci/lib-travisci.sh' to make sure that it exists at the very beginning of every build job. Signed-off-by: SZEDER Gábor Reviewed-by: Lars Schneider Signed-off-by: Junio C Hamano --- ci/lib-travisci.sh | 2 ++ ci/run-tests.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index 9d379db8a..197aa14c1 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -27,6 +27,8 @@ skip_branch_tip_with_tag () { # and installing dependencies. set -ex +mkdir -p "$HOME/travis-cache" + skip_branch_tip_with_tag if test -z "$jobname" diff --git a/ci/run-tests.sh b/ci/run-tests.sh index f0c743de9..ccdfc2b9d 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -5,6 +5,5 @@ . ${0%/*}/lib-travisci.sh -mkdir -p $HOME/travis-cache ln -s $HOME/travis-cache/.prove t/.prove make --quiet test -- cgit v1.2.1 From 9cc2c76f5eaab557c42f67b9d574db89fd0865c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SZEDER=20G=C3=A1bor?= Date: Sun, 31 Dec 2017 11:12:05 +0100 Subject: travis-ci: record and skip successfully built trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Travis CI dutifully builds and tests each new branch tip, even if its tree has previously been successfully built and tested. This happens often enough in contributors' workflows, when a work-in-progress branch is rebased changing e.g. only commit messages or the order or number of commits while leaving the resulting code intact, and is then pushed to a Travis CI-enabled GitHub fork. This is wasting Travis CI's resources and is sometimes scary-annoying when the new tip commit with a tree identical to the previous, successfully tested one is suddenly reported in red, because one of the OSX build jobs happened to exceed the time limit yet again. So extend our Travis CI build scripts to skip building commits whose trees have previously been successfully built and tested. Use the Travis CI cache feature to keep a record of the object names of trees that tested successfully, in a plain and simple flat text file, one line per tree object name. Append the current tree's object name at the end of every successful build job to this file, along with a bit of additional info about the build job (commit object name, Travis CI job number and id). Limit the size of this file to 1000 records, to prevent it from growing too large for git/git's forever living integration branches. Check, using a simple grep invocation, in each build job whether the current commit's tree is already in there, and skip the build if it is. Include a message in the skipped build job's trace log, containing the URL to the build job successfully testing that tree for the first time and instructions on how to force a re-build. Catch the case when a build job, which successfully built and tested a particular tree for the first time, is restarted and omit the URL of the previous build job's trace log, as in this case it's the same build job and the trace log has just been overwritten. Note: this won't kick in if two identical trees are on two different branches, because Travis CI caches are not shared between build jobs of different branches. Signed-off-by: SZEDER Gábor Reviewed-by: Lars Schneider Signed-off-by: Junio C Hamano --- ci/lib-travisci.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++++ ci/run-linux32-docker.sh | 2 ++ ci/run-static-analysis.sh | 2 ++ ci/run-tests.sh | 2 ++ ci/run-windows-build.sh | 2 ++ ci/test-documentation.sh | 2 ++ 6 files changed, 57 insertions(+) diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index 197aa14c1..bade71617 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -21,6 +21,52 @@ skip_branch_tip_with_tag () { fi } +good_trees_file="$HOME/travis-cache/good-trees" + +# Save some info about the current commit's tree, so we can skip the build +# job if we encounter the same tree again and can provide a useful info +# message. +save_good_tree () { + echo "$(git rev-parse $TRAVIS_COMMIT^{tree}) $TRAVIS_COMMIT $TRAVIS_JOB_NUMBER $TRAVIS_JOB_ID" >>"$good_trees_file" + # limit the file size + tail -1000 "$good_trees_file" >"$good_trees_file".tmp + mv "$good_trees_file".tmp "$good_trees_file" +} + +# Skip the build job if the same tree has already been built and tested +# successfully before (e.g. because the branch got rebased, changing only +# the commit messages). +skip_good_tree () { + if ! good_tree_info="$(grep "^$(git rev-parse $TRAVIS_COMMIT^{tree}) " "$good_trees_file")" + then + # Haven't seen this tree yet, or no cached good trees file yet. + # Continue the build job. + return + fi + + echo "$good_tree_info" | { + read tree prev_good_commit prev_good_job_number prev_good_job_id + + if test "$TRAVIS_JOB_ID" = "$prev_good_job_id" + then + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit has already been built and tested successfully by this build job. + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + else + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. + The log of that build job is available at https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$prev_good_job_id + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + fi + } + + exit 0 +} + # Set 'exit on error' for all CI scripts to let the caller know that # something went wrong. # Set tracing executed commands, primarily setting environment variables @@ -30,6 +76,7 @@ set -ex mkdir -p "$HOME/travis-cache" skip_branch_tip_with_tag +skip_good_tree if test -z "$jobname" then diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index 3a8b2ba42..870a41246 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -22,3 +22,5 @@ docker run \ --volume "${HOME}/travis-cache:/tmp/travis-cache" \ daald/ubuntu32:xenial \ /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) + +save_good_tree diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index 68dd0f080..fe4ee4e06 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -6,3 +6,5 @@ . ${0%/*}/lib-travisci.sh make coccicheck + +save_good_tree diff --git a/ci/run-tests.sh b/ci/run-tests.sh index ccdfc2b9d..eb5ba4058 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -7,3 +7,5 @@ ln -s $HOME/travis-cache/.prove t/.prove make --quiet test + +save_good_tree diff --git a/ci/run-windows-build.sh b/ci/run-windows-build.sh index 86999268a..d99a180e5 100755 --- a/ci/run-windows-build.sh +++ b/ci/run-windows-build.sh @@ -99,3 +99,5 @@ gfwci "action=log&buildId=$BUILD_ID" | cut -c 30- # Set exit code for TravisCI test "$RESULT" = "success" + +save_good_tree diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index 7a0a848e8..3d62e6c95 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -25,3 +25,5 @@ sed '/^GIT_VERSION = / d' stderr.log ! test -s stderr.log test -s Documentation/git.html grep '