v3 of boolean folder: patches

Subject: v3 of boolean folder: patches

Date: Sat, 8 Mar 2014 17:19:30 -0400

Cc:

This is a followup to 

     id:cover.1393105055.git.jani@nikula.org

The first 11 patches are very close to that series; in the last 4 I
have added some minimalist infrastructure to download and verify
pre-built test databases.

I decided that the benefits of being to really apply and test the
patch series outweighed the fact that one of patchs is about
150K. Apologies to those of you on GSM modems and the like.

Interdiff follows 

diff --git a/Makefile b/Makefile
index 0428160..97084b1 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,8 @@ all:
 # List all subdirectories here. Each contains its own Makefile.local.
 # Use of '=', without '+=', seems to be required for out-of-tree
 # builds to work.
-subdirs = compat completion emacs lib man parse-time-string performance-test util test
+subdirs = compat completion emacs lib man parse-time-string performance-test util test \
+	test/test-databases
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/devel/gen-testdb.sh b/devel/gen-testdb.sh
index c291dff..621b31e 100755
--- a/devel/gen-testdb.sh
+++ b/devel/gen-testdb.sh
@@ -81,6 +81,10 @@ shift `expr $OPTIND - 1`
 
 . ./test-lib.sh
 
+SHORT_CORPUS=$(basename ${CORPUS:-database})
+DBNAME=${SHORT_CORPUS}${SUFFIX}
+TARBALLNAME=${DBNAME}.tar.xz
+
 CORPUS=${CORPUS:-${TEST_DIRECTORY}/corpus}
 
 test_expect_code 0 "notmuch version specified on the command line" \
@@ -112,13 +116,16 @@ cp -a ${CORPUS} ${MAIL_DIR}
 test_expect_code 0 "index the corpus" \
     "notmuch new"
 
-# finally, wrap the resulting mail store and database in a tarball
-DBNAME=database${SUFFIX}
+# wrap the resulting mail store and database in a tarball
+
 cp -a ${MAIL_DIR} ${TMP_DIRECTORY}/${DBNAME}
-tar zcf ${TMP_DIRECTORY}/${DBNAME}.tar.gz -C ${TMP_DIRECTORY} ${DBNAME}
+tar Jcf ${TMP_DIRECTORY}/${TARBALLNAME} -C ${TMP_DIRECTORY} ${DBNAME}
 mkdir -p  ${TEST_DIRECTORY}/test-databases
-cp -a ${TMP_DIRECTORY}/${DBNAME}.tar.gz ${TEST_DIRECTORY}/test-databases
-test_expect_code 0 "create the output tarball ${DBNAME}.tar.gz" \
-    "test -f ${TEST_DIRECTORY}/test-databases/${DBNAME}.tar.gz"
+cp -a ${TMP_DIRECTORY}/${TARBALLNAME} ${TEST_DIRECTORY}/test-databases
+test_expect_code 0 "create the output tarball ${TARBALLNAME}" \
+    "test -f ${TEST_DIRECTORY}/test-databases/${TARBALLNAME}"
 
+# generate a checksum file
+test_expect_code 0 "compute checksum" \
+    "(cd ${TEST_DIRECTORY}/test-databases/ && sha256sum ${TARBALLNAME} > ${TARBALLNAME}.sha256)"
 test_done
diff --git a/test/Makefile.local b/test/Makefile.local
index 99324ba..bfabd15 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -2,6 +2,8 @@
 
 dir := test
 
+# save against changes in $(dir)
+test_src_dir := $(dir)
 extra_cflags += -I.
 
 smtp_dummy_srcs =		\
@@ -44,12 +46,12 @@ TEST_BINARIES=$(dir)/arg-test \
 
 test-binaries: $(TEST_BINARIES)
 
-test:	all test-binaries
-	@${dir}/notmuch-test $(OPTIONS)
+test:	all test-binaries fetch-test-databases
+	@${test_src_dir}/notmuch-test $(OPTIONS)
 
 check: test
 
 SRCS := $(SRCS) $(smtp_dummy_srcs)
 CLEAN += $(TEST_BINARIES) $(addsuffix .o,$(TEST_BINARIES)) \
 	 $(dir)/database-test.o \
-	 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
+	 $(dir)/corpus.mail.* $(dir)/test-results $(dir)/tmp.*
diff --git a/test/README b/test/README
index 79a9b1b..81a1c82 100644
--- a/test/README
+++ b/test/README
@@ -64,6 +64,14 @@ The following command-line options are available when running tests:
 	Pointing this argument at a tmpfs filesystem can improve the
 	speed of the test suite for some users.
 
+Certain tests require precomputed databases to complete. You can fetch these
+databases with
+
+	make download-test-databases
+
+If you do not download the test databases, the relevant tests will be
+skipped.
+
 When invoking the test suite via "make test" any of the above options
 can be specified as follows:
 
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
index cf9914e..e1e8ac5 100755
--- a/test/T530-upgrade.sh
+++ b/test/T530-upgrade.sh
@@ -3,7 +3,19 @@ test_description="database upgrade"
 
 . ./test-lib.sh
 
-tar zxf $TEST_DIRECTORY/test-databases/database-v1.tar.gz -C ${MAIL_DIR} --strip-components=1
+dbtarball=folders-v1.tar.xz
+
+# XXX: Accomplish the same with test lib helpers
+if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
+    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
+fi
+
+test_expect_success \
+    'database checksum' \
+    '( cd $TEST_DIRECTORY/test-databases &&
+       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
+
+tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
 
 test_begin_subtest "folder: search does not work with old database version"
 output=$(notmuch search folder:foo)
diff --git a/test/test-databases/.gitignore b/test/test-databases/.gitignore
new file mode 100644
index 0000000..b5624b7
--- /dev/null
+++ b/test/test-databases/.gitignore
@@ -0,0 +1 @@
+*.tar.xz
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
new file mode 100644
index 0000000..b250a8b
--- /dev/null
+++ b/test/test-databases/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C ../.. all
+
+.DEFAULT:
+	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
new file mode 100644
index 0000000..e777ada
--- /dev/null
+++ b/test/test-databases/Makefile.local
@@ -0,0 +1,14 @@
+# -*- makefile -*-
+
+TEST_DATABASE_MIRROR=http://notmuchmail.org/releases/test-databases
+
+dir := test/test-databases
+
+test_databases := $(dir)/folders-v1.tar.xz
+
+%.tar.xz:
+	wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@);
+
+download-test-databases: ${test_databases}
+
+DISTCLEAN := $(DISTCLEAN) ${test_databases}
diff --git a/test/test-databases/README b/test/test-databases/README
deleted file mode 100644
index af5defe..0000000
--- a/test/test-databases/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Notmuch test databases
-======================
-
-This directory contains pre-generated databases with their source
-corpus, chiefly for the purpose of testing database upgrade.
diff --git a/test/test-databases/database-v1.tar.gz b/test/test-databases/database-v1.tar.gz
deleted file mode 100644
index bb4df4d..0000000
Binary files a/test/test-databases/database-v1.tar.gz and /dev/null differ
diff --git a/test/test-databases/folders-v1.tar.xz.sha256 b/test/test-databases/folders-v1.tar.xz.sha256
new file mode 100644
index 0000000..01ad79d
--- /dev/null
+++ b/test/test-databases/folders-v1.tar.xz.sha256
@@ -0,0 +1 @@
+ace8a61216756b90a421e23d03910e1228bcb910e197c35e51e29f2cf57b37d9  folders-v1.tar.xz

Previous message (by thread): Re: v3 of boolean folder: patches

Thread:

David Bremner—v3 of boolean folder: patches [inbox, unread]
- David Bremner—[Patch v3 01/15] lib: refactor folder term update after filename removal [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 02/15] lib: add support for path: prefix searches [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 03/15] test: make insert test use the path: prefix [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 04/15] lib: make folder: prefix literal [inbox, notmuch::obsolete, notmuch::patch, unread]
  - Austin Clements—Re: [Patch v3 04/15] lib: make folder: prefix literal [inbox, unread]
    - Jani Nikula—Re: [Patch v3 04/15] lib: make folder: prefix literal [inbox, unread]
      - Austin Clements—Re: [Patch v3 04/15] lib: make folder: prefix literal [inbox, unread]
- David Bremner—[Patch v3 05/15] test: fix test for literal folder: search [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 08/15] test: add tests for the new boolean folder: and path: prefixes [inbox, notmuch::obsolete, notmuch::patch, unread]
  - Austin Clements—Re: [Patch v3 08/15] test: add tests for the new boolean folder: and path: prefixes [inbox, unread]
- David Bremner—[Patch v3 09/15] test: add database upgrade test from format version 1 to 2 [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 10/15] man: update man pages for folder: and path: search terms [inbox, notmuch::obsolete, notmuch::patch, unread]
  - Austin Clements—Re: [Patch v3 10/15] man: update man pages for folder: and path: search terms [inbox, unread]
- David Bremner—[Patch v3 11/15] man: try to clarify the folder: and path: vs. --output=files confusion [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 12/15] test: don't use $(dir) in recipes. [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 13/15] devel: add script to generate test databases [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 14/15] test: commit folders-v1.tar.xz checksum, ignore actual databases [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—[Patch v3 15/15] test: add machinery to download and verify databases [inbox, notmuch::obsolete, notmuch::patch, unread]
- David Bremner—Re: v3 of boolean folder: patches [inbox, unread]
  - W. Trevor King—Re: v3 of boolean folder: patches [inbox, signed, unread]
    - Jani Nikula—Re: v3 of boolean folder: patches [inbox, unread]
      - W. Trevor King—Re: v3 of boolean folder: patches [inbox, signed, unread]