[PATCH v2 3/4] cli/git-remote: add import command

Subject: [PATCH v2 3/4] cli/git-remote: add import command

Date: Sun, 8 Sep 2024 08:27:31 -0300

To: notmuch@notmuchmail.org

Cc:

From: David Bremner


The output in default.import is based on a modified version
of Felipe's git-remote-nm with Blake2 hashing replaced by SHA1
(for portability). This enable fetch/pull/clone, so test that as well.
---
 git-remote-notmuch.c                          | 108 +++++++++
 performance-test/M07-git-remote.sh            |  16 ++
 performance-test/T08-git-remote.sh            |  12 +
 test/T860-git-remote.sh                       |  69 ++++++
 .../git-remote.expected-output/default.import | 229 ++++++++++++++++++
 5 files changed, 434 insertions(+)
 create mode 100755 performance-test/M07-git-remote.sh
 create mode 100755 performance-test/T08-git-remote.sh
 create mode 100644 test/git-remote.expected-output/default.import

diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index 4afd198c..39ba6bf3 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -155,6 +155,30 @@ read_lastmod (const char *dir, char **uuid_out, unsigned long *counter_out)
 
 }
 
+static void
+store_lastmod (notmuch_database_t *notmuch, const char *dir)
+{
+    char *filename = NULL;
+    FILE *out;
+    unsigned long lastmod;
+    const char *uuid;
+
+    ASSERT (filename = talloc_asprintf (notmuch, "%s/lastmod", dir));
+
+    out = fopen (filename, "w");
+    ensure (out, "error opening %s for writing: %s", filename, strerror (errno));
+
+    lastmod = notmuch_database_get_revision (notmuch, &uuid);
+    ASSERT (fprintf (out, "%s\t%zu\n", uuid, lastmod) > 0);
+}
+
+static void
+write_data (const char *data)
+{
+    printf ("data %zu\n", strlen (data));
+    fputs (data, stdout);
+}
+
 static void
 cmd_capabilities ()
 {
@@ -174,6 +198,88 @@ cmd_list (notmuch_database_t *db, const char *uuid, unsigned long lastmod)
 	    equal_lastmod (uuid, lastmod, db_uuid, db_lastmod) ? " unchanged" : "");
 }
 
+static void
+cmd_import (notmuch_database_t *notmuch,
+	    const char *nm_dir,
+	    const char *uuid,
+	    unsigned long lastmod)
+{
+    const char *ident = NULL;
+    const char *lastmod_str = NULL;
+    notmuch_messages_t *messages;
+    notmuch_status_t status;
+    notmuch_query_t *query;
+    char *mid_buf = NULL;
+    size_t mid_buf_len = 0;
+
+    ident = talloc_asprintf (notmuch, "%s <%s> %zu +0000",
+			     notmuch_config_get (notmuch, NOTMUCH_CONFIG_USER_NAME),
+			     notmuch_config_get (notmuch, NOTMUCH_CONFIG_PRIMARY_EMAIL),
+			     time (NULL));
+
+
+    printf ("feature done\ncommit refs/notmuch/master\nmark :1\ncommitter %s\n", ident);
+
+    ASSERT (lastmod_str = talloc_asprintf (notmuch, "lastmod: %zu\n", lastmod));
+    write_data (lastmod_str);
+    if (uuid)
+	puts ("from refs/notmuch/master^0");
+    puts ("deleteall");
+
+    status = notmuch_query_create_with_syntax (notmuch,
+					       "",
+					       NOTMUCH_QUERY_SYNTAX_XAPIAN,
+					       &query);
+
+    if (print_status_database ("git-remote-nm", notmuch, status))
+	exit (EXIT_FAILURE);
+
+    if (debug_flags && strchr (debug_flags, 's'))
+	notmuch_query_set_sort (query, NOTMUCH_SORT_NEWEST_FIRST);
+    else
+	notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (print_status_query ("git-remote-nm", query, status))
+	exit (EXIT_FAILURE);
+
+    for (;
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	const char *tag_buf = "";
+	const char *mid;
+	const char *hash;
+	int ret;
+
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	mid = notmuch_message_get_message_id (message);
+
+	ret = hex_encode (notmuch, mid, &mid_buf, &mid_buf_len);
+	ensure (ret == HEX_SUCCESS, "failed to hex-encode message-id %s\n", mid);
+
+	/* we can't use _notmuch_sha1_from_string because we don't want
+	 * to include the null terminator */
+	g_autoptr (GChecksum) sha1 = NULL;
+	sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+	g_checksum_update (sha1, (const guchar *) mid, strlen (mid));
+	hash = g_checksum_get_string (sha1);
+	printf ("M 644 inline %2.2s/%2.2s/%s/tags\n", hash, hash + 2, mid_buf);
+
+	for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+	     notmuch_tags_valid (tags);
+	     notmuch_tags_move_to_next (tags)) {
+	    const char *tag_str = notmuch_tags_get (tags);
+	    ASSERT (tag_buf = talloc_asprintf (message, "%s%s\n", tag_buf, tag_str));
+	}
+	write_data (tag_buf);
+	notmuch_message_destroy (message);
+    }
+    puts ("");
+    puts ("done");
+    fflush (stdout);
+    store_lastmod (notmuch, nm_dir);
+}
+
 /* stubs since we cannot link with notmuch.o */
 const notmuch_opt_desc_t notmuch_shared_options[] = {
     { }
@@ -302,6 +408,8 @@ main (int argc, char *argv[])
 
 	if (STRNCMP_LITERAL (s, "capabilities") == 0)
 	    cmd_capabilities ();
+	else if (STRNCMP_LITERAL (s, "import") == 0)
+	    cmd_import (db, nm_dir, uuid, lastmod);
 	else if (STRNCMP_LITERAL (s, "list") == 0)
 	    cmd_list (db, uuid, lastmod);
 
diff --git a/performance-test/M07-git-remote.sh b/performance-test/M07-git-remote.sh
new file mode 100755
index 00000000..efce18a6
--- /dev/null
+++ b/performance-test/M07-git-remote.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+test_description='search'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+mkdir repo
+export GIT_DIR=`pwd`/repo
+
+memory_start
+
+echo "import refs/heads/master" > import.in
+
+memory_run "import" "git-remote-notmuch origin notmuch:// >import.out <import.in"
+
+memory_done
diff --git a/performance-test/T08-git-remote.sh b/performance-test/T08-git-remote.sh
new file mode 100755
index 00000000..df03d978
--- /dev/null
+++ b/performance-test/T08-git-remote.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+test_description='git-remote-notmuch'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run 'clone --bare' "git clone --quiet --bare -b master notmuch::default default.git"
+time_run 'clone' "git clone --quiet -b master notmuch:// repo"
+
+time_done
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 1a087611..97b8e4f9 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -43,4 +43,73 @@ cat <<EOF > EXPECTED
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
+test_begin_subtest 'import writes lastmod file'
+echo import | run_helper dummy-alias dummy-url > /dev/null
+lastmod=$(notmuch count --lastmod '*' | cut -f2-)
+test_expect_equal "${lastmod}" "$(cat < ${git_tmp}/notmuch/lastmod)"
+
+# note that this test must not be the first time import is run,
+# because it depends on the lastmod file
+test_begin_subtest 'import produces expected output'
+echo import | run_helper | notmuch_sanitize_git > OUTPUT
+test_expect_equal_file $EXPECTED/default.import OUTPUT
+
+test_begin_subtest "clone notmuch://"
+test_expect_success "git clone notmuch:// $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?config=notmuch-config"
+test_expect_success "git clone notmuch://?config=notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?profile=default"
+test_expect_success "git clone notmuch://?profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch://?config=notmuch-config&profile=default"
+test_expect_success "git clone notmuch://?config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://`pwd`/mail'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch://.../mail/?config=.../notmuch-config&profile=default'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config\&profile=default $(mktemp -d clone XXX)"
+
+test_begin_subtest 'clone notmuch://?path=.../mail/&config=.../notmuch-config&profile=default'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://?path=`pwd`/mail\&config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone notmuch::"
+test_expect_success "git clone notmuch:: $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch::`pwd`/mail'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest 'clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config'
+test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)"
+
+test_begin_subtest "clone has every message"
+git clone notmuch:: repo
+find repo -name tags -type f | sed -e s,repo/../../,id:, -e s,/tags$,, | sort > OUTPUT
+notmuch search --output=messages '*' | sort > EXPECTED
+test_expect_equal_file EXPECTED OUTPUT
+
+test_begin_subtest "pull get new tag"
+notmuch tag +zznew -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+unread
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
+test_begin_subtest "pull sees deletion"
+notmuch tag -unread -- id:4EFC743A.3060609@april.org
+git -C repo pull
+cat<<EOF >EXPECTED
+inbox
+zznew
+EOF
+test_expect_equal_file EXPECTED repo/$TAG_FILE
+
 test_done
diff --git a/test/git-remote.expected-output/default.import b/test/git-remote.expected-output/default.import
new file mode 100644
index 00000000..2d59861e
--- /dev/null
+++ b/test/git-remote.expected-output/default.import
@@ -0,0 +1,229 @@
+feature done
+commit refs/notmuch/master
+mark :1
+committer Notmuch Test Suite <test_suite@notmuchmail.org> TIMESTAMP TIMEZONE
+data 12
+lastmod: 53
+from refs/notmuch/master^0
+deleteall
+M 644 inline 87/b1/4EFC743A.3060609@april.org/tags
+data 13
+inbox
+unread
+M 644 inline 9a/a0/877h1wv7mg.fsf@inf-8657.int-evry.fr/tags
+data 13
+inbox
+unread
+M 644 inline a8/61/1258544095-16616-1-git-send-email-chris@chris-wilson.co.uk/tags
+data 13
+inbox
+unread
+M 644 inline 50/8c/877htoqdbo.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 50/69/878we4qdqf.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 94/67/87aaykqe24.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline bd/10/87bpj0qeng.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 27/84/87fx8cqf8v.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 3c/98/87hbssqfix.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline dd/a4/87iqd8qgiz.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 02/28/87k4xoqgnl.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 61/f6/87ocn0qh6d.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline 9a/6b/87pr7gqidx.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline cc/ab/867hto2p0t.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline ba/76/1258532999-9316-1-git-send-email-keithp@keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline ec/65/86aayk2rbj.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 15/a0/86d43g2w3y.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 54/e7/ddd65cda0911172214t60d22b63hcfeb5a19ab54a39b@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline 60/cc/86einw2xof.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags
+data 13
+inbox
+unread
+M 644 inline 68/39/736613.51770.qm@web113505.mail.gq1.yahoo.com/tags
+data 13
+inbox
+unread
+M 644 inline 1c/7b/1258520223-15328-1-git-send-email-jan@ryngle.com/tags
+data 13
+inbox
+unread
+M 644 inline 46/60/ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline 78/87/1258510940-7018-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline 8c/92/yunzl6kd1w0.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline cf/e1/yun1vjwegii.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline d3/55/yun3a4cegoa.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline fa/e4/1258509400-32511-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline c2/7f/1258506353-20352-1-git-send-email-stewart@flamingspork.com/tags
+data 13
+inbox
+unread
+M 644 inline 13/dd/20091118010116.GC25380@dottiness.seas.harvard.edu/tags
+data 31
+attachment
+inbox
+signed
+unread
+M 644 inline 97/cf/20091118005829.GB25380@dottiness.seas.harvard.edu/tags
+data 31
+attachment
+inbox
+signed
+unread
+M 644 inline cf/81/20091118005040.GA25380@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline ca/dc/cf0c4d610911171623q3e27a0adx802e47039b57604b@mail.gmail.com/tags
+data 24
+attachment
+inbox
+unread
+M 644 inline 42/c5/1258500222-32066-1-git-send-email-ingmar@exherbo.org/tags
+data 13
+inbox
+unread
+M 644 inline 47/c1/20091117232137.GA7669@griffis1.net/tags
+data 13
+inbox
+unread
+M 644 inline d7/7c/20091118002059.067214ed@hikari/tags
+data 20
+inbox
+signed
+unread
+M 644 inline ae/4b/1258498485-sup-142@elly/tags
+data 13
+inbox
+unread
+M 644 inline 41/fa/f35dbb950911171438k5df6eb56k77b6c0944e2e79ae@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline e4/81/f35dbb950911171435ieecd458o853c873e35f4be95@mail.gmail.com/tags
+data 13
+inbox
+unread
+M 644 inline fe/4b/1258496327-12086-1-git-send-email-jan@ryngle.com/tags
+data 13
+inbox
+unread
+M 644 inline 2c/3b/1258493565-13508-1-git-send-email-keithp@keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline 0c/a5/yunaayketfm.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline 21/a3/yunbpj0etua.fsf@aiko.keithp.com/tags
+data 13
+inbox
+unread
+M 644 inline b8/fa/1258491078-29658-1-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+M 644 inline 08/50/87fx8can9z.fsf@vertex.dottedmag/tags
+data 13
+inbox
+unread
+M 644 inline d3/76/20091117203301.GV3165@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline 52/bb/87lji4lx9v.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline df/26/cf0c4d610911171136h1713aa59w9cf9aa31f052ad0a@mail.gmail.com/tags
+data 24
+attachment
+inbox
+unread
+M 644 inline ab/f7/87iqd9rn3l.fsf@vertex.dottedmag/tags
+data 20
+inbox
+signed
+unread
+M 644 inline c5/19/20091117190054.GU3165@dottiness.seas.harvard.edu/tags
+data 20
+inbox
+signed
+unread
+M 644 inline 15/c2/87lji5cbwo.fsf@yoom.home.cworth.org/tags
+data 13
+inbox
+unread
+M 644 inline de/77/1258471718-6781-2-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+M 644 inline 77/76/1258471718-6781-1-git-send-email-dottedmag@dottedmag.net/tags
+data 13
+inbox
+unread
+
+done
-- 
2.43.0

_______________________________________________
notmuch mailing list -- notmuch@notmuchmail.org
To unsubscribe send an email to notmuch-leave@notmuchmail.org

Thread: