This is just documentation and the config/indexopts interfaces, actual
implementation will be added in following commits.
A basic performance test is also added here, to easily verify that the
following changes do not cost anything when not enabled.
---
bindings/python-cffi/notmuch2/_build.py | 5 +++
bindings/python-cffi/notmuch2/_database.py | 21 +++++++++++++
bindings/python-cffi/tests/test_database.py | 16 ++++++++++
doc/man1/notmuch-config.rst | 27 ++++++++++++++++
lib/config.cc | 3 ++
lib/indexopts.c | 34 +++++++++++++++++++++
lib/notmuch.h | 23 ++++++++++++++
performance-test/T09-attachment-index.sh | 18 +++++++++++
test/T590-libconfig.sh | 5 +++
9 files changed, 152 insertions(+)
create mode 100755 performance-test/T09-attachment-index.sh
diff --git a/bindings/python-cffi/notmuch2/_build.py b/bindings/python-cffi/notmuch2/_build.py
index 2f3152c6..f918f96f 100644
--- a/bindings/python-cffi/notmuch2/_build.py
+++ b/bindings/python-cffi/notmuch2/_build.py
@@ -328,6 +328,11 @@ ffibuilder.cdef(
notmuch_decryption_policy_t decrypt_policy);
notmuch_decryption_policy_t
notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts);
+ notmuch_status_t
+ notmuch_indexopts_set_filter (notmuch_indexopts_t *indexopts,
+ const char *filter_cmd);
+ const char *
+ notmuch_indexopts_get_filter (const notmuch_indexopts_t *indexopts);
void
notmuch_indexopts_destroy (notmuch_indexopts_t *options);
diff --git a/bindings/python-cffi/notmuch2/_database.py b/bindings/python-cffi/notmuch2/_database.py
index ba389a42..78a43d72 100644
--- a/bindings/python-cffi/notmuch2/_database.py
+++ b/bindings/python-cffi/notmuch2/_database.py
@@ -878,3 +878,24 @@ class IndexOptions(base.NotmuchObject):
self._opts_p, val.value)
if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
raise errors.NotmuchError(ret)
+
+ @property
+ def filter_cmd(self):
+ """Filtering program to extract text from non-text MIME parts.
+
+ CAUTION: improper use of this option may lead to remote code
+ execution on the user's machine. See the `index.filter` section
+ in :any:`notmuch-config(1)` for details. Make sure you read and
+ understand it before setting this property.
+
+ The value is a string that is split in a shell-like manner and
+ executed.
+ """
+ raw = capi.lib.notmuch_indexopts_get_filter(self._opts_p)
+ return base.BinString.from_cffi(raw)
+
+ @filter_cmd.setter
+ def filter_cmd(self, val):
+ ret = capi.lib.notmuch_indexopts_set_filter(self._opts_p, val)
+ if ret != capi.lib.NOTMUCH_STATUS_SUCCESS:
+ raise errors.NotmuchError(ret)
diff --git a/bindings/python-cffi/tests/test_database.py b/bindings/python-cffi/tests/test_database.py
index ece838ee..6173d8df 100644
--- a/bindings/python-cffi/tests/test_database.py
+++ b/bindings/python-cffi/tests/test_database.py
@@ -114,6 +114,22 @@ class TestCreate:
assert db.needs_upgrade in (True, False)
+class TestIndexOptions:
+
+ @pytest.fixture
+ def indexopts(self, db):
+ return db.default_indexopts()
+
+ def test_filter_cmd(self, indexopts):
+ filter_cmd = b'some filter command'
+ indexopts.filter_cmd = filter_cmd
+ assert bytes(indexopts.filter_cmd) == filter_cmd
+
+ def test_filter_cmd_invalid(self, indexopts):
+ with pytest.raises(TypeError):
+ indexopts.filter_cmd = {}
+
+
class TestAtomic:
def test_exit_early(self, db):
diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 6a63e457..b8c13462 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -154,6 +154,33 @@ paths are presumed relative to `$HOME` for items in section
History: This configuration value was introduced in notmuch 0.38.
+.. nmconfig:: index.filter
+
+ Filtering program to convert non-text MIME parts to a text
+ representation for indexing. Will only be applied to those parts that
+ match ``index.as_text``.
+
+ CAUTION: It is very common for hostile actors to send emails with
+ crafted attachments that exploit bugs in common parsing libraries. It
+ is thus IMPERATIVE that your filtering program uses some sort of a
+ sandboxing mechanism, so that it cannot be subverted to attack your
+ system or steal your data.
+
+ The filter is a commandline split in a POSIX shell-like manner (without
+ actually invoking the shell, so shell expansions are not performed and shell
+ constructs are not allowed) and executed. The payload of the MIME part to be
+ filtered will be supplied on its `stdin`, it is expected to write the text
+ output to its `stdout`. The following environment variables will be set:
+
+ * :envvar:`NOTMUCH_FILTER_MIME_TYPE` - the ``type/subtype`` part of the
+ "content-type" header
+ * :envvar:`NOTMUCH_FILTER_FILENAME` - the file name associated with the
+ attachment, if present
+ * :envvar:`NOTMUCH_FILTER_MESSAGE_ID` - the message ID, without enclosing
+ angle brackets <>
+
+ History: This configuration value was introduced in notmuch 0.41.
+
.. nmconfig:: index.decrypt
Policy for decrypting encrypted messages during indexing. Must be
diff --git a/lib/config.cc b/lib/config.cc
index d231c893..9bbbe467 100644
--- a/lib/config.cc
+++ b/lib/config.cc
@@ -614,6 +614,8 @@ _notmuch_config_key_to_string (notmuch_config_key_t key)
return "git.metadata_prefix";
case NOTMUCH_CONFIG_GIT_REF:
return "git.ref";
+ case NOTMUCH_CONFIG_INDEX_FILTER:
+ return "index.filter";
default:
return NULL;
}
@@ -675,6 +677,7 @@ _notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key)
case NOTMUCH_CONFIG_HOOK_DIR:
case NOTMUCH_CONFIG_BACKUP_DIR:
case NOTMUCH_CONFIG_OTHER_EMAIL:
+ case NOTMUCH_CONFIG_INDEX_FILTER:
return NULL;
default:
case NOTMUCH_CONFIG_LAST:
diff --git a/lib/indexopts.c b/lib/indexopts.c
index da378700..8e497ba8 100644
--- a/lib/indexopts.c
+++ b/lib/indexopts.c
@@ -22,6 +22,8 @@
struct _notmuch_indexopts {
_notmuch_crypto_t crypto;
+
+ char *filter_cmd;
};
notmuch_indexopts_t *
@@ -53,6 +55,21 @@ notmuch_database_get_default_indexopts (notmuch_database_t *db)
}
free (decrypt_policy);
+
+ char *filter_cmd;
+
+ err = notmuch_database_get_config (db, "index.filter", &filter_cmd);
+ if (err)
+ goto FAIL;
+
+ if (filter_cmd && *filter_cmd) {
+ ret->filter_cmd = talloc_strdup (ret, filter_cmd);
+ free (filter_cmd);
+ if (!ret->filter_cmd)
+ goto FAIL;
+ } else
+ free (filter_cmd);
+
return ret;
FAIL:
@@ -78,6 +95,23 @@ notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts)
return indexopts->crypto.decrypt;
}
+notmuch_status_t
+notmuch_indexopts_set_filter (notmuch_indexopts_t *indexopts,
+ const char *filter_cmd)
+{
+ talloc_free (indexopts->filter_cmd);
+ indexopts->filter_cmd = talloc_strdup (indexopts, filter_cmd);
+ if (!indexopts->filter_cmd)
+ return NOTMUCH_STATUS_OUT_OF_MEMORY;
+ return NOTMUCH_STATUS_SUCCESS;
+}
+
+const char *
+notmuch_indexopts_get_filter (const notmuch_indexopts_t *indexopts)
+{
+ return indexopts ? indexopts->filter_cmd : NULL;
+}
+
void
notmuch_indexopts_destroy (notmuch_indexopts_t *indexopts)
{
diff --git a/lib/notmuch.h b/lib/notmuch.h
index c403a348..33c7ed8c 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -2644,6 +2644,7 @@ typedef enum {
NOTMUCH_CONFIG_GIT_FAIL_ON_MISSING,
NOTMUCH_CONFIG_GIT_METADATA_PREFIX,
NOTMUCH_CONFIG_GIT_REF,
+ NOTMUCH_CONFIG_INDEX_FILTER,
NOTMUCH_CONFIG_LAST
} notmuch_config_key_t;
@@ -2931,6 +2932,28 @@ notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts);
*
* @since libnotmuch 5.1 (notmuch 0.26)
*/
+
+/**
+ * Set a filtering program to extract text from non-text MIME parts.
+ *
+ * CAUTION: improper use of this option may lead to remote code
+ * execution on the user's machine. See the `index.filter` section in
+ * `notmuch-config(1)` for details. Make sure you read and understand
+ * it before calling this function.
+ */
+notmuch_status_t
+notmuch_indexopts_set_filter (notmuch_indexopts_t *indexopts,
+ const char *filter_cmd);
+
+/**
+ * Return currently configured filtering program, or NULL if one is not
+ * configured.
+ *
+ * see notmuch_indexopts_set_filter
+ */
+const char *
+notmuch_indexopts_get_filter (const notmuch_indexopts_t *indexopts);
+
void
notmuch_indexopts_destroy (notmuch_indexopts_t *options);
diff --git a/performance-test/T09-attachment-index.sh b/performance-test/T09-attachment-index.sh
new file mode 100755
index 00000000..e2a5b4e3
--- /dev/null
+++ b/performance-test/T09-attachment-index.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+test_description='attachment content indexing'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+notmuch config set index.as_text ".*"
+
+time_run 'reindex w/o filter' "notmuch reindex tag:attachment"
+time_run 'reindex w/o filter' "notmuch reindex tag:attachment"
+
+notmuch config set index.filter "/bin/true"
+
+time_run 'reindex with filter' "notmuch reindex tag:attachment"
+
+time_done
diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
index e893fd1f..554da268 100755
--- a/test/T590-libconfig.sh
+++ b/test/T590-libconfig.sh
@@ -446,6 +446,7 @@ cat <<'EOF' >EXPECTED
16: 'true'
17: '_notmuch_metadata'
18: 'refs/heads/master'
+19: 'NULL'
== stderr ==
EOF
unset MAILDIR
@@ -734,6 +735,7 @@ notmuch config set search.authors_matched_separator "| "
notmuch config set search.authors_separator ", "
notmuch config set new.ignore "sekrit_junk"
notmuch config set index.as_text "text/"
+notmuch config set index.filter "filter"
cat c_head2 - c_tail <<'EOF' | test_C ${MAIL_DIR} %NULL% %NULL%
{
notmuch_config_key_t key;
@@ -766,6 +768,7 @@ cat <<'EOF' >EXPECTED
16: 'true'
17: '_notmuch_metadata'
18: 'refs/heads/master'
+19: 'filter'
== stderr ==
EOF
test_expect_equal_file EXPECTED OUTPUT
@@ -806,6 +809,7 @@ cat <<'EOF' >EXPECTED
16: 'true'
17: '_notmuch_metadata'
18: 'refs/heads/master'
+19: 'NULL'
== stderr ==
EOF
test_expect_equal_file EXPECTED OUTPUT.clean
@@ -881,6 +885,7 @@ git.fail_on_missing true
git.metadata_prefix _notmuch_metadata
git.ref refs/heads/master
index.as_text text/
+index.filter filter
key with spaces value, with, spaces!
maildir.synchronize_flags true
new.ignore sekrit_junk
--
2.47.3
_______________________________________________
notmuch mailing list -- notmuch@notmuchmail.org
To unsubscribe send an email to notmuch-leave@notmuchmail.org