From: Anton Khirnov <anton@khirnov.net>
To: notmuch@notmuchmail.org
Subject: [PATCH v5 6/6] doc/examples: add an attachment filtering program
Date: Sat, 15 Aug 2026 16:01:28 +0200
Message-ID: <20260815140127.26199-8-anton@khirnov.net>
In-Reply-To: <20260815140127.26199-2-anton@khirnov.net>
References: <20260815140127.26199-2-anton@khirnov.net>
MIME-Version: 1.0
Message-ID-Hash: VIRJ2GIALDXTMVFXZGVCG5C7D2WB5BEU
Precedence: list
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit

---
 doc/Makefile.local             |   4 +-
 doc/examples/README.filter.rst |  23 ++
 doc/examples/filter.py         | 385 +++++++++++++++++++++++++++++++++
 doc/man1/notmuch-config.rst    |   7 +
 4 files changed, 417 insertions(+), 2 deletions(-)
 create mode 100644 doc/examples/README.filter.rst
 create mode 100755 doc/examples/filter.py

diff --git a/doc/Makefile.local b/doc/Makefile.local
index 64e51475..c9a67f94 100644
--- a/doc/Makefile.local
+++ b/doc/Makefile.local
@@ -38,8 +38,8 @@ endif
 COPY_INFO1 := $(patsubst $(DOCBUILDDIR)/man/man1/%.1,$(DOCBUILDDIR)/texinfo/%.info,$(COPY_ROFF1))
 INFO_INFO_FILES := $(INFO_TEXI_FILES:.texi=.info) $(COPY_INFO1)
 
-EXAMPLES_FILES := doc/examples/firejail.profile
-EXAMPLES_FILES_EXEC := doc/examples/bwrap.sh
+EXAMPLES_FILES := doc/examples/firejail.profile doc/examples/README.filter.rst
+EXAMPLES_FILES_EXEC := doc/examples/bwrap.sh doc/examples/filter.py
 
 .PHONY: sphinx-html sphinx-texinfo sphinx-info
 
diff --git a/doc/examples/README.filter.rst b/doc/examples/README.filter.rst
new file mode 100644
index 00000000..afa5ca19
--- /dev/null
+++ b/doc/examples/README.filter.rst
@@ -0,0 +1,23 @@
+``filter.py`` is a Python script implementing an external attachment filter for
+notmuch indexing (see ``index.filter`` in :any:`notmuch-config(1)` for details).
+
+The filter supports the following data types:
+
+* PDF, converted with ``pdftotext``;
+
+* HTML, converted with ``w3m`` or ``elinks``;
+
+* office formats (MS Office, ODF, RTF), converted with ``soffice`` (part of
+  ``LibreOffice``);
+
+* a variety of text-like formats (scripts, source code, diffs, etc.) are passed
+  through unchanged;
+
+* archives supported by libarchive are read recursively; this requires the
+  libarchive Python bindings (``python3-libarchive-c`` package on Debian);
+
+* email messages, either standalone or in mbox format, are parsed and converted
+  recursively;
+
+To avoid recursion bombs, recursion depth for archives and email messages is by
+default limited to 8, use the ``--max-recurse`` switch to change it.
diff --git a/doc/examples/filter.py b/doc/examples/filter.py
new file mode 100755
index 00000000..52923e84
--- /dev/null
+++ b/doc/examples/filter.py
@@ -0,0 +1,385 @@
+#!/usr/bin/python3
+
+import argparse
+from collections import namedtuple
+from contextlib import ExitStack, closing
+import email
+import email.policy as email_policy
+import logging
+import mimetypes
+import os
+import re
+import shutil
+import subprocess
+import sys
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+
+try:
+    import libarchive
+except ImportError:
+    libarchive = None
+
+try:
+    import magic
+except ImportError:
+    magic = None
+
+MAX_RECURSE_DEFAULT = 8
+
+# archives, extracted with libarchive
+mimetypes_archive = {
+    'application/zip',
+    'application/x-zip',
+    'application/x-zip-compressed',
+    'application/rar',
+    'application/vnd.rar',
+    'application/x-rar',
+    'application/x-rar-compressed',
+    'application/x-7z-compressed',
+    'application/x-tar',
+    'application/x-tar-gz',
+    'application/x-compressed-tar',
+    'application/x-compressed',
+    'application/x-bzip-compressed-tar',
+    'application/x-gtar',
+    'application/x-gtar-compressed',
+    'application/x-tbz',
+    'application/vnd.ms-cab-compressed',
+    'application/x-ace-compressed',
+}
+
+# these may be compressed individual files rather than archives (handled
+# via libarchive's 'raw' format), but sometimes they are mistyped compressed
+# tar archives
+mimetypes_compressed = {
+    'application/gzip',
+    'application/x-gzip',
+    'application/x-gunzip',
+    'application/x-xz',
+    'application/x-bzip',
+    'application/x-bzip2',
+    'application/bzip2',
+    'application/x-bzip',
+}
+
+# types that can be indexed directly as text
+mimetypes_passthrough = {
+    'application/x-patch',
+    'application/x-diff',
+    'application/x-sh',
+    'application/x-shellscript',
+    'application/x-csh',
+    'application/x-ruby',
+    'application/x-tex',
+    'application/x-perl',
+    'application/x-httpd-php',
+    'application/javascript',
+    'application/x-javascript',
+    'application/emacs-lisp',
+    'application/json',
+    'application/x-subrip',
+    'application/x-config',
+    'application/xml',
+    'application/ics',
+    'application/x-po',
+    'application/x-info',
+    'application/x-texinfo',
+    'application/vnd.lotus-organizer',
+}
+
+# generic or unreliable mime types;
+# guess the real type from file contents or filename extension
+mimetypes_guess = {
+    'application/octet-stream',
+    'application/octetstream',
+    'octet/stream',
+    'application/mac-binary',
+    'application/macbinary',
+    'application/text',
+    'application/text-plain',
+    'application/x-download',
+    'application/force-download',
+}
+
+class InputData:
+    mime_type = None
+
+    logger    = None
+
+    # original filename associated with the data
+    # NOT what is returned by get_filename()
+    _orig_filename = None
+
+    _bytes      = None
+    _stream     = None
+    _tempfile   = None
+
+    def __init__(self, log_parent,
+                 bytes = None, stream = None,
+                 filename = None, mime_type = None):
+        if int(bytes is not None) + int(stream is not None) != 1:
+            raise ValueError('Exactly one of bytes/stream must be provided')
+
+        if bytes is not None:
+            self._bytes = bytes
+        elif stream is not None:
+            self._stream = stream
+
+        self._orig_filename = filename
+
+        if mime_type:
+            mime_type = mime_type.lower()
+
+        if mime_type is None or mime_type in mimetypes_guess:
+            mime_type = self._guess_mimetype(filename)
+
+        self.mime_type = mime_type
+
+        self.logger = log_parent.getChild('|' + (filename or self.mime_type) + '|')
+
+    def _guess_mimetype(self, filename = None):
+        mime_type = None
+
+        if magic:
+            t = magic.from_buffer(self.bytes(), mime = True)
+
+            if t != 'application/octet-stream':
+                mime_type = t
+
+        if not mime_type and filename:
+            t = mimetypes.guess_file_type(filename, strict = False)
+            if t:
+                mime_type = t[0]
+
+        return mime_type or 'application/octet-stream'
+
+    def get_filename(self):
+        if not self._tempfile:
+            ext = os.path.splitext(self._orig_filename)[1] if self._orig_filename else None
+
+            self._tempfile = NamedTemporaryFile(suffix = ext)
+            self._tempfile.write(self.bytes())
+            self._tempfile.flush()
+            os.fsync(self._tempfile.fileno())
+
+        return self._tempfile.name
+
+    def bytes(self):
+        if self._stream:
+            b = self._stream.read()
+            self._stream = None
+            self._bytes = (self._bytes or b'') + b
+        return self._bytes
+
+    def close(self):
+        if self._tempfile:
+            self._tempfile.close()
+            self._tempfile = None
+
+
+Handler = namedtuple('Handler', ('cmdline', 'need_tempdir', 'need_file_in'),
+                     defaults = (False, False))
+handler_map = {
+    'application/(x-)?pdf' : (
+        Handler(cmdline = ('pdftotext', '-', '-')),
+    ),
+    '(application|text)/html' : (
+        Handler(cmdline = ('elinks', '-force-html', '-dump')),
+        Handler(cmdline = ('w3m', '-T', 'text/html', '-dump')),
+    ),
+    r'application/(vnd\.openxmlformats-officedocument\..*|'
+                 r'vnd\.oasis\.opendocument\..*|'
+                 r'vnd\.ms-powerpoint|vnd\.ms-excel|'
+                  '(ms)?word|docx?|rtf)': (
+        Handler(cmdline = ('soffice', '-env:UserInstallation=file://{tempdir}', '--cat', '{file_in}'),
+                           need_tempdir = True, need_file_in = True),
+    ),
+}
+
+def get_handler(mime_type):
+    for pat, handlers in handler_map.items():
+        if not re.fullmatch(pat, mime_type):
+            continue
+
+        for handler in handlers:
+            if shutil.which(handler.cmdline[0]):
+                return handler
+
+def handle_default(data, outfile):
+    handler = get_handler(data.mime_type)
+
+    if not handler:
+        if (data.mime_type.startswith('text/') or
+            data.mime_type in mimetypes_passthrough):
+            data.logger.debug('passthrough: %s', data.mime_type)
+            outfile.write(data.bytes())
+            outfile.flush()
+        else:
+            data.logger.debug('no handler, skipping: %s', data.mime_type)
+
+        return
+
+    cmdline = list(handler.cmdline)
+
+    with ExitStack() as stack:
+        tempdir = None
+        if handler.need_tempdir:
+            tempdir = stack.enter_context(TemporaryDirectory())
+
+        args = { 'cwd' : tempdir }
+
+        file_in = None
+        if handler.need_file_in:
+            file_in = data.get_filename()
+        else:
+            args['input'] = data.bytes()
+
+        for i in range(len(cmdline)):
+            cmdline[i] = cmdline[i].format(file_in = file_in, tempdir = tempdir)
+
+        data.logger.debug('executing external handler: %s', ' '.join(cmdline))
+
+        ret = subprocess.run(cmdline, stdout = outfile, **args)
+
+def handle_archive(data, outfile, depth, max_depth):
+    # mime types in mimetypes_compressed may be either an archive (a compressed
+    # tarball) or a compressed invididual file;
+    # the following heuristic first tries the former, falling back on the latter
+    # on error
+    format_name = 'all'
+    if data.mime_type in mimetypes_compressed:
+        try:
+            with libarchive.memory_reader(data.bytes(), format_name = format_name):
+                pass
+        except libarchive.ArchiveError:
+            format_name = 'raw'
+
+    try:
+        with libarchive.memory_reader(data.bytes(),
+                                      format_name = format_name) as archive:
+            for entry in archive:
+                name = None
+                if str(archive.format_name) == 'raw':
+                    # for raw formats the filename is typically
+                    # something like foo.pdf.gz
+                    if filename:
+                        name = os.path.splitext(filename)[0]
+                elif entry.name:
+                    name = entry.name
+                    # this should not happen, but apparently sometimes does
+                    if isinstance(name, bytes):
+                        name = name.decode(errors = 'replace')
+
+                if name:
+                    outfile.write(name.encode(errors = 'replace') + b'\n')
+                    outfile.flush()
+
+                if not entry.isfile:
+                    continue
+
+                blocks = []
+                for b in entry.get_blocks():
+                    # force data copy, as the buffer underlying b may be reused
+                    blocks.append(b[:])
+
+                with closing(InputData(data.logger,
+                                       bytes = b''.join(blocks),
+                                       filename = name)) as data_e:
+                    handle_data(data_e, outfile, depth + 1, max_depth)
+                    outfile.write(b'\n')
+                    outfile.flush()
+    except libarchive.exception.ArchiveError as e:
+        data.logger.error('Error reading archive: %s', str(e))
+
+def handle_email(msg, outfile, depth, max_depth, logger):
+    headers = ('From', 'To', 'Subject', 'Date')
+    for header in headers:
+        val = msg.get(header)
+        if val:
+            line = '%s: %s\n' % (header, val)
+            outfile.write(line.encode(errors = 'replace'))
+            outfile.flush()
+
+    outfile.write(b'\n')
+    outfile.flush()
+
+    for part in msg.walk():
+        if not part.is_multipart():
+            payload = part.get_content()
+            if isinstance(payload, str):
+                payload = payload.encode(errors = 'replace')
+
+            data = InputData(logger,
+                             bytes = payload,
+                             filename = part.get_filename(),
+                             mime_type = part.get_content_type())
+
+            handle_data(data, outfile, depth + 1, max_depth)
+
+def handle_mbox(data, outfile, depth, max_depth):
+    # unfortunately we cannot use python's mailbox handling, because it is
+    # file-based; so instead we roll a very simple mbox parser below
+    lines = data.bytes().decode(errors = 'replace').splitlines()
+
+    def flush_msg(msg_lines):
+        if not msg_lines:
+            return
+
+        msg = email.message_from_string('\r\n'.join(msg_lines),
+                                        policy = email_policy.SMTP)
+        handle_email(msg, outfile, depth, max_depth, data.logger)
+
+    buffered_lines = []
+    for line in lines:
+        if line.startswith('From '):
+            outfile.write(line.encode() + b'\n')
+            outfile.flush()
+            flush_msg(buffered_lines)
+            continue
+
+        # add current line to current-message buffer
+        buffered_lines.append(line)
+
+    flush_msg(buffered_lines)
+
+def handle_data(data, outfile, depth, max_depth):
+    if depth > max_depth:
+        data.logger.error('Maximum recursion depth %d reached, discarding %s',
+                          max_depth, data.mime_type)
+        return False
+
+    if data.mime_type in mimetypes_compressed | mimetypes_archive:
+        if not libarchive:
+            data.logger.error('libarchive not available, cannot handle %s',
+                              data.mime_type)
+            return False
+
+        return handle_archive(data, outfile, depth, max_depth)
+    elif data.mime_type == 'application/mbox':
+        return handle_mbox(data, outfile, depth, max_depth)
+    elif data.mime_type == 'message/rfc822':
+        msg = email.message_from_bytes(data.bytes(), policy = email_policy.SMTP)
+        return handle_email(msg, outfile, depth, max_depth, data.logger)
+
+    return handle_default(data, outfile)
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('-v', '--verbose', action = 'count', default = 0)
+parser.add_argument('-q', '--quiet',   action = 'count', default = 0)
+parser.add_argument('-m', '--max-recurse', default = MAX_RECURSE_DEFAULT)
+
+args = parser.parse_args()
+
+msgid     = os.environ.get('NOTMUCH_FILTER_MESSAGE_ID', '<unknown>')
+mime_type = os.environ.get('NOTMUCH_FILTER_MIME_TYPE')
+filename  = os.environ.get('NOTMUCH_FILTER_FILENAME')
+
+logging.basicConfig(level = max(2 - args.verbose + args.quiet, 0) * 10)
+
+outfile = sys.stdout.buffer
+
+with closing(InputData(logging.getLogger(msgid),
+                       stream = sys.stdin.buffer,
+                       filename = filename, mime_type = mime_type)) as data:
+    handle_data(data, outfile, 0, args.max_recurse)
diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
index 29f8f77e..189639ec 100644
--- a/doc/man1/notmuch-config.rst
+++ b/doc/man1/notmuch-config.rst
@@ -191,6 +191,13 @@ paths are presumed relative to `$HOME` for items in section
    * :envvar:`NOTMUCH_FILTER_MESSAGE_ID` - the message ID, without enclosing
      angle brackets <>
 
+   A sample filtering script is shipped with notmuch documentation (typically in
+   ``/usr/share/doc/notmuch``) as ``examples/filter.py``. You may use it as e.g.:
+
+   * ``notmuch config set index.as_text text/html application/pdf``
+   * ``notmuch config set index.filter '<your sandbox>
+     <.../contrib/filter/filter.py>'``
+
    History: This configuration value was introduced in notmuch 0.41.
 
 .. nmconfig:: index.decrypt
-- 
2.47.3

_______________________________________________
notmuch mailing list -- notmuch@notmuchmail.org
To unsubscribe send an email to notmuch-leave@notmuchmail.org