OpenSMTPD queue Support

__init__.py

@@ -1,8 +1,9 @@
 # coding: utf-8
 #
-#    Postfix queue control python tool (pymailq)
+#    Postfix/OpenSMTPD queue control python tool (pymailq)
 #
 #    Copyright (C) 2014 Denis Pompilio (jawa) <denis.pompilio@gmail.com>
+#                  2020 Yo                    <johan@nosd.in> - OpenSMTPD compatibility
 #
 #    This file is part of pymailq
 #
@@ -33,11 +34,12 @@ except ImportError:
 DEBUG = False
 
 #: Current version of the package as :class:`str`.
-VERSION = "0.9.0"
+VERSION = "0.10.0"
 
 #: Module configuration as :class:`dict`.
 CONFIG = {
     "core": {
+        "smtpd_type": "postfix",
         "postfix_spool": "/var/spool/postfix"
     },
     "commands": {
@@ -105,6 +107,8 @@ def load_config(cfg_file):
     if "core" in cfg.sections():
         if cfg.has_option("core", "postfix_spool"):
             CONFIG["core"]["postfix_spool"] = cfg.get("core", "postfix_spool")
+        if cfg.has_option("core", "smtpd_type"):
+            CONFIG["core"]["smtpd_type"] = cfg.get("core", "smtpd_type")
 
     if "commands" in cfg.sections():
         for key in cfg.options("commands"):

control.py

@@ -26,7 +26,7 @@ from pymailq import CONFIG, debug
 
 class QueueControl(object):
     """
-    Postfix queue control using postsuper command.
+    Postfix/openSMTPD queue control using postsuper/smtpctl command.
 
     The :class:`~control.QueueControl` instance defines the following
     attributes:
@@ -109,6 +109,7 @@ class QueueControl(object):
         # It should not be possible to inject commands, but who knows...
         # https://www.kevinlondon.com/2015/07/26/dangerous-python-functions.html
         # And consider the use of sh module: https://amoffat.github.io/sh/
+        if 'postfix' in CONFIG['core']['smtpd_type']:
             operation_cmd = self.get_operation_cmd(operation) + ['-']
             try:
                 child = subprocess.Popen(operation_cmd,
@@ -135,6 +136,29 @@ class QueueControl(object):
             
             return [line.strip() for line in stderr.decode().split('\n')]
              
+        # smtpctl do not use stdin, have to call the binary for each message :-(
+        else:
+            result = []
+            for msg in messages:
+                operation_cmd = self.get_operation_cmd(operation) + [msg.qid]
+                try:
+                    child = subprocess.Popen(operation_cmd,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.PIPE)
+                except EnvironmentError as exc:
+                    command_str = " ".join(operation_cmd)
+                    error_msg = "Unable to call '%s': %s" % (command_str, str(exc))
+                    raise RuntimeError(error_msg)
+                
+                stdout, stderr = child.communicate()
+                if not len(stdout):
+                    return [line.strip() for line in stderr.decode().split('\n')]
+                
+                result.append(stdout.decode())
+                
+            return result
+
+
     def delete_messages(self, messages):
         """
         Delete several messages from postfix mail queue.

store.py

@@ -3,6 +3,7 @@
 #    Postfix queue control python tool (pymailq)
 #
 #    Copyright (C) 2014 Denis Pompilio (jawa) <denis.pompilio@gmail.com>
+#                  2020 Yo                    <johan@nosd.in> - OpenSMTPD compatibility
 #
 #    This file is part of pymailq
 #
@@ -150,17 +151,21 @@ class Mail(object):
                 :ref:`pymailq-configuration`
     """
 
-    def __init__(self, mail_id, size=0, date=None, sender=""):
+    def __init__(self, mail_id, size=0, date=None, exp_date=None, last_delivery=None,
+                 nr_delivery=None, status=None, sender="", recipients=[], errors=[]):
         """Init method"""
         self.parsed = False
         self.parse_error = ""
         self.qid = mail_id
         self.date = date
-        self.status = ""
+        self.exp_date = exp_date
+        self.last_delivery = last_delivery
+        self.nr_delivery = nr_delivery
+        self.status = status
         self.size = int(size)
         self.sender = sender
-        self.recipients = []
+        self.recipients = recipients
-        self.errors = []
+        self.errors = errors
         self.head = MailHeaders()
 
         # Getting optionnal status from postqueue mail_id
@@ -169,6 +174,9 @@ class Mail(object):
             self.qid = mail_id[:-1]
         self.status = postqueue_status.get(mail_id[-1], "deferred")
 
+        # We want to keep compatibility, so "postfix" is considered default
+        if not 'smtpd_type' in CONFIG['core']: CONFIG['core']['smtpd_type'] = 'postfix'
+
     @property
     def postcat_cmd(self):
         """
@@ -210,7 +218,7 @@ class Mail(object):
         """
         Parse message content.
 
-        This method use Postfix mails content parsing command defined in
+        This method use Postfix/OpenSMTPD mails content parsing command defined in
         :attr:`~Mail.postcat_cmd` attribute.
         This command is runned using :class:`subprocess.Popen` instance.
 
@@ -222,8 +230,12 @@ class Mail(object):
 
             Postfix manual:
                 `postcat`_ -- Show Postfix queue file contents
+            OpenSMTPD manual:
+                `smtpctl`_ -- Show OpenSMTPD queue file contents
 
         """
+        if 'opensmtpd' in CONFIG['core']['smtpd_type']:
+            is_postfix = False
         # Reset parsing error message
         self.parse_error = ""
 
@@ -233,6 +245,9 @@ class Mail(object):
         stdout, stderr = child.communicate()
 
         if not len(stdout):
+            if 'opensmtpd' in CONFIG['core']['smtpd_type']:
+                self.parse_error = "\n".join(stderr.decode().split('\n'))
+            else:
                 # Ignore first 3 line on stderr which are:
                 #   postcat: name_mask: all
                 #   postcat: inet_addr_local: configured 3 IPv4 addresses
@@ -242,15 +257,17 @@ class Mail(object):
 
         raw_content = ""
         for line in stdout.decode('utf-8', errors='replace').split('\n'):
-            if self.size == 0 and line.startswith("message_size: "):
+            if is_postfix and self.size == 0 and line.startswith("message_size: "):
                 self.size = int(line[14:].strip().split()[0])
-            elif self.date is None and line.startswith("create_time: "):
+            elif is_postfix and self.date is None and line.startswith("create_time: "):
                 self.date = datetime.strptime(line[13:].strip(),
                                               "%a %b %d %H:%M:%S %Y")
-            elif not len(self.sender) and line.startswith("sender: "):
+            elif is_postfix and not len(self.sender) and line.startswith("sender: "):
                 self.sender = line[8:].strip()
-            elif line.startswith("regular_text: "):
+            elif is_postfix and line.startswith("regular_text: "):
                 raw_content += "%s\n" % (line[14:],)
+            elif is_postfix == False:
+                raw_content += "%s\n" % (line,)
 
         # For python2.7 compatibility, encode unicode to str
         if not isinstance(raw_content, str):
@@ -378,7 +395,9 @@ class PostqueueStore(object):
     postqueue_cmd = None
     spool_path = None
     postqueue_mailstatus = ['active', 'deferred', 'hold']
-    mail_id_re = re.compile(r"^([A-F0-9]{8,12}|[B-Zb-z0-9]{11,16})[*!]?$")
+    # Just a little change to match OpenSMTPD envelope id
+    #mail_id_re = re.compile(r"^([A-F0-9]{8,12}|[B-Zb-z0-9]{11,16})[*!]?$")
+    mail_id_re = re.compile(r"^([A-F0-9]{8,12}|[B-Za-z0-9]{11,16})[*!]?$")
     mail_addr_re = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+$")
     MailClass = Mail
     
@@ -392,6 +411,10 @@ class PostqueueStore(object):
         self.loaded_at = None
         self.mails = []
         
+        # We want to keep compatibility, so "postfix" is considered default
+        if not 'smtpd_type' in CONFIG['core']: CONFIG['core']['smtpd_type'] = 'postfix'
+
+
     @property
     @debug
     def known_headers(self):
@@ -430,8 +453,11 @@ class PostqueueStore(object):
                                  stdout=subprocess.PIPE)
         stdout = child.communicate()[0]
 
+        if 'postfix' in CONFIG['core']['smtpd_type']:
             # return lines list without the headers and footers
             return [line.strip() for line in stdout.decode().split('\n')][1:-2]
+        else:
+            return [line.strip() for line in stdout.decode().split('\n')]
 
     def _is_mail_id(self, mail_id):
         """
@@ -451,7 +477,7 @@ class PostqueueStore(object):
         return True
 
     @debug
-    def _load_from_postqueue(self, filename=None, parse=False):
+    def _load_from_postfix_postqueue(self, filename=None, parse=False):
         """
         Load content from postfix queue using postqueue command output.
 
@@ -547,6 +573,70 @@ class PostqueueStore(object):
             #print("parsing mails")
             [mail.parse() for mail in self.mails]
             
+    @debug
+    def _load_from_opensmtpd_postqueue(self, filename=None, parse=False):
+        """
+        Load content from opensmtpd queue using "smtpctl show queue" command output.
+
+        Output lines from :attr:`~store.PostqueueStore._get_postqueue_output`
+        are parsed to build :class:`~store.Mail` objects. Sample Postfix queue
+        control tool (`postqueue`_) output::
+
+            fab15d0efd0b9489|inet4|mta||ToddCookolnuo@psi.br|john.galt@post.lu|john.galt@post.lu|1606747552|1607093152|0|16|pending|8464|450 4.1.8 <ToddCookolnuo@psi.br>: Sender address rejected: Domain not found
+            e9eb7c58be75d090|inet4|mta||fyxt@snowcopolo.com|marco.polo@vuvuzela.br|marco.polo@vuvuzela.br|1606645251|1606990851|0|23|pending|15363|450 4.1.8 <fyxt@snowcopolo.com>: Sender address rejected: Domain not found
+
+        Parsing rules are pretty simple
+
+        Optionnal argument ``filename`` can be set with a file containing
+        output of the `postqueue`_ command. In this case, output lines of
+        `postqueue`_ command are directly read from ``filename`` and parsed,
+        the `postqueue`_ command is never used.
+
+        Optionnal argument ``parse`` controls whether mails are parsed or not.
+        This is useful to load every known mail headers for later filtering.
+
+        :param str filename: File to load mails from
+        :param bool parse: Controls whether loaded mails are parsed or not.
+        """
+        if filename is None:
+            postqueue_output = self._get_postqueue_output()
+        else:
+            postqueue_output = open(filename).readlines()
+
+        mail = None
+        for line in postqueue_output:
+            line = line.strip()
+
+            # Headers and footers start with dash (-)
+            #if line.startswith('-'):
+            #    continue
+            # Mails are blank line separated
+            #if not len(line):
+            #    continue
+
+            fields = line.split('|')
+            #if "(" == fields[0][0]:
+                # Store error message without parenthesis: [1:-1]
+                # gathered errors must be associated with specific recipients
+                # TODO: change recipients or errors structures to link these
+                #       objects together.
+                
+
+            if self._is_mail_id(fields[0]):
+                date = datetime.fromtimestamp(int(fields[7]))
+                exp_date = datetime.fromtimestamp(int(fields[8]))
+                lst_date = datetime.fromtimestamp(int(fields[9]))
+                nr_delivery = fields[10]
+                mail = self.MailClass(fields[0], sender=fields[4], recipients=[fields[5]],
+                                      date=date, exp_date=exp_date, last_delivery=lst_date,
+                                      nr_delivery=nr_delivery, status=fields[11],
+                                      errors=[fields[13]])
+                self.mails.append(mail)
+
+        if parse:
+            #print("parsing mails")
+            [mail.parse() for mail in self.mails]
+
     @debug
     def _load_from_spool(self, parse=True):
         """
@@ -587,13 +677,13 @@ class PostqueueStore(object):
     @debug
     def load(self, method="postqueue", filename=None, parse=False):
         """
-        Load content from postfix mails queue.
+        Load content from postfix/OpenSMTPD mails queue.
 
-        Mails are loaded using postqueue command line tool or reading directly
+        Mails are loaded using postqueue/smtpctl command line tool or reading
-        from spool. The optionnal argument, if present, is a method string and
+        directly from spool. The optionnal argument, if present, is a 
-        specifies the method used to gather mails informations. By default,
+        method string and specifies the method used to gather mails informations.
-        method is set to ``"postqueue"`` and the standard Postfix queue
+        By default,  method is set to ``"postqueue"`` and the standard queue 
-        control tool: `postqueue`_ is used.
+        control tool: `postqueue`_ or `smtpctl show queue`is used.
 
         Optionnal argument ``parse`` controls whether mails are parsed or not.
         This is useful to load every known mail headers for later filtering.
@@ -610,10 +700,13 @@ class PostqueueStore(object):
         gc.collect()
 
         self.mails = []
+        
+        smtpd_type = CONFIG['core']['smtpd_type']
+        
         if filename is None:
-            getattr(self, "_load_from_{0}".format(method))(parse=parse)
+            getattr(self, "_load_from_{0}_{1}".format(smtpd_type, method))(parse=parse)
         else:
-            getattr(self, "_load_from_{0}".format(method))(filename, parse)
+            getattr(self, "_load_from_{0}_{1}".format(smtpd_type, method))(filename, parse)
         self.loaded_at = datetime.now()
 
     @debug