830 lines
31 KiB
Python
830 lines
31 KiB
Python
# coding: utf-8
|
|
#
|
|
# Postfix queue control python tool (pymailq)
|
|
#
|
|
# Copyright (C) 2014 Denis Pompilio (jawa) <denis.pompilio@gmail.com>
|
|
# 2020 Yo <johan@nosd.in> - OpenSMTPD compatibility
|
|
#
|
|
# This file is part of pymailq
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
import sys
|
|
import os
|
|
import gc
|
|
import re
|
|
import subprocess
|
|
import email
|
|
from email import header
|
|
from collections import Counter
|
|
from datetime import datetime, timedelta
|
|
from pymailq import CONFIG, debug
|
|
|
|
|
|
class MailHeaders(object):
|
|
"""
|
|
Simple object to store mail headers.
|
|
|
|
Object's attributes are dynamically created when parent :class:`~store.Mail`
|
|
object's method :meth:`~store.Mail.parse` is called. Those attributes are
|
|
retrieved with help of :func:`~email.message_from_string` method provided
|
|
by the :mod:`email` module.
|
|
|
|
Standard RFC *822-style* mail headers becomes attributes including but not
|
|
limited to:
|
|
|
|
- :mailheader:`Received`
|
|
- :mailheader:`From`
|
|
- :mailheader:`To`
|
|
- :mailheader:`Cc`
|
|
- :mailheader:`Bcc`
|
|
- :mailheader:`Sender`
|
|
- :mailheader:`Reply-To`
|
|
- :mailheader:`Subject`
|
|
|
|
Case is kept while creating attribute and access will be made with
|
|
:attr:`Mail.From` or :attr:`Mail.Received` for example. All those
|
|
attributes will return *list* of values.
|
|
|
|
.. seealso::
|
|
|
|
Python modules:
|
|
:mod:`email` -- An email and MIME handling package
|
|
|
|
:class:`email.message.Message` -- Representing an email message
|
|
|
|
:rfc:`822` -- Standard for ARPA Internet Text Messages
|
|
"""
|
|
|
|
|
|
class Mail(object):
|
|
"""
|
|
Simple object to manipulate email messages.
|
|
|
|
This class provides the necessary methods to load and inspect mails
|
|
content. This object functionnalities are mainly based on :mod:`email`
|
|
module's provided class and methods. However,
|
|
:class:`email.message.Message` instance's stored informations are
|
|
extracted to extend :class:`~store.Mail` instances attributes.
|
|
|
|
Initialization of :class:`~store.Mail` instances are made the following
|
|
way:
|
|
|
|
:param str mail_id: Mail's queue ID string
|
|
:param int size: Mail size in Bytes (Default: ``0``)
|
|
:param datetime.datetime date: Acceptance date and time in mails queue.
|
|
(Default: :data:`None`)
|
|
:param str sender: Mail sender string as seen in mails queue.
|
|
(Default: empty :func:`str`)
|
|
|
|
The :class:`~pymailq.Mail` class defines the following attributes:
|
|
|
|
.. attribute:: qid
|
|
|
|
Mail Postfix queue ID string, validated by
|
|
:meth:`~store.PostqueueStore._is_mail_id` method.
|
|
|
|
.. attribute:: size
|
|
|
|
Mail size in bytes. Expected type is :func:`int`.
|
|
|
|
.. attribute:: parsed
|
|
|
|
:func:`bool` value to track if mail's content has been loaded from
|
|
corresponding spool file.
|
|
|
|
.. attribute:: parse_error
|
|
|
|
Last encountered parse error message :func:`str`.
|
|
|
|
.. attribute:: date
|
|
|
|
:class:`~datetime.datetime` object of acceptance date and time in
|
|
mails queue.
|
|
|
|
.. attribute:: status
|
|
|
|
Mail's queue status :func:`str`.
|
|
|
|
.. attribute:: sender
|
|
|
|
Mail's sender :func:`str` as seen in mails queue.
|
|
|
|
.. attribute:: recipients
|
|
|
|
Recipients :func:`list` as seen in mails queue.
|
|
|
|
.. attribute:: errors
|
|
|
|
Mail deliver errors :func:`list` as seen in mails queue.
|
|
|
|
.. attribute:: head
|
|
|
|
Mail's headers :class:`~store.MailHeaders` structure.
|
|
|
|
.. attribute:: raw_content
|
|
|
|
Mail's raw content :func:`str`.
|
|
|
|
.. attribute:: postcat_cmd
|
|
|
|
This property use Postfix mails content parsing command defined in
|
|
:attr:`pymailq.CONFIG` attribute under the key 'cat_message'.
|
|
Command and arguments list is build on call with the configuration
|
|
data.
|
|
|
|
.. seealso::
|
|
|
|
:ref:`pymailq-configuration`
|
|
"""
|
|
|
|
def __init__(self, mail_id, size=0, date=None, exp_date=None, last_delivery=None,
|
|
nr_delivery=None, status=None, sender="", recipients=[], errors=[]):
|
|
"""Init method"""
|
|
self.parsed = False
|
|
self.parse_error = ""
|
|
self.qid = mail_id
|
|
self.date = date
|
|
self.exp_date = exp_date
|
|
self.last_delivery = last_delivery
|
|
self.nr_delivery = nr_delivery
|
|
self.status = status
|
|
self.size = int(size)
|
|
self.sender = sender
|
|
self.recipients = recipients
|
|
self.errors = errors
|
|
self.head = MailHeaders()
|
|
|
|
# Getting optionnal status from postqueue mail_id
|
|
postqueue_status = {'*': "active", '!': "hold"}
|
|
if mail_id[-1] in postqueue_status:
|
|
self.qid = mail_id[:-1]
|
|
self.status = postqueue_status.get(mail_id[-1], "deferred")
|
|
|
|
# We want to keep compatibility, so "postfix" is considered default
|
|
if not 'smtpd_type' in CONFIG['core']: CONFIG['core']['smtpd_type'] = 'postfix'
|
|
|
|
@property
|
|
def postcat_cmd(self):
|
|
"""
|
|
Get the cat_message command from configuration
|
|
:return: Command as :class:`list`
|
|
"""
|
|
postcat_cmd = CONFIG['commands']['cat_message'] + [self.qid]
|
|
if CONFIG['commands']['use_sudo']:
|
|
postcat_cmd.insert(0, 'sudo')
|
|
return postcat_cmd
|
|
|
|
def show(self):
|
|
"""
|
|
Return mail detailled representation for printing
|
|
|
|
:return: Representation as :class:`str`
|
|
"""
|
|
output = "=== Mail %s ===\n" % (self.qid,)
|
|
for attr in sorted(dir(self.head)):
|
|
if attr.startswith("_"):
|
|
continue
|
|
|
|
value = getattr(self.head, attr)
|
|
if not isinstance(value, str):
|
|
value = ", ".join(value)
|
|
|
|
if attr == "Subject":
|
|
#print(attr, value)
|
|
value, enc = header.decode_header(value)[0]
|
|
#print(enc, attr, value)
|
|
if sys.version_info[0] == 2:
|
|
value = value.decode(enc) if enc else unicode(value)
|
|
|
|
output += "%s: %s\n" % (attr, value)
|
|
return output
|
|
|
|
@debug
|
|
def parse(self):
|
|
"""
|
|
Parse message content.
|
|
|
|
This method use Postfix/OpenSMTPD mails content parsing command defined in
|
|
:attr:`~Mail.postcat_cmd` attribute.
|
|
This command is runned using :class:`subprocess.Popen` instance.
|
|
|
|
Parsed headers become attributes and are retrieved with help of
|
|
:func:`~email.message_from_string` function provided by the
|
|
:mod:`email` module.
|
|
|
|
.. seealso::
|
|
|
|
Postfix manual:
|
|
`postcat`_ -- Show Postfix queue file contents
|
|
OpenSMTPD manual:
|
|
`smtpctl`_ -- Show OpenSMTPD queue file contents
|
|
|
|
"""
|
|
if 'opensmtpd' in CONFIG['core']['smtpd_type']:
|
|
is_postfix = False
|
|
else:
|
|
is_postfix = True
|
|
# Reset parsing error message
|
|
self.parse_error = ""
|
|
|
|
child = subprocess.Popen(self.postcat_cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
stdout, stderr = child.communicate()
|
|
|
|
if not len(stdout):
|
|
if 'opensmtpd' in CONFIG['core']['smtpd_type']:
|
|
self.parse_error = "\n".join(stderr.decode().split('\n'))
|
|
else:
|
|
# Ignore first 3 line on stderr which are:
|
|
# postcat: name_mask: all
|
|
# postcat: inet_addr_local: configured 3 IPv4 addresses
|
|
# postcat: inet_addr_local: configured 3 IPv6 addresses
|
|
self.parse_error = "\n".join(stderr.decode().split('\n')[3:])
|
|
return
|
|
|
|
raw_content = ""
|
|
for line in stdout.decode('utf-8', errors='replace').split('\n'):
|
|
if is_postfix and self.size == 0 and line.startswith("message_size: "):
|
|
self.size = int(line[14:].strip().split()[0])
|
|
elif is_postfix and self.date is None and line.startswith("create_time: "):
|
|
self.date = datetime.strptime(line[13:].strip(),
|
|
"%a %b %d %H:%M:%S %Y")
|
|
elif is_postfix and not len(self.sender) and line.startswith("sender: "):
|
|
self.sender = line[8:].strip()
|
|
elif is_postfix and line.startswith("regular_text: "):
|
|
raw_content += "%s\n" % (line[14:],)
|
|
elif is_postfix == False:
|
|
raw_content += "%s\n" % (line,)
|
|
|
|
# For python2.7 compatibility, encode unicode to str
|
|
if not isinstance(raw_content, str):
|
|
raw_content = raw_content.encode('utf-8')
|
|
|
|
message = email.message_from_string(raw_content)
|
|
|
|
for mailheader in set(message.keys()):
|
|
value = message.get_all(mailheader)
|
|
setattr(self.head, mailheader, value)
|
|
|
|
self.raw_content = raw_content
|
|
|
|
self.parsed = True
|
|
|
|
@debug
|
|
def dump(self):
|
|
"""
|
|
Dump mail's gathered informations to a :class:`dict` object.
|
|
|
|
Mails informations are splitted in two parts in dictionnary.
|
|
``postqueue`` key regroups every informations directly gathered from
|
|
Postfix queue, while ``headers`` regroups :class:`~store.MailHeaders`
|
|
attributes converted from mail content with the
|
|
:meth:`~store.Mail.parse` method.
|
|
|
|
If mail has not been parsed with the :meth:`~store.Mail.parse` method,
|
|
informations under the ``headers`` key will be empty.
|
|
|
|
:return: Mail gathered informations
|
|
:rtype: :class:`dict`
|
|
"""
|
|
datas = {'postqueue': {},
|
|
'headers': {}}
|
|
|
|
for attr in self.__dict__:
|
|
if attr[0] != "_" and attr != 'head':
|
|
datas['postqueue'].update({attr: getattr(self, attr)})
|
|
|
|
for mailheader in self.head.__dict__:
|
|
if mailheader[0] != "_":
|
|
datas['headers'].update(
|
|
{mailheader: getattr(self.head, mailheader)}
|
|
)
|
|
|
|
return datas
|
|
|
|
|
|
class PostqueueStore(object):
|
|
"""
|
|
Postfix mails queue informations storage.
|
|
|
|
The :class:`~store.PostqueueStore` provides methods to load Postfix
|
|
queued mails informations into Python structures. Thoses structures are
|
|
based on :class:`~store.Mail` and :class:`~store.MailHeaders` classes
|
|
which can be processed by a :class:`~selector.MailSelector` instance.
|
|
|
|
The :class:`~store.PostqueueStore` class defines the following attributes:
|
|
|
|
.. attribute:: mails
|
|
|
|
Loaded :class:`MailClass` objects :func:`list`.
|
|
|
|
.. attribute:: loaded_at
|
|
|
|
:class:`datetime.datetime` instance to store load date and time
|
|
informations, useful for datas deprecation tracking. Updated on
|
|
:meth:`~store.PostqueueStore.load` call with
|
|
:meth:`datetime.datetime.now` method.
|
|
|
|
.. attribute:: postqueue_cmd
|
|
|
|
:obj:`list` object to store Postfix command and arguments to view
|
|
the mails queue content. This property use Postfix mails content
|
|
parsing command defined in :attr:`pymailq.CONFIG` attribute under
|
|
the key 'list_queue'. Command and arguments list is build on call
|
|
with the configuration data.
|
|
|
|
.. attribute:: spool_path
|
|
|
|
Postfix spool path string.
|
|
Default is ``"/var/spool/postfix"``.
|
|
|
|
.. attribute:: postqueue_mailstatus
|
|
|
|
Postfix known queued mail status list.
|
|
Default is ``['active', 'deferred', 'hold']``.
|
|
|
|
.. attribute:: mail_id_re
|
|
|
|
Python compiled regular expression object (:class:`re.RegexObject`)
|
|
provided by :func:`re.compile` method to match postfix IDs.
|
|
Recognized IDs are either:
|
|
- hexadecimals, 8 to 12 chars length (regular queue IDs)
|
|
- encoded in a 52-character alphabet, 11 to 16 chars length
|
|
(long queue IDs)
|
|
They can be followed with ``*`` or ``!``.
|
|
Default used regular expression is:
|
|
``r"^([A-F0-9]{8,12}|[B-Zb-z0-9]{11,16})[*!]?$"``.
|
|
|
|
.. attribute:: mail_addr_re
|
|
|
|
Python compiled regular expression object (:class:`re.RegexObject`)
|
|
provided by :func:`re.compile` method to match email addresses.
|
|
Default used regular expression is:
|
|
``r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+$"``
|
|
|
|
.. attribute:: MailClass
|
|
|
|
The class used to manipulate/parse mails individually.
|
|
Default is :class:`~store.Mail`.
|
|
|
|
.. seealso::
|
|
|
|
Python modules:
|
|
:mod:`datetime` -- Basic date and time types
|
|
|
|
:mod:`re` -- Regular expression operations
|
|
|
|
Postfix manual:
|
|
`postqueue`_ -- Postfix queue control
|
|
|
|
:rfc:`3696` -- Checking and Transformation of Names
|
|
"""
|
|
postqueue_cmd = None
|
|
spool_path = None
|
|
postqueue_mailstatus = ['active', 'deferred', 'hold']
|
|
# Just a little change to match OpenSMTPD envelope id
|
|
#mail_id_re = re.compile(r"^([A-F0-9]{8,12}|[B-Zb-z0-9]{11,16})[*!]?$")
|
|
mail_id_re = re.compile(r"^([A-F0-9]{8,12}|[B-Za-z0-9]{11,16})[*!]?$")
|
|
mail_addr_re = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]+$")
|
|
MailClass = Mail
|
|
|
|
def __init__(self):
|
|
"""Init method"""
|
|
self.spool_path = CONFIG['core']['postfix_spool']
|
|
self.postqueue_cmd = CONFIG['commands']['list_queue']
|
|
if CONFIG['commands']['use_sudo']:
|
|
self.postqueue_cmd.insert(0, 'sudo')
|
|
|
|
self.loaded_at = None
|
|
self.mails = []
|
|
|
|
# We want to keep compatibility, so "postfix" is considered default
|
|
if not 'smtpd_type' in CONFIG['core']: CONFIG['core']['smtpd_type'] = 'postfix'
|
|
|
|
|
|
@property
|
|
@debug
|
|
def known_headers(self):
|
|
"""Return known headers from loaded mails
|
|
|
|
:return: headers as :func:`set`
|
|
"""
|
|
headers = set()
|
|
for mail in self.mails:
|
|
for mailheader in dir(mail.head):
|
|
if not mailheader.startswith("_"):
|
|
headers.add(mailheader)
|
|
return headers
|
|
|
|
@debug
|
|
def _get_postqueue_output(self):
|
|
"""
|
|
Get Postfix postqueue command output.
|
|
|
|
This method used the postfix command defined in
|
|
:attr:`~PostqueueStore.postqueue_cmd` attribute to view the mails queue
|
|
content.
|
|
|
|
Command defined in :attr:`~PostqueueStore.postqueue_cmd` attribute is
|
|
runned using a :class:`subprocess.Popen` instance.
|
|
|
|
:return: Command's output lines.
|
|
:rtype: :func:`list`
|
|
|
|
.. seealso::
|
|
|
|
Python module:
|
|
:mod:`subprocess` -- Subprocess management
|
|
"""
|
|
child = subprocess.Popen(self.postqueue_cmd,
|
|
stdout=subprocess.PIPE)
|
|
stdout = child.communicate()[0]
|
|
|
|
if 'postfix' in CONFIG['core']['smtpd_type']:
|
|
# return lines list without the headers and footers
|
|
return [line.strip() for line in stdout.decode().split('\n')][1:-2]
|
|
else:
|
|
return [line.strip() for line in stdout.decode().split('\n')]
|
|
|
|
def _is_mail_id(self, mail_id):
|
|
"""
|
|
Check mail_id for a valid postfix queued mail ID.
|
|
|
|
Validation is made using a :class:`re.RegexObject` stored in
|
|
the :attr:`~PostqueueStore.mail_id_re` attribute of the
|
|
:class:`~store.PostqueueStore` instance.
|
|
|
|
:param str mail_id: Mail Postfix queue ID string
|
|
:return: True or false
|
|
:rtype: :func:`bool`
|
|
"""
|
|
|
|
if self.mail_id_re.match(mail_id) is None:
|
|
return False
|
|
return True
|
|
|
|
@debug
|
|
def _load_from_postfix_postqueue(self, filename=None, parse=False):
|
|
"""
|
|
Load content from postfix queue using postqueue command output.
|
|
|
|
Output lines from :attr:`~store.PostqueueStore._get_postqueue_output`
|
|
are parsed to build :class:`~store.Mail` objects. Sample Postfix queue
|
|
control tool (`postqueue`_) output::
|
|
|
|
C0004979687 4769 Tue Apr 29 06:35:05 sender@domain.com
|
|
(error message from mx.remote1.org with parenthesis)
|
|
first.rcpt@remote1.org
|
|
(error message from mx.remote2.org with parenthesis)
|
|
second.rcpt@remote2.org
|
|
third.rcpt@remote2.org
|
|
|
|
Parsing rules are pretty simple:
|
|
|
|
- Line starts with a valid :attr:`Mail.qid`: create new
|
|
:class:`~store.Mail` object with :attr:`~Mail.qid`,
|
|
:attr:`~Mail.size`, :attr:`~Mail.date` and :attr:`~Mail.sender`
|
|
informations from line.
|
|
|
|
+-------------+------+---------------------------+-----------------+
|
|
| Queue ID | Size | Reception date and time | Sender |
|
|
+-------------+------+-----+-----+----+----------+-----------------+
|
|
| C0004979687 | 4769 | Tue | Apr | 29 | 06:35:05 | user@domain.com |
|
|
+-------------+------+-----+-----+----+----------+-----------------+
|
|
|
|
- Line starts with a parenthesis: store error messages to last created
|
|
:class:`~store.Mail` object's :attr:`~Mail.errors` attribute.
|
|
|
|
- Any other matches: add new recipient to the :attr:`~Mail.recipients`
|
|
attribute of the last created :class:`~store.Mail` object.
|
|
|
|
Optionnal argument ``filename`` can be set with a file containing
|
|
output of the `postqueue`_ command. In this case, output lines of
|
|
`postqueue`_ command are directly read from ``filename`` and parsed,
|
|
the `postqueue`_ command is never used.
|
|
|
|
Optionnal argument ``parse`` controls whether mails are parsed or not.
|
|
This is useful to load every known mail headers for later filtering.
|
|
|
|
:param str filename: File to load mails from
|
|
:param bool parse: Controls whether loaded mails are parsed or not.
|
|
"""
|
|
if filename is None:
|
|
postqueue_output = self._get_postqueue_output()
|
|
else:
|
|
postqueue_output = open(filename).readlines()
|
|
|
|
mail = None
|
|
for line in postqueue_output:
|
|
line = line.strip()
|
|
|
|
# Headers and footers start with dash (-)
|
|
if line.startswith('-'):
|
|
continue
|
|
# Mails are blank line separated
|
|
if not len(line):
|
|
continue
|
|
|
|
fields = line.split()
|
|
if "(" == fields[0][0]:
|
|
# Store error message without parenthesis: [1:-1]
|
|
# gathered errors must be associated with specific recipients
|
|
# TODO: change recipients or errors structures to link these
|
|
# objects together.
|
|
mail.errors.append(" ".join(fields)[1:-1])
|
|
else:
|
|
if self._is_mail_id(fields[0]):
|
|
# postfix does not precise year in mails timestamps so
|
|
# we consider mails have been sent this year.
|
|
# If gathered date is in the future:
|
|
# mail has been received last year (or NTP problem).
|
|
now = datetime.now()
|
|
datestr = "{0} {1}".format(" ".join(fields[2:-1]), now.year)
|
|
date = datetime.strptime(datestr, "%a %b %d %H:%M:%S %Y")
|
|
if date > now:
|
|
date = date - timedelta(days=365)
|
|
|
|
mail = self.MailClass(fields[0], size=fields[1],
|
|
date=date,
|
|
sender=fields[-1],
|
|
recipients=[])
|
|
self.mails.append(mail)
|
|
else:
|
|
# Email address validity check can be tricky. RFC3696 talks
|
|
# about. Fow now, we use a simple regular expression to
|
|
# match most of email addresses.
|
|
rcpt_email_addr = " ".join(fields)
|
|
if self.mail_addr_re.match(rcpt_email_addr):
|
|
mail.recipients.append(rcpt_email_addr)
|
|
|
|
if parse:
|
|
#print("parsing mails")
|
|
[mail.parse() for mail in self.mails]
|
|
|
|
@debug
|
|
def _load_from_opensmtpd_postqueue(self, filename=None, parse=False):
|
|
"""
|
|
Load content from opensmtpd queue using "smtpctl show queue" command output.
|
|
|
|
Output lines from :attr:`~store.PostqueueStore._get_postqueue_output`
|
|
are parsed to build :class:`~store.Mail` objects. Sample Postfix queue
|
|
control tool (`postqueue`_) output::
|
|
|
|
fab15d0efd0b9489|inet4|mta||ToddCookolnuo@psi.br|john.galt@post.lu|john.galt@post.lu|1606747552|1607093152|0|16|pending|8464|450 4.1.8 <ToddCookolnuo@psi.br>: Sender address rejected: Domain not found
|
|
e9eb7c58be75d090|inet4|mta||fyxt@snowcopolo.com|marco.polo@vuvuzela.br|marco.polo@vuvuzela.br|1606645251|1606990851|0|23|pending|15363|450 4.1.8 <fyxt@snowcopolo.com>: Sender address rejected: Domain not found
|
|
|
|
Parsing rules are pretty simple
|
|
|
|
Optionnal argument ``filename`` can be set with a file containing
|
|
output of the `postqueue`_ command. In this case, output lines of
|
|
`postqueue`_ command are directly read from ``filename`` and parsed,
|
|
the `postqueue`_ command is never used.
|
|
|
|
Optionnal argument ``parse`` controls whether mails are parsed or not.
|
|
This is useful to load every known mail headers for later filtering.
|
|
|
|
:param str filename: File to load mails from
|
|
:param bool parse: Controls whether loaded mails are parsed or not.
|
|
"""
|
|
if filename is None:
|
|
postqueue_output = self._get_postqueue_output()
|
|
else:
|
|
postqueue_output = open(filename).readlines()
|
|
|
|
mail = None
|
|
for line in postqueue_output:
|
|
line = line.strip()
|
|
|
|
# Headers and footers start with dash (-)
|
|
#if line.startswith('-'):
|
|
# continue
|
|
# Mails are blank line separated
|
|
#if not len(line):
|
|
# continue
|
|
|
|
fields = line.split('|')
|
|
#if "(" == fields[0][0]:
|
|
# Store error message without parenthesis: [1:-1]
|
|
# gathered errors must be associated with specific recipients
|
|
# TODO: change recipients or errors structures to link these
|
|
# objects together.
|
|
|
|
|
|
if self._is_mail_id(fields[0]):
|
|
date = datetime.fromtimestamp(int(fields[7]))
|
|
exp_date = datetime.fromtimestamp(int(fields[8]))
|
|
lst_date = datetime.fromtimestamp(int(fields[9]))
|
|
nr_delivery = fields[10]
|
|
mail = self.MailClass(fields[0], sender=fields[4], recipients=[fields[5]],
|
|
date=date, exp_date=exp_date, last_delivery=lst_date,
|
|
nr_delivery=nr_delivery, status=fields[11],
|
|
errors=[fields[13]])
|
|
self.mails.append(mail)
|
|
|
|
if parse:
|
|
#print("parsing mails")
|
|
[mail.parse() for mail in self.mails]
|
|
|
|
@debug
|
|
def _load_from_spool(self, parse=True):
|
|
"""
|
|
Load content from postfix queue using files from spool.
|
|
|
|
Mails are loaded using the command defined in
|
|
:attr:`~PostqueueStore.postqueue_cmd` attribute. Some informations may
|
|
be missing using the :meth:`~store.PostqueueStore._load_from_spool`
|
|
method, including at least :attr:`Mail.status` field.
|
|
|
|
Optionnal argument ``parse`` controls whether mails are parsed or not.
|
|
This is useful to load every known mail headers for later filtering.
|
|
|
|
Loaded mails are stored as :class:`~store.Mail` objects in
|
|
:attr:`~PostqueueStore.mails` attribute.
|
|
|
|
:param bool parse: Controls whether loaded mails are parsed or not.
|
|
|
|
.. warning::
|
|
|
|
Be aware that parsing mails on disk is slow and can lead to
|
|
high load usage on system with large mails queue.
|
|
"""
|
|
for status in self.postqueue_mailstatus:
|
|
for fs_data in os.walk("%s/%s" % (self.spool_path, status)):
|
|
for mail_id in fs_data[2]:
|
|
mail = self.MailClass(mail_id)
|
|
mail.status = status
|
|
|
|
mail.parse()
|
|
|
|
self.mails.append(mail)
|
|
|
|
@debug
|
|
def _load_from_file(self, filename):
|
|
"""Unimplemented method"""
|
|
|
|
@debug
|
|
def load(self, method="postqueue", filename=None, parse=False):
|
|
"""
|
|
Load content from postfix/OpenSMTPD mails queue.
|
|
|
|
Mails are loaded using postqueue/smtpctl command line tool or reading
|
|
directly from spool. The optionnal argument, if present, is a
|
|
method string and specifies the method used to gather mails informations.
|
|
By default, method is set to ``"postqueue"`` and the standard queue
|
|
control tool: `postqueue`_ or `smtpctl show queue`is used.
|
|
|
|
Optionnal argument ``parse`` controls whether mails are parsed or not.
|
|
This is useful to load every known mail headers for later filtering.
|
|
|
|
:param str method: Method used to load mails from Postfix queue
|
|
:param str filename: File to load mails from
|
|
:param bool parse: Controls whether loaded mails are parsed or not.
|
|
|
|
Provided method :func:`str` name is directly used with :func:`getattr`
|
|
to find a *self._load_from_<method>* method.
|
|
"""
|
|
# releasing memory
|
|
del self.mails
|
|
gc.collect()
|
|
|
|
self.mails = []
|
|
|
|
smtpd_type = CONFIG['core']['smtpd_type']
|
|
|
|
if filename is None:
|
|
getattr(self, "_load_from_{0}_{1}".format(smtpd_type, method))(parse=parse)
|
|
else:
|
|
getattr(self, "_load_from_{0}_{1}".format(smtpd_type, method))(filename, parse)
|
|
self.loaded_at = datetime.now()
|
|
|
|
@debug
|
|
def summary(self):
|
|
"""
|
|
Summarize the mails queue content.
|
|
|
|
:return: Mail queue summary as :class:`dict`
|
|
|
|
Sizes are in bytes.
|
|
|
|
Example response::
|
|
|
|
{
|
|
'total_mails': 500,
|
|
'total_mails_size': 709750,
|
|
'average_mail_size': 1419.5,
|
|
'max_mail_size': 2414,
|
|
'min_mail_size': 423,
|
|
'top_errors': [
|
|
('mail transport unavailable', 484),
|
|
('Test error message', 16)
|
|
],
|
|
'top_recipient_domains': [
|
|
('test-domain.tld', 500)
|
|
],
|
|
'top_recipients': [
|
|
('user-3@test-domain.tld', 200),
|
|
('user-2@test-domain.tld', 200),
|
|
('user-1@test-domain.tld', 100)
|
|
],
|
|
'top_sender_domains': [
|
|
('test-domain.tld', 500)
|
|
],
|
|
'top_senders': [
|
|
('sender-1@test-domain.tld', 100),
|
|
('sender-2@test-domain.tld', 100),
|
|
('sender-7@test-domain.tld', 50),
|
|
('sender-4@test-domain.tld', 50),
|
|
('sender-5@test-domain.tld', 50)
|
|
],
|
|
'top_status': [
|
|
('deferred', 500),
|
|
('active', 0),
|
|
('hold', 0)
|
|
],
|
|
'unique_recipient_domains': 1,
|
|
'unique_recipients': 3,
|
|
'unique_sender_domains': 1,
|
|
'unique_senders': 8
|
|
}
|
|
"""
|
|
senders = Counter()
|
|
sender_domains = Counter()
|
|
recipients = Counter()
|
|
recipient_domains = Counter()
|
|
status = Counter(active=0, hold=0, deferred=0)
|
|
errors = Counter()
|
|
total_mails_size = 0
|
|
average_mail_size = 0
|
|
max_mail_size = 0
|
|
min_mail_size = 0
|
|
mails_by_age = {
|
|
'last_24h': 0,
|
|
'1_to_4_days_ago': 0,
|
|
'older_than_4_days': 0
|
|
}
|
|
|
|
for mail in self.mails:
|
|
status[mail.status] += 1
|
|
senders[mail.sender] += 1
|
|
if '@' in mail.sender:
|
|
sender_domains[mail.sender.split('@', 1)[1]] += 1
|
|
for recipient in mail.recipients:
|
|
recipients[recipient] += 1
|
|
if '@' in recipient:
|
|
recipient_domains[recipient.split('@', 1)[1]] += 1
|
|
for error in mail.errors:
|
|
errors[error] += 1
|
|
total_mails_size += mail.size
|
|
if mail.size > max_mail_size:
|
|
max_mail_size = mail.size
|
|
if min_mail_size == 0:
|
|
min_mail_size = mail.size
|
|
elif mail.size < min_mail_size:
|
|
min_mail_size = mail.size
|
|
|
|
mail_age = datetime.now() - mail.date
|
|
if mail_age.days >= 4:
|
|
mails_by_age['older_than_4_days'] += 1
|
|
elif mail_age.days == 1:
|
|
mails_by_age['1_to_4_days_ago'] += 1
|
|
elif mail_age.days == 0:
|
|
mails_by_age['last_24h'] += 1
|
|
|
|
if len(self.mails):
|
|
average_mail_size = total_mails_size / len(self.mails)
|
|
|
|
summary = {
|
|
'total_mails': len(self.mails),
|
|
'mails_by_age': mails_by_age,
|
|
'total_mails_size': total_mails_size,
|
|
'average_mail_size': average_mail_size,
|
|
'max_mail_size': max_mail_size,
|
|
'min_mail_size': min_mail_size,
|
|
'top_status': status.most_common()[:5],
|
|
'unique_senders': len(list(senders)),
|
|
'unique_sender_domains': len(list(sender_domains)),
|
|
'unique_recipients': len(list(recipients)),
|
|
'unique_recipient_domains': len(list(recipient_domains)),
|
|
'top_senders': senders.most_common()[:5],
|
|
'top_sender_domains': sender_domains.most_common()[:5],
|
|
'top_recipients': recipients.most_common()[:5],
|
|
'top_recipient_domains': recipient_domains.most_common()[:5],
|
|
'top_errors': errors.most_common()[:5]
|
|
}
|
|
return summary
|