source: announcerplugin/trunk/announcer/distributors/mail.py @ 16128

Last change on this file since 16128 was 16128, checked in by Ryan J Ollos, 7 years ago

1.0dev: Remove compatibility code and conform to PEP8

Refs #12120.
File size: 24.8 KB
Line 
1# -*- coding: utf-8 -*-
2#
3# Copyright (c) 2008, Stephen Hansen
4# Copyright (c) 2009, Robert Corsaro
5# Copyright (c) 2010,2012 Steffen Hoffmann
6#
7# This software is licensed as described in the file COPYING, which
8# you should have received as part of this distribution.
9#
10
11# TODO: pick format based on subscription.  For now users will use the same
12#       format for all announcements, but in the future we can make this more
13#       flexible, since it's in the subscription table.
14
15import Queue
16import hashlib
17import random
18import re
19import smtplib
20import sys
21import threading
22import time
23try:
24    from email.header import Header
25    from email.charset import Charset, QP, BASE64
26    from email.mimemultipart import MIMEMultipart
27    from email.mimetext import MIMEText
28    from email.utils import formatdate, formataddr
29except ImportError:
30    from email.Header import Header
31    from email.Charset import Charset, QP, BASE64
32    from email.MIMEMultipart import MIMEMultipart
33    from email.MIMEText import MIMEText
34    from email.Utils import formatdate, formataddr
35from subprocess import Popen, PIPE
36
37from trac.config import BoolOption, ExtensionOption, IntOption, Option, \
38                        OrderedExtensionsOption
39from trac.core import Component, ExtensionPoint, Interface, TracError, \
40                      implements
41from trac.util.text import CRLF
42
43from announcer.api import _, IAnnouncementAddressResolver, \
44                          IAnnouncementDistributor, IAnnouncementFormatter
45from announcer.model import Subscription
46from announcer.util.mail import set_header
47from announcer.util.mail_crypto import CryptoTxt
48
49
50class IEmailSender(Interface):
51    """Extension point interface for components that allow sending e-mail."""
52
53    def send(self, from_addr, recipients, message):
54        """Send message to recipients."""
55
56
57class IAnnouncementEmailDecorator(Interface):
58
59    def decorate_message(event, message, decorators):
60        """Manipulate the message before it is sent on it's way.  The callee
61        should call the next decorator by popping decorators and calling the
62        popped decorator.  If decorators is empty, don't worry about it.
63        """
64
65
66class EmailDistributor(Component):
67
68    implements(IAnnouncementDistributor)
69
70    formatters = ExtensionPoint(IAnnouncementFormatter)
71    decorators = ExtensionPoint(IAnnouncementEmailDecorator)
72
73    resolvers = OrderedExtensionsOption('announcer',
74        'email_address_resolvers', IAnnouncementAddressResolver,
75        'SpecifiedEmailResolver, SessionEmailResolver, '
76        'DefaultDomainEmailResolver',
77        """Comma seperated list of email resolver components in the order
78        they will be called.  If an email address is resolved, the remaining
79        resolvers will not be called.
80        """)
81
82    email_sender = ExtensionOption('announcer', 'email_sender',
83        IEmailSender, 'SmtpEmailSender',
84        """Name of the component implementing `IEmailSender`.
85
86        This component is used by the announcer system to send emails.
87        Currently, `SmtpEmailSender` and `SendmailEmailSender` are provided.
88        """)
89
90    enabled = BoolOption('announcer', 'email_enabled', True,
91        """Enable email notification.""")
92
93    email_from = Option('announcer', 'email_from', 'trac@localhost',
94        """Sender address to use in notification emails.""")
95
96    from_name = Option('announcer', 'email_from_name', '',
97        """Sender name to use in notification emails.""")
98
99    reply_to = Option('announcer', 'email_replyto', 'trac@localhost',
100        """Reply-To address to use in notification emails.""")
101
102    mime_encoding = Option('announcer', 'mime_encoding', 'base64',
103        """Specifies the MIME encoding scheme for emails.
104
105        Valid options are 'base64' for Base64 encoding, 'qp' for
106        Quoted-Printable, and 'none' for no encoding. Note that the no
107        encoding means that non-ASCII characters in text are going to cause
108        problems with notifications.
109        """)
110
111    use_public_cc = BoolOption('announcer', 'use_public_cc', 'false',
112        """Recipients can see email addresses of other CC'ed recipients.
113
114        If this option is disabled (the default), recipients are put on BCC
115        """)
116
117    # used in email decorators, but not here
118    subject_prefix = Option('announcer', 'email_subject_prefix',
119        '__default__',
120        """Text to prepend to subject line of notification emails.
121
122        If the setting is not defined, then the [$project_name] prefix.
123        If no prefix is desired, then specifying an empty option
124        will disable it.
125        """)
126
127    to_default = 'undisclosed-recipients: ;'
128    to = Option('announcer', 'email_to', to_default, 'Default To: field')
129
130    use_threaded_delivery = BoolOption('announcer', 'use_threaded_delivery',
131        False,
132        """Do message delivery in a separate thread.
133
134        Enabling this will improve responsiveness for requests that end up
135        with an announcement being sent over email. It requires building
136        Python with threading support enabled-- which is usually the case.
137        To test, start Python and type 'import threading' to see
138        if it raises an error.
139        """)
140
141    default_email_format = Option('announcer', 'default_email_format',
142        'text/plain',
143        """The default mime type of the email notifications.
144
145        This can be overridden on a per user basis through the announcer
146        preferences panel.
147        """)
148
149    rcpt_allow_regexp = Option('announcer', 'rcpt_allow_regexp', '',
150        """A whitelist pattern to match any address to before adding to
151        recipients list.
152        """)
153
154    rcpt_local_regexp = Option('announcer', 'rcpt_local_regexp', '',
155        """A whitelist pattern to match any address, that should be
156        considered local.
157
158        This will be evaluated only if msg encryption is set too.
159        Recipients with matching email addresses will continue to
160        receive unencrypted email messages.
161        """)
162
163    crypto = Option('announcer', 'email_crypto', '',
164        """Enable cryptographically operation on email msg body.
165
166        Empty string, the default for unset, disables all crypto operations.
167        Valid values are:
168            sign          sign msg body with given privkey
169            encrypt       encrypt msg body with pubkeys of all recipients
170            sign,encrypt  sign, than encrypt msg body
171        """)
172
173    # get GnuPG configuration options
174    gpg_binary = Option('announcer', 'gpg_binary', 'gpg',
175        """GnuPG binary name, allows for full path too.
176
177        Value 'gpg' is same default as in python-gnupg itself.
178        For usual installations location of the gpg binary is auto-detected.
179        """)
180
181    gpg_home = Option('announcer', 'gpg_home', '',
182        """Directory containing keyring files.
183
184        In case of wrong configuration missing keyring files without content
185        will be created in the configured location, provided necessary
186        write permssion is granted for the corresponding parent directory.
187        """)
188
189    private_key = Option('announcer', 'gpg_signing_key', None,
190         """Keyid of private key (last 8 chars or more) used for signing.
191
192         If unset, a private key will be selected from keyring automagicly.
193         The password must be available i.e. provided by running gpg-agent
194         or empty (bad security). On failing to unlock the private key,
195         msg body will get emptied.
196         """)
197
198    def __init__(self):
199        self.enigma = None
200        self.delivery_queue = None
201        self._init_pref_encoding()
202
203    def get_delivery_queue(self):
204        if not self.delivery_queue:
205            self.delivery_queue = Queue.Queue()
206            thread = DeliveryThread(self.delivery_queue, self.send)
207            thread.start()
208        return self.delivery_queue
209
210    # IAnnouncementDistributor methods
211
212    def transports(self):
213        yield 'email'
214
215    def formats(self, transport, realm):
216        """Find valid formats for transport and realm."""
217        formats = {}
218        for f in self.formatters:
219            for style in f.styles(transport, realm):
220                formats[style] = f
221        self.log.debug("EmailDistributor has found the following formats "
222                       "capable of handling '%s' of '%s': %s",
223                       transport, realm, ', '.join(formats.keys()))
224        if not formats:
225            self.log.error("EmailDistributor is unable to continue without "
226                           "supporting formatters.")
227        return formats
228
229    def distribute(self, transport, recipients, event):
230        found = False
231        for supported_transport in self.transports():
232            if supported_transport == transport:
233                found = True
234        if not self.enabled or not found:
235            self.log.debug("EmailDistributor email_enabled set to false")
236            return
237        formats = self.formats(transport, event.realm)
238        if not formats:
239            self.log.error("EmailDistributor No formats found for %s %s",
240                           transport, event.realm)
241            return
242        msgdict = {}
243        msgdict_encrypt = {}
244        msg_pubkey_ids = []
245        # compile pattern before use for better performance
246        rcpt_allow_re = re.compile(self.rcpt_allow_regexp)
247        rcpt_local_re = re.compile(self.rcpt_local_regexp)
248
249        if self.crypto != '':
250            self.log.debug("EmailDistributor attempts crypto operation.")
251            self.enigma = CryptoTxt(self.gpg_binary, self.gpg_home)
252
253        for name, authed, address in recipients:
254            fmt = name and \
255                  self._get_preferred_format(event.realm, name, authed) or \
256                  self._get_default_format()
257            old_fmt = fmt
258            if fmt not in formats:
259                self.log.debug("EmailDistributor format %s not available "
260                               "for %s %s, looking for an alternative",
261                               fmt, transport, event.realm)
262                # If the fmt is not available for this realm, then try to find
263                # an alternative
264                fmt = None
265                for f in formats.values():
266                    fmt = f.alternative_style_for(
267                        transport, event.realm, old_fmt)
268                    if fmt:
269                        break
270            if not fmt:
271                self.log.error("EmailDistributor was unable to find a "
272                               "formatter for format %s", old_fmt)
273                continue
274            resolver = None
275            if name and not address:
276                # figure out what the addr should be if it's not defined
277                for resolver in self.resolvers:
278                    address = resolver.get_address_for_name(name, authed)
279                    if address:
280                        break
281            if address:
282                self.log.debug("EmailDistributor found the address '%s' "
283                               "for '%s (%s)' via: %s", address, name,
284                               authed and 'authenticated' or
285                               'not authenticated',
286                               resolver.__class__.__name__)
287
288                # ok, we found an addr, add the message
289                # but wait, check for allowed rcpt first, if set
290                if rcpt_allow_re.search(address) is not None:
291                    # check for local recipients now
292                    local_match = rcpt_local_re.search(address)
293                    if self.crypto in ['encrypt', 'sign,encrypt'] and \
294                            local_match is None:
295                        # search available public keys for matching UID
296                        pubkey_ids = self.enigma.get_pubkey_ids(address)
297                        if pubkey_ids > 0:
298                            msgdict_encrypt.setdefault(fmt, set())\
299                                .add((name, authed, address))
300                            msg_pubkey_ids[len(msg_pubkey_ids):] = pubkey_ids
301                            self.log.debug("EmailDistributor got pubkeys "
302                                           "for %s: %s", address, pubkey_ids)
303                        else:
304                            self.log.debug("EmailDistributor dropped %s "
305                                           "after missing pubkey with "
306                                           "corresponding address %s in any "
307                                           "UID", name, address)
308                    else:
309                        msgdict.setdefault(fmt, set())\
310                            .add((name, authed, address))
311                        if local_match is not None:
312                            self.log.debug("EmailDistributor expected local "
313                                           "delivery for %s to: %s", name,
314                                           address)
315                else:
316                    self.log.debug("EmailDistributor dropped %s for not "
317                                   "matching allowed recipient pattern %s",
318                                   address, self.rcpt_allow_regexp)
319            else:
320                self.log.debug("EmailDistributor was unable to find an "
321                               "address for: %s (%s)", name, authed and
322                               'authenticated' or 'not authenticated')
323        for k, v in msgdict.items():
324            if not v or not formats.get(k):
325                continue
326            fmt = formats[k]
327            self.log.debug("EmailDistributor is sending event as '%s' to: "
328                           "%s", fmt, ', '.join(x[2] for x in v))
329            self._do_send(transport, event, k, v, fmt)
330        for k, v in msgdict_encrypt.items():
331            if not v or not formats.get(k):
332                continue
333            fmt = formats[k]
334            self.log.debug("EmailDistributor is sending encrypted info on "
335                           "event as '%s' to: %s", fmt,
336                           ', '.join(x[2] for x in v))
337            self._do_send(transport, event, k, v, formats[k], msg_pubkey_ids)
338
339    def _get_default_format(self):
340        return self.default_email_format
341
342    def _get_preferred_format(self, realm, sid, authenticated):
343        if authenticated is None:
344            authenticated = 0
345        # Format is unified for all subscriptions of a user.
346        result = Subscription.find_by_sid_and_distributor(
347            self.env, sid, authenticated, 'email')
348        if result:
349            chosen = result[0]['format']
350            self.log.debug("EmailDistributor determined the preferred format"
351                           " for '%s (%s)' is: %s", sid, authenticated and
352                           'authenticated' or 'not authenticated', chosen)
353            return chosen
354        else:
355            return self._get_default_format()
356
357    def _init_pref_encoding(self):
358        self._charset = Charset()
359        self._charset.input_charset = 'utf-8'
360        pref = self.mime_encoding.lower()
361        if pref == 'base64':
362            self._charset.header_encoding = BASE64
363            self._charset.body_encoding = BASE64
364            self._charset.output_charset = 'utf-8'
365            self._charset.input_codec = 'utf-8'
366            self._charset.output_codec = 'utf-8'
367        elif pref in ['qp', 'quoted-printable']:
368            self._charset.header_encoding = QP
369            self._charset.body_encoding = QP
370            self._charset.output_charset = 'utf-8'
371            self._charset.input_codec = 'utf-8'
372            self._charset.output_codec = 'utf-8'
373        elif pref == 'none':
374            self._charset.header_encoding = None
375            self._charset.body_encoding = None
376            self._charset.input_codec = None
377            self._charset.output_charset = 'ascii'
378        else:
379            raise TracError(_("Invalid email encoding setting: %(pref)s",
380                              pref=pref))
381
382    def _message_id(self, realm):
383        """Generate a predictable, but sufficiently unique message ID."""
384        modtime = time.time()
385        rand = random.randint(0, 32000)
386        s = '%s.%d.%d.%s' % (self.env.project_url,
387                             modtime, rand,
388                             realm.encode('ascii', 'ignore'))
389        dig = hashlib.md5(s).hexdigest()
390        host = self.email_from[self.email_from.find('@') + 1:]
391        msgid = '<%03d.%s@%s>' % (len(s), dig, host)
392        return msgid
393
394    def _filter_recipients(self, rcpt):
395        return rcpt
396
397    def _do_send(self, transport, event, format, recipients, formatter,
398                 pubkey_ids=None):
399        pubkey_ids = pubkey_ids or []
400        # Prepare sender for use in IEmailSender component and message header.
401        from_header = formataddr(
402            (self.from_name and self.from_name or self.env.project_name,
403             self.email_from)
404        )
405        headers = dict()
406        headers['Message-ID'] = self._message_id(event.realm)
407        headers['Date'] = formatdate()
408        headers['From'] = from_header
409        headers['Reply-To'] = self.reply_to
410
411        recip_adds = [x[2] for x in recipients if x]
412
413        if self.use_public_cc:
414            headers['Cc'] = ', '.join(recip_adds)
415        else:
416            # Use localized Bcc: hint for default To: content.
417            if self.to == self.to_default:
418                headers['To'] = _("undisclosed-recipients: ;")
419            else:
420                headers['To'] = '"%s"' % self.to
421                if self.to:
422                    recip_adds += [self.to]
423        if not recip_adds:
424            self.log.debug("EmailDistributor stopped (no recipients).")
425            return
426        self.log.debug("All email recipients: %s", recip_adds)
427
428        root_message = MIMEMultipart('related')
429
430        # Write header data into message object.
431        for k, v in headers.iteritems():
432            set_header(root_message, k, v)
433
434        output = formatter.format(transport, event.realm, format, event)
435
436        # DEVEL: Currently crypto operations work with format text/plain only.
437        alternate_output = None
438        alternate_style = []
439        if self.crypto != '' and pubkey_ids:
440            if self.crypto == 'sign':
441                output = self.enigma.sign(output, self.private_key)
442            elif self.crypto == 'encrypt':
443                output = self.enigma.encrypt(output, pubkey_ids)
444            elif self.crypto == 'sign,encrypt':
445                output = self.enigma.sign_encrypt(output, pubkey_ids,
446                                                  self.private_key)
447            self.log.debug(output)
448            self.log.debug("EmailDistributor crypto operation successful.")
449        else:
450            alternate_style = formatter.alternative_style_for(
451                transport,
452                event.realm,
453                format
454            )
455            if alternate_style:
456                alternate_output = formatter.format(
457                    transport,
458                    event.realm,
459                    alternate_style,
460                    event
461                )
462
463        # Sanity check for suitable encoding setting.
464        if not self._charset.body_encoding:
465            try:
466                output.encode('ascii')
467            except UnicodeDecodeError:
468                raise TracError(_("Ticket contains non-ASCII chars. Please "
469                                  "change encoding setting"))
470
471        root_message.preamble = "This is a multi-part message in MIME format."
472        if alternate_output:
473            parent_message = MIMEMultipart('alternative')
474            root_message.attach(parent_message)
475
476            alt_msg_format = 'html' in alternate_style and 'html' or 'plain'
477            if isinstance(alternate_output, unicode):
478                alternate_output = alternate_output.encode('utf-8')
479            msg_text = MIMEText(alternate_output, alt_msg_format)
480            msg_text.set_charset(self._charset)
481            parent_message.attach(msg_text)
482        else:
483            parent_message = root_message
484
485        msg_format = 'html' in format and 'html' or 'plain'
486        if isinstance(output, unicode):
487            output = output.encode('utf-8')
488        msg_text = MIMEText(output, msg_format)
489        del msg_text['Content-Transfer-Encoding']
490        msg_text.set_charset(self._charset)
491        # According to RFC 2046, the last part of a multipart message is best
492        #   and preferred.
493        parent_message.attach(msg_text)
494
495        # DEVEL: Decorators can interfere with crypto operation here. Fix it.
496        decorators = self._get_decorators()
497        if decorators:
498            decorator = decorators.pop()
499            decorator.decorate_message(event, root_message, decorators)
500
501        package = (from_header, recip_adds, root_message.as_string())
502        start = time.time()
503        if self.use_threaded_delivery:
504            self.get_delivery_queue().put(package)
505        else:
506            self.send(*package)
507        stop = time.time()
508        self.log.debug("EmailDistributor took %s seconds to send.",
509                       round(stop - start, 2))
510
511    def send(self, from_addr, recipients, message):
512        """Send message to recipients via e-mail."""
513        # Ensure the message complies with RFC2822: use CRLF line endings
514        message = CRLF.join(re.split('\r?\n', message))
515        self.email_sender.send(from_addr, recipients, message)
516
517    def _get_decorators(self):
518        return self.decorators[:]
519
520
521class SmtpEmailSender(Component):
522    """E-mail sender connecting to an SMTP server."""
523
524    implements(IEmailSender)
525
526    server = Option('smtp', 'server', 'localhost',
527        """SMTP server hostname to use for email notifications.""")
528
529    timeout = IntOption('smtp', 'timeout', 10,
530        """SMTP server connection timeout. (requires python-2.6)""")
531
532    port = IntOption('smtp', 'port', 25,
533        """SMTP server port to use for email notification.""")
534
535    user = Option('smtp', 'user', '', "Username for SMTP server.")
536
537    password = Option('smtp', 'password', '', "Password for SMTP server.")
538
539    use_tls = BoolOption('smtp', 'use_tls', False,
540        """Use SSL/TLS to send notifications over SMTP.""")
541
542    use_ssl = BoolOption('smtp', 'use_ssl', False,
543        """Use ssl for smtp connection.""")
544
545    debuglevel = IntOption('smtp', 'debuglevel', 0,
546        """Set to 1 for useful smtp debugging on stdout.""")
547
548    def send(self, from_addr, recipients, message):
549        # use defaults to make sure connect() is called in the constructor
550        smtpclass = smtplib.SMTP
551        if self.use_ssl:
552            smtpclass = smtplib.SMTP_SSL
553
554        args = {
555            'host': self.server,
556            'port': self.port
557        }
558        # timeout isn't supported until python 2.6
559        vparts = sys.version_info[0:2]
560        if vparts[0] >= 2 and vparts[1] >= 6:
561            args['timeout'] = self.timeout
562
563        smtp = smtpclass(**args)
564        smtp.set_debuglevel(self.debuglevel)
565        if self.use_tls:
566            smtp.ehlo()
567            if 'starttls' not in smtp.esmtp_features:
568                raise TracError(_("TLS enabled but server does not support "
569                                  "TLS"))
570            smtp.starttls()
571            smtp.ehlo()
572        if self.user:
573            smtp.login(
574                self.user.encode('utf-8'),
575                self.password.encode('utf-8')
576            )
577        smtp.sendmail(from_addr, recipients, message)
578        if self.use_tls or self.use_ssl:
579            # avoid false failure detection when the server closes
580            # the SMTP connection with TLS/SSL enabled
581            import socket
582            try:
583                smtp.quit()
584            except socket.sslerror:
585                pass
586        else:
587            smtp.quit()
588
589
590class SendmailEmailSender(Component):
591    """E-mail sender using a locally-installed sendmail program."""
592
593    implements(IEmailSender)
594
595    sendmail_path = Option('sendmail', 'sendmail_path', 'sendmail',
596        """Path to the sendmail executable.
597
598        The sendmail program must accept the `-i` and `-f` options.
599        """)
600
601    def send(self, from_addr, recipients, message):
602        self.log.info("Sending notification through sendmail at %s to %s",
603                      self.sendmail_path, recipients)
604        cmdline = [self.sendmail_path, '-i', '-f', from_addr]
605        cmdline.extend(recipients)
606        self.log.debug("Sendmail command line: %s", ' '.join(cmdline))
607        try:
608            child = Popen(cmdline, bufsize=-1, stdin=PIPE, stdout=PIPE,
609                          stderr=PIPE)
610            (out, err) = child.communicate(message)
611            if child.returncode or err:
612                raise Exception("Sendmail failed with (%s, %s), command: "
613                                "'%s'", child.returncode, err.strip(),
614                                cmdline)
615        except OSError, e:
616            self.log.error("Failed to run sendmail[%s] with error %s",
617                           self.sendmail_path, e)
618
619
620class DeliveryThread(threading.Thread):
621    def __init__(self, queue, sender):
622        threading.Thread.__init__(self)
623        self._sender = sender
624        self._queue = queue
625        self.setDaemon(True)
626
627    def run(self):
628        while 1:
629            send_from, recipients, message = self._queue.get()
630            self._sender(send_from, recipients, message)
Note: See TracBrowser for help on using the repository browser.