Merge branch 'release/2.0.4-1'2.0.4-1

1 files changed, 182 insertions, 301 deletions

diff --git a/mwc.py b/mwc.py
index a0635a1..6a48317 100755
--- a/mwc.py
+++ b/mwc.py
@@ -1,14 +1,12 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
-# Copyright: (2013-2015) Michael Till Beck <Debianguru@gmx.de>
+# Copyright: (2013-2017) Michael Till Beck <Debianguru@gmx.de>
 # License: GPL-2.0+
 
-import urllib.request, urllib.error, urllib.parse
-import urllib.parse
-from lxml import etree
-from cssselect import GenericTranslator
-import re
 import io
+from lxml import etree
+import hashlib
 
 import smtplib
 from email.mime.text import MIMEText
@@ -20,8 +18,6 @@ import sys
 import getopt
 import traceback
 
-import subprocess
-
 import time
 from time import strftime
 import random
@@ -30,330 +26,215 @@ import importlib
 config = None
 
 defaultEncoding = 'utf-8'
-maxTitleLength = 150
 
 # this is how an empty RSS feed looks like
 emptyfeed = """<?xml version="1.0"?>
 <rss version="2.0">
  <channel>
   <title>MailWebsiteChanges Feed</title>
-  <link>https://github.com/Debianguru/MailWebsiteChanges</link>
+  <link>https://github.com/mtill/MailWebsiteChanges</link>
   <description>MailWebsiteChanges Feed</description>
  </channel>
 </rss>"""
 
-# Attributes in HTML files storing URI values. These values are automatically translated to absolute URIs.
-uriAttributes = [['//img[@src]', 'src'], ['//a[@href]', 'href']]
-cmdscheme = 'cmd://'
-
 mailsession = None
 
 
-# translates all relative URIs found in trees to absolute URIs
-def toAbsoluteURIs(trees, baseuri):
-        for tree in trees:
-                if isinstance(tree, str):
-                        continue
-                for uriAttribute in uriAttributes:
-                        tags = tree.xpath(uriAttribute[0])
-                        for tag in tags:
-                                if tag.attrib.get(uriAttribute[1]) != None:
-                                        if urllib.parse.urlparse(tag.attrib[uriAttribute[1]]).scheme == '':
-                                                tag.attrib[uriAttribute[1]] = urllib.parse.urljoin(baseuri, tag.attrib[uriAttribute[1]])
-
-
-def parseSite(site):
-        file, content, titles, warning = None, None, None, None
-
-        uri = site['uri']
-        contenttype = site.get('type', 'html')
-        contentregex = site.get('contentregex', '')
-        titleregex = site.get('titleregex', '')
-        enc = site.get('encoding', defaultEncoding)
-
-        contentxpath = site.get('contentxpath', '')
-        if contentxpath == '' and site.get('contentcss', '') != '':
-                # CSS
-                contentxpath = GenericTranslator().css_to_xpath(site.get('contentcss'))
-        titlexpath = site.get('titlexpath', '')
-        if titlexpath == '' and site.get('titlecss', '') != '':
-                titlexpath = GenericTranslator().css_to_xpath(site.get('titlecss'))
-
-        try:
-
-                if uri.startswith(cmdscheme):
-                        # run command and retrieve output
-                        process = subprocess.Popen(uri[len(cmdscheme):], stdout=subprocess.PIPE, shell=True, close_fds=True)
-                        file = process.stdout
-                else:
-                        # open website
-                        req = urllib.request.Request(uri)
-                        if 'user-agent' in site:
-                            req.add_header('User-Agent', site['user-agent'])
-                        if 'accept' in site:
-                            req.add_header('Accept', site['accept'])
-                        file = urllib.request.urlopen(req)
-
-
-                if contenttype == 'text' or (contentxpath == '' and titlexpath == ''):
-                        contents = [file.read().decode(enc)]
-                        titles = []
-                else:
-                        baseuri = uri
-                        if contenttype == 'html':
-                                parser = etree.HTMLParser(encoding=enc)
-                        else:
-                                parser = etree.XMLParser(recover=True, encoding=enc)
-
-                        tree = etree.parse(file, parser)
-
-                        # xpath
-                        contentresult = tree.xpath(contentxpath) if contentxpath else []
-                        titleresult = tree.xpath(titlexpath) if titlexpath else []
-
-                        # translate relative URIs to absolute URIs
-                        if contenttype == 'html':
-                                basetaglist = tree.xpath('/html/head/base')
-                                if len(basetaglist) != 0:
-                                        baseuri = basetaglist[0].attrib['href']
-                                if len(contentresult) != 0:
-                                        toAbsoluteURIs(contentresult, baseuri)
-                                if len(titleresult) != 0:
-                                        toAbsoluteURIs(titleresult, baseuri)
-
-                        if contentxpath != '' and titlexpath != '' and len(contentresult) != len(titleresult):
-                                warning = 'WARNING: number of title blocks (' + str(len(titleresult)) + ') does not match number of content blocks (' + str(len(contentresult)) + ')'
-                        elif contentxpath and len(contentresult) == 0:
-                                warning = 'WARNING: content selector became invalid!'
-                        elif titlexpath and len(titleresult) == 0:
-                                warning = 'WARNING: title selector became invalid!'
-                        else:
-                                if len(contentresult) == 0:
-                                        contentresult = titleresult
-                                if len(titleresult) == 0:
-                                        titleresult = contentresult
-
-                        if isinstance(contentresult, str):
-                                contents = [contentresult]
-                        else:
-                                contents = [etree.tostring(s, encoding=defaultEncoding, pretty_print=True).decode(defaultEncoding) for s in contentresult]
-                        if isinstance(titleresult, str):
-                                titles = [getSubject(titleresult)]
-                        else:
-                                titles = [getSubject(' '.join(s.xpath('.//text()'))) for s in titleresult]
-
-        except IOError as e:
-                warning = 'WARNING: could not open URL; maybe content was moved?\n\n' + str(e)
-
-        if file is not None:
-                file.close()
-
-        if uri.startswith(cmdscheme) and process.wait() != 0:
-                warning = 'WARNING: process terminated with an error'
-
-        if warning:
-                return {'content': content, 'titles': titles, 'warning': warning}
-
-        # parse regex
-        if contentregex:
-                contents = [x for y in [re.findall(r'' + contentregex, c, re.S) for c in contents] for x in y]
-        if titleregex:
-                titles = [x for y in [re.findall(r'' + titleregex, c, re.S) for c in titles] for x in y]
-
-        if contentregex and titleregex and len(contents) != len(titles):
-                warning = 'WARNING: number of title blocks (' + str(len(titles)) + ') does not match number of content blocks (' + str(len(contents)) + ') after regex'
-        elif contentregex and len(contents) == 0:
-                warning = 'WARNING: content regex became invalid!'
-        elif titleregex and len(titles) == 0:
-                warning = 'WARNING: title regex became invalid!'
-        else:
-                if len(contents) == 0:
-                        contents = titles
-                if len(titles) == 0:
-                        titles = [getSubject(c) for c in contents]
-
-        return {'contents': contents, 'titles': titles, 'warning': warning}
-
-
-# returns a short subject line
-def getSubject(textContent):
-        if textContent == None or textContent == '':
-                return config.subjectPostfix
-        textContent = re.sub(' +', ' ', re.sub('\s', ' ', textContent)).strip()
-        return (textContent[:maxTitleLength] + ' [..]') if len(textContent) > maxTitleLength else textContent
-
-
 # generates a new RSS feed item
 def genFeedItem(subject, content, link, change):
-        feeditem = etree.Element('item')
-        titleitem = etree.Element('title')
-        titleitem.text = subject + ' #' + str(change)
-        feeditem.append(titleitem)
-        linkitem = etree.Element('link')
-        linkitem.text = link
-        feeditem.append(linkitem)
-        descriptionitem = etree.Element('description')
-        descriptionitem.text = content
-        feeditem.append(descriptionitem)
-        guiditem = etree.Element('guid')
-        guiditem.text = str(random.getrandbits(32))
-        feeditem.append(guiditem)
-        dateitem = etree.Element('pubDate')
-        dateitem.text = strftime("%a, %d %b %Y %H:%M:%S %Z", time.localtime())
-        feeditem.append(dateitem)
-
-        return feeditem
+    feeditem = etree.Element('item')
+    titleitem = etree.Element('title')
+    titleitem.text = subject + ' #' + str(change)
+    feeditem.append(titleitem)
+    linkitem = etree.Element('link')
+    linkitem.text = link
+    feeditem.append(linkitem)
+    descriptionitem = etree.Element('description')
+    descriptionitem.text = content
+    feeditem.append(descriptionitem)
+    guiditem = etree.Element('guid')
+    guiditem.text = str(random.getrandbits(32))
+    feeditem.append(guiditem)
+    dateitem = etree.Element('pubDate')
+    dateitem.text = strftime("%a, %d %b %Y %H:%M:%S %Z", time.localtime())
+    feeditem.append(dateitem)
+
+    return feeditem
 
 
 # sends mail notification
-def sendmail(receiver, subject, content, sendAsHtml, link):
-        global mailsession
-
-        if sendAsHtml:
-                baseurl = None
-                if link != None:
-                        content = '<p><a href="' + link + '">' + subject + '</a></p>\n' + content
-                        baseurl = urljoin(link, '/')
-                mail = MIMEText('<html><head><title>' + subject + '</title>' + ('<base href="' + baseurl + '">' if baseurl else '') + '</head><body>' + content + '</body></html>', 'html', defaultEncoding)
-        else:
-                if link != None:
-                        content = link + '\n\n' + content
-                mail = MIMEText(content, 'text', defaultEncoding)
+def sendmail(receiver, subject, content, sendAsHtml, link, encoding=None):
+    global mailsession, defaultEncoding
+
+    if encoding is None:
+        encoding = defaultEncoding
+
+    if sendAsHtml:
+        baseurl = None
+        if link is not None:
+            content = '<p><a href="' + link + '">' + subject + '</a></p>\n' + content
+            baseurl = urljoin(link, '/')
+        mail = MIMEText('<html><head><title>' + subject + '</title>' + ('<base href="' + baseurl + '">' if baseurl else '') + '</head><body>' + content + '</body></html>', 'html', encoding)
+    else:
+        if link is not None:
+            content = link + '\n\n' + content
+        mail = MIMEText(content, 'text', encoding)
+
+    mail['From'] = config.sender
+    mail['To'] = receiver
+    mail['Subject'] = Header(subject, encoding)
+
+    # initialize session once, not each time this method gets called
+    if mailsession is None:
+        mailsession = smtplib.SMTP(config.smtphost, config.smtpport)
+        if config.useTLS:
+            mailsession.ehlo()
+            mailsession.starttls()
+        if config.smtpusername is not None:
+            mailsession.login(config.smtpusername, config.smtppwd)
+
+    mailsession.sendmail(config.sender, receiver.split(','), mail.as_string())
 
-        mail['From'] = config.sender
-        mail['To'] = receiver
-        mail['Subject'] = Header(subject, defaultEncoding)
 
-        # initialize session once, not each time this method gets called
-        if mailsession is None:
-                mailsession = smtplib.SMTP(config.smtphost, config.smtpport)
-                if config.useTLS:
-                        mailsession.ehlo()
-                        mailsession.starttls()
-                if config.smtpusername is not None:
-                    mailsession.login(config.smtpusername, config.smtppwd)
+# returns a list of all content that is stored locally for a specific site
+def getStoredHashes(name):
+    result = []
+    filename = os.path.join(config.workingDirectory, name + ".txt")
+    if os.path.exists(filename):
+        with open(filename, 'r') as thefile:
+            for line in thefile:
+                result.append(line.rstrip())
 
-        mailsession.sendmail(config.sender, receiver.split(','), mail.as_string())
+    return result
 
 
-# returns a list of all content that is stored locally for a specific site
-def getFileContents(shortname):
-        result = []
-        for f in os.listdir('.'):
-                if f.startswith(shortname + '.') and f.endswith('.txt'):
-                        file = open(f, 'rb')
-                        result.append(file.read().decode('utf-8'))
-                        file.close()
-        return result
+# updates list of content that is stored locally for a specific site
+def storeHashes(name, contentHashes):
+    with open(os.path.join(config.workingDirectory, name + '.txt'), 'w') as thefile:
+        for h in contentHashes:
+            thefile.write(h + "\n")
 
 
-# updates list of content that is stored locally for a specific site
-def storeFileContents(shortname, parseResult):
-        for f in os.listdir('.'):
-                if f.startswith(shortname + '.') and f.endswith('.txt'):
-                        os.remove(f)
+def runParsers(parsers, contentList=None):
+    if contentList is None:
+        contentList = []
 
-        i = 0
-        for c in parseResult['contents']:
-                file = open(shortname + '.' + str(i) + '.txt', 'wb')
-                file.write(c.encode('utf-8'))
-                file.close()
-                i += 1
+    for parser in parsers:
+        contentList = parser.performAction(contentList)
 
+    return contentList
 
-def pollWebsites():
 
-        # parse existing feed or create a new one
-        if config.enableRSSFeed:
-                if os.path.isfile(config.rssfile):
-                        feedXML = etree.parse(config.rssfile)
-                else:
-                        feedXML = etree.parse(io.StringIO(emptyfeed))
-
-        # start polling sites
-        for site in config.sites:
-
-                print('polling site [' + site['shortname'] + '] ...')
-                parseResult = parseSite(site)
-                receiver = site.get('receiver', config.receiver)
-
-                # if something went wrong, notify the user
-                if parseResult['warning']:
-                        subject = '[' + site['shortname'] + '] WARNING'
-                        print('WARNING: ' + parseResult['warning'])
-                        if config.enableMailNotifications:
-                                sendmail(receiver, subject, parseResult['warning'], False, None)
-                        if config.enableRSSFeed:
-                                feedXML.xpath('//channel')[0].append(genFeedItem(subject, parseResult['warning'], site['uri'], 0))
-                else:
-                        # otherwise, check which parts of the site were updated
-                        changes = 0
-                        fileContents = getFileContents(site['shortname'])
-                        i = 0
-                        for content in parseResult['contents']:
-                                if content not in fileContents:
-                                        changes += 1
-
-                                        subject = '[' + site['shortname'] + '] ' + parseResult['titles'][i]
-                                        print('    ' + subject)
-                                        if config.enableMailNotifications and len(fileContents) > 0:
-                                                sendmail(receiver, subject, content, (site.get('type', 'html') == 'html'), site['uri'])
-
-                                        if config.enableRSSFeed:
-                                                feedXML.xpath('//channel')[0].append(genFeedItem(subject, content, site['uri'], changes))
-                                i += 1
-
-
-                        if changes > 0:
-                                storeFileContents(site['shortname'], parseResult)
-                                print('        ' + str(changes) + ' updates')
- 
-        # store feed
-        if config.enableRSSFeed:
-                for o in feedXML.xpath('//channel/item[position()<last()-' + str(config.maxFeeds - 1) + ']'):
-                        o.getparent().remove(o)
-                file = open(config.rssfile, 'w')
-                file.write(etree.tostring(feedXML, pretty_print=True, xml_declaration=True, encoding=defaultEncoding).decode(defaultEncoding))
-                file.close()
+def pollWebsites():
+    global defaultEncoding
 
+    # parse existing feed or create a new one
+    rssfile = config.rssfile
+    if not os.path.isabs(rssfile):
+        rssfile = os.path.join(config.workingDirectory, rssfile)
 
-if __name__ == "__main__":
+    if config.enableRSSFeed:
+        if os.path.isfile(rssfile):
+            feedXML = etree.parse(rssfile)
+        else:
+            feedXML = etree.parse(io.StringIO(emptyfeed))
 
-        configMod = 'config'
-        dryrun = None
+    # start polling sites
+    mailsSent = 0
+    for site in config.sites:
+        print('polling site [' + site['name'] + '] ...')
+        receiver = site.get('receiver', config.receiver)
 
         try:
-                opts, args = getopt.getopt(sys.argv[1:], 'hc:d:', ['help', 'config=', 'dry-run='])
-        except getopt.GetoptError:
-                print('Usage: mwc.py --config=config --dry-run=shortname')
-                sys.exit(1)
-        for opt, arg in opts:
-                if opt == '-h':
-                        print('Usage: mwc.py --config=config')
-                        exit()
-                elif opt in ('-c', '--config'):
-                        configMod = arg
-                elif opt in ('-d', '--dry-run'):
-                        dryrun = arg
-
-        config = importlib.import_module(configMod)
-
-        if dryrun:
-                for site in config.sites:
-                        if site['shortname'] == dryrun:
-                                parseResult = parseSite(site)
-                                print(parseResult)
-                                break
-        else:
-                try:
-                        pollWebsites()
-                except:
-                        msg = str(sys.exc_info()[0]) + '\n\n' + traceback.format_exc()
-                        print(msg)
-                        if config.receiver != '':
-                                sendmail(config.receiver, '[mwc] Something went wrong ...', msg, False, None)
-
-                if mailsession:
-                        mailsession.quit()
-                        mailsession = None
+            contentList = runParsers(site['parsers'])
+        except Exception as e:
+            # if something went wrong, notify the user
+            subject = '[' + site['name'] + '] WARNING'
+            print('WARNING: ' + str(e))
+            if config.enableMailNotifications:
+                if config.maxMailsPerSession == -1 or mailsSent < config.maxMailsPerSession:
+                    sendmail(receiver=receiver, subject=subject, content=str(e), sendAsHtml=False, link=None)
+                    mailsSent = mailsSent + 1
+            if config.enableRSSFeed:
+                feedXML.xpath('//channel')[0].append(genFeedItem(subject, str(e), "", 0))
+            continue
+
+        sessionHashes = []
+        changedContents = []
+        fileHashes = getStoredHashes(site['name'])
+        for content in contentList:
+
+            contenthash = hashlib.md5(content.content.encode(content.encoding)).hexdigest()
+            if contenthash not in fileHashes:
+                if config.maxMailsPerSession == -1 or mailsSent < config.maxMailsPerSession:
+                    sessionHashes.append(contenthash)
+                    changedContents.append(content)
+
+                    subject = '[' + site['name'] + '] ' + content.title
+                    print('    ' + subject)
+                    if config.enableMailNotifications and len(fileHashes) > 0:
+                        sendAsHtml = (content.contenttype == 'html')
+                        sendmail(receiver=receiver, subject=subject, content=content.content, sendAsHtml=sendAsHtml, link=content.uri, encoding=content.encoding)
+                        mailsSent = mailsSent + 1
+
+                    if config.enableRSSFeed:
+                        feedXML.xpath('//channel')[0].append(genFeedItem(subject, content.content, content.uri, len(changedContents)))
+            else:
+                sessionHashes.append(contenthash)
+
+        if 'postRun' in site:
+            runParsers(site['postRun'], changedContents)
+
+        if len(changedContents) > 0:
+            storeHashes(site['name'], sessionHashes)
+            print('        ' + str(len(changedContents)) + ' updates')
+
+    # store feed
+    if config.enableRSSFeed:
+        for o in feedXML.xpath('//channel/item[position()<last()-' + str(config.maxFeeds - 1) + ']'):
+            o.getparent().remove(o)
+        with open(rssfile, 'w') as thefile:
+            thefile.write(etree.tostring(feedXML, pretty_print=True, xml_declaration=True, encoding=defaultEncoding).decode(defaultEncoding, errors='ignore'))
 
+
+if __name__ == "__main__":
+    configMod = 'config'
+    dryrun = None
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'hc:d:', ['help', 'config=', 'dry-run='])
+    except getopt.GetoptError:
+        print('Usage: mwc.py --config=config --dry-run=name')
+        sys.exit(1)
+    for opt, arg in opts:
+        if opt == '-h':
+            print('Usage: mwc.py --config=config')
+            exit()
+        elif opt in ('-c', '--config'):
+            configMod = arg
+        elif opt in ('-d', '--dry-run'):
+            dryrun = arg
+
+    config = importlib.import_module(configMod)
+
+    if dryrun:
+        for thesite in config.sites:
+            if thesite['name'] == dryrun:
+                parseResult = runParsers(thesite['parsers'])
+                for p in parseResult:
+                    print(p.title)
+                    print(p.content)
+                print(str(len(parseResult)) + " results")
+                break
+    else:
+        try:
+            pollWebsites()
+        except:
+            msg = str(sys.exc_info()[0]) + '\n\n' + traceback.format_exc()
+            print(msg)
+            if config.receiver != '':
+                sendmail(receiver=config.receiver, subject='[mwc] Something went wrong ...', content=msg, sendAsHtml=False, link=None)
+
+        if mailsession:
+            mailsession.quit()
+            mailsession = None