1
=== modified file 'ibid/plugins/feeds.py'
2
--- ibid/plugins/feeds.py	2009-03-01 23:01:30 +0000
3
+++ ibid/plugins/feeds.py	2009-03-07 18:41:10 +0000
4
@@ -1,6 +1,9 @@
5
1
import re
1
import re
6
2
from datetime import datetime
2
from datetime import datetime
7
3
import logging
3
import logging
8
4
from urllib2 import urlopen, URLError
9
5
from urlparse import urljoin
10
6
from html5lib import HTMLParser, treebuilders
11
4
7
12
5
from sqlalchemy import Column, Integer, Unicode, DateTime, UnicodeText, ForeignKey, Table
8
from sqlalchemy import Column, Integer, Unicode, DateTime, UnicodeText, ForeignKey, Table
13
6
from sqlalchemy.sql import func
9
from sqlalchemy.sql import func
14
@@ -10,6 +13,7 @@
15
10
import ibid
13
import ibid
16
11
from ibid.plugins import Processor, match, authorise
14
from ibid.plugins import Processor, match, authorise
17
12
from ibid.models import Base
15
from ibid.models import Base
18
16
from ibid.utils import cacheable_download, get_soup
19
13
17
20
14
help = {'feeds': u'Displays articles from RSS and Atom feeds'}
18
help = {'feeds': u'Displays articles from RSS and Atom feeds'}
21
15
19
22
@@ -32,15 +36,11 @@
23
32
        self.url = url
36
        self.url = url
24
33
        self.identity_id = identity_id
37
        self.identity_id = identity_id
25
34
        self.time = datetime.now()
38
        self.time = datetime.now()
26
35
27
36
    def is_valid(self):
28
37
        self.update()
39
        self.update()
29
38
        if self.feed['version']:
30
39
            return True
31
40
        return False
32
41
40
33
42
    def update(self):
41
    def update(self):
35
43
        self.feed = feedparser.parse(self.url)
42
        feedfile = cacheable_download(self.url, "feeds/%s-%i.xml" % (re.sub(r'\W+', '_', self.name), self.identity_id))
36
43
        self.feed = feedparser.parse(feedfile)
37
44
        self.entries = self.feed['entries']
44
        self.entries = self.feed['entries']
38
45
45
39
46
class Manage(Processor):
46
class Manage(Processor):
40
@@ -59,18 +59,31 @@
41
59
59
42
60
        if feed:
60
        if feed:
43
61
            event.addresponse(u"I already have the %s feed" % name)
61
            event.addresponse(u"I already have the %s feed" % name)
53
62
        else:
62
            return
54
63
            feed = Feed(unicode(name), unicode(url), event.identity)
63
        
55
64
64
        valid = bool(feedparser.parse(url)["version"])
56
65
        if feed.is_valid():
65
57
66
            session.save(feed)
66
        if not valid:
58
67
            session.flush()
67
            soup = get_soup(url)
59
68
            event.addresponse(True)
68
            for alternate in soup.findAll('link', {'rel': 'alternate',
60
69
            log.info(u"Added feed '%s' by %s/%s (%s): %s (Found %s entries)", name, event.account, event.identity, event.sender['connection'], url, len(feed.entries))
69
                    'type': re.compile(r'^application/(atom|rss)\+xml$'),
61
70
        else:
70
                    'href': re.compile(r'.+')}):
62
71
                newurl = urljoin(url, alternate["href"])
63
72
                valid = bool(feedparser.parse(newurl)["version"])
64
73
65
74
                if valid:
66
75
                    url = newurl
67
76
                    break
68
77
69
78
        if not valid:
70
71
            event.addresponse(u"Sorry, I could not add the %s feed. %s is not a valid feed" % (name,url))
79
            event.addresponse(u"Sorry, I could not add the %s feed. %s is not a valid feed" % (name,url))
71
80
            return
72
72
81
74
73
        session.close()
82
        feed = Feed(unicode(name), unicode(url), event.identity)
75
83
        session.save(feed)
76
84
        session.flush()
77
85
        event.addresponse(True)
78
86
        log.info(u"Added feed '%s' by %s/%s (%s): %s (Found %s entries)", name, event.account, event.identity, event.sender['connection'], url, len(feed.entries))
79
74
87
80
75
    @match(r'^(?:list\s+)?feeds$')
88
    @match(r'^(?:list\s+)?feeds$')
81
76
    def list(self, event):
89
    def list(self, event):
82
77
90
83
=== modified file 'ibid/utils.py'
84
--- ibid/utils.py	2009-03-05 16:33:12 +0000
85
+++ ibid/utils.py	2009-03-07 18:41:10 +0000
86
@@ -1,10 +1,16 @@
87
1
import cgi
88
2
from gzip import GzipFile
89
1
from htmlentitydefs import name2codepoint
3
from htmlentitydefs import name2codepoint
90
2
import os
4
import os
91
3
import os.path
5
import os.path
92
4
from pkg_resources import resource_exists, resource_string
6
from pkg_resources import resource_exists, resource_string
93
5
import re
7
import re
94
8
from StringIO import StringIO
95
6
import time
9
import time
96
7
import urllib2
10
import urllib2
97
11
import zlib
98
12
99
13
from html5lib import HTMLParser, treebuilders
100
8
14
101
9
import ibid
15
import ibid
102
10
16
103
@@ -80,10 +86,21 @@
104
80
    # Download into a temporary file, in case something goes wrong
86
    # Download into a temporary file, in case something goes wrong
105
81
    downloadfile = os.path.join(plugindir, ".download." + os.path.basename(cachefile))
87
    downloadfile = os.path.join(plugindir, ".download." + os.path.basename(cachefile))
106
82
    outfile = file(downloadfile, "wb")
88
    outfile = file(downloadfile, "wb")
111
83
    buf = "x"
89
    data = connection.read()
112
84
    while len(buf) > 0:
90
113
85
        buf = connection.read(1024)
91
    compression = connection.headers.get('content-encoding')
114
86
        outfile.write(buf)
92
    if compression:
115
93
        if compression.lower() == "deflate":
116
94
            try:
117
95
                data = zlib.decompress(data)
118
96
            except zlib.error:
119
97
                data = zlib.decompress(data, -zlib.MAX_WBITS)
120
98
        elif compression.lower() == "gzip":
121
99
            compressedstream = StringIO(data)
122
100
            gzipper = GzipFile(fileobj=compressedstream)
123
101
            data = gzipper.read()
124
102
125
103
    outfile.write(data)
126
87
    
104
    
127
88
    outfile.close()
105
    outfile.close()
128
89
106
129
@@ -112,4 +129,34 @@
130
112
def ibid_version():
129
def ibid_version():
131
113
    return resource_exists(__name__, '.version') and resource_string(__name__, '.version').strip() or None
130
    return resource_exists(__name__, '.version') and resource_string(__name__, '.version').strip() or None
132
114
131
133
132
def get_soup(url, data=None, headers={}):
134
133
    "Request a URL and create a BeautifulSoup parse tree from it"
135
134
136
135
    req = urllib2.Request(url, data, headers)
137
136
    f = urllib2.urlopen(req)
138
137
    data = f.read()
139
138
    f.close()
140
139
141
140
    encoding = None
142
141
    contentType = f.headers.get('content-type')
143
142
    if contentType:
144
143
        (mediaType, params) = cgi.parse_header(contentType)
145
144
        encoding = params.get('charset')
146
145
147
146
    compression = f.headers.get('content-encoding')
148
147
    if compression.lower() == "deflate":
149
148
        try:
150
149
            data = zlib.decompress(data)
151
150
        except zlib.error:
152
151
            data = zlib.decompress(data, -zlib.MAX_WBITS)
153
152
    elif compression.lower() == "gzip":
154
153
        compressedstream = StringIO(data)
155
154
        gzipper = GzipFile(fileobj=compressedstream)
156
155
        data = gzipper.read()
157
156
158
157
    treebuilder = treebuilders.getTreeBuilder("beautifulsoup")
159
158
    parser = HTMLParser(tree=treebuilder)
160
159
161
160
    return parser.parse(data, encoding=encoding)
162
161
163
115
# vi: set et sta sw=4 ts=4:
162
# vi: set et sta sw=4 ts=4:
Reviewer	Date Requested	Status
Michael Gorven		Approve on 2009-03-10
Jonathan Hitchcock	2009-03-07	Approve on 2009-03-08
Review via email: mp+4271@code.launchpad.net