Mercurial > libervia-backend
comparison sat/plugins/plugin_blog_import_dokuwiki.py @ 2624:56f94936df1e
code style reformatting using black
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 27 Jun 2018 20:14:46 +0200 |
parents | 26edcf3a30eb |
children | 003b8b4b56a7 |
comparison
equal
deleted
inserted
replaced
2623:49533de4540b | 2624:56f94936df1e |
---|---|
19 # along with this program. If not, see <http://www.gnu.org/licenses/>. | 19 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
20 | 20 |
21 from sat.core.i18n import _, D_ | 21 from sat.core.i18n import _, D_ |
22 from sat.core.constants import Const as C | 22 from sat.core.constants import Const as C |
23 from sat.core.log import getLogger | 23 from sat.core.log import getLogger |
24 | |
24 log = getLogger(__name__) | 25 log = getLogger(__name__) |
25 from sat.core import exceptions | 26 from sat.core import exceptions |
26 from sat.tools import xml_tools | 27 from sat.tools import xml_tools |
27 from twisted.internet import threads | 28 from twisted.internet import threads |
28 from collections import OrderedDict | 29 from collections import OrderedDict |
31 import urlparse | 32 import urlparse |
32 import tempfile | 33 import tempfile |
33 import re | 34 import re |
34 import time | 35 import time |
35 import os.path | 36 import os.path |
37 | |
36 try: | 38 try: |
37 from dokuwiki import DokuWiki, DokuWikiError # this is a new dependency | 39 from dokuwiki import DokuWiki, DokuWikiError # this is a new dependency |
38 except ImportError: | 40 except ImportError: |
39 raise exceptions.MissingModule(u'Missing module dokuwiki, please install it with "pip install dokuwiki"') | 41 raise exceptions.MissingModule( |
42 u'Missing module dokuwiki, please install it with "pip install dokuwiki"' | |
43 ) | |
40 try: | 44 try: |
41 from PIL import Image # this is already needed by plugin XEP-0054 | 45 from PIL import Image # this is already needed by plugin XEP-0054 |
42 except: | 46 except: |
43 raise exceptions.MissingModule(u"Missing module pillow, please download/install it from https://python-pillow.github.io") | 47 raise exceptions.MissingModule( |
48 u"Missing module pillow, please download/install it from https://python-pillow.github.io" | |
49 ) | |
44 | 50 |
45 PLUGIN_INFO = { | 51 PLUGIN_INFO = { |
46 C.PI_NAME: "Dokuwiki import", | 52 C.PI_NAME: "Dokuwiki import", |
47 C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI", | 53 C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI", |
48 C.PI_TYPE: C.PLUG_TYPE_BLOG, | 54 C.PI_TYPE: C.PLUG_TYPE_BLOG, |
49 C.PI_DEPENDENCIES: ["BLOG_IMPORT"], | 55 C.PI_DEPENDENCIES: ["BLOG_IMPORT"], |
50 C.PI_MAIN: "DokuwikiImport", | 56 C.PI_MAIN: "DokuwikiImport", |
51 C.PI_HANDLER: "no", | 57 C.PI_HANDLER: "no", |
52 C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine.""") | 58 C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine."""), |
53 } | 59 } |
54 | 60 |
55 SHORT_DESC = D_(u"import posts from Dokuwiki blog engine") | 61 SHORT_DESC = D_(u"import posts from Dokuwiki blog engine") |
56 | 62 |
57 LONG_DESC = D_(u"""This importer handle Dokuwiki blog engine. | 63 LONG_DESC = D_( |
64 u"""This importer handle Dokuwiki blog engine. | |
58 | 65 |
59 To use it, you need an admin access to a running Dokuwiki website | 66 To use it, you need an admin access to a running Dokuwiki website |
60 (local or on the Internet). The importer retrieves the data using | 67 (local or on the Internet). The importer retrieves the data using |
61 the XMLRPC Dokuwiki API. | 68 the XMLRPC Dokuwiki API. |
62 | 69 |
89 This retrieves the 100 last blog posts from http://127.0.1.1 that | 96 This retrieves the 100 last blog posts from http://127.0.1.1 that |
90 are inside the namespace "public:2015:10" using the Dokuwiki user | 97 are inside the namespace "public:2015:10" using the Dokuwiki user |
91 "souliane", and it imports them to sat profile dave's microblog node. | 98 "souliane", and it imports them to sat profile dave's microblog node. |
92 Internal Dokuwiki media that were hosted on http://127.0.1.1 are now | 99 Internal Dokuwiki media that were hosted on http://127.0.1.1 are now |
93 pointing to http://media.diekulturvermittlung.at. | 100 pointing to http://media.diekulturvermittlung.at. |
94 """) | 101 """ |
102 ) | |
95 DEFAULT_MEDIA_REPO = "" | 103 DEFAULT_MEDIA_REPO = "" |
96 DEFAULT_NAMESPACE = "/" | 104 DEFAULT_NAMESPACE = "/" |
97 DEFAULT_LIMIT = 100 # you might get a DBUS timeout (no reply) if it lasts too long | 105 DEFAULT_LIMIT = 100 # you might get a DBUS timeout (no reply) if it lasts too long |
98 | 106 |
99 | 107 |
100 class Importer(DokuWiki): | 108 class Importer(DokuWiki): |
101 | 109 def __init__( |
102 def __init__(self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT): | 110 self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT |
111 ): | |
103 """ | 112 """ |
104 | 113 |
105 @param url (unicode): DokuWiki site URL | 114 @param url (unicode): DokuWiki site URL |
106 @param user (unicode): DokuWiki admin user | 115 @param user (unicode): DokuWiki admin user |
107 @param passwd (unicode): DokuWiki admin password | 116 @param passwd (unicode): DokuWiki admin password |
118 """Return a unique and constant post id | 127 """Return a unique and constant post id |
119 | 128 |
120 @param post(dict): parsed post data | 129 @param post(dict): parsed post data |
121 @return (unicode): post unique item id | 130 @return (unicode): post unique item id |
122 """ | 131 """ |
123 return unicode(post['id']) | 132 return unicode(post["id"]) |
124 | 133 |
125 def getPostUpdated(self, post): | 134 def getPostUpdated(self, post): |
126 """Return the update date. | 135 """Return the update date. |
127 | 136 |
128 @param post(dict): parsed post data | 137 @param post(dict): parsed post data |
129 @return (unicode): update date | 138 @return (unicode): update date |
130 """ | 139 """ |
131 return unicode(post['mtime']) | 140 return unicode(post["mtime"]) |
132 | 141 |
133 def getPostPublished(self, post): | 142 def getPostPublished(self, post): |
134 """Try to parse the date from the message ID, else use "mtime". | 143 """Try to parse the date from the message ID, else use "mtime". |
135 | 144 |
136 The date can be extracted if the message ID looks like one of: | 145 The date can be extracted if the message ID looks like one of: |
177 | 186 |
178 # XXX: title is already in content_xhtml and difficult to remove, so leave it | 187 # XXX: title is already in content_xhtml and difficult to remove, so leave it |
179 # title = content.split("\n")[0].strip(u"\ufeff= ") | 188 # title = content.split("\n")[0].strip(u"\ufeff= ") |
180 | 189 |
181 # build the extra data dictionary | 190 # build the extra data dictionary |
182 mb_data = {"id": id_, | 191 mb_data = { |
183 "published": published, | 192 "id": id_, |
184 "updated": updated, | 193 "published": published, |
185 "author": profile_jid.user, | 194 "updated": updated, |
186 # "content": content, # when passed, it is displayed in Libervia instead of content_xhtml | 195 "author": profile_jid.user, |
187 "content_xhtml": content_xhtml, | 196 # "content": content, # when passed, it is displayed in Libervia instead of content_xhtml |
188 # "title": title, | 197 "content_xhtml": content_xhtml, |
189 "allow_comments": "true", | 198 # "title": title, |
190 } | 199 "allow_comments": "true", |
200 } | |
191 | 201 |
192 # find out if the message access is public or restricted | 202 # find out if the message access is public or restricted |
193 namespace = id_.split(":")[0] | 203 namespace = id_.split(":")[0] |
194 if namespace and namespace.lower() not in ("public", "/"): | 204 if namespace and namespace.lower() not in ("public", "/"): |
195 mb_data["group"] = namespace # roster group must exist | 205 mb_data["group"] = namespace # roster group must exist |
196 | 206 |
197 self.posts_data[id_] = {'blog': mb_data, 'comments':[[]]} | 207 self.posts_data[id_] = {"blog": mb_data, "comments": [[]]} |
198 | 208 |
199 def process(self, client, namespace=DEFAULT_NAMESPACE): | 209 def process(self, client, namespace=DEFAULT_NAMESPACE): |
200 """Process a namespace or a single page. | 210 """Process a namespace or a single page. |
201 | 211 |
202 @param namespace (unicode): DokuWiki namespace (or page) to import | 212 @param namespace (unicode): DokuWiki namespace (or page) to import |
204 profile_jid = client.jid | 214 profile_jid = client.jid |
205 log.info("Importing data from DokuWiki %s" % self.version) | 215 log.info("Importing data from DokuWiki %s" % self.version) |
206 try: | 216 try: |
207 pages_list = self.pages.list(namespace) | 217 pages_list = self.pages.list(namespace) |
208 except DokuWikiError: | 218 except DokuWikiError: |
209 log.warning('Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.') | 219 log.warning( |
220 'Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.' | |
221 ) | |
210 return | 222 return |
211 | 223 |
212 if not pages_list: # namespace is actually a page? | 224 if not pages_list: # namespace is actually a page? |
213 names = namespace.split(":") | 225 names = namespace.split(":") |
214 real_namespace = ":".join(names[0:-1]) | 226 real_namespace = ":".join(names[0:-1]) |
218 | 230 |
219 count = 0 | 231 count = 0 |
220 for page in pages_list: | 232 for page in pages_list: |
221 self.processPost(page, profile_jid) | 233 self.processPost(page, profile_jid) |
222 count += 1 | 234 count += 1 |
223 if count >= self.limit : | 235 if count >= self.limit: |
224 break | 236 break |
225 | 237 |
226 return (self.posts_data.itervalues(), len(self.posts_data)) | 238 return (self.posts_data.itervalues(), len(self.posts_data)) |
227 | 239 |
228 def processContent(self, text, backlinks, profile_jid): | 240 def processContent(self, text, backlinks, profile_jid): |
332 log.debug("DokuWiki media thumbnail created: %s" % dest) | 344 log.debug("DokuWiki media thumbnail created: %s" % dest) |
333 except IOError: | 345 except IOError: |
334 log.error("Cannot create DokuWiki media thumbnail %s" % dest) | 346 log.error("Cannot create DokuWiki media thumbnail %s" % dest) |
335 | 347 |
336 | 348 |
337 | |
338 class DokuwikiImport(object): | 349 class DokuwikiImport(object): |
339 | |
340 def __init__(self, host): | 350 def __init__(self, host): |
341 log.info(_("plugin Dokuwiki Import initialization")) | 351 log.info(_("plugin Dokuwiki Import initialization")) |
342 self.host = host | 352 self.host = host |
343 self._blog_import = host.plugins['BLOG_IMPORT'] | 353 self._blog_import = host.plugins["BLOG_IMPORT"] |
344 self._blog_import.register('dokuwiki', self.DkImport, SHORT_DESC, LONG_DESC) | 354 self._blog_import.register("dokuwiki", self.DkImport, SHORT_DESC, LONG_DESC) |
345 | 355 |
346 def DkImport(self, client, location, options=None): | 356 def DkImport(self, client, location, options=None): |
347 """Import from DokuWiki to PubSub | 357 """Import from DokuWiki to PubSub |
348 | 358 |
349 @param location (unicode): DokuWiki site URL | 359 @param location (unicode): DokuWiki site URL |
365 | 375 |
366 opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None) | 376 opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None) |
367 try: | 377 try: |
368 media_repo = options["media_repo"] | 378 media_repo = options["media_repo"] |
369 if opt_upload_images: | 379 if opt_upload_images: |
370 options[self._blog_import.OPT_UPLOAD_IMAGES] = False # force using --no-images-upload | 380 options[ |
371 info_msg = _("DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}") | 381 self._blog_import.OPT_UPLOAD_IMAGES |
382 ] = False # force using --no-images-upload | |
383 info_msg = _( | |
384 "DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}" | |
385 ) | |
372 except KeyError: | 386 except KeyError: |
373 media_repo = DEFAULT_MEDIA_REPO | 387 media_repo = DEFAULT_MEDIA_REPO |
374 if opt_upload_images: | 388 if opt_upload_images: |
375 info_msg = _("DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though.") | 389 info_msg = _( |
390 "DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though." | |
391 ) | |
376 else: | 392 else: |
377 info_msg = _("DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible.") | 393 info_msg = _( |
394 "DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible." | |
395 ) | |
378 | 396 |
379 try: | 397 try: |
380 namespace = options["namespace"] | 398 namespace = options["namespace"] |
381 except KeyError: | 399 except KeyError: |
382 namespace = DEFAULT_NAMESPACE | 400 namespace = DEFAULT_NAMESPACE |
384 limit = options["limit"] | 402 limit = options["limit"] |
385 except KeyError: | 403 except KeyError: |
386 limit = DEFAULT_LIMIT | 404 limit = DEFAULT_LIMIT |
387 | 405 |
388 dk_importer = Importer(location, user, passwd, media_repo, limit) | 406 dk_importer = Importer(location, user, passwd, media_repo, limit) |
389 info_msg = info_msg.format(temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location) | 407 info_msg = info_msg.format( |
390 self.host.actionNew({'xmlui': xml_tools.note(info_msg).toXml()}, profile=client.profile) | 408 temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location |
409 ) | |
410 self.host.actionNew( | |
411 {"xmlui": xml_tools.note(info_msg).toXml()}, profile=client.profile | |
412 ) | |
391 d = threads.deferToThread(dk_importer.process, client, namespace) | 413 d = threads.deferToThread(dk_importer.process, client, namespace) |
392 return d | 414 return d |