Mercurial > libervia-backend

comparison sat/plugins/plugin_blog_import_dokuwiki.py @ 2624:56f94936df1e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
code style reformatting using black
author Goffi <goffi@goffi.org>
date Wed, 27 Jun 2018 20:14:46 +0200
parents 26edcf3a30eb
children 003b8b4b56a7
comparison
equal deleted inserted replaced
2623:49533de4540b 2624:56f94936df1e
19 # along with this program. If not, see <http://www.gnu.org/licenses/>. 19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 20
21 from sat.core.i18n import _, D_ 21 from sat.core.i18n import _, D_
22 from sat.core.constants import Const as C 22 from sat.core.constants import Const as C
23 from sat.core.log import getLogger 23 from sat.core.log import getLogger
24
24 log = getLogger(__name__) 25 log = getLogger(__name__)
25 from sat.core import exceptions 26 from sat.core import exceptions
26 from sat.tools import xml_tools 27 from sat.tools import xml_tools
27 from twisted.internet import threads 28 from twisted.internet import threads
28 from collections import OrderedDict 29 from collections import OrderedDict
31 import urlparse 32 import urlparse
32 import tempfile 33 import tempfile
33 import re 34 import re
34 import time 35 import time
35 import os.path 36 import os.path
37
36 try: 38 try:
37 from dokuwiki import DokuWiki, DokuWikiError # this is a new dependency 39 from dokuwiki import DokuWiki, DokuWikiError # this is a new dependency
38 except ImportError: 40 except ImportError:
39 raise exceptions.MissingModule(u'Missing module dokuwiki, please install it with "pip install dokuwiki"') 41 raise exceptions.MissingModule(
42 u'Missing module dokuwiki, please install it with "pip install dokuwiki"'
43 )
40 try: 44 try:
41 from PIL import Image # this is already needed by plugin XEP-0054 45 from PIL import Image # this is already needed by plugin XEP-0054
42 except: 46 except:
43 raise exceptions.MissingModule(u"Missing module pillow, please download/install it from https://python-pillow.github.io") 47 raise exceptions.MissingModule(
48 u"Missing module pillow, please download/install it from https://python-pillow.github.io"
49 )
44 50
45 PLUGIN_INFO = { 51 PLUGIN_INFO = {
46 C.PI_NAME: "Dokuwiki import", 52 C.PI_NAME: "Dokuwiki import",
47 C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI", 53 C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI",
48 C.PI_TYPE: C.PLUG_TYPE_BLOG, 54 C.PI_TYPE: C.PLUG_TYPE_BLOG,
49 C.PI_DEPENDENCIES: ["BLOG_IMPORT"], 55 C.PI_DEPENDENCIES: ["BLOG_IMPORT"],
50 C.PI_MAIN: "DokuwikiImport", 56 C.PI_MAIN: "DokuwikiImport",
51 C.PI_HANDLER: "no", 57 C.PI_HANDLER: "no",
52 C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine.""") 58 C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine."""),
53 } 59 }
54 60
55 SHORT_DESC = D_(u"import posts from Dokuwiki blog engine") 61 SHORT_DESC = D_(u"import posts from Dokuwiki blog engine")
56 62
57 LONG_DESC = D_(u"""This importer handle Dokuwiki blog engine. 63 LONG_DESC = D_(
64 u"""This importer handle Dokuwiki blog engine.
58 65
59 To use it, you need an admin access to a running Dokuwiki website 66 To use it, you need an admin access to a running Dokuwiki website
60 (local or on the Internet). The importer retrieves the data using 67 (local or on the Internet). The importer retrieves the data using
61 the XMLRPC Dokuwiki API. 68 the XMLRPC Dokuwiki API.
62 69
89 This retrieves the 100 last blog posts from http://127.0.1.1 that 96 This retrieves the 100 last blog posts from http://127.0.1.1 that
90 are inside the namespace "public:2015:10" using the Dokuwiki user 97 are inside the namespace "public:2015:10" using the Dokuwiki user
91 "souliane", and it imports them to sat profile dave's microblog node. 98 "souliane", and it imports them to sat profile dave's microblog node.
92 Internal Dokuwiki media that were hosted on http://127.0.1.1 are now 99 Internal Dokuwiki media that were hosted on http://127.0.1.1 are now
93 pointing to http://media.diekulturvermittlung.at. 100 pointing to http://media.diekulturvermittlung.at.
94 """) 101 """
102 )
95 DEFAULT_MEDIA_REPO = "" 103 DEFAULT_MEDIA_REPO = ""
96 DEFAULT_NAMESPACE = "/" 104 DEFAULT_NAMESPACE = "/"
97 DEFAULT_LIMIT = 100 # you might get a DBUS timeout (no reply) if it lasts too long 105 DEFAULT_LIMIT = 100 # you might get a DBUS timeout (no reply) if it lasts too long
98 106
99 107
100 class Importer(DokuWiki): 108 class Importer(DokuWiki):
101 109 def __init__(
102 def __init__(self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT): 110 self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT
111 ):
103 """ 112 """
104 113
105 @param url (unicode): DokuWiki site URL 114 @param url (unicode): DokuWiki site URL
106 @param user (unicode): DokuWiki admin user 115 @param user (unicode): DokuWiki admin user
107 @param passwd (unicode): DokuWiki admin password 116 @param passwd (unicode): DokuWiki admin password
118 """Return a unique and constant post id 127 """Return a unique and constant post id
119 128
120 @param post(dict): parsed post data 129 @param post(dict): parsed post data
121 @return (unicode): post unique item id 130 @return (unicode): post unique item id
122 """ 131 """
123 return unicode(post['id']) 132 return unicode(post["id"])
124 133
125 def getPostUpdated(self, post): 134 def getPostUpdated(self, post):
126 """Return the update date. 135 """Return the update date.
127 136
128 @param post(dict): parsed post data 137 @param post(dict): parsed post data
129 @return (unicode): update date 138 @return (unicode): update date
130 """ 139 """
131 return unicode(post['mtime']) 140 return unicode(post["mtime"])
132 141
133 def getPostPublished(self, post): 142 def getPostPublished(self, post):
134 """Try to parse the date from the message ID, else use "mtime". 143 """Try to parse the date from the message ID, else use "mtime".
135 144
136 The date can be extracted if the message ID looks like one of: 145 The date can be extracted if the message ID looks like one of:
177 186
178 # XXX: title is already in content_xhtml and difficult to remove, so leave it 187 # XXX: title is already in content_xhtml and difficult to remove, so leave it
179 # title = content.split("\n")[0].strip(u"\ufeff= ") 188 # title = content.split("\n")[0].strip(u"\ufeff= ")
180 189
181 # build the extra data dictionary 190 # build the extra data dictionary
182 mb_data = {"id": id_, 191 mb_data = {
183 "published": published, 192 "id": id_,
184 "updated": updated, 193 "published": published,
185 "author": profile_jid.user, 194 "updated": updated,
186 # "content": content, # when passed, it is displayed in Libervia instead of content_xhtml 195 "author": profile_jid.user,
187 "content_xhtml": content_xhtml, 196 # "content": content, # when passed, it is displayed in Libervia instead of content_xhtml
188 # "title": title, 197 "content_xhtml": content_xhtml,
189 "allow_comments": "true", 198 # "title": title,
190 } 199 "allow_comments": "true",
200 }
191 201
192 # find out if the message access is public or restricted 202 # find out if the message access is public or restricted
193 namespace = id_.split(":")[0] 203 namespace = id_.split(":")[0]
194 if namespace and namespace.lower() not in ("public", "/"): 204 if namespace and namespace.lower() not in ("public", "/"):
195 mb_data["group"] = namespace # roster group must exist 205 mb_data["group"] = namespace # roster group must exist
196 206
197 self.posts_data[id_] = {'blog': mb_data, 'comments':[[]]} 207 self.posts_data[id_] = {"blog": mb_data, "comments": [[]]}
198 208
199 def process(self, client, namespace=DEFAULT_NAMESPACE): 209 def process(self, client, namespace=DEFAULT_NAMESPACE):
200 """Process a namespace or a single page. 210 """Process a namespace or a single page.
201 211
202 @param namespace (unicode): DokuWiki namespace (or page) to import 212 @param namespace (unicode): DokuWiki namespace (or page) to import
204 profile_jid = client.jid 214 profile_jid = client.jid
205 log.info("Importing data from DokuWiki %s" % self.version) 215 log.info("Importing data from DokuWiki %s" % self.version)
206 try: 216 try:
207 pages_list = self.pages.list(namespace) 217 pages_list = self.pages.list(namespace)
208 except DokuWikiError: 218 except DokuWikiError:
209 log.warning('Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.') 219 log.warning(
220 'Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.'
221 )
210 return 222 return
211 223
212 if not pages_list: # namespace is actually a page? 224 if not pages_list: # namespace is actually a page?
213 names = namespace.split(":") 225 names = namespace.split(":")
214 real_namespace = ":".join(names[0:-1]) 226 real_namespace = ":".join(names[0:-1])
218 230
219 count = 0 231 count = 0
220 for page in pages_list: 232 for page in pages_list:
221 self.processPost(page, profile_jid) 233 self.processPost(page, profile_jid)
222 count += 1 234 count += 1
223 if count >= self.limit : 235 if count >= self.limit:
224 break 236 break
225 237
226 return (self.posts_data.itervalues(), len(self.posts_data)) 238 return (self.posts_data.itervalues(), len(self.posts_data))
227 239
228 def processContent(self, text, backlinks, profile_jid): 240 def processContent(self, text, backlinks, profile_jid):
332 log.debug("DokuWiki media thumbnail created: %s" % dest) 344 log.debug("DokuWiki media thumbnail created: %s" % dest)
333 except IOError: 345 except IOError:
334 log.error("Cannot create DokuWiki media thumbnail %s" % dest) 346 log.error("Cannot create DokuWiki media thumbnail %s" % dest)
335 347
336 348
337
338 class DokuwikiImport(object): 349 class DokuwikiImport(object):
339
340 def __init__(self, host): 350 def __init__(self, host):
341 log.info(_("plugin Dokuwiki Import initialization")) 351 log.info(_("plugin Dokuwiki Import initialization"))
342 self.host = host 352 self.host = host
343 self._blog_import = host.plugins['BLOG_IMPORT'] 353 self._blog_import = host.plugins["BLOG_IMPORT"]
344 self._blog_import.register('dokuwiki', self.DkImport, SHORT_DESC, LONG_DESC) 354 self._blog_import.register("dokuwiki", self.DkImport, SHORT_DESC, LONG_DESC)
345 355
346 def DkImport(self, client, location, options=None): 356 def DkImport(self, client, location, options=None):
347 """Import from DokuWiki to PubSub 357 """Import from DokuWiki to PubSub
348 358
349 @param location (unicode): DokuWiki site URL 359 @param location (unicode): DokuWiki site URL
365 375
366 opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None) 376 opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None)
367 try: 377 try:
368 media_repo = options["media_repo"] 378 media_repo = options["media_repo"]
369 if opt_upload_images: 379 if opt_upload_images:
370 options[self._blog_import.OPT_UPLOAD_IMAGES] = False # force using --no-images-upload 380 options[
371 info_msg = _("DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}") 381 self._blog_import.OPT_UPLOAD_IMAGES
382 ] = False # force using --no-images-upload
383 info_msg = _(
384 "DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}"
385 )
372 except KeyError: 386 except KeyError:
373 media_repo = DEFAULT_MEDIA_REPO 387 media_repo = DEFAULT_MEDIA_REPO
374 if opt_upload_images: 388 if opt_upload_images:
375 info_msg = _("DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though.") 389 info_msg = _(
390 "DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though."
391 )
376 else: 392 else:
377 info_msg = _("DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible.") 393 info_msg = _(
394 "DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible."
395 )
378 396
379 try: 397 try:
380 namespace = options["namespace"] 398 namespace = options["namespace"]
381 except KeyError: 399 except KeyError:
382 namespace = DEFAULT_NAMESPACE 400 namespace = DEFAULT_NAMESPACE
384 limit = options["limit"] 402 limit = options["limit"]
385 except KeyError: 403 except KeyError:
386 limit = DEFAULT_LIMIT 404 limit = DEFAULT_LIMIT
387 405
388 dk_importer = Importer(location, user, passwd, media_repo, limit) 406 dk_importer = Importer(location, user, passwd, media_repo, limit)
389 info_msg = info_msg.format(temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location) 407 info_msg = info_msg.format(
390 self.host.actionNew({'xmlui': xml_tools.note(info_msg).toXml()}, profile=client.profile) 408 temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location
409 )
410 self.host.actionNew(
411 {"xmlui": xml_tools.note(info_msg).toXml()}, profile=client.profile
412 )
391 d = threads.deferToThread(dk_importer.process, client, namespace) 413 d = threads.deferToThread(dk_importer.process, client, namespace)
392 return d 414 return d