Mercurial > libervia-backend
view sat/plugins/plugin_blog_import_dokuwiki.py @ 3024:c2e7bb251e3e
install: added doc to MANIFEST.in
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 24 Jul 2019 19:54:29 +0200 |
parents | 003b8b4b56a7 |
children | ab2696e34d29 |
line wrap: on
line source
#!/usr/bin/env python2 # -*- coding: utf-8 -*- # SàT plugin to import dokuwiki blogs # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org) # Copyright (C) 2013-2016 Adrien Cossa (souliane@mailoo.org) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from sat.core.i18n import _, D_ from sat.core.constants import Const as C from sat.core.log import getLogger log = getLogger(__name__) from sat.core import exceptions from sat.tools import xml_tools from twisted.internet import threads from collections import OrderedDict import calendar import urllib import urlparse import tempfile import re import time import os.path try: from dokuwiki import DokuWiki, DokuWikiError # this is a new dependency except ImportError: raise exceptions.MissingModule( u'Missing module dokuwiki, please install it with "pip install dokuwiki"' ) try: from PIL import Image # this is already needed by plugin XEP-0054 except: raise exceptions.MissingModule( u"Missing module pillow, please download/install it from https://python-pillow.github.io" ) PLUGIN_INFO = { C.PI_NAME: "Dokuwiki import", C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI", C.PI_TYPE: C.PLUG_TYPE_BLOG, C.PI_DEPENDENCIES: ["BLOG_IMPORT"], C.PI_MAIN: "DokuwikiImport", C.PI_HANDLER: "no", C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine."""), } SHORT_DESC = D_(u"import posts from Dokuwiki blog engine") LONG_DESC = D_( u"""This importer handle Dokuwiki blog engine. To use it, you need an admin access to a running Dokuwiki website (local or on the Internet). The importer retrieves the data using the XMLRPC Dokuwiki API. You can specify a namespace (that could be a namespace directory or a single post) or leave it empty to use the root namespace "/" and import all the posts. You can specify a new media repository to modify the internal media links and make them point to the URL of your choice, but note that the upload is not done automatically: a temporary directory will be created on your local drive and you will need to upload it yourself to your repository via SSH or FTP. Following options are recognized: location: DokuWiki site URL user: DokuWiki admin user passwd: DokuWiki admin password namespace: DokuWiki namespace to import (default: root namespace "/") media_repo: URL to the new remote media repository (default: none) limit: maximal number of posts to import (default: 100) Example of usage (with jp frontend): jp import dokuwiki -p dave --pwd xxxxxx --connect http://127.0.1.1 -o user souliane -o passwd qwertz -o namespace public:2015:10 -o media_repo http://media.diekulturvermittlung.at This retrieves the 100 last blog posts from http://127.0.1.1 that are inside the namespace "public:2015:10" using the Dokuwiki user "souliane", and it imports them to sat profile dave's microblog node. Internal Dokuwiki media that were hosted on http://127.0.1.1 are now pointing to http://media.diekulturvermittlung.at. """ ) DEFAULT_MEDIA_REPO = "" DEFAULT_NAMESPACE = "/" DEFAULT_LIMIT = 100 # you might get a DBUS timeout (no reply) if it lasts too long class Importer(DokuWiki): def __init__( self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT ): """ @param url (unicode): DokuWiki site URL @param user (unicode): DokuWiki admin user @param passwd (unicode): DokuWiki admin password @param media_repo (unicode): New remote media repository """ DokuWiki.__init__(self, url, user, passwd) self.url = url self.media_repo = media_repo self.temp_dir = tempfile.mkdtemp() if self.media_repo else None self.limit = limit self.posts_data = OrderedDict() def getPostId(self, post): """Return a unique and constant post id @param post(dict): parsed post data @return (unicode): post unique item id """ return unicode(post["id"]) def getPostUpdated(self, post): """Return the update date. @param post(dict): parsed post data @return (unicode): update date """ return unicode(post["mtime"]) def getPostPublished(self, post): """Try to parse the date from the message ID, else use "mtime". The date can be extracted if the message ID looks like one of: - namespace:YYMMDD_short_title - namespace:YYYYMMDD_short_title @param post (dict): parsed post data @return (unicode): publication date """ id_, default = unicode(post["id"]), unicode(post["mtime"]) try: date = id_.split(":")[-1].split("_")[0] except KeyError: return default try: time_struct = time.strptime(date, "%y%m%d") except ValueError: try: time_struct = time.strptime(date, "%Y%m%d") except ValueError: return default return unicode(calendar.timegm(time_struct)) def processPost(self, post, profile_jid): """Process a single page. @param post (dict): parsed post data @param profile_jid """ # get main information id_ = self.getPostId(post) updated = self.getPostUpdated(post) published = self.getPostPublished(post) # manage links backlinks = self.pages.backlinks(id_) for link in self.pages.links(id_): if link["type"] != "extern": assert link["type"] == "local" page = link["page"] backlinks.append(page[1:] if page.startswith(":") else page) self.pages.get(id_) content_xhtml = self.processContent(self.pages.html(id_), backlinks, profile_jid) # XXX: title is already in content_xhtml and difficult to remove, so leave it # title = content.split("\n")[0].strip(u"\ufeff= ") # build the extra data dictionary mb_data = { "id": id_, "published": published, "updated": updated, "author": profile_jid.user, # "content": content, # when passed, it is displayed in Libervia instead of content_xhtml "content_xhtml": content_xhtml, # "title": title, "allow_comments": "true", } # find out if the message access is public or restricted namespace = id_.split(":")[0] if namespace and namespace.lower() not in ("public", "/"): mb_data["group"] = namespace # roster group must exist self.posts_data[id_] = {"blog": mb_data, "comments": [[]]} def process(self, client, namespace=DEFAULT_NAMESPACE): """Process a namespace or a single page. @param namespace (unicode): DokuWiki namespace (or page) to import """ profile_jid = client.jid log.info("Importing data from DokuWiki %s" % self.version) try: pages_list = self.pages.list(namespace) except DokuWikiError: log.warning( 'Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.' ) return if not pages_list: # namespace is actually a page? names = namespace.split(":") real_namespace = ":".join(names[0:-1]) pages_list = self.pages.list(real_namespace) pages_list = [page for page in pages_list if page["id"] == namespace] namespace = real_namespace count = 0 for page in pages_list: self.processPost(page, profile_jid) count += 1 if count >= self.limit: break return (self.posts_data.itervalues(), len(self.posts_data)) def processContent(self, text, backlinks, profile_jid): """Do text substitutions and file copy. @param text (unicode): message content @param backlinks (list[unicode]): list of backlinks """ text = text.strip(u"\ufeff") # this is at the beginning of the file (BOM) for backlink in backlinks: src = '/doku.php?id=%s"' % backlink tgt = '/blog/%s/%s" target="#"' % (profile_jid.user, backlink) text = text.replace(src, tgt) subs = {} link_pattern = r"""<(img|a)[^>]* (src|href)="([^"]+)"[^>]*>""" for tag in re.finditer(link_pattern, text): type_, attr, link = tag.group(1), tag.group(2), tag.group(3) assert (type_ == "img" and attr == "src") or (type_ == "a" and attr == "href") if re.match(r"^\w*://", link): # absolute URL to link directly continue if self.media_repo: self.moveMedia(link, subs) elif link not in subs: subs[link] = urlparse.urljoin(self.url, link) for url, new_url in subs.iteritems(): text = text.replace(url, new_url) return text def moveMedia(self, link, subs): """Move a media from the DokuWiki host to the new repository. This also updates the hyperlinks to internal media files. @param link (unicode): media link @param subs (dict): substitutions data """ url = urlparse.urljoin(self.url, link) user_media = re.match(r"(/lib/exe/\w+.php\?)(.*)", link) thumb_width = None if user_media: # media that has been added by the user params = urlparse.parse_qs(urlparse.urlparse(url).query) try: media = params["media"][0] except KeyError: log.warning("No media found in fetch URL: %s" % user_media.group(2)) return if re.match(r"^\w*://", media): # external URL to link directly subs[link] = media return try: # create thumbnail thumb_width = params["w"][0] except KeyError: pass filename = media.replace(":", "/") # XXX: avoid "precondition failed" error (only keep the media parameter) url = urlparse.urljoin(self.url, "/lib/exe/fetch.php?media=%s" % media) elif link.startswith("/lib/plugins/"): # other link added by a plugin or something else filename = link[13:] else: # fake alert... there's no media (or we don't handle it yet) return filepath = os.path.join(self.temp_dir, filename) self.downloadMedia(url, filepath) if thumb_width: filename = os.path.join("thumbs", thumb_width, filename) thumbnail = os.path.join(self.temp_dir, filename) self.createThumbnail(filepath, thumbnail, thumb_width) new_url = os.path.join(self.media_repo, filename) subs[link] = new_url def downloadMedia(self, source, dest): """Copy media to localhost. @param source (unicode): source url @param dest (unicode): target path """ dirname = os.path.dirname(dest) if not os.path.exists(dest): if not os.path.exists(dirname): os.makedirs(dirname) urllib.urlretrieve(source, dest) log.debug("DokuWiki media file copied to %s" % dest) def createThumbnail(self, source, dest, width): """Create a thumbnail. @param source (unicode): source file path @param dest (unicode): destination file path @param width (unicode): thumbnail's width """ thumb_dir = os.path.dirname(dest) if not os.path.exists(thumb_dir): os.makedirs(thumb_dir) try: im = Image.open(source) im.thumbnail((width, int(width) * im.size[0] / im.size[1])) im.save(dest) log.debug("DokuWiki media thumbnail created: %s" % dest) except IOError: log.error("Cannot create DokuWiki media thumbnail %s" % dest) class DokuwikiImport(object): def __init__(self, host): log.info(_("plugin Dokuwiki Import initialization")) self.host = host self._blog_import = host.plugins["BLOG_IMPORT"] self._blog_import.register("dokuwiki", self.DkImport, SHORT_DESC, LONG_DESC) def DkImport(self, client, location, options=None): """Import from DokuWiki to PubSub @param location (unicode): DokuWiki site URL @param options (dict, None): DokuWiki import parameters - user (unicode): DokuWiki admin user - passwd (unicode): DokuWiki admin password - namespace (unicode): DokuWiki namespace to import - media_repo (unicode): New remote media repository """ options[self._blog_import.OPT_HOST] = location try: user = options["user"] except KeyError: raise exceptions.DataError('parameter "user" is required') try: passwd = options["passwd"] except KeyError: raise exceptions.DataError('parameter "passwd" is required') opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None) try: media_repo = options["media_repo"] if opt_upload_images: options[ self._blog_import.OPT_UPLOAD_IMAGES ] = False # force using --no-images-upload info_msg = _( "DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}" ) except KeyError: media_repo = DEFAULT_MEDIA_REPO if opt_upload_images: info_msg = _( "DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though." ) else: info_msg = _( "DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible." ) try: namespace = options["namespace"] except KeyError: namespace = DEFAULT_NAMESPACE try: limit = options["limit"] except KeyError: limit = DEFAULT_LIMIT dk_importer = Importer(location, user, passwd, media_repo, limit) info_msg = info_msg.format( temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location ) self.host.actionNew( {"xmlui": xml_tools.note(info_msg).toXml()}, profile=client.profile ) d = threads.deferToThread(dk_importer.process, client, namespace) return d