Mercurial > libervia-backend
annotate src/plugins/plugin_blog_import.py @ 2535:a19b2c43e719
plugin XEP-0234: fixed regex import
author | Goffi <goffi@goffi.org> |
---|---|
date | Sun, 25 Mar 2018 20:51:02 +0200 |
parents | 0046283a285d |
children |
rev | line source |
---|---|
1934
2daf7b4c6756
use of /usr/bin/env instead of /usr/bin/python in shebang
Goffi <goffi@goffi.org>
parents:
1852
diff
changeset
|
1 #!/usr/bin/env python2 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SàT plugin for import external blogs |
2483 | 5 # Copyright (C) 2009-2018 Jérôme Poisson (goffi@goffi.org) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
20 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 from sat.core.i18n import _ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.core.constants import Const as C |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from sat.core.log import getLogger |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 log = getLogger(__name__) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 from twisted.internet import defer |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 from twisted.web import client as web_client |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 from twisted.words.xish import domish |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 from sat.core import exceptions |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 from sat.tools import xml_tools |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 import os |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 import os.path |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 import tempfile |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 import urlparse |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
34 import shortuuid |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 PLUGIN_INFO = { |
2145
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
38 C.PI_NAME: "blog import", |
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
39 C.PI_IMPORT_NAME: "BLOG_IMPORT", |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
40 C.PI_TYPE: (C.PLUG_TYPE_BLOG, C.PLUG_TYPE_IMPORT), |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
41 C.PI_DEPENDENCIES: ["IMPORT", "XEP-0060", "XEP-0277", "TEXT-SYNTAXES", "UPLOAD"], |
2145
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
42 C.PI_MAIN: "BlogImportPlugin", |
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
43 C.PI_HANDLER: "no", |
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
44 C.PI_DESCRIPTION: _(u"""Blog import management: |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
45 This plugin manage the different blog importers which can register to it, and handle generic importing tasks.""") |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 } |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 OPT_HOST = 'host' |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 OPT_UPLOAD_IMAGES = 'upload_images' |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 OPT_UPLOAD_IGNORE_HOST = 'upload_ignore_host' |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 OPT_IGNORE_TLS = 'ignore_tls_errors' |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
52 URL_REDIRECT_PREFIX = 'url_redirect_' |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
55 class BlogImportPlugin(object): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
56 BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS) |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
57 JSON_OPTIONS = () |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
58 OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
59 OPT_IGNORE_TLS: False} |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
61 def __init__(self, host): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 log.info(_("plugin Blog Import initialization")) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 self.host = host |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 self._u = host.plugins['UPLOAD'] |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
65 self._p = host.plugins['XEP-0060'] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 self._m = host.plugins['XEP-0277'] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 self._s = self.host.plugins['TEXT-SYNTAXES'] |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
68 host.plugins['IMPORT'].initialize(self, u'blog') |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
69 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
70 def importItem(self, client, item_import_data, session, options, return_data, service, node): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
71 """importItem specialized for blog import |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
73 @param item_import_data(dict): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
74 * mandatory keys: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
75 'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
76 the importer MUST NOT create node or call XEP-0277 plugin itself |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
77 'comments*' key MUST NOT be used in this microblog_data, see bellow for comments |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
78 It is recommanded to use a unique id in the "id" key which is constant per blog item, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
79 so if the import fail, a new import will overwrite the failed items and avoid duplicates. |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
80 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
81 'comments' (list[list[dict]],None): Dictionaries must have the same keys as main item (i.e. 'blog' and 'comments') |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
82 a list of list is used because XEP-0277 can handler several comments nodes, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
83 but in most cases, there will we only one item it the first list (something like [[{comment1_data},{comment2_data}, ...]]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
84 blog['allow_comments'] must be True if there is any comment, and False (or not present) if comments are not allowed. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
85 If allow_comments is False and some comments are present, an exceptions.DataError will be raised |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
86 * optional keys: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
87 'url' (unicode): former url of the post (only the path, without host part) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
88 if present the association to the new path will be displayed to user, so it can make redirections if necessary |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
89 @param options(dict, None): Below are the generic options, |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
90 blog importer can have specific ones. All options have unicode values |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
91 generic options: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
92 - OPT_HOST (unicode): original host |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
93 - OPT_UPLOAD_IMAGES (bool): upload images to XMPP server if True |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
94 see OPT_UPLOAD_IGNORE_HOST. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
95 Default: True |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
96 - OPT_UPLOAD_IGNORE_HOST (unicode): don't upload images from this host |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
97 - OPT_IGNORE_TLS (bool): ignore TLS error for image upload. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
98 Default: False |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
99 @param return_data(dict): will contain link between former posts and new items |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
100 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
101 """ |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
102 mb_data = item_import_data['blog'] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
103 try: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
104 item_id = mb_data['id'] |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
105 except KeyError: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
106 item_id = mb_data['id'] = unicode(shortuuid.uuid()) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
107 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
108 try: |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
109 # we keep the link between old url and new blog item |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
110 # so the user can redirect its former blog urls |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
111 old_uri = item_import_data['url'] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 except KeyError: |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
113 pass |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
114 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
115 new_uri = return_data[URL_REDIRECT_PREFIX + old_uri] = self._p.getNodeURI( |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
116 service if service is not None else client.jid.userhostJID(), |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
117 node or self._m.namespace, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
118 item_id) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
119 log.info(u"url link from {old} to {new}".format( |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
120 old=old_uri, new=new_uri)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
121 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
122 return mb_data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
123 |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
124 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
125 def importSubItems(self, client, item_import_data, mb_data, session, options): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
126 # comments data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
127 if len(item_import_data['comments']) != 1: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
128 raise NotImplementedError(u"can't manage multiple comment links") |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
129 allow_comments = C.bool(mb_data.get('allow_comments', C.BOOL_FALSE)) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
130 if allow_comments: |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
131 comments_service = yield self._m.getCommentsService(client) |
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
132 comments_node = self._m.getCommentsNode(mb_data['id']) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
133 mb_data['comments_service'] = comments_service.full() |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
134 mb_data['comments_node'] = comments_node |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
135 recurse_kwargs = { |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
136 'items_import_data':item_import_data['comments'][0], |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
137 'service':comments_service, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
138 'node':comments_node} |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
139 defer.returnValue(recurse_kwargs) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
140 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
141 if item_import_data['comments'][0]: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
142 raise exceptions.DataError(u"allow_comments set to False, but comments are there") |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
143 defer.returnValue(None) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
144 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
145 def publishItem(self, client, mb_data, service, node, session): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
146 log.debug(u"uploading item [{id}]: {title}".format(id=mb_data['id'], title=mb_data.get('title',''))) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
147 return self._m.send(client, mb_data, service, node) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
148 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
149 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
150 def itemFilters(self, client, mb_data, session, options): |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 """Apply filters according to options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
152 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
153 modify mb_data in place |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
154 @param posts_data(list[dict]): data as returned by importer callback |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
155 @param options(dict): dict as given in [blogImport] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
156 """ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 # FIXME: blog filters don't work on text content |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 # TODO: text => XHTML conversion should handler links with <a/> |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
159 # filters can then be used by converting text to XHTML |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
160 if not options: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
161 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
162 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
163 # we want only XHTML content |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
164 for prefix in ('content',): # a tuple is use, if title need to be added in the future |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
165 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
166 rich = mb_data['{}_rich'.format(prefix)] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
167 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
168 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
169 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
170 if '{}_xhtml'.format(prefix) in mb_data: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
171 raise exceptions.DataError(u"importer gave {prefix}_rich and {prefix}_xhtml at the same time, this is not allowed".format(prefix=prefix)) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
172 # we convert rich syntax to XHTML here, so we can handle filters easily |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
173 converted = yield self._s.convert(rich, self._s.getCurrentSyntax(client.profile), safe=False) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
174 mb_data['{}_xhtml'.format(prefix)] = converted |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
175 del mb_data['{}_rich'.format(prefix)] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
176 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
177 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
178 mb_data['txt'] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
179 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
180 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
181 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
182 if '{}_xhtml'.format(prefix) in mb_data: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
183 log.warning(u"{prefix}_text will be replaced by converted {prefix}_xhtml, so filters can be handled".format(prefix=prefix)) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
184 del mb_data['{}_text'.format(prefix)] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
185 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
186 log.warning(u"importer gave a text {prefix}, blog filters don't work on text {prefix}".format(prefix=prefix)) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
187 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
188 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
189 # at this point, we have only XHTML version of content |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
190 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
191 top_elt = xml_tools.ElementParser()(mb_data['content_xhtml'], namespace=C.NS_XHTML) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
192 except domish.ParserError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
193 # we clean the xml and try again our luck |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
194 cleaned = yield self._s.cleanXHTML(mb_data['content_xhtml']) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
195 top_elt = xml_tools.ElementParser()(cleaned, namespace=C.NS_XHTML) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
196 opt_host = options.get(OPT_HOST) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
197 if opt_host: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
198 # we normalise the domain |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
199 parsed_host = urlparse.urlsplit(opt_host) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
200 opt_host = urlparse.urlunsplit((parsed_host.scheme or 'http', parsed_host.netloc or parsed_host.path, '', '', '')) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
201 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
202 tmp_dir = tempfile.mkdtemp() |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
203 try: |
1844
489b968b3723
plugin blog_import_dokuwiki: also uses the generic image uploader from blog_import (when media_repo is empty and OPT_UPLOAD_IMAGES is True)
souliane <souliane@mailoo.org>
parents:
1839
diff
changeset
|
204 # TODO: would be nice to also update the hyperlinks to these images, e.g. when you have <a href="{url}"><img src="{url}"></a> |
2108 | 205 for img_elt in xml_tools.findAll(top_elt, names=[u'img']): |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
206 yield self.imgFilters(client, img_elt, options, opt_host, tmp_dir) |
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
207 finally: |
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
208 os.rmdir(tmp_dir) # XXX: tmp_dir should be empty, or something went wrong |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
209 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
210 # we now replace the content with filtered one |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
211 mb_data['content_xhtml'] = top_elt.toXml() |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
212 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
213 @defer.inlineCallbacks |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
214 def imgFilters(self, client, img_elt, options, opt_host, tmp_dir): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
215 """Filters handling images |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
216 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
217 url without host are fixed (if possible) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
218 according to options, images are uploaded to XMPP server |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
219 @param img_elt(domish.Element): <img/> element to handle |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
220 @param options(dict): filters options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
221 @param opt_host(unicode): normalised host given in options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
222 @param tmp_dir(str): path to temp directory |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
223 """ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
224 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
225 url = img_elt['src'] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 if url[0] == u'/': |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 if not opt_host: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
228 log.warning(u"host was not specified, we can't deal with src without host ({url}) and have to ignore the following <img/>:\n{xml}" |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
229 .format(url=url, xml=img_elt.toXml())) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
230 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
231 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
232 url = urlparse.urljoin(opt_host, url) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
233 filename = url.rsplit('/',1)[-1].strip() |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
234 if not filename: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
235 raise KeyError |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 except (KeyError, IndexError): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
237 log.warning(u"ignoring invalid img element: {}".format(img_elt.toXml())) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
238 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
239 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
240 # we change the url for the normalized one |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
241 img_elt['src'] = url |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
242 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 if options.get(OPT_UPLOAD_IMAGES, False): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
244 # upload is requested |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
245 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
246 ignore_host = options[OPT_UPLOAD_IGNORE_HOST] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
247 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
248 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
249 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
250 # host is the ignored one, we skip |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
251 parsed_url = urlparse.urlsplit(url) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
252 if ignore_host in parsed_url.hostname: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
253 log.info(u"Don't upload image at {url} because of {opt} option".format( |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
254 url=url, opt=OPT_UPLOAD_IGNORE_HOST)) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
255 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
256 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
257 # we download images and re-upload them via XMPP |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
258 tmp_file = os.path.join(tmp_dir, filename).encode('utf-8') |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
259 upload_options = {'ignore_tls_errors': options.get(OPT_IGNORE_TLS, False)} |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
260 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
261 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
262 yield web_client.downloadPage(url.encode('utf-8'), tmp_file) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
263 filename = filename.replace(u'%', u'_') # FIXME: tmp workaround for a bug in prosody http upload |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
264 dummy, download_d = yield self._u.upload(client, tmp_file, filename, options=upload_options) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
265 download_url = yield download_d |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
266 except Exception as e: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
267 log.warning(u"can't download image at {url}: {reason}".format(url=url, reason=e)) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
268 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
269 img_elt['src'] = download_url |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
270 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
271 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
272 os.unlink(tmp_file) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
273 except OSError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
274 pass |