Mercurial > libervia-backend
annotate sat/plugins/plugin_blog_import.py @ 3319:3a15e76a694e
core (memory): `setFileAccessModel` implementation:
this method let change read permission using access model similar to what is used with
Pubsub.
author | Goffi <goffi@goffi.org> |
---|---|
date | Sat, 01 Aug 2020 16:07:39 +0200 |
parents | 559a625a236b |
children | be6d91572633 |
rev | line source |
---|---|
3028 | 1 #!/usr/bin/env python3 |
3137 | 2 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SàT plugin for import external blogs |
3136 | 5 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
20 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 from sat.core.i18n import _ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.core.constants import Const as C |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from sat.core.log import getLogger |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
24 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 log = getLogger(__name__) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 from twisted.internet import defer |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 from twisted.web import client as web_client |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 from twisted.words.xish import domish |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 from sat.core import exceptions |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 from sat.tools import xml_tools |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 import os |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 import os.path |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 import tempfile |
3028 | 34 import urllib.parse |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
35 import shortuuid |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 PLUGIN_INFO = { |
2145
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
39 C.PI_NAME: "blog import", |
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
40 C.PI_IMPORT_NAME: "BLOG_IMPORT", |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
41 C.PI_TYPE: (C.PLUG_TYPE_BLOG, C.PLUG_TYPE_IMPORT), |
2780
85d3240a400f
plugin text syntaxes: changed import name to TEXT_SYNTAX (better with underscore for autocompletion)
Goffi <goffi@goffi.org>
parents:
2771
diff
changeset
|
42 C.PI_DEPENDENCIES: ["IMPORT", "XEP-0060", "XEP-0277", "TEXT_SYNTAXES", "UPLOAD"], |
2145
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
43 C.PI_MAIN: "BlogImportPlugin", |
33c8c4973743
core (plugins): added missing contants + use of new constants in PLUGIN_INFO
Goffi <goffi@goffi.org>
parents:
2108
diff
changeset
|
44 C.PI_HANDLER: "no", |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
45 C.PI_DESCRIPTION: _( |
3028 | 46 """Blog import management: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
47 This plugin manage the different blog importers which can register to it, and handle generic importing tasks.""" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
48 ), |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 } |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
51 OPT_HOST = "host" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
52 OPT_UPLOAD_IMAGES = "upload_images" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
53 OPT_UPLOAD_IGNORE_HOST = "upload_ignore_host" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
54 OPT_IGNORE_TLS = "ignore_tls_errors" |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
55 URL_REDIRECT_PREFIX = "url_redirect_" |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
58 class BlogImportPlugin(object): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
59 BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS) |
2396
66baa687c682
plugins tickets import, jp (ticket/import): implemented mapping:
Goffi <goffi@goffi.org>
parents:
2370
diff
changeset
|
60 JSON_OPTIONS = () |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
61 OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True, OPT_IGNORE_TLS: False} |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 def __init__(self, host): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 log.info(_("plugin Blog Import initialization")) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 self.host = host |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
66 self._u = host.plugins["UPLOAD"] |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
67 self._p = host.plugins["XEP-0060"] |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
68 self._m = host.plugins["XEP-0277"] |
2780
85d3240a400f
plugin text syntaxes: changed import name to TEXT_SYNTAX (better with underscore for autocompletion)
Goffi <goffi@goffi.org>
parents:
2771
diff
changeset
|
69 self._s = self.host.plugins["TEXT_SYNTAXES"] |
3028 | 70 host.plugins["IMPORT"].initialize(self, "blog") |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
71 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
72 def importItem( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
73 self, client, item_import_data, session, options, return_data, service, node |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
74 ): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
75 """importItem specialized for blog import |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
77 @param item_import_data(dict): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
78 * mandatory keys: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
79 'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
80 the importer MUST NOT create node or call XEP-0277 plugin itself |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
81 'comments*' key MUST NOT be used in this microblog_data, see bellow for comments |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
82 It is recommanded to use a unique id in the "id" key which is constant per blog item, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
83 so if the import fail, a new import will overwrite the failed items and avoid duplicates. |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
85 'comments' (list[list[dict]],None): Dictionaries must have the same keys as main item (i.e. 'blog' and 'comments') |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
86 a list of list is used because XEP-0277 can handler several comments nodes, |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
87 but in most cases, there will we only one item it the first list (something like [[{comment1_data},{comment2_data}, ...]]) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
88 blog['allow_comments'] must be True if there is any comment, and False (or not present) if comments are not allowed. |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
89 If allow_comments is False and some comments are present, an exceptions.DataError will be raised |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
90 * optional keys: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
91 'url' (unicode): former url of the post (only the path, without host part) |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
92 if present the association to the new path will be displayed to user, so it can make redirections if necessary |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
93 @param options(dict, None): Below are the generic options, |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
94 blog importer can have specific ones. All options have unicode values |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
95 generic options: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
96 - OPT_HOST (unicode): original host |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
97 - OPT_UPLOAD_IMAGES (bool): upload images to XMPP server if True |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
98 see OPT_UPLOAD_IGNORE_HOST. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
99 Default: True |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
100 - OPT_UPLOAD_IGNORE_HOST (unicode): don't upload images from this host |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
101 - OPT_IGNORE_TLS (bool): ignore TLS error for image upload. |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
102 Default: False |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
103 @param return_data(dict): will contain link between former posts and new items |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
104 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
105 """ |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
106 mb_data = item_import_data["blog"] |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
107 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
108 item_id = mb_data["id"] |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
109 except KeyError: |
3028 | 110 item_id = mb_data["id"] = str(shortuuid.uuid()) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
111 |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 try: |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
113 # we keep the link between old url and new blog item |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
114 # so the user can redirect its former blog urls |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
115 old_uri = item_import_data["url"] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 except KeyError: |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
117 pass |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
118 else: |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
119 new_uri = return_data[URL_REDIRECT_PREFIX + old_uri] = self._p.getNodeURI( |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
120 service if service is not None else client.jid.userhostJID(), |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
121 node or self._m.namespace, |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
122 item_id, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
123 ) |
3028 | 124 log.info("url link from {old} to {new}".format(old=old_uri, new=new_uri)) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
125 |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
126 return mb_data |
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
127 |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
128 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
129 def importSubItems(self, client, item_import_data, mb_data, session, options): |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
130 # comments data |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
131 if len(item_import_data["comments"]) != 1: |
3028 | 132 raise NotImplementedError("can't manage multiple comment links") |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
133 allow_comments = C.bool(mb_data.get("allow_comments", C.BOOL_FALSE)) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
134 if allow_comments: |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
135 comments_service = yield self._m.getCommentsService(client) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
136 comments_node = self._m.getCommentsNode(mb_data["id"]) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
137 mb_data["comments_service"] = comments_service.full() |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
138 mb_data["comments_node"] = comments_node |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
139 recurse_kwargs = { |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
140 "items_import_data": item_import_data["comments"][0], |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
141 "service": comments_service, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
142 "node": comments_node, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
143 } |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
144 defer.returnValue(recurse_kwargs) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
145 else: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
146 if item_import_data["comments"][0]: |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
147 raise exceptions.DataError( |
3028 | 148 "allow_comments set to False, but comments are there" |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
149 ) |
2437
91bbad17fd53
plugins blog import, tickets import, XEP-0277: fixed call to getCommentsService (it is now returning a Deferred)
Goffi <goffi@goffi.org>
parents:
2414
diff
changeset
|
150 defer.returnValue(None) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
151 |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
152 def publishItem(self, client, mb_data, service, node, session): |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
153 log.debug( |
3028 | 154 "uploading item [{id}]: {title}".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
155 id=mb_data["id"], title=mb_data.get("title", "") |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
156 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
157 ) |
2369
cdaa58e14553
plugin import: generic data import plugin:
Goffi <goffi@goffi.org>
parents:
2145
diff
changeset
|
158 return self._m.send(client, mb_data, service, node) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
159 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
160 @defer.inlineCallbacks |
2370
2c2b826b0bb3
plugin import: node can now be specified + added a "session" dict to keep import session data:
Goffi <goffi@goffi.org>
parents:
2369
diff
changeset
|
161 def itemFilters(self, client, mb_data, session, options): |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
162 """Apply filters according to options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
163 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
164 modify mb_data in place |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
165 @param posts_data(list[dict]): data as returned by importer callback |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
166 @param options(dict): dict as given in [blogImport] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
167 """ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
168 # FIXME: blog filters don't work on text content |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
169 # TODO: text => XHTML conversion should handler links with <a/> |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
170 # filters can then be used by converting text to XHTML |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
171 if not options: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
172 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
173 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
174 # we want only XHTML content |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
175 for prefix in ( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
176 "content", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
177 ): # a tuple is use, if title need to be added in the future |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
178 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
179 rich = mb_data["{}_rich".format(prefix)] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
180 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
181 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
182 else: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
183 if "{}_xhtml".format(prefix) in mb_data: |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
184 raise exceptions.DataError( |
3028 | 185 "importer gave {prefix}_rich and {prefix}_xhtml at the same time, this is not allowed".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
186 prefix=prefix |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
187 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
188 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
189 # we convert rich syntax to XHTML here, so we can handle filters easily |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
190 converted = yield self._s.convert( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
191 rich, self._s.getCurrentSyntax(client.profile), safe=False |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
192 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
193 mb_data["{}_xhtml".format(prefix)] = converted |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
194 del mb_data["{}_rich".format(prefix)] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
195 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
196 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
197 mb_data["txt"] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
198 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
199 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
200 else: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
201 if "{}_xhtml".format(prefix) in mb_data: |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
202 log.warning( |
3028 | 203 "{prefix}_text will be replaced by converted {prefix}_xhtml, so filters can be handled".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
204 prefix=prefix |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
205 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
206 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
207 del mb_data["{}_text".format(prefix)] |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
208 else: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
209 log.warning( |
3028 | 210 "importer gave a text {prefix}, blog filters don't work on text {prefix}".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
211 prefix=prefix |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
212 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
213 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
214 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
215 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
216 # at this point, we have only XHTML version of content |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
217 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
218 top_elt = xml_tools.ElementParser()( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
219 mb_data["content_xhtml"], namespace=C.NS_XHTML |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
220 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
221 except domish.ParserError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
222 # we clean the xml and try again our luck |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
223 cleaned = yield self._s.cleanXHTML(mb_data["content_xhtml"]) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
224 top_elt = xml_tools.ElementParser()(cleaned, namespace=C.NS_XHTML) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
225 opt_host = options.get(OPT_HOST) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 if opt_host: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 # we normalise the domain |
3028 | 228 parsed_host = urllib.parse.urlsplit(opt_host) |
229 opt_host = urllib.parse.urlunsplit( | |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
230 ( |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
231 parsed_host.scheme or "http", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
232 parsed_host.netloc or parsed_host.path, |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
233 "", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
234 "", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
235 "", |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
236 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
237 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
238 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
239 tmp_dir = tempfile.mkdtemp() |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
240 try: |
1844
489b968b3723
plugin blog_import_dokuwiki: also uses the generic image uploader from blog_import (when media_repo is empty and OPT_UPLOAD_IMAGES is True)
souliane <souliane@mailoo.org>
parents:
1839
diff
changeset
|
241 # TODO: would be nice to also update the hyperlinks to these images, e.g. when you have <a href="{url}"><img src="{url}"></a> |
3028 | 242 for img_elt in xml_tools.findAll(top_elt, names=["img"]): |
1831
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
243 yield self.imgFilters(client, img_elt, options, opt_host, tmp_dir) |
68c0dc13d821
plugin blog import, XEP-0277: progress + redirect:
Goffi <goffi@goffi.org>
parents:
1825
diff
changeset
|
244 finally: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
245 os.rmdir(tmp_dir) # XXX: tmp_dir should be empty, or something went wrong |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
246 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
247 # we now replace the content with filtered one |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
248 mb_data["content_xhtml"] = top_elt.toXml() |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
249 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
250 @defer.inlineCallbacks |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
251 def imgFilters(self, client, img_elt, options, opt_host, tmp_dir): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
252 """Filters handling images |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
253 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
254 url without host are fixed (if possible) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
255 according to options, images are uploaded to XMPP server |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
256 @param img_elt(domish.Element): <img/> element to handle |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
257 @param options(dict): filters options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
258 @param opt_host(unicode): normalised host given in options |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
259 @param tmp_dir(str): path to temp directory |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
260 """ |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
261 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
262 url = img_elt["src"] |
3028 | 263 if url[0] == "/": |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
264 if not opt_host: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
265 log.warning( |
3028 | 266 "host was not specified, we can't deal with src without host ({url}) and have to ignore the following <img/>:\n{xml}".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
267 url=url, xml=img_elt.toXml() |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
268 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
269 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
270 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
271 else: |
3028 | 272 url = urllib.parse.urljoin(opt_host, url) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
273 filename = url.rsplit("/", 1)[-1].strip() |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
274 if not filename: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
275 raise KeyError |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
276 except (KeyError, IndexError): |
3028 | 277 log.warning("ignoring invalid img element: {}".format(img_elt.toXml())) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
278 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
279 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
280 # we change the url for the normalized one |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
281 img_elt["src"] = url |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
282 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
283 if options.get(OPT_UPLOAD_IMAGES, False): |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
284 # upload is requested |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
285 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
286 ignore_host = options[OPT_UPLOAD_IGNORE_HOST] |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
287 except KeyError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
288 pass |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
289 else: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
290 # host is the ignored one, we skip |
3028 | 291 parsed_url = urllib.parse.urlsplit(url) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
292 if ignore_host in parsed_url.hostname: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
293 log.info( |
3028 | 294 "Don't upload image at {url} because of {opt} option".format( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
295 url=url, opt=OPT_UPLOAD_IGNORE_HOST |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
296 ) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
297 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
298 return |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
299 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
300 # we download images and re-upload them via XMPP |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
301 tmp_file = os.path.join(tmp_dir, filename).encode("utf-8") |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
302 upload_options = {"ignore_tls_errors": options.get(OPT_IGNORE_TLS, False)} |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
303 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
304 try: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
305 yield web_client.downloadPage(url.encode("utf-8"), tmp_file) |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
306 filename = filename.replace( |
3028 | 307 "%", "_" |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
308 ) # FIXME: tmp workaround for a bug in prosody http upload |
2765
378188abe941
misc: replaced all "dummy" by the more conventional and readable "__" ("_" being used for gettext)
Goffi <goffi@goffi.org>
parents:
2624
diff
changeset
|
309 __, download_d = yield self._u.upload( |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
310 client, tmp_file, filename, options=upload_options |
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
311 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
312 download_url = yield download_d |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
313 except Exception as e: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
314 log.warning( |
3028 | 315 "can't download image at {url}: {reason}".format(url=url, reason=e) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
316 ) |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
317 else: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
318 img_elt["src"] = download_url |
1825
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
319 |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
320 try: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
321 os.unlink(tmp_file) |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
322 except OSError: |
4e51f21c687f
plugin blog import: this plugin is the base handling blog importers:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
323 pass |