Mercurial > libervia-backend
annotate sat/plugins/plugin_misc_download.py @ 3381:127dde80a0a5
docker: dockers images, first draft:
This patch introduce new docker images, replacing the unmaintained ones which were in
sat_docs.
Images + docker-compose files are provided to use SàT backend and Libervia, but also to
integrate and configure Prosody and SàT PubSub to run end-to-end tests.
author | Goffi <goffi@goffi.org> |
---|---|
date | Mon, 19 Oct 2020 09:30:45 +0200 |
parents | 4252176ad993 |
children | be6d91572633 |
rev | line source |
---|---|
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python3 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 # SAT plugin for downloading files |
3136 | 4 # Copyright (C) 2009-2020 Jérôme Poisson (goffi@goffi.org) |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 # This program is free software: you can redistribute it and/or modify |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # it under the terms of the GNU Affero General Public License as published by |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # the Free Software Foundation, either version 3 of the License, or |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # (at your option) any later version. |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 # This program is distributed in the hope that it will be useful, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # GNU Affero General Public License for more details. |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 # You should have received a copy of the GNU Affero General Public License |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 from pathlib import Path |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
20 from urllib.parse import urlparse, unquote |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
21 import hashlib |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 import treq |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from twisted.internet import defer |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 from twisted.words.protocols.jabber import error as jabber_error |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 from sat.core.i18n import _, D_ |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 from sat.core.constants import Const as C |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 from sat.core.log import getLogger |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 from sat.core import exceptions |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 from sat.tools import xml_tools |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 from sat.tools.common import data_format |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 from sat.tools import stream |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
32 from sat.tools.web import treq_client_no_ssl |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 log = getLogger(__name__) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 PLUGIN_INFO = { |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 C.PI_NAME: "File Download", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 C.PI_IMPORT_NAME: "DOWNLOAD", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 C.PI_TYPE: C.PLUG_TYPE_MISC, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 C.PI_MAIN: "DownloadPlugin", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 C.PI_HANDLER: "no", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 C.PI_DESCRIPTION: _("""File download management"""), |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
44 } |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 class DownloadPlugin(object): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 def __init__(self, host): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 log.info(_("plugin Download initialization")) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 self.host = host |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 host.bridge.addMethod( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 "fileDownload", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 ".plugin", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
55 in_sign="ssss", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 out_sign="a{ss}", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 method=self._fileDownload, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
58 async_=True, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
59 ) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 host.bridge.addMethod( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
61 "fileDownloadComplete", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 ".plugin", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 in_sign="ssss", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 out_sign="s", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 method=self._fileDownloadComplete, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 async_=True, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 ) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 self._download_callbacks = {} |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 self.registerScheme('http', self.downloadHTTP) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 self.registerScheme('https', self.downloadHTTP) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 def _fileDownload(self, uri, dest_path, options_s, profile): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
73 client = self.host.getClient(profile) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
74 options = data_format.deserialise(options_s) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
75 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 return defer.ensureDeferred(self.fileDownload( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
77 client, uri, Path(dest_path), options |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
78 )) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
79 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
80 async def fileDownload(self, client, uri, dest_path, options=None): |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
81 """Download a file using best available method |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
82 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
83 parameters are the same as for [download] |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 @return (dict): action dictionary, with progress id in case of success, else xmlui |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
85 message |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
86 """ |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
87 try: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
88 progress_id, __ = await self.download(client, uri, dest_path, options) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
89 except Exception as e: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
90 if (isinstance(e, jabber_error.StanzaError) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
91 and e.condition == 'not-acceptable'): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
92 reason = e.text |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
93 else: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
94 reason = str(e) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
95 msg = D_("Can't download file: {reason}").format(reason=reason) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
96 log.warning(msg) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
97 return { |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
98 "xmlui": xml_tools.note( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
99 msg, D_("Can't download file"), C.XMLUI_DATA_LVL_WARNING |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
100 ).toXml() |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
101 } |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
102 else: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
103 return {"progress": progress_id} |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
104 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
105 def _fileDownloadComplete(self, uri, dest_path, options_s, profile): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
106 client = self.host.getClient(profile) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
107 options = data_format.deserialise(options_s) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
108 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
109 d = defer.ensureDeferred(self.fileDownloadComplete( |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
110 client, uri, dest_path, options |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
111 )) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 d.addCallback(lambda path: str(path)) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 return d |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
114 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
115 async def fileDownloadComplete(self, client, uri, dest_path, options=None): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 """Helper method to fully download a file and return its path |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
117 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
118 parameters are the same as for [download] |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
119 @return (str): path to the downloaded file |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
120 use empty string to store the file in cache |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
121 """ |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
122 __, download_d = await self.download(client, uri, dest_path, options) |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
123 dest_path = await download_d |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
124 return dest_path |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
125 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
126 async def download(self, client, uri, dest_path, options=None): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
127 """Send a file using best available method |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
128 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
129 @param uri(str): URI to the file to download |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
130 @param dest_path(str, Path): where the file must be downloaded |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
131 if empty string, the file will be stored in local path |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
132 @param options(dict, None): options depending on scheme handler |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
133 Some common options: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
134 - ignore_tls_errors(bool): True to ignore SSL/TLS certificate verification |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
135 used only if HTTPS transport is needed |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
136 @return (tuple[unicode,D(unicode)]): progress_id and a Deferred which fire |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
137 download URL when download is finished |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
138 progress_id can be empty string if the file already exist and is not |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
139 downloaded again (can happen if cache is used with empty dest_path) |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
140 """ |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
141 if options is None: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
142 options = {} |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
143 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
144 uri_parsed = urlparse(uri, 'http') |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
145 if dest_path: |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
146 dest_path = Path(dest_path) |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
147 cache_uid = None |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
148 else: |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
149 filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
150 # we don't use Path.suffixes because we don't want to have more than 2 |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
151 # suffixes, but we still want to handle suffixes like "tar.gz". |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
152 stem, *suffixes = filename.rsplit('.', 2) |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
153 # we hash the URL to have an unique identifier, and avoid double download |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
154 url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest() |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
155 cache_uid = f"{stem}_{url_hash}" |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
156 cache_data = client.cache.getMetadata(cache_uid) |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
157 if cache_data is not None: |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
158 # file is already in cache, we return it |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
159 download_d = defer.succeed(cache_data['path']) |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
160 return '', download_d |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
161 else: |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
162 # the file is not in cache |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
163 unique_name = '.'.join([cache_uid] + suffixes) |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
164 with client.cache.cacheData( |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
165 "DOWNLOAD", cache_uid, filename=unique_name) as f: |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
166 # we close the file and only use its name, the file will be opened |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
167 # by the registered callback |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
168 dest_path = Path(f.name) |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
169 |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
170 # should we check certificates? |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
171 check_certificate = self.host.memory.getParamA( |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
172 "check_certificate", "Connection", profile_key=client.profile) |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
173 if not check_certificate: |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
174 options['ignore_tls_errors'] = True |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
175 log.warning( |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
176 _("certificate check disabled for download, this is dangerous!")) |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
177 |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
178 try: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
179 callback = self._download_callbacks[uri_parsed.scheme] |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
180 except KeyError: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
181 raise exceptions.NotFound(f"Can't find any handler for uri {uri}") |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
182 else: |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
183 try: |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
184 progress_id, download_d = await callback( |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
185 client, uri_parsed, dest_path, options) |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
186 except Exception as e: |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
187 log.warning(_( |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
188 "Can't download URI {uri}: {reason}").format( |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
189 uri=uri, reason=e)) |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
190 if cache_uid is not None: |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
191 client.cache.removeFromCache(cache_uid) |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
192 elif dest_path.exists(): |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
193 dest_path.unlink() |
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
194 raise e |
3187
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
195 download_d.addCallback(lambda __: dest_path) |
d92a144f3589
plugin download: use cache if dest_path is empty:
Goffi <goffi@goffi.org>
parents:
3186
diff
changeset
|
196 return progress_id, download_d |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
197 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
198 def registerScheme(self, scheme, download_cb): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
199 """Register an URI scheme handler |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
200 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
201 @param scheme(unicode): URI scheme this callback is handling |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
202 @param download_cb(callable): callback to download a file |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
203 arguments are: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
204 - (SatXMPPClient) client |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
205 - (urllib.parse.SplitResult) parsed URI |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
206 - (Path) destination path where the file must be downloaded |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
207 - (dict) options |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
208 must return a tuple with progress_id and a Deferred which fire when download |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
209 is finished |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
210 """ |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
211 if scheme in self._download_callbacks: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
212 raise exceptions.ConflictError( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
213 f"A method with scheme {scheme!r} is already registered" |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
214 ) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
215 self._download_callbacks[scheme] = download_cb |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
216 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
217 def unregister(self, scheme): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
218 try: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
219 del self._download_callbacks[scheme] |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
220 except KeyError: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
221 raise exceptions.NotFound(f"No callback registered for scheme {scheme!r}") |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
222 |
3186
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
223 def errbackDownload(self, file_obj, download_d, resp): |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
224 """Set file_obj and download deferred appropriatly after a network error |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
225 |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
226 @param file_obj(SatFile): file where the download must be done |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
227 @param download_d(Deferred): deffered which must be fired on complete download |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
228 @param resp(treq.response.IResponse): treq response |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
229 """ |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
230 msg = f"HTTP error ({resp.code}): {resp.phrase.decode()}" |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
231 file_obj.close(error=msg) |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
232 download_d.errback(exceptions.NetworkError(msg)) |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
233 |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
234 async def downloadHTTP(self, client, uri_parsed, dest_path, options): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
235 url = uri_parsed.geturl() |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
237 if options.get('ignore_tls_errors', False): |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
238 log.warning( |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
239 "TLS certificate check disabled, this is highly insecure" |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
240 ) |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
241 treq_client = treq_client_no_ssl |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
242 else: |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
243 treq_client = treq |
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
244 |
3211
4252176ad993
plugin download: clean unfinished files and re-raise exception in case of download error
Goffi <goffi@goffi.org>
parents:
3205
diff
changeset
|
245 head_data = await treq_client.head(url) |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
246 try: |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
247 content_length = int(head_data.headers.getRawHeaders('content-length')[0]) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
248 except (KeyError, TypeError, IndexError): |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
249 content_length = None |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
250 log.debug(f"No content lenght found at {url}") |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
251 file_obj = stream.SatFile( |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
252 self.host, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
253 client, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
254 dest_path, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
255 mode="wb", |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
256 size = content_length, |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
257 ) |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
258 |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
259 progress_id = file_obj.uid |
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
260 |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3187
diff
changeset
|
261 resp = await treq_client.get(url, unbuffered=True) |
3186
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
262 if resp.code == 200: |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
263 d = treq.collect(resp, file_obj.write) |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
264 d.addBoth(lambda _: file_obj.close()) |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
265 else: |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
266 d = defer.Deferred() |
84b0c8b4dee0
plugin download, aesgcm: fixed handling of HTTP errors
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
267 self.errbackDownload(file_obj, d, resp) |
3088
d1464548055a
plugin file download: meta plugin to handle file download:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
268 return progress_id, d |