Mercurial > libervia-backend
comparison libervia/backend/plugins/plugin_misc_download.py @ 4270:0d7bb4df2343
Reformatted code base using black.
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 19 Jun 2024 18:44:57 +0200 |
parents | 4b842c1fb686 |
children |
comparison
equal
deleted
inserted
replaced
4269:64a85ce8be70 | 4270:0d7bb4df2343 |
---|---|
70 method=self._file_download_complete, | 70 method=self._file_download_complete, |
71 async_=True, | 71 async_=True, |
72 ) | 72 ) |
73 self._download_callbacks = {} | 73 self._download_callbacks = {} |
74 self._scheme_callbacks = {} | 74 self._scheme_callbacks = {} |
75 self.register_scheme('http', self.download_http) | 75 self.register_scheme("http", self.download_http) |
76 self.register_scheme('https', self.download_http) | 76 self.register_scheme("https", self.download_http) |
77 | 77 |
78 def _file_download( | 78 def _file_download( |
79 self, attachment_s: str, dest_path: str, extra_s: str, profile: str | 79 self, attachment_s: str, dest_path: str, extra_s: str, profile: str |
80 ) -> defer.Deferred: | 80 ) -> defer.Deferred: |
81 d = defer.ensureDeferred(self.file_download( | 81 d = defer.ensureDeferred( |
82 self.host.get_client(profile), | 82 self.file_download( |
83 data_format.deserialise(attachment_s), | 83 self.host.get_client(profile), |
84 Path(dest_path), | 84 data_format.deserialise(attachment_s), |
85 data_format.deserialise(extra_s) | 85 Path(dest_path), |
86 )) | 86 data_format.deserialise(extra_s), |
87 ) | |
88 ) | |
87 d.addCallback(lambda ret: data_format.serialise(ret)) | 89 d.addCallback(lambda ret: data_format.serialise(ret)) |
88 return d | 90 return d |
89 | 91 |
90 async def file_download( | 92 async def file_download( |
91 self, | 93 self, |
92 client: SatXMPPEntity, | 94 client: SatXMPPEntity, |
93 attachment: Dict[str, Any], | 95 attachment: Dict[str, Any], |
94 dest_path: Path, | 96 dest_path: Path, |
95 extra: Optional[Dict[str, Any]] = None | 97 extra: Optional[Dict[str, Any]] = None, |
96 ) -> Dict[str, Any]: | 98 ) -> Dict[str, Any]: |
97 """Download a file using best available method | 99 """Download a file using best available method |
98 | 100 |
99 parameters are the same as for [download] | 101 parameters are the same as for [download] |
100 @return (dict): action dictionary, with progress id in case of success, else xmlui | 102 @return (dict): action dictionary, with progress id in case of success, else xmlui |
101 message | 103 message |
102 """ | 104 """ |
103 try: | 105 try: |
104 progress_id, __ = await self.download(client, attachment, dest_path, extra) | 106 progress_id, __ = await self.download(client, attachment, dest_path, extra) |
105 except Exception as e: | 107 except Exception as e: |
106 if (isinstance(e, jabber_error.StanzaError) | 108 if ( |
107 and e.condition == 'not-acceptable'): | 109 isinstance(e, jabber_error.StanzaError) |
110 and e.condition == "not-acceptable" | |
111 ): | |
108 reason = e.text | 112 reason = e.text |
109 else: | 113 else: |
110 reason = str(e) | 114 reason = str(e) |
111 msg = D_("Can't download file: {reason}").format(reason=reason) | 115 msg = D_("Can't download file: {reason}").format(reason=reason) |
112 log.warning(msg) | 116 log.warning(msg) |
117 } | 121 } |
118 else: | 122 else: |
119 return {"progress": progress_id} | 123 return {"progress": progress_id} |
120 | 124 |
121 def _file_download_complete( | 125 def _file_download_complete( |
122 self, attachment_s: str, dest_path: str, extra_s: str, profile: str | 126 self, attachment_s: str, dest_path: str, extra_s: str, profile: str |
123 ) -> defer.Deferred: | 127 ) -> defer.Deferred: |
124 d = defer.ensureDeferred(self.file_download_complete( | 128 d = defer.ensureDeferred( |
125 self.host.get_client(profile), | 129 self.file_download_complete( |
126 data_format.deserialise(attachment_s), | 130 self.host.get_client(profile), |
127 Path(dest_path), | 131 data_format.deserialise(attachment_s), |
128 data_format.deserialise(extra_s) | 132 Path(dest_path), |
129 )) | 133 data_format.deserialise(extra_s), |
134 ) | |
135 ) | |
130 d.addCallback(lambda path: str(path)) | 136 d.addCallback(lambda path: str(path)) |
131 return d | 137 return d |
132 | 138 |
133 async def file_download_complete( | 139 async def file_download_complete( |
134 self, | 140 self, |
135 client: SatXMPPEntity, | 141 client: SatXMPPEntity, |
136 attachment: Dict[str, Any], | 142 attachment: Dict[str, Any], |
137 dest_path: Path, | 143 dest_path: Path, |
138 extra: Optional[Dict[str, Any]] = None | 144 extra: Optional[Dict[str, Any]] = None, |
139 ) -> str: | 145 ) -> str: |
140 """Helper method to fully download a file and return its path | 146 """Helper method to fully download a file and return its path |
141 | 147 |
142 parameters are the same as for [download] | 148 parameters are the same as for [download] |
143 @return (str): path to the downloaded file | 149 @return (str): path to the downloaded file |
150 async def download_uri( | 156 async def download_uri( |
151 self, | 157 self, |
152 client: SatXMPPEntity, | 158 client: SatXMPPEntity, |
153 uri: str, | 159 uri: str, |
154 dest_path: Union[Path, str], | 160 dest_path: Union[Path, str], |
155 extra: Optional[Dict[str, Any]] = None | 161 extra: Optional[Dict[str, Any]] = None, |
156 ) -> Tuple[str, defer.Deferred]: | 162 ) -> Tuple[str, defer.Deferred]: |
157 if extra is None: | 163 if extra is None: |
158 extra = {} | 164 extra = {} |
159 uri_parsed = urlparse(uri, 'http') | 165 uri_parsed = urlparse(uri, "http") |
160 if dest_path: | 166 if dest_path: |
161 dest_path = Path(dest_path) | 167 dest_path = Path(dest_path) |
162 cache_uid = None | 168 cache_uid = None |
163 else: | 169 else: |
164 filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME | 170 filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME |
165 # we don't use Path.suffixes because we don't want to have more than 2 | 171 # we don't use Path.suffixes because we don't want to have more than 2 |
166 # suffixes, but we still want to handle suffixes like "tar.gz". | 172 # suffixes, but we still want to handle suffixes like "tar.gz". |
167 stem, *suffixes = filename.rsplit('.', 2) | 173 stem, *suffixes = filename.rsplit(".", 2) |
168 # we hash the URL to have an unique identifier, and avoid double download | 174 # we hash the URL to have an unique identifier, and avoid double download |
169 url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest() | 175 url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest() |
170 cache_uid = f"{stem}_{url_hash}" | 176 cache_uid = f"{stem}_{url_hash}" |
171 cache_data = client.cache.get_metadata(cache_uid) | 177 cache_data = client.cache.get_metadata(cache_uid) |
172 if cache_data is not None: | 178 if cache_data is not None: |
173 # file is already in cache, we return it | 179 # file is already in cache, we return it |
174 download_d = defer.succeed(cache_data['path']) | 180 download_d = defer.succeed(cache_data["path"]) |
175 return '', download_d | 181 return "", download_d |
176 else: | 182 else: |
177 # the file is not in cache | 183 # the file is not in cache |
178 unique_name = '.'.join([cache_uid] + suffixes) | 184 unique_name = ".".join([cache_uid] + suffixes) |
179 with client.cache.cache_data( | 185 with client.cache.cache_data( |
180 "DOWNLOAD", cache_uid, filename=unique_name) as f: | 186 "DOWNLOAD", cache_uid, filename=unique_name |
187 ) as f: | |
181 # we close the file and only use its name, the file will be opened | 188 # we close the file and only use its name, the file will be opened |
182 # by the registered callback | 189 # by the registered callback |
183 dest_path = Path(f.name) | 190 dest_path = Path(f.name) |
184 | 191 |
185 # should we check certificates? | 192 # should we check certificates? |
186 check_certificate = self.host.memory.param_get_a( | 193 check_certificate = self.host.memory.param_get_a( |
187 "check_certificate", "Connection", profile_key=client.profile) | 194 "check_certificate", "Connection", profile_key=client.profile |
195 ) | |
188 if not check_certificate: | 196 if not check_certificate: |
189 extra['ignore_tls_errors'] = True | 197 extra["ignore_tls_errors"] = True |
190 log.warning( | 198 log.warning(_("certificate check disabled for download, this is dangerous!")) |
191 _("certificate check disabled for download, this is dangerous!")) | |
192 | 199 |
193 try: | 200 try: |
194 callback = self._scheme_callbacks[uri_parsed.scheme] | 201 callback = self._scheme_callbacks[uri_parsed.scheme] |
195 except KeyError: | 202 except KeyError: |
196 raise exceptions.NotFound(f"Can't find any handler for uri {uri}") | 203 raise exceptions.NotFound(f"Can't find any handler for uri {uri}") |
197 else: | 204 else: |
198 try: | 205 try: |
199 progress_id, download_d = await callback( | 206 progress_id, download_d = await callback( |
200 client, uri_parsed, dest_path, extra) | 207 client, uri_parsed, dest_path, extra |
208 ) | |
201 except Exception as e: | 209 except Exception as e: |
202 log.warning(_( | 210 log.warning( |
203 "Can't download URI {uri}: {reason}").format( | 211 _("Can't download URI {uri}: {reason}").format(uri=uri, reason=e) |
204 uri=uri, reason=e)) | 212 ) |
205 if cache_uid is not None: | 213 if cache_uid is not None: |
206 client.cache.remove_from_cache(cache_uid) | 214 client.cache.remove_from_cache(cache_uid) |
207 elif dest_path.exists(): | 215 elif dest_path.exists(): |
208 dest_path.unlink() | 216 dest_path.unlink() |
209 raise e | 217 raise e |
210 download_d.addCallback(lambda __: dest_path) | 218 download_d.addCallback(lambda __: dest_path) |
211 return progress_id, download_d | 219 return progress_id, download_d |
212 | 220 |
213 | |
214 async def download( | 221 async def download( |
215 self, | 222 self, |
216 client: SatXMPPEntity, | 223 client: SatXMPPEntity, |
217 attachment: Dict[str, Any], | 224 attachment: Dict[str, Any], |
218 dest_path: Union[Path, str], | 225 dest_path: Union[Path, str], |
219 extra: Optional[Dict[str, Any]] = None | 226 extra: Optional[Dict[str, Any]] = None, |
220 ) -> Tuple[str, defer.Deferred]: | 227 ) -> Tuple[str, defer.Deferred]: |
221 """Download a file from URI using suitable method | 228 """Download a file from URI using suitable method |
222 | 229 |
223 @param uri: URI to the file to download | 230 @param uri: URI to the file to download |
224 @param dest_path: where the file must be downloaded | 231 @param dest_path: where the file must be downloaded |
245 ) | 252 ) |
246 continue | 253 continue |
247 try: | 254 try: |
248 cb = self._download_callbacks[source_type] | 255 cb = self._download_callbacks[source_type] |
249 except KeyError: | 256 except KeyError: |
250 log.warning( | 257 log.warning(f"no source handler registered for {source_type!r}") |
251 f"no source handler registered for {source_type!r}" | |
252 ) | |
253 else: | 258 else: |
254 try: | 259 try: |
255 return await cb(client, attachment, source, dest_path, extra) | 260 return await cb(client, attachment, source, dest_path, extra) |
256 except exceptions.CancelError as e: | 261 except exceptions.CancelError as e: |
257 # the handler can't or doesn't want to handle this source | 262 # the handler can't or doesn't want to handle this source |
269 def register_download_handler( | 274 def register_download_handler( |
270 self, | 275 self, |
271 source_type: str, | 276 source_type: str, |
272 callback: Callable[ | 277 callback: Callable[ |
273 [ | 278 [ |
274 SatXMPPEntity, Dict[str, Any], Dict[str, Any], Union[str, Path], | 279 SatXMPPEntity, |
275 Dict[str, Any] | 280 Dict[str, Any], |
281 Dict[str, Any], | |
282 Union[str, Path], | |
283 Dict[str, Any], | |
276 ], | 284 ], |
277 Tuple[str, defer.Deferred] | 285 Tuple[str, defer.Deferred], |
278 ] | 286 ], |
279 ) -> None: | 287 ) -> None: |
280 """Register a handler to manage a type of attachment source | 288 """Register a handler to manage a type of attachment source |
281 | 289 |
282 @param source_type: ``type`` of source handled | 290 @param source_type: ``type`` of source handled |
283 This is usually the namespace of the protocol used | 291 This is usually the namespace of the protocol used |
332 download_d.errback(exceptions.NetworkError(msg)) | 340 download_d.errback(exceptions.NetworkError(msg)) |
333 | 341 |
334 async def download_http(self, client, uri_parsed, dest_path, options): | 342 async def download_http(self, client, uri_parsed, dest_path, options): |
335 url = uri_parsed.geturl() | 343 url = uri_parsed.geturl() |
336 | 344 |
337 if options.get('ignore_tls_errors', False): | 345 if options.get("ignore_tls_errors", False): |
338 log.warning( | 346 log.warning("TLS certificate check disabled, this is highly insecure") |
339 "TLS certificate check disabled, this is highly insecure" | |
340 ) | |
341 treq_client = treq_client_no_ssl | 347 treq_client = treq_client_no_ssl |
342 else: | 348 else: |
343 treq_client = treq | 349 treq_client = treq |
344 | 350 |
345 head_data = await treq_client.head(url) | 351 head_data = await treq_client.head(url) |
346 try: | 352 try: |
347 content_length = int(head_data.headers.getRawHeaders('content-length')[0]) | 353 content_length = int(head_data.headers.getRawHeaders("content-length")[0]) |
348 except (KeyError, TypeError, IndexError): | 354 except (KeyError, TypeError, IndexError): |
349 content_length = None | 355 content_length = None |
350 log.debug(f"No content lenght found at {url}") | 356 log.debug(f"No content lenght found at {url}") |
351 file_obj = stream.SatFile( | 357 file_obj = stream.SatFile( |
352 self.host, | 358 self.host, |
353 client, | 359 client, |
354 dest_path, | 360 dest_path, |
355 mode="wb", | 361 mode="wb", |
356 size = content_length, | 362 size=content_length, |
357 ) | 363 ) |
358 | 364 |
359 progress_id = file_obj.uid | 365 progress_id = file_obj.uid |
360 | 366 |
361 resp = await treq_client.get(url, unbuffered=True) | 367 resp = await treq_client.get(url, unbuffered=True) |