Mercurial > libervia-backend

comparison libervia/backend/plugins/plugin_misc_download.py @ 4270:0d7bb4df2343

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Reformatted code base using black.
author Goffi <goffi@goffi.org>
date Wed, 19 Jun 2024 18:44:57 +0200
parents 4b842c1fb686
children
comparison
equal deleted inserted replaced
4269:64a85ce8be70 4270:0d7bb4df2343
70 method=self._file_download_complete, 70 method=self._file_download_complete,
71 async_=True, 71 async_=True,
72 ) 72 )
73 self._download_callbacks = {} 73 self._download_callbacks = {}
74 self._scheme_callbacks = {} 74 self._scheme_callbacks = {}
75 self.register_scheme('http', self.download_http) 75 self.register_scheme("http", self.download_http)
76 self.register_scheme('https', self.download_http) 76 self.register_scheme("https", self.download_http)
77 77
78 def _file_download( 78 def _file_download(
79 self, attachment_s: str, dest_path: str, extra_s: str, profile: str 79 self, attachment_s: str, dest_path: str, extra_s: str, profile: str
80 ) -> defer.Deferred: 80 ) -> defer.Deferred:
81 d = defer.ensureDeferred(self.file_download( 81 d = defer.ensureDeferred(
82 self.host.get_client(profile), 82 self.file_download(
83 data_format.deserialise(attachment_s), 83 self.host.get_client(profile),
84 Path(dest_path), 84 data_format.deserialise(attachment_s),
85 data_format.deserialise(extra_s) 85 Path(dest_path),
86 )) 86 data_format.deserialise(extra_s),
87 )
88 )
87 d.addCallback(lambda ret: data_format.serialise(ret)) 89 d.addCallback(lambda ret: data_format.serialise(ret))
88 return d 90 return d
89 91
90 async def file_download( 92 async def file_download(
91 self, 93 self,
92 client: SatXMPPEntity, 94 client: SatXMPPEntity,
93 attachment: Dict[str, Any], 95 attachment: Dict[str, Any],
94 dest_path: Path, 96 dest_path: Path,
95 extra: Optional[Dict[str, Any]] = None 97 extra: Optional[Dict[str, Any]] = None,
96 ) -> Dict[str, Any]: 98 ) -> Dict[str, Any]:
97 """Download a file using best available method 99 """Download a file using best available method
98 100
99 parameters are the same as for [download] 101 parameters are the same as for [download]
100 @return (dict): action dictionary, with progress id in case of success, else xmlui 102 @return (dict): action dictionary, with progress id in case of success, else xmlui
101 message 103 message
102 """ 104 """
103 try: 105 try:
104 progress_id, __ = await self.download(client, attachment, dest_path, extra) 106 progress_id, __ = await self.download(client, attachment, dest_path, extra)
105 except Exception as e: 107 except Exception as e:
106 if (isinstance(e, jabber_error.StanzaError) 108 if (
107 and e.condition == 'not-acceptable'): 109 isinstance(e, jabber_error.StanzaError)
110 and e.condition == "not-acceptable"
111 ):
108 reason = e.text 112 reason = e.text
109 else: 113 else:
110 reason = str(e) 114 reason = str(e)
111 msg = D_("Can't download file: {reason}").format(reason=reason) 115 msg = D_("Can't download file: {reason}").format(reason=reason)
112 log.warning(msg) 116 log.warning(msg)
117 } 121 }
118 else: 122 else:
119 return {"progress": progress_id} 123 return {"progress": progress_id}
120 124
121 def _file_download_complete( 125 def _file_download_complete(
122 self, attachment_s: str, dest_path: str, extra_s: str, profile: str 126 self, attachment_s: str, dest_path: str, extra_s: str, profile: str
123 ) -> defer.Deferred: 127 ) -> defer.Deferred:
124 d = defer.ensureDeferred(self.file_download_complete( 128 d = defer.ensureDeferred(
125 self.host.get_client(profile), 129 self.file_download_complete(
126 data_format.deserialise(attachment_s), 130 self.host.get_client(profile),
127 Path(dest_path), 131 data_format.deserialise(attachment_s),
128 data_format.deserialise(extra_s) 132 Path(dest_path),
129 )) 133 data_format.deserialise(extra_s),
134 )
135 )
130 d.addCallback(lambda path: str(path)) 136 d.addCallback(lambda path: str(path))
131 return d 137 return d
132 138
133 async def file_download_complete( 139 async def file_download_complete(
134 self, 140 self,
135 client: SatXMPPEntity, 141 client: SatXMPPEntity,
136 attachment: Dict[str, Any], 142 attachment: Dict[str, Any],
137 dest_path: Path, 143 dest_path: Path,
138 extra: Optional[Dict[str, Any]] = None 144 extra: Optional[Dict[str, Any]] = None,
139 ) -> str: 145 ) -> str:
140 """Helper method to fully download a file and return its path 146 """Helper method to fully download a file and return its path
141 147
142 parameters are the same as for [download] 148 parameters are the same as for [download]
143 @return (str): path to the downloaded file 149 @return (str): path to the downloaded file
150 async def download_uri( 156 async def download_uri(
151 self, 157 self,
152 client: SatXMPPEntity, 158 client: SatXMPPEntity,
153 uri: str, 159 uri: str,
154 dest_path: Union[Path, str], 160 dest_path: Union[Path, str],
155 extra: Optional[Dict[str, Any]] = None 161 extra: Optional[Dict[str, Any]] = None,
156 ) -> Tuple[str, defer.Deferred]: 162 ) -> Tuple[str, defer.Deferred]:
157 if extra is None: 163 if extra is None:
158 extra = {} 164 extra = {}
159 uri_parsed = urlparse(uri, 'http') 165 uri_parsed = urlparse(uri, "http")
160 if dest_path: 166 if dest_path:
161 dest_path = Path(dest_path) 167 dest_path = Path(dest_path)
162 cache_uid = None 168 cache_uid = None
163 else: 169 else:
164 filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME 170 filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME
165 # we don't use Path.suffixes because we don't want to have more than 2 171 # we don't use Path.suffixes because we don't want to have more than 2
166 # suffixes, but we still want to handle suffixes like "tar.gz". 172 # suffixes, but we still want to handle suffixes like "tar.gz".
167 stem, *suffixes = filename.rsplit('.', 2) 173 stem, *suffixes = filename.rsplit(".", 2)
168 # we hash the URL to have an unique identifier, and avoid double download 174 # we hash the URL to have an unique identifier, and avoid double download
169 url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest() 175 url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest()
170 cache_uid = f"{stem}_{url_hash}" 176 cache_uid = f"{stem}_{url_hash}"
171 cache_data = client.cache.get_metadata(cache_uid) 177 cache_data = client.cache.get_metadata(cache_uid)
172 if cache_data is not None: 178 if cache_data is not None:
173 # file is already in cache, we return it 179 # file is already in cache, we return it
174 download_d = defer.succeed(cache_data['path']) 180 download_d = defer.succeed(cache_data["path"])
175 return '', download_d 181 return "", download_d
176 else: 182 else:
177 # the file is not in cache 183 # the file is not in cache
178 unique_name = '.'.join([cache_uid] + suffixes) 184 unique_name = ".".join([cache_uid] + suffixes)
179 with client.cache.cache_data( 185 with client.cache.cache_data(
180 "DOWNLOAD", cache_uid, filename=unique_name) as f: 186 "DOWNLOAD", cache_uid, filename=unique_name
187 ) as f:
181 # we close the file and only use its name, the file will be opened 188 # we close the file and only use its name, the file will be opened
182 # by the registered callback 189 # by the registered callback
183 dest_path = Path(f.name) 190 dest_path = Path(f.name)
184 191
185 # should we check certificates? 192 # should we check certificates?
186 check_certificate = self.host.memory.param_get_a( 193 check_certificate = self.host.memory.param_get_a(
187 "check_certificate", "Connection", profile_key=client.profile) 194 "check_certificate", "Connection", profile_key=client.profile
195 )
188 if not check_certificate: 196 if not check_certificate:
189 extra['ignore_tls_errors'] = True 197 extra["ignore_tls_errors"] = True
190 log.warning( 198 log.warning(_("certificate check disabled for download, this is dangerous!"))
191 _("certificate check disabled for download, this is dangerous!"))
192 199
193 try: 200 try:
194 callback = self._scheme_callbacks[uri_parsed.scheme] 201 callback = self._scheme_callbacks[uri_parsed.scheme]
195 except KeyError: 202 except KeyError:
196 raise exceptions.NotFound(f"Can't find any handler for uri {uri}") 203 raise exceptions.NotFound(f"Can't find any handler for uri {uri}")
197 else: 204 else:
198 try: 205 try:
199 progress_id, download_d = await callback( 206 progress_id, download_d = await callback(
200 client, uri_parsed, dest_path, extra) 207 client, uri_parsed, dest_path, extra
208 )
201 except Exception as e: 209 except Exception as e:
202 log.warning(_( 210 log.warning(
203 "Can't download URI {uri}: {reason}").format( 211 _("Can't download URI {uri}: {reason}").format(uri=uri, reason=e)
204 uri=uri, reason=e)) 212 )
205 if cache_uid is not None: 213 if cache_uid is not None:
206 client.cache.remove_from_cache(cache_uid) 214 client.cache.remove_from_cache(cache_uid)
207 elif dest_path.exists(): 215 elif dest_path.exists():
208 dest_path.unlink() 216 dest_path.unlink()
209 raise e 217 raise e
210 download_d.addCallback(lambda __: dest_path) 218 download_d.addCallback(lambda __: dest_path)
211 return progress_id, download_d 219 return progress_id, download_d
212 220
213
214 async def download( 221 async def download(
215 self, 222 self,
216 client: SatXMPPEntity, 223 client: SatXMPPEntity,
217 attachment: Dict[str, Any], 224 attachment: Dict[str, Any],
218 dest_path: Union[Path, str], 225 dest_path: Union[Path, str],
219 extra: Optional[Dict[str, Any]] = None 226 extra: Optional[Dict[str, Any]] = None,
220 ) -> Tuple[str, defer.Deferred]: 227 ) -> Tuple[str, defer.Deferred]:
221 """Download a file from URI using suitable method 228 """Download a file from URI using suitable method
222 229
223 @param uri: URI to the file to download 230 @param uri: URI to the file to download
224 @param dest_path: where the file must be downloaded 231 @param dest_path: where the file must be downloaded
245 ) 252 )
246 continue 253 continue
247 try: 254 try:
248 cb = self._download_callbacks[source_type] 255 cb = self._download_callbacks[source_type]
249 except KeyError: 256 except KeyError:
250 log.warning( 257 log.warning(f"no source handler registered for {source_type!r}")
251 f"no source handler registered for {source_type!r}"
252 )
253 else: 258 else:
254 try: 259 try:
255 return await cb(client, attachment, source, dest_path, extra) 260 return await cb(client, attachment, source, dest_path, extra)
256 except exceptions.CancelError as e: 261 except exceptions.CancelError as e:
257 # the handler can't or doesn't want to handle this source 262 # the handler can't or doesn't want to handle this source
269 def register_download_handler( 274 def register_download_handler(
270 self, 275 self,
271 source_type: str, 276 source_type: str,
272 callback: Callable[ 277 callback: Callable[
273 [ 278 [
274 SatXMPPEntity, Dict[str, Any], Dict[str, Any], Union[str, Path], 279 SatXMPPEntity,
275 Dict[str, Any] 280 Dict[str, Any],
281 Dict[str, Any],
282 Union[str, Path],
283 Dict[str, Any],
276 ], 284 ],
277 Tuple[str, defer.Deferred] 285 Tuple[str, defer.Deferred],
278 ] 286 ],
279 ) -> None: 287 ) -> None:
280 """Register a handler to manage a type of attachment source 288 """Register a handler to manage a type of attachment source
281 289
282 @param source_type: ``type`` of source handled 290 @param source_type: ``type`` of source handled
283 This is usually the namespace of the protocol used 291 This is usually the namespace of the protocol used
332 download_d.errback(exceptions.NetworkError(msg)) 340 download_d.errback(exceptions.NetworkError(msg))
333 341
334 async def download_http(self, client, uri_parsed, dest_path, options): 342 async def download_http(self, client, uri_parsed, dest_path, options):
335 url = uri_parsed.geturl() 343 url = uri_parsed.geturl()
336 344
337 if options.get('ignore_tls_errors', False): 345 if options.get("ignore_tls_errors", False):
338 log.warning( 346 log.warning("TLS certificate check disabled, this is highly insecure")
339 "TLS certificate check disabled, this is highly insecure"
340 )
341 treq_client = treq_client_no_ssl 347 treq_client = treq_client_no_ssl
342 else: 348 else:
343 treq_client = treq 349 treq_client = treq
344 350
345 head_data = await treq_client.head(url) 351 head_data = await treq_client.head(url)
346 try: 352 try:
347 content_length = int(head_data.headers.getRawHeaders('content-length')[0]) 353 content_length = int(head_data.headers.getRawHeaders("content-length")[0])
348 except (KeyError, TypeError, IndexError): 354 except (KeyError, TypeError, IndexError):
349 content_length = None 355 content_length = None
350 log.debug(f"No content lenght found at {url}") 356 log.debug(f"No content lenght found at {url}")
351 file_obj = stream.SatFile( 357 file_obj = stream.SatFile(
352 self.host, 358 self.host,
353 client, 359 client,
354 dest_path, 360 dest_path,
355 mode="wb", 361 mode="wb",
356 size = content_length, 362 size=content_length,
357 ) 363 )
358 364
359 progress_id = file_obj.uid 365 progress_id = file_obj.uid
360 366
361 resp = await treq_client.get(url, unbuffered=True) 367 resp = await treq_client.get(url, unbuffered=True)