Mercurial > libervia-backend
annotate libervia/backend/tools/web.py @ 4304:92a886f31581 default tip @
doc (components): new Email gateway documentation:
fix 449
author | Goffi <goffi@goffi.org> |
---|---|
date | Fri, 06 Sep 2024 18:07:44 +0200 |
parents | 0d7bb4df2343 |
children |
rev | line source |
---|---|
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python3 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 |
3480
7550ae9cfbac
Renamed the project from "Salut à Toi" to "Libervia":
Goffi <goffi@goffi.org>
parents:
3479
diff
changeset
|
3 # Libervia: an XMPP client |
3479 | 4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 # This program is free software: you can redistribute it and/or modify |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # it under the terms of the GNU Affero General Public License as published by |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # the Free Software Foundation, either version 3 of the License, or |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # (at your option) any later version. |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 # This program is distributed in the hope that it will be useful, |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # GNU Affero General Public License for more details. |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 # You should have received a copy of the GNU Affero General Public License |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
19 from typing import Optional, Union |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
20 from pathlib import Path |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
21 from io import BufferedIOBase |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
22 |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from OpenSSL import SSL |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
24 import treq |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 from treq.client import HTTPClient |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
26 from twisted.internet import reactor, ssl |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 from twisted.web import iweb |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 from twisted.web import client as http_client |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
30 from zope.interface import implementer |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
31 |
4071
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
32 from libervia.backend.core import exceptions |
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
33 from libervia.backend.core.log import getLogger |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 log = getLogger(__name__) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 SSLError = SSL.Error |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 @implementer(IOpenSSLClientConnectionCreator) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 class NoCheckConnectionCreator(object): |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
44 def __init__(self, hostname, ctx): |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 self._ctx = ctx |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 def clientConnectionForTLS(self, tlsProtocol): |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 context = self._ctx |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 connection = SSL.Connection(context, None) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 connection.set_app_data(tlsProtocol) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 return connection |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 @implementer(iweb.IPolicyForHTTPS) |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
55 class NoCheckContextFactory: |
3089
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 """Context factory which doesn't do TLS certificate check |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
58 /!\\ it's obvisously a security flaw to use this class, |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
59 and it should be used only with explicit agreement from the end used |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 """ |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
61 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 def creatorForNetloc(self, hostname, port): |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 log.warning( |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 "TLS check disabled for {host} on port {port}".format( |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 host=hostname, port=port |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 ) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 ) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 certificateOptions = ssl.CertificateOptions(trustRoot=None) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 return NoCheckConnectionCreator(hostname, certificateOptions.getContext()) |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 #: following treq doesn't check TLS, obviously it is unsecure and should not be used |
e75024e41f81
plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
73 #: without explicit warning |
3205
2c0628f3927e
plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents:
3136
diff
changeset
|
74 treq_client_no_ssl = HTTPClient(http_client.Agent(reactor, NoCheckContextFactory())) |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
75 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
76 |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3822
diff
changeset
|
77 async def download_file( |
4270
0d7bb4df2343
Reformatted code base using black.
Goffi <goffi@goffi.org>
parents:
4071
diff
changeset
|
78 url: str, dest: Union[str, Path, BufferedIOBase], max_size: Optional[int] = None |
3822
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
79 ) -> None: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
80 """Helper method to download a file |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
81 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
82 This is for internal download, for high level download with progression, use |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
83 ``plugin_misc_download``. |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
84 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
85 Inspired from |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
86 https://treq.readthedocs.io/en/latest/howto.html#handling-streaming-responses |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
87 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
88 @param dest: destination filename or file-like object |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
89 of it's a file-like object, you'll have to close it yourself |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
90 @param max_size: if set, an exceptions.DataError will be raised if the downloaded file |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
91 is bigger that given value (in bytes). |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
92 """ |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
93 if isinstance(dest, BufferedIOBase): |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
94 f = dest |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
95 must_close = False |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
96 else: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
97 dest = Path(dest) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
98 f = dest.open("wb") |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
99 must_close = True |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
100 d = treq.get(url, unbuffered=True) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
101 written = 0 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
102 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
103 def write(data: bytes): |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
104 if max_size is not None: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
105 nonlocal written |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
106 written += len(data) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
107 if written > max_size: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
108 raise exceptions.DataError( |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
109 "downloaded file is bigger than expected ({max_size})" |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
110 ) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
111 f.write(data) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
112 |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
113 d.addCallback(treq.collect, f.write) |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
114 try: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
115 await d |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
116 except exceptions.DataError as e: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
117 log.warning("download cancelled due to file oversized") |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
118 raise e |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
119 except Exception as e: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
120 log.error(f"Can't write file {dest}: {e}") |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
121 raise e |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
122 finally: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
123 if must_close: |
65bac82e4049
core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents:
3480
diff
changeset
|
124 f.close() |