Mercurial > libervia-backend

annotate sat/tools/web.py @ 3971:9b1d74a6b48c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tests (e2e/cli): add tests for JET: rel 378
author Goffi <goffi@goffi.org>
date Mon, 31 Oct 2022 04:09:38 +0100
parents 65bac82e4049
children 524856bd7b19
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python3
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
2
3480
7550ae9cfbac Renamed the project from "Salut à Toi" to "Libervia":
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
3 # Libervia: an XMPP client
3479
be6d91572633 date update
Goffi <goffi@goffi.org>
parents: 3205
diff changeset
4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
5
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
6 # This program is free software: you can redistribute it and/or modify
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # it under the terms of the GNU Affero General Public License as published by
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # the Free Software Foundation, either version 3 of the License, or
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # (at your option) any later version.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
10
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
11 # This program is distributed in the hope that it will be useful,
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # GNU Affero General Public License for more details.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
15
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
16 # You should have received a copy of the GNU Affero General Public License
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
18
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
19 from typing import Optional, Union
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
20 from pathlib import Path
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
21 from io import BufferedIOBase
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
22
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 from OpenSSL import SSL
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
24 import treq
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from treq.client import HTTPClient
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
26 from twisted.internet import reactor, ssl
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 from twisted.web import iweb
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 from twisted.web import client as http_client
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
30 from zope.interface import implementer
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
31
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
32 from sat.core import exceptions
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
33 from sat.core.log import getLogger
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
34
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
35
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 log = getLogger(__name__)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
37
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
38
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 SSLError = SSL.Error
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
40
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
41
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 @implementer(IOpenSSLClientConnectionCreator)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
43 class NoCheckConnectionCreator(object):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
44 def __init__(self, hostname, ctx):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
45 self._ctx = ctx
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 def clientConnectionForTLS(self, tlsProtocol):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 context = self._ctx
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 connection = SSL.Connection(context, None)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 connection.set_app_data(tlsProtocol)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
51 return connection
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
52
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
53
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
54 @implementer(iweb.IPolicyForHTTPS)
3205
2c0628f3927e plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
55 class NoCheckContextFactory:
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 """Context factory which doesn't do TLS certificate check
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
57
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
58 /!\\ it's obvisously a security flaw to use this class,
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
59 and it should be used only with explicit agreement from the end used
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
60 """
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
61
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
62 def creatorForNetloc(self, hostname, port):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 log.warning(
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 "TLS check disabled for {host} on port {port}".format(
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 host=hostname, port=port
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 )
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
67 )
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 certificateOptions = ssl.CertificateOptions(trustRoot=None)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 return NoCheckConnectionCreator(hostname, certificateOptions.getContext())
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
70
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
71
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
72 #: following treq doesn't check TLS, obviously it is unsecure and should not be used
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
73 #: without explicit warning
3205
2c0628f3927e plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
74 treq_client_no_ssl = HTTPClient(http_client.Agent(reactor, NoCheckContextFactory()))
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
75
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
76
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
77 async def downloadFile(
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
78 url: str,
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
79 dest: Union[str, Path, BufferedIOBase],
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
80 max_size: Optional[int] = None
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
81 ) -> None:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
82 """Helper method to download a file
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
83
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
84 This is for internal download, for high level download with progression, use
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
85 ``plugin_misc_download``.
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
86
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
87 Inspired from
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
88 https://treq.readthedocs.io/en/latest/howto.html#handling-streaming-responses
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
89
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
90 @param dest: destination filename or file-like object
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
91 of it's a file-like object, you'll have to close it yourself
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
92 @param max_size: if set, an exceptions.DataError will be raised if the downloaded file
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
93 is bigger that given value (in bytes).
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
94 """
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
95 if isinstance(dest, BufferedIOBase):
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
96 f = dest
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
97 must_close = False
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
98 else:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
99 dest = Path(dest)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
100 f = dest.open("wb")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
101 must_close = True
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
102 d = treq.get(url, unbuffered=True)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
103 written = 0
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
104
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
105 def write(data: bytes):
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
106 if max_size is not None:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
107 nonlocal written
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
108 written += len(data)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
109 if written > max_size:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
110 raise exceptions.DataError(
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
111 "downloaded file is bigger than expected ({max_size})"
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
112 )
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
113 f.write(data)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
114
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
115 d.addCallback(treq.collect, f.write)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
116 try:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
117 await d
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
118 except exceptions.DataError as e:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
119 log.warning("download cancelled due to file oversized")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
120 raise e
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
121 except Exception as e:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
122 log.error(f"Can't write file {dest}: {e}")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
123 raise e
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
124 finally:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
125 if must_close:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
126 f.close()