Mercurial > libervia-backend

annotate libervia/backend/tools/web.py @ 4100:810921c33a47

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
tools (common/template): add filter to get media types: Add 2 filters to get main type and subtype of media type. Jinja2 and Nunjucks don't handle slices in the same way (Python way for Jinja2, JS way for Nunjucks), making it difficult to retrieve main type of a media from media type. Thoses filters work in both cases.
author Goffi <goffi@goffi.org>
date Thu, 22 Jun 2023 15:49:06 +0200
parents 4b842c1fb686
children 0d7bb4df2343
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python3
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
2
3480
7550ae9cfbac Renamed the project from "Salut à Toi" to "Libervia":
Goffi <goffi@goffi.org>
parents: 3479
diff changeset
3 # Libervia: an XMPP client
3479
be6d91572633 date update
Goffi <goffi@goffi.org>
parents: 3205
diff changeset
4 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
5
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
6 # This program is free software: you can redistribute it and/or modify
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # it under the terms of the GNU Affero General Public License as published by
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # the Free Software Foundation, either version 3 of the License, or
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # (at your option) any later version.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
10
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
11 # This program is distributed in the hope that it will be useful,
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # GNU Affero General Public License for more details.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
15
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
16 # You should have received a copy of the GNU Affero General Public License
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
18
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
19 from typing import Optional, Union
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
20 from pathlib import Path
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
21 from io import BufferedIOBase
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
22
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 from OpenSSL import SSL
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
24 import treq
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 from treq.client import HTTPClient
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
26 from twisted.internet import reactor, ssl
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 from twisted.web import iweb
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
29 from twisted.web import client as http_client
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
30 from zope.interface import implementer
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
31
4071
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
32 from libervia.backend.core import exceptions
4b842c1fb686 refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents: 4037
diff changeset
33 from libervia.backend.core.log import getLogger
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
34
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
35
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 log = getLogger(__name__)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
37
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
38
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 SSLError = SSL.Error
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
40
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
41
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 @implementer(IOpenSSLClientConnectionCreator)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
43 class NoCheckConnectionCreator(object):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
44 def __init__(self, hostname, ctx):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
45 self._ctx = ctx
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 def clientConnectionForTLS(self, tlsProtocol):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 context = self._ctx
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 connection = SSL.Connection(context, None)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 connection.set_app_data(tlsProtocol)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
51 return connection
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
52
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
53
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
54 @implementer(iweb.IPolicyForHTTPS)
3205
2c0628f3927e plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
55 class NoCheckContextFactory:
3089
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 """Context factory which doesn't do TLS certificate check
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
57
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
58 /!\\ it's obvisously a security flaw to use this class,
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
59 and it should be used only with explicit agreement from the end used
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
60 """
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
61
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
62 def creatorForNetloc(self, hostname, port):
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 log.warning(
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 "TLS check disabled for {host} on port {port}".format(
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 host=hostname, port=port
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 )
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
67 )
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 certificateOptions = ssl.CertificateOptions(trustRoot=None)
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 return NoCheckConnectionCreator(hostname, certificateOptions.getContext())
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
70
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
71
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
72 #: following treq doesn't check TLS, obviously it is unsecure and should not be used
e75024e41f81 plugin upload, XEP-0363: code modernisation + preparation for extension:
Goffi <goffi@goffi.org>
parents:
diff changeset
73 #: without explicit warning
3205
2c0628f3927e plugin download, aesgcm: disable TLS check if `check_certificate` setting is disabled
Goffi <goffi@goffi.org>
parents: 3136
diff changeset
74 treq_client_no_ssl = HTTPClient(http_client.Agent(reactor, NoCheckContextFactory()))
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
75
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
76
4037
524856bd7b19 massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents: 3822
diff changeset
77 async def download_file(
3822
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
78 url: str,
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
79 dest: Union[str, Path, BufferedIOBase],
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
80 max_size: Optional[int] = None
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
81 ) -> None:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
82 """Helper method to download a file
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
83
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
84 This is for internal download, for high level download with progression, use
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
85 ``plugin_misc_download``.
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
86
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
87 Inspired from
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
88 https://treq.readthedocs.io/en/latest/howto.html#handling-streaming-responses
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
89
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
90 @param dest: destination filename or file-like object
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
91 of it's a file-like object, you'll have to close it yourself
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
92 @param max_size: if set, an exceptions.DataError will be raised if the downloaded file
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
93 is bigger that given value (in bytes).
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
94 """
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
95 if isinstance(dest, BufferedIOBase):
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
96 f = dest
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
97 must_close = False
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
98 else:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
99 dest = Path(dest)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
100 f = dest.open("wb")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
101 must_close = True
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
102 d = treq.get(url, unbuffered=True)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
103 written = 0
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
104
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
105 def write(data: bytes):
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
106 if max_size is not None:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
107 nonlocal written
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
108 written += len(data)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
109 if written > max_size:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
110 raise exceptions.DataError(
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
111 "downloaded file is bigger than expected ({max_size})"
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
112 )
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
113 f.write(data)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
114
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
115 d.addCallback(treq.collect, f.write)
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
116 try:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
117 await d
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
118 except exceptions.DataError as e:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
119 log.warning("download cancelled due to file oversized")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
120 raise e
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
121 except Exception as e:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
122 log.error(f"Can't write file {dest}: {e}")
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
123 raise e
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
124 finally:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
125 if must_close:
65bac82e4049 core (tools/web): helped method to download files:
Goffi <goffi@goffi.org>
parents: 3480
diff changeset
126 f.close()