Mercurial > libervia-backend
changeset 3528:849374e59178
component file sharing: quotas implementation:
quotas can now be specified using the `quotas_json` option of `component file_sharing`
section in settings. This must be a dict where:
- `users` key contains default quotas for all users
- `admins` key contains quotas for administrators (not implemented yet)
- `jids` contain bare JID to quota mapping, to have user-specific quota
The value can be either a int for quota in bytes, or a case insensitive string with an
optional multiplier symbol (e.g. "500 Mio"). `None` can be used for explicit unlimited
quota (which is the default is `users` is not set).
When a file size is too big for quota, upload is refused with an error message indicating
allowed quota, used space, and the size of the file that user wants to upload.
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 05 May 2021 15:37:33 +0200 (2021-05-05) |
parents | bbf92ef05f38 |
children | 698579bedd6f |
files | sat/plugins/plugin_comp_file_sharing.py sat/plugins/plugin_misc_file.py sat/plugins/plugin_xep_0096.py |
diffstat | 3 files changed, 68 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/sat/plugins/plugin_comp_file_sharing.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_comp_file_sharing.py Wed May 05 15:37:33 2021 +0200 @@ -25,7 +25,7 @@ import unicodedata from urllib.parse import urljoin, urlparse, quote, unquote from pathlib import Path -from sat.core.i18n import _ +from sat.core.i18n import _, D_ from sat.core.constants import Const as C from sat.core import exceptions from sat.core.log import getLogger @@ -34,6 +34,7 @@ from sat.tools.common import regex from sat.tools.common import uri from sat.tools.common import files_utils +from sat.tools.common import utils from sat.tools.common import tls from twisted.internet import defer, reactor from twisted.words.protocols.jabber import error @@ -75,6 +76,10 @@ # Directory used to buffer request body (i.e. file in case of PUT) we use more than one @ # there, to be sure than it's not conflicting with a JID TMP_BUFFER_DIR = "@@tmp@@" +OVER_QUOTA_TXT = D_( + "You are over quota, your maximum allowed size is {quota} and you are already using " + "{used_space}, you can't upload {file_size} more." +) server.version = unicodedata.normalize( 'NFKD', @@ -242,7 +247,7 @@ if self._upload_data is not None: return self._upload_data - # self.path is not available if we are easly in the request (e.g. when gotLength + # self.path is not available if we are early in the request (e.g. when gotLength # is called), in which case channel._path must be used. On the other hand, when # render_[VERB] is called, only self.path is available path = self.channel._path if self.path is None else self.path @@ -347,7 +352,8 @@ self._jf = self.host.plugins["XEP-0234"] self._h = self.host.plugins["XEP-0300"] self._t = self.host.plugins["XEP-0264"] - self.host.plugins["XEP-0363"].registerHandler(self._onHTTPUpload) + self._hu = self.host.plugins["XEP-0363"] + self._hu.registerHandler(self._onHTTPUpload) self.host.trigger.add("FILE_getDestDir", self._getDestDirTrigger) self.host.trigger.add( "XEP-0234_fileSendingRequest", self._fileSendingRequestTrigger, priority=1000 @@ -400,6 +406,17 @@ if not os.path.exists(path): os.makedirs(path) + def getQuota(self, client, entity): + """Return maximum size allowed for all files for entity""" + # TODO: handle special entities like admins + quotas = self.host.memory.getConfig("component file_sharing", "quotas_json", {}) + entity_bare_s = entity.userhost() + try: + quota = quotas["jids"][entity_bare_s] + except KeyError: + quota = quotas.get("users") + return None if quota is None else utils.parseSize(quota) + async def generate_thumbnails(self, extra: dict, image_path: Path): thumbnails = extra.setdefault(C.KEY_THUMBNAILS, []) for max_thumb_size in self._t.SIZES: @@ -485,7 +502,7 @@ extra=extra, ) - def _getDestDirTrigger( + async def _getDestDirTrigger( self, client, peer_jid, transfer_data, file_data, stream_object ): """This trigger accept file sending request, and store file locally""" @@ -496,6 +513,19 @@ assert C.KEY_PROGRESS_ID in file_data filename = file_data["name"] assert filename and not "/" in filename + quota = self.getQuota(client, peer_jid) + if quota is not None: + used_space = await self.host.memory.fileGetUsedSpace(client, peer_jid) + + if (used_space + file_data["size"]) > quota: + raise error.StanzaError( + "not-acceptable", + text=OVER_QUOTA_TXT.format( + quota=utils.getHumanSize(quota), + used_space=utils.getHumanSize(used_space), + file_size=utils.getHumanSize(file_data['size']) + ) + ) file_tmp_dir = self.host.getLocalPath( None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True, profile=False ) @@ -597,19 +627,34 @@ except KeyError: log.error(f"trying to purge an inexisting upload slot ({upload_id})") - def _onHTTPUpload(self, client, request): + async def _onHTTPUpload(self, client, request): # filename should be already cleaned, but it's better to double check assert '/' not in request.filename # client._file_sharing_allowed_hosts is set in plugin XEP-0329 if request.from_.host not in client._file_sharing_allowed_hosts: raise error.StanzaError("forbidden") + quota = self.getQuota(client, request.from_) + if quota is not None: + used_space = await self.host.memory.fileGetUsedSpace(client, request.from_) + + if (used_space + request.size) > quota: + raise error.StanzaError( + "not-acceptable", + text=OVER_QUOTA_TXT.format( + quota=utils.getHumanSize(quota), + used_space=utils.getHumanSize(used_space), + file_size=utils.getHumanSize(request.size) + ), + appCondition = self._hu.getFileTooLargeElt(max(quota - used_space, 0)) + ) + upload_id = shortuuid.ShortUUID().random(length=30) assert '/' not in upload_id timer = reactor.callLater(30, self._purge_slot, upload_id) self.expected_uploads[upload_id] = (client, request, timer) url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}") - slot = self.host.plugins["XEP-0363"].Slot( + slot = self._hu.Slot( put=url, get=url, headers=[],
--- a/sat/plugins/plugin_misc_file.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_misc_file.py Wed May 05 15:37:33 2021 +0200 @@ -266,7 +266,9 @@ ) return True else: - return self.getDestDir(client, peer_jid, transfer_data, file_data) + return defer.ensureDeferred( + self.getDestDir(client, peer_jid, transfer_data, file_data) + ) exists_d = xml_tools.deferConfirm( self.host, @@ -286,7 +288,9 @@ self.openFileWrite(client, file_path, transfer_data, file_data, stream_object) return True - def getDestDir(self, client, peer_jid, transfer_data, file_data, stream_object=False): + async def getDestDir( + self, client, peer_jid, transfer_data, file_data, stream_object=False + ): """Request confirmation and destination dir to user Overwrite confirmation is managed. @@ -313,7 +317,7 @@ a stream.FileStreamObject will be used return (defer.Deferred): True if transfer is accepted """ - cont, ret_value = self.host.trigger.returnPoint( + cont, ret_value = await self.host.trigger.asyncReturnPoint( "FILE_getDestDir", client, peer_jid, transfer_data, file_data, stream_object ) if not cont:
--- a/sat/plugins/plugin_xep_0096.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_xep_0096.py Wed May 05 15:37:33 2021 +0200 @@ -17,18 +17,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +from twisted.words.xish import domish +from twisted.words.protocols.jabber import jid +from twisted.words.protocols.jabber import error +from twisted.internet import defer from sat.core.i18n import _, D_ from sat.core.constants import Const as C from sat.core.log import getLogger - -log = getLogger(__name__) from sat.core import exceptions from sat.tools import xml_tools from sat.tools import stream -from twisted.words.xish import domish -from twisted.words.protocols.jabber import jid -from twisted.words.protocols.jabber import error -import os + +log = getLogger(__name__) NS_SI_FT = "http://jabber.org/protocol/si/profile/file-transfer" @@ -205,7 +206,9 @@ "stream_plugin": plugin, } - d = self._f.getDestDir(client, peer_jid, data, data, stream_object=True) + d = defer.ensureDeferred( + self._f.getDestDir(client, peer_jid, data, data, stream_object=True) + ) d.addCallback(self.confirmationCb, client, iq_elt, data) def confirmationCb(self, accepted, client, iq_elt, data):