Mercurial > libervia-backend
changeset 3528:849374e59178
component file sharing: quotas implementation:
quotas can now be specified using the `quotas_json` option of `component file_sharing`
section in settings. This must be a dict where:
- `users` key contains default quotas for all users
- `admins` key contains quotas for administrators (not implemented yet)
- `jids` contain bare JID to quota mapping, to have user-specific quota
The value can be either a int for quota in bytes, or a case insensitive string with an
optional multiplier symbol (e.g. "500 Mio"). `None` can be used for explicit unlimited
quota (which is the default is `users` is not set).
When a file size is too big for quota, upload is refused with an error message indicating
allowed quota, used space, and the size of the file that user wants to upload.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Wed, 05 May 2021 15:37:33 +0200 |
| parents | bbf92ef05f38 |
| children | 698579bedd6f |
| files | sat/plugins/plugin_comp_file_sharing.py sat/plugins/plugin_misc_file.py sat/plugins/plugin_xep_0096.py |
| diffstat | 3 files changed, 68 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/sat/plugins/plugin_comp_file_sharing.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_comp_file_sharing.py Wed May 05 15:37:33 2021 +0200 @@ -25,7 +25,7 @@ import unicodedata from urllib.parse import urljoin, urlparse, quote, unquote from pathlib import Path -from sat.core.i18n import _ +from sat.core.i18n import _, D_ from sat.core.constants import Const as C from sat.core import exceptions from sat.core.log import getLogger @@ -34,6 +34,7 @@ from sat.tools.common import regex from sat.tools.common import uri from sat.tools.common import files_utils +from sat.tools.common import utils from sat.tools.common import tls from twisted.internet import defer, reactor from twisted.words.protocols.jabber import error @@ -75,6 +76,10 @@ # Directory used to buffer request body (i.e. file in case of PUT) we use more than one @ # there, to be sure than it's not conflicting with a JID TMP_BUFFER_DIR = "@@tmp@@" +OVER_QUOTA_TXT = D_( + "You are over quota, your maximum allowed size is {quota} and you are already using " + "{used_space}, you can't upload {file_size} more." +) server.version = unicodedata.normalize( 'NFKD', @@ -242,7 +247,7 @@ if self._upload_data is not None: return self._upload_data - # self.path is not available if we are easly in the request (e.g. when gotLength + # self.path is not available if we are early in the request (e.g. when gotLength # is called), in which case channel._path must be used. On the other hand, when # render_[VERB] is called, only self.path is available path = self.channel._path if self.path is None else self.path @@ -347,7 +352,8 @@ self._jf = self.host.plugins["XEP-0234"] self._h = self.host.plugins["XEP-0300"] self._t = self.host.plugins["XEP-0264"] - self.host.plugins["XEP-0363"].registerHandler(self._onHTTPUpload) + self._hu = self.host.plugins["XEP-0363"] + self._hu.registerHandler(self._onHTTPUpload) self.host.trigger.add("FILE_getDestDir", self._getDestDirTrigger) self.host.trigger.add( "XEP-0234_fileSendingRequest", self._fileSendingRequestTrigger, priority=1000 @@ -400,6 +406,17 @@ if not os.path.exists(path): os.makedirs(path) + def getQuota(self, client, entity): + """Return maximum size allowed for all files for entity""" + # TODO: handle special entities like admins + quotas = self.host.memory.getConfig("component file_sharing", "quotas_json", {}) + entity_bare_s = entity.userhost() + try: + quota = quotas["jids"][entity_bare_s] + except KeyError: + quota = quotas.get("users") + return None if quota is None else utils.parseSize(quota) + async def generate_thumbnails(self, extra: dict, image_path: Path): thumbnails = extra.setdefault(C.KEY_THUMBNAILS, []) for max_thumb_size in self._t.SIZES: @@ -485,7 +502,7 @@ extra=extra, ) - def _getDestDirTrigger( + async def _getDestDirTrigger( self, client, peer_jid, transfer_data, file_data, stream_object ): """This trigger accept file sending request, and store file locally""" @@ -496,6 +513,19 @@ assert C.KEY_PROGRESS_ID in file_data filename = file_data["name"] assert filename and not "/" in filename + quota = self.getQuota(client, peer_jid) + if quota is not None: + used_space = await self.host.memory.fileGetUsedSpace(client, peer_jid) + + if (used_space + file_data["size"]) > quota: + raise error.StanzaError( + "not-acceptable", + text=OVER_QUOTA_TXT.format( + quota=utils.getHumanSize(quota), + used_space=utils.getHumanSize(used_space), + file_size=utils.getHumanSize(file_data['size']) + ) + ) file_tmp_dir = self.host.getLocalPath( None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True, profile=False ) @@ -597,19 +627,34 @@ except KeyError: log.error(f"trying to purge an inexisting upload slot ({upload_id})") - def _onHTTPUpload(self, client, request): + async def _onHTTPUpload(self, client, request): # filename should be already cleaned, but it's better to double check assert '/' not in request.filename # client._file_sharing_allowed_hosts is set in plugin XEP-0329 if request.from_.host not in client._file_sharing_allowed_hosts: raise error.StanzaError("forbidden") + quota = self.getQuota(client, request.from_) + if quota is not None: + used_space = await self.host.memory.fileGetUsedSpace(client, request.from_) + + if (used_space + request.size) > quota: + raise error.StanzaError( + "not-acceptable", + text=OVER_QUOTA_TXT.format( + quota=utils.getHumanSize(quota), + used_space=utils.getHumanSize(used_space), + file_size=utils.getHumanSize(request.size) + ), + appCondition = self._hu.getFileTooLargeElt(max(quota - used_space, 0)) + ) + upload_id = shortuuid.ShortUUID().random(length=30) assert '/' not in upload_id timer = reactor.callLater(30, self._purge_slot, upload_id) self.expected_uploads[upload_id] = (client, request, timer) url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}") - slot = self.host.plugins["XEP-0363"].Slot( + slot = self._hu.Slot( put=url, get=url, headers=[],
--- a/sat/plugins/plugin_misc_file.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_misc_file.py Wed May 05 15:37:33 2021 +0200 @@ -266,7 +266,9 @@ ) return True else: - return self.getDestDir(client, peer_jid, transfer_data, file_data) + return defer.ensureDeferred( + self.getDestDir(client, peer_jid, transfer_data, file_data) + ) exists_d = xml_tools.deferConfirm( self.host, @@ -286,7 +288,9 @@ self.openFileWrite(client, file_path, transfer_data, file_data, stream_object) return True - def getDestDir(self, client, peer_jid, transfer_data, file_data, stream_object=False): + async def getDestDir( + self, client, peer_jid, transfer_data, file_data, stream_object=False + ): """Request confirmation and destination dir to user Overwrite confirmation is managed. @@ -313,7 +317,7 @@ a stream.FileStreamObject will be used return (defer.Deferred): True if transfer is accepted """ - cont, ret_value = self.host.trigger.returnPoint( + cont, ret_value = await self.host.trigger.asyncReturnPoint( "FILE_getDestDir", client, peer_jid, transfer_data, file_data, stream_object ) if not cont:
--- a/sat/plugins/plugin_xep_0096.py Wed May 05 15:37:33 2021 +0200 +++ b/sat/plugins/plugin_xep_0096.py Wed May 05 15:37:33 2021 +0200 @@ -17,18 +17,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +from twisted.words.xish import domish +from twisted.words.protocols.jabber import jid +from twisted.words.protocols.jabber import error +from twisted.internet import defer from sat.core.i18n import _, D_ from sat.core.constants import Const as C from sat.core.log import getLogger - -log = getLogger(__name__) from sat.core import exceptions from sat.tools import xml_tools from sat.tools import stream -from twisted.words.xish import domish -from twisted.words.protocols.jabber import jid -from twisted.words.protocols.jabber import error -import os + +log = getLogger(__name__) NS_SI_FT = "http://jabber.org/protocol/si/profile/file-transfer" @@ -205,7 +206,9 @@ "stream_plugin": plugin, } - d = self._f.getDestDir(client, peer_jid, data, data, stream_object=True) + d = defer.ensureDeferred( + self._f.getDestDir(client, peer_jid, data, data, stream_object=True) + ) d.addCallback(self.confirmationCb, client, iq_elt, data) def confirmationCb(self, accepted, client, iq_elt, data):