changeset 3528:849374e59178

component file sharing: quotas implementation: quotas can now be specified using the `quotas_json` option of `component file_sharing` section in settings. This must be a dict where: - `users` key contains default quotas for all users - `admins` key contains quotas for administrators (not implemented yet) - `jids` contain bare JID to quota mapping, to have user-specific quota The value can be either a int for quota in bytes, or a case insensitive string with an optional multiplier symbol (e.g. "500 Mio"). `None` can be used for explicit unlimited quota (which is the default is `users` is not set). When a file size is too big for quota, upload is refused with an error message indicating allowed quota, used space, and the size of the file that user wants to upload.
author Goffi <goffi@goffi.org>
date Wed, 05 May 2021 15:37:33 +0200 (2021-05-05)
parents bbf92ef05f38
children 698579bedd6f
files sat/plugins/plugin_comp_file_sharing.py sat/plugins/plugin_misc_file.py sat/plugins/plugin_xep_0096.py
diffstat 3 files changed, 68 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/sat/plugins/plugin_comp_file_sharing.py	Wed May 05 15:37:33 2021 +0200
+++ b/sat/plugins/plugin_comp_file_sharing.py	Wed May 05 15:37:33 2021 +0200
@@ -25,7 +25,7 @@
 import unicodedata
 from urllib.parse import urljoin, urlparse, quote, unquote
 from pathlib import Path
-from sat.core.i18n import _
+from sat.core.i18n import _, D_
 from sat.core.constants import Const as C
 from sat.core import exceptions
 from sat.core.log import getLogger
@@ -34,6 +34,7 @@
 from sat.tools.common import regex
 from sat.tools.common import uri
 from sat.tools.common import files_utils
+from sat.tools.common import utils
 from sat.tools.common import tls
 from twisted.internet import defer, reactor
 from twisted.words.protocols.jabber import error
@@ -75,6 +76,10 @@
 # Directory used to buffer request body (i.e. file in case of PUT) we use more than one @
 # there, to be sure than it's not conflicting with a JID
 TMP_BUFFER_DIR = "@@tmp@@"
+OVER_QUOTA_TXT = D_(
+    "You are over quota, your maximum allowed size is {quota} and you are already using "
+    "{used_space}, you can't upload {file_size} more."
+)
 
 server.version = unicodedata.normalize(
     'NFKD',
@@ -242,7 +247,7 @@
         if self._upload_data is not None:
             return self._upload_data
 
-        # self.path is not available if we are easly in the request (e.g. when gotLength
+        # self.path is not available if we are early in the request (e.g. when gotLength
         # is called), in which case channel._path must be used. On the other hand, when
         # render_[VERB] is called, only self.path is available
         path = self.channel._path if self.path is None else self.path
@@ -347,7 +352,8 @@
         self._jf = self.host.plugins["XEP-0234"]
         self._h = self.host.plugins["XEP-0300"]
         self._t = self.host.plugins["XEP-0264"]
-        self.host.plugins["XEP-0363"].registerHandler(self._onHTTPUpload)
+        self._hu = self.host.plugins["XEP-0363"]
+        self._hu.registerHandler(self._onHTTPUpload)
         self.host.trigger.add("FILE_getDestDir", self._getDestDirTrigger)
         self.host.trigger.add(
             "XEP-0234_fileSendingRequest", self._fileSendingRequestTrigger, priority=1000
@@ -400,6 +406,17 @@
         if not os.path.exists(path):
             os.makedirs(path)
 
+    def getQuota(self, client, entity):
+        """Return maximum size allowed for all files for entity"""
+        # TODO: handle special entities like admins
+        quotas = self.host.memory.getConfig("component file_sharing", "quotas_json", {})
+        entity_bare_s = entity.userhost()
+        try:
+            quota = quotas["jids"][entity_bare_s]
+        except KeyError:
+            quota = quotas.get("users")
+        return None if quota is None else utils.parseSize(quota)
+
     async def generate_thumbnails(self, extra: dict, image_path: Path):
         thumbnails = extra.setdefault(C.KEY_THUMBNAILS, [])
         for max_thumb_size in self._t.SIZES:
@@ -485,7 +502,7 @@
             extra=extra,
         )
 
-    def _getDestDirTrigger(
+    async def _getDestDirTrigger(
         self, client, peer_jid, transfer_data, file_data, stream_object
     ):
         """This trigger accept file sending request, and store file locally"""
@@ -496,6 +513,19 @@
         assert C.KEY_PROGRESS_ID in file_data
         filename = file_data["name"]
         assert filename and not "/" in filename
+        quota = self.getQuota(client, peer_jid)
+        if quota is not None:
+            used_space = await self.host.memory.fileGetUsedSpace(client, peer_jid)
+
+            if (used_space + file_data["size"]) > quota:
+                raise error.StanzaError(
+                    "not-acceptable",
+                    text=OVER_QUOTA_TXT.format(
+                        quota=utils.getHumanSize(quota),
+                        used_space=utils.getHumanSize(used_space),
+                        file_size=utils.getHumanSize(file_data['size'])
+                    )
+                )
         file_tmp_dir = self.host.getLocalPath(
             None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True, profile=False
         )
@@ -597,19 +627,34 @@
         except KeyError:
             log.error(f"trying to purge an inexisting upload slot ({upload_id})")
 
-    def _onHTTPUpload(self, client, request):
+    async def _onHTTPUpload(self, client, request):
         # filename should be already cleaned, but it's better to double check
         assert '/' not in request.filename
         # client._file_sharing_allowed_hosts is set in plugin XEP-0329
         if request.from_.host not in client._file_sharing_allowed_hosts:
             raise error.StanzaError("forbidden")
 
+        quota = self.getQuota(client, request.from_)
+        if quota is not None:
+            used_space = await self.host.memory.fileGetUsedSpace(client, request.from_)
+
+            if (used_space + request.size) > quota:
+                raise error.StanzaError(
+                    "not-acceptable",
+                    text=OVER_QUOTA_TXT.format(
+                        quota=utils.getHumanSize(quota),
+                        used_space=utils.getHumanSize(used_space),
+                        file_size=utils.getHumanSize(request.size)
+                    ),
+                    appCondition = self._hu.getFileTooLargeElt(max(quota - used_space, 0))
+                )
+
         upload_id = shortuuid.ShortUUID().random(length=30)
         assert '/' not in upload_id
         timer = reactor.callLater(30, self._purge_slot, upload_id)
         self.expected_uploads[upload_id] = (client, request, timer)
         url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}")
-        slot = self.host.plugins["XEP-0363"].Slot(
+        slot = self._hu.Slot(
             put=url,
             get=url,
             headers=[],
--- a/sat/plugins/plugin_misc_file.py	Wed May 05 15:37:33 2021 +0200
+++ b/sat/plugins/plugin_misc_file.py	Wed May 05 15:37:33 2021 +0200
@@ -266,7 +266,9 @@
                     )
                     return True
                 else:
-                    return self.getDestDir(client, peer_jid, transfer_data, file_data)
+                    return defer.ensureDeferred(
+                        self.getDestDir(client, peer_jid, transfer_data, file_data)
+                    )
 
             exists_d = xml_tools.deferConfirm(
                 self.host,
@@ -286,7 +288,9 @@
         self.openFileWrite(client, file_path, transfer_data, file_data, stream_object)
         return True
 
-    def getDestDir(self, client, peer_jid, transfer_data, file_data, stream_object=False):
+    async def getDestDir(
+        self, client, peer_jid, transfer_data, file_data, stream_object=False
+    ):
         """Request confirmation and destination dir to user
 
         Overwrite confirmation is managed.
@@ -313,7 +317,7 @@
             a stream.FileStreamObject will be used
         return (defer.Deferred): True if transfer is accepted
         """
-        cont, ret_value = self.host.trigger.returnPoint(
+        cont, ret_value = await self.host.trigger.asyncReturnPoint(
             "FILE_getDestDir", client, peer_jid, transfer_data, file_data, stream_object
         )
         if not cont:
--- a/sat/plugins/plugin_xep_0096.py	Wed May 05 15:37:33 2021 +0200
+++ b/sat/plugins/plugin_xep_0096.py	Wed May 05 15:37:33 2021 +0200
@@ -17,18 +17,19 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+from twisted.words.xish import domish
+from twisted.words.protocols.jabber import jid
+from twisted.words.protocols.jabber import error
+from twisted.internet import defer
 from sat.core.i18n import _, D_
 from sat.core.constants import Const as C
 from sat.core.log import getLogger
-
-log = getLogger(__name__)
 from sat.core import exceptions
 from sat.tools import xml_tools
 from sat.tools import stream
-from twisted.words.xish import domish
-from twisted.words.protocols.jabber import jid
-from twisted.words.protocols.jabber import error
-import os
+
+log = getLogger(__name__)
 
 
 NS_SI_FT = "http://jabber.org/protocol/si/profile/file-transfer"
@@ -205,7 +206,9 @@
             "stream_plugin": plugin,
         }
 
-        d = self._f.getDestDir(client, peer_jid, data, data, stream_object=True)
+        d = defer.ensureDeferred(
+            self._f.getDestDir(client, peer_jid, data, data, stream_object=True)
+        )
         d.addCallback(self.confirmationCb, client, iq_elt, data)
 
     def confirmationCb(self, accepted, client, iq_elt, data):