Mercurial > libervia-backend

comparison sat/plugins/plugin_comp_file_sharing.py @ 3528:849374e59178

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
component file sharing: quotas implementation: quotas can now be specified using the `quotas_json` option of `component file_sharing` section in settings. This must be a dict where: - `users` key contains default quotas for all users - `admins` key contains quotas for administrators (not implemented yet) - `jids` contain bare JID to quota mapping, to have user-specific quota The value can be either a int for quota in bytes, or a case insensitive string with an optional multiplier symbol (e.g. "500 Mio"). `None` can be used for explicit unlimited quota (which is the default is `users` is not set). When a file size is too big for quota, upload is refused with an error message indicating allowed quota, used space, and the size of the file that user wants to upload.
author Goffi <goffi@goffi.org>
date Wed, 05 May 2021 15:37:33 +0200
parents 6d9c9e2dca0a
children ab72b8ac3bd2
comparison
equal deleted inserted replaced
3527:bbf92ef05f38 3528:849374e59178
23 from functools import partial 23 from functools import partial
24 import shortuuid 24 import shortuuid
25 import unicodedata 25 import unicodedata
26 from urllib.parse import urljoin, urlparse, quote, unquote 26 from urllib.parse import urljoin, urlparse, quote, unquote
27 from pathlib import Path 27 from pathlib import Path
28 from sat.core.i18n import _ 28 from sat.core.i18n import _, D_
29 from sat.core.constants import Const as C 29 from sat.core.constants import Const as C
30 from sat.core import exceptions 30 from sat.core import exceptions
31 from sat.core.log import getLogger 31 from sat.core.log import getLogger
32 from sat.tools import stream 32 from sat.tools import stream
33 from sat.tools import video 33 from sat.tools import video
34 from sat.tools.common import regex 34 from sat.tools.common import regex
35 from sat.tools.common import uri 35 from sat.tools.common import uri
36 from sat.tools.common import files_utils 36 from sat.tools.common import files_utils
37 from sat.tools.common import utils
37 from sat.tools.common import tls 38 from sat.tools.common import tls
38 from twisted.internet import defer, reactor 39 from twisted.internet import defer, reactor
39 from twisted.words.protocols.jabber import error 40 from twisted.words.protocols.jabber import error
40 from twisted.web import server, resource, static, http 41 from twisted.web import server, resource, static, http
41 from wokkel import pubsub 42 from wokkel import pubsub
73 NS_FS_AFFILIATION = "org.salut-a-toi.file-sharing-affiliation" 74 NS_FS_AFFILIATION = "org.salut-a-toi.file-sharing-affiliation"
74 COMMENT_NODE_PREFIX = "org.salut-a-toi.file_comments/" 75 COMMENT_NODE_PREFIX = "org.salut-a-toi.file_comments/"
75 # Directory used to buffer request body (i.e. file in case of PUT) we use more than one @ 76 # Directory used to buffer request body (i.e. file in case of PUT) we use more than one @
76 # there, to be sure than it's not conflicting with a JID 77 # there, to be sure than it's not conflicting with a JID
77 TMP_BUFFER_DIR = "@@tmp@@" 78 TMP_BUFFER_DIR = "@@tmp@@"
79 OVER_QUOTA_TXT = D_(
80 "You are over quota, your maximum allowed size is {quota} and you are already using "
81 "{used_space}, you can't upload {file_size} more."
82 )
78 83
79 server.version = unicodedata.normalize( 84 server.version = unicodedata.normalize(
80 'NFKD', 85 'NFKD',
81 f"{C.APP_NAME} file sharing {C.APP_VERSION}" 86 f"{C.APP_NAME} file sharing {C.APP_VERSION}"
82 ).encode('ascii','ignore') 87 ).encode('ascii','ignore')
240 def upload_data(self): 245 def upload_data(self):
241 """A tuple with upload_id and filename retrieve from requested path""" 246 """A tuple with upload_id and filename retrieve from requested path"""
242 if self._upload_data is not None: 247 if self._upload_data is not None:
243 return self._upload_data 248 return self._upload_data
244 249
245 # self.path is not available if we are easly in the request (e.g. when gotLength 250 # self.path is not available if we are early in the request (e.g. when gotLength
246 # is called), in which case channel._path must be used. On the other hand, when 251 # is called), in which case channel._path must be used. On the other hand, when
247 # render_[VERB] is called, only self.path is available 252 # render_[VERB] is called, only self.path is available
248 path = self.channel._path if self.path is None else self.path 253 path = self.channel._path if self.path is None else self.path
249 # we normalise the path 254 # we normalise the path
250 path = urlparse(path.decode()).path 255 path = urlparse(path.decode()).path
345 log.info(_("File Sharing initialization")) 350 log.info(_("File Sharing initialization"))
346 self._f = self.host.plugins["FILE"] 351 self._f = self.host.plugins["FILE"]
347 self._jf = self.host.plugins["XEP-0234"] 352 self._jf = self.host.plugins["XEP-0234"]
348 self._h = self.host.plugins["XEP-0300"] 353 self._h = self.host.plugins["XEP-0300"]
349 self._t = self.host.plugins["XEP-0264"] 354 self._t = self.host.plugins["XEP-0264"]
350 self.host.plugins["XEP-0363"].registerHandler(self._onHTTPUpload) 355 self._hu = self.host.plugins["XEP-0363"]
356 self._hu.registerHandler(self._onHTTPUpload)
351 self.host.trigger.add("FILE_getDestDir", self._getDestDirTrigger) 357 self.host.trigger.add("FILE_getDestDir", self._getDestDirTrigger)
352 self.host.trigger.add( 358 self.host.trigger.add(
353 "XEP-0234_fileSendingRequest", self._fileSendingRequestTrigger, priority=1000 359 "XEP-0234_fileSendingRequest", self._fileSendingRequestTrigger, priority=1000
354 ) 360 )
355 self.host.trigger.add("XEP-0234_buildFileElement", self._addFileMetadataElts) 361 self.host.trigger.add("XEP-0234_buildFileElement", self._addFileMetadataElts)
398 regex.pathEscape(client.profile), 404 regex.pathEscape(client.profile),
399 ) 405 )
400 if not os.path.exists(path): 406 if not os.path.exists(path):
401 os.makedirs(path) 407 os.makedirs(path)
402 408
409 def getQuota(self, client, entity):
410 """Return maximum size allowed for all files for entity"""
411 # TODO: handle special entities like admins
412 quotas = self.host.memory.getConfig("component file_sharing", "quotas_json", {})
413 entity_bare_s = entity.userhost()
414 try:
415 quota = quotas["jids"][entity_bare_s]
416 except KeyError:
417 quota = quotas.get("users")
418 return None if quota is None else utils.parseSize(quota)
419
403 async def generate_thumbnails(self, extra: dict, image_path: Path): 420 async def generate_thumbnails(self, extra: dict, image_path: Path):
404 thumbnails = extra.setdefault(C.KEY_THUMBNAILS, []) 421 thumbnails = extra.setdefault(C.KEY_THUMBNAILS, [])
405 for max_thumb_size in self._t.SIZES: 422 for max_thumb_size in self._t.SIZES:
406 try: 423 try:
407 thumb_size, thumb_id = await self._t.generateThumbnail( 424 thumb_size, thumb_id = await self._t.generateThumbnail(
483 public_id=public_id, 500 public_id=public_id,
484 owner=peer_jid, 501 owner=peer_jid,
485 extra=extra, 502 extra=extra,
486 ) 503 )
487 504
488 def _getDestDirTrigger( 505 async def _getDestDirTrigger(
489 self, client, peer_jid, transfer_data, file_data, stream_object 506 self, client, peer_jid, transfer_data, file_data, stream_object
490 ): 507 ):
491 """This trigger accept file sending request, and store file locally""" 508 """This trigger accept file sending request, and store file locally"""
492 if not client.is_component: 509 if not client.is_component:
493 return True, None 510 return True, None
494 assert stream_object 511 assert stream_object
495 assert "stream_object" not in transfer_data 512 assert "stream_object" not in transfer_data
496 assert C.KEY_PROGRESS_ID in file_data 513 assert C.KEY_PROGRESS_ID in file_data
497 filename = file_data["name"] 514 filename = file_data["name"]
498 assert filename and not "/" in filename 515 assert filename and not "/" in filename
516 quota = self.getQuota(client, peer_jid)
517 if quota is not None:
518 used_space = await self.host.memory.fileGetUsedSpace(client, peer_jid)
519
520 if (used_space + file_data["size"]) > quota:
521 raise error.StanzaError(
522 "not-acceptable",
523 text=OVER_QUOTA_TXT.format(
524 quota=utils.getHumanSize(quota),
525 used_space=utils.getHumanSize(used_space),
526 file_size=utils.getHumanSize(file_data['size'])
527 )
528 )
499 file_tmp_dir = self.host.getLocalPath( 529 file_tmp_dir = self.host.getLocalPath(
500 None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True, profile=False 530 None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True, profile=False
501 ) 531 )
502 file_tmp_path = file_data['file_path'] = files_utils.get_unique_name( 532 file_tmp_path = file_data['file_path'] = files_utils.get_unique_name(
503 file_tmp_dir/filename) 533 file_tmp_dir/filename)
595 try: 625 try:
596 del self.expected_uploads[upload_id] 626 del self.expected_uploads[upload_id]
597 except KeyError: 627 except KeyError:
598 log.error(f"trying to purge an inexisting upload slot ({upload_id})") 628 log.error(f"trying to purge an inexisting upload slot ({upload_id})")
599 629
600 def _onHTTPUpload(self, client, request): 630 async def _onHTTPUpload(self, client, request):
601 # filename should be already cleaned, but it's better to double check 631 # filename should be already cleaned, but it's better to double check
602 assert '/' not in request.filename 632 assert '/' not in request.filename
603 # client._file_sharing_allowed_hosts is set in plugin XEP-0329 633 # client._file_sharing_allowed_hosts is set in plugin XEP-0329
604 if request.from_.host not in client._file_sharing_allowed_hosts: 634 if request.from_.host not in client._file_sharing_allowed_hosts:
605 raise error.StanzaError("forbidden") 635 raise error.StanzaError("forbidden")
606 636
637 quota = self.getQuota(client, request.from_)
638 if quota is not None:
639 used_space = await self.host.memory.fileGetUsedSpace(client, request.from_)
640
641 if (used_space + request.size) > quota:
642 raise error.StanzaError(
643 "not-acceptable",
644 text=OVER_QUOTA_TXT.format(
645 quota=utils.getHumanSize(quota),
646 used_space=utils.getHumanSize(used_space),
647 file_size=utils.getHumanSize(request.size)
648 ),
649 appCondition = self._hu.getFileTooLargeElt(max(quota - used_space, 0))
650 )
651
607 upload_id = shortuuid.ShortUUID().random(length=30) 652 upload_id = shortuuid.ShortUUID().random(length=30)
608 assert '/' not in upload_id 653 assert '/' not in upload_id
609 timer = reactor.callLater(30, self._purge_slot, upload_id) 654 timer = reactor.callLater(30, self._purge_slot, upload_id)
610 self.expected_uploads[upload_id] = (client, request, timer) 655 self.expected_uploads[upload_id] = (client, request, timer)
611 url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}") 656 url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}")
612 slot = self.host.plugins["XEP-0363"].Slot( 657 slot = self._hu.Slot(
613 put=url, 658 put=url,
614 get=url, 659 get=url,
615 headers=[], 660 headers=[],
616 ) 661 )
617 return slot 662 return slot