comparison sat/memory/memory.py @ 3028:ab2696e34d29

Python 3 port: /!\ this is a huge commit /!\ starting from this commit, SàT is needs Python 3.6+ /!\ SàT maybe be instable or some feature may not work anymore, this will improve with time This patch port backend, bridge and frontends to Python 3. Roughly this has been done this way: - 2to3 tools has been applied (with python 3.7) - all references to python2 have been replaced with python3 (notably shebangs) - fixed files not handled by 2to3 (notably the shell script) - several manual fixes - fixed issues reported by Python 3 that where not handled in Python 2 - replaced "async" with "async_" when needed (it's a reserved word from Python 3.7) - replaced zope's "implements" with @implementer decorator - temporary hack to handle data pickled in database, as str or bytes may be returned, to be checked later - fixed hash comparison for password - removed some code which is not needed anymore with Python 3 - deactivated some code which needs to be checked (notably certificate validation) - tested with jp, fixed reported issues until some basic commands worked - ported Primitivus (after porting dependencies like urwid satext) - more manual fixes
author Goffi <goffi@goffi.org>
date Tue, 13 Aug 2019 19:08:41 +0200
parents 94708a7d3ecf
children 98d1f34ce5b9
comparison
equal deleted inserted replaced
3027:ff5bcb12ae60 3028:ab2696e34d29
1 #!/usr/bin/env python2 1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 3
4 # SAT: a jabber client 4 # SAT: a jabber client
5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org) 5 # Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org)
6 6
24 log = getLogger(__name__) 24 log = getLogger(__name__)
25 25
26 import os.path 26 import os.path
27 import copy 27 import copy
28 from collections import namedtuple 28 from collections import namedtuple
29 from ConfigParser import SafeConfigParser, NoOptionError, NoSectionError 29 from configparser import SafeConfigParser, NoOptionError, NoSectionError
30 from uuid import uuid4 30 from uuid import uuid4
31 from twisted.python import failure 31 from twisted.python import failure
32 from twisted.internet import defer, reactor, error 32 from twisted.internet import defer, reactor, error
33 from twisted.words.protocols.jabber import jid 33 from twisted.words.protocols.jabber import jid
34 from sat.core import exceptions 34 from sat.core import exceptions
74 """ 74 """
75 if session_id is None: 75 if session_id is None:
76 session_id = str(uuid4()) 76 session_id = str(uuid4())
77 elif session_id in self._sessions: 77 elif session_id in self._sessions:
78 raise exceptions.ConflictError( 78 raise exceptions.ConflictError(
79 u"Session id {} is already used".format(session_id) 79 "Session id {} is already used".format(session_id)
80 ) 80 )
81 timer = reactor.callLater(self.timeout, self._purgeSession, session_id) 81 timer = reactor.callLater(self.timeout, self._purgeSession, session_id)
82 if session_data is None: 82 if session_data is None:
83 session_data = {} 83 session_data = {}
84 self._sessions[session_id] = ( 84 self._sessions[session_id] = (
97 except error.AlreadyCalled: 97 except error.AlreadyCalled:
98 # if the session is time-outed, the timer has been called 98 # if the session is time-outed, the timer has been called
99 pass 99 pass
100 del self._sessions[session_id] 100 del self._sessions[session_id]
101 log.debug( 101 log.debug(
102 u"Session {} purged{}".format( 102 "Session {} purged{}".format(
103 session_id, 103 session_id,
104 u" (profile {})".format(profile) if profile is not None else u"", 104 " (profile {})".format(profile) if profile is not None else "",
105 ) 105 )
106 ) 106 )
107 107
108 def __len__(self): 108 def __len__(self):
109 return len(self._sessions) 109 return len(self._sessions)
145 def __delitem__(self, session_id): 145 def __delitem__(self, session_id):
146 """ delete the session data """ 146 """ delete the session data """
147 self._purgeSession(session_id) 147 self._purgeSession(session_id)
148 148
149 def keys(self): 149 def keys(self):
150 return self._sessions.keys() 150 return list(self._sessions.keys())
151 151
152 def iterkeys(self): 152 def iterkeys(self):
153 return self._sessions.iterkeys() 153 return iter(self._sessions.keys())
154 154
155 155
156 class ProfileSessions(Sessions): 156 class ProfileSessions(Sessions):
157 """ProfileSessions extends the Sessions class, but here the profile can be 157 """ProfileSessions extends the Sessions class, but here the profile can be
158 used as the key to retrieve data or delete a session (instead of session id). 158 used as the key to retrieve data or delete a session (instead of session id).
163 163
164 @param profile: %(doc_profile)s 164 @param profile: %(doc_profile)s
165 @return: a list containing the sessions ids 165 @return: a list containing the sessions ids
166 """ 166 """
167 ret = [] 167 ret = []
168 for session_id in self._sessions.iterkeys(): 168 for session_id in self._sessions.keys():
169 try: 169 try:
170 timer, session_data, profile_set = self._sessions[session_id] 170 timer, session_data, profile_set = self._sessions[session_id]
171 except ValueError: 171 except ValueError:
172 continue 172 continue
173 if profile == profile_set: 173 if profile == profile_set:
243 old_default = "~/.sat" 243 old_default = "~/.sat"
244 if os.path.isfile(os.path.expanduser(old_default) + "/" + C.SAVEFILE_DATABASE): 244 if os.path.isfile(os.path.expanduser(old_default) + "/" + C.SAVEFILE_DATABASE):
245 if not silent: 245 if not silent:
246 log.warning( 246 log.warning(
247 _( 247 _(
248 u"A database has been found in the default local_dir for previous versions (< 0.5)" 248 "A database has been found in the default local_dir for previous versions (< 0.5)"
249 ) 249 )
250 ) 250 )
251 tools_config.fixConfigOption("", "local_dir", old_default, silent) 251 tools_config.fixConfigOption("", "local_dir", old_default, silent)
252 252
253 253
304 return False 304 return False
305 filename = os.path.expanduser(filename) 305 filename = os.path.expanduser(filename)
306 if os.path.exists(filename): 306 if os.path.exists(filename):
307 try: 307 try:
308 self.params.load_xml(filename) 308 self.params.load_xml(filename)
309 log.debug(_(u"Parameters loaded from file: %s") % filename) 309 log.debug(_("Parameters loaded from file: %s") % filename)
310 return True 310 return True
311 except Exception as e: 311 except Exception as e:
312 log.error(_(u"Can't load parameters from file: %s") % e) 312 log.error(_("Can't load parameters from file: %s") % e)
313 return False 313 return False
314 314
315 def save_xml(self, filename): 315 def save_xml(self, filename):
316 """Save parameters template to xml file 316 """Save parameters template to xml file
317 317
322 return False 322 return False
323 # TODO: need to encrypt files (at least passwords !) and set permissions 323 # TODO: need to encrypt files (at least passwords !) and set permissions
324 filename = os.path.expanduser(filename) 324 filename = os.path.expanduser(filename)
325 try: 325 try:
326 self.params.save_xml(filename) 326 self.params.save_xml(filename)
327 log.debug(_(u"Parameters saved to file: %s") % filename) 327 log.debug(_("Parameters saved to file: %s") % filename)
328 return True 328 return True
329 except Exception as e: 329 except Exception as e:
330 log.error(_(u"Can't save parameters to file: %s") % e) 330 log.error(_("Can't save parameters to file: %s") % e)
331 return False 331 return False
332 332
333 def load(self): 333 def load(self):
334 """Load parameters and all memory things from db""" 334 """Load parameters and all memory things from db"""
335 # parameters data 335 # parameters data
354 profile = self.getProfileName(profile) 354 profile = self.getProfileName(profile)
355 355
356 def createSession(__): 356 def createSession(__):
357 """Called once params are loaded.""" 357 """Called once params are loaded."""
358 self._entities_cache[profile] = {} 358 self._entities_cache[profile] = {}
359 log.info(u"[{}] Profile session started".format(profile)) 359 log.info("[{}] Profile session started".format(profile))
360 return False 360 return False
361 361
362 def backendInitialised(__): 362 def backendInitialised(__):
363 def doStartSession(__=None): 363 def doStartSession(__=None):
364 if self.isSessionStarted(profile): 364 if self.isSessionStarted(profile):
390 """Delete a profile session 390 """Delete a profile session
391 391
392 @param profile: %(doc_profile)s 392 @param profile: %(doc_profile)s
393 """ 393 """
394 if self.host.isConnected(profile): 394 if self.host.isConnected(profile):
395 log.debug(u"Disconnecting profile because of session stop") 395 log.debug("Disconnecting profile because of session stop")
396 self.host.disconnect(profile) 396 self.host.disconnect(profile)
397 self.auth_sessions.profileDelUnique(profile) 397 self.auth_sessions.profileDelUnique(profile)
398 try: 398 try:
399 self._entities_cache[profile] 399 self._entities_cache[profile]
400 except KeyError: 400 except KeyError:
401 log.warning(u"Profile was not in cache") 401 log.warning("Profile was not in cache")
402 402
403 def _isSessionStarted(self, profile_key): 403 def _isSessionStarted(self, profile_key):
404 return self.isSessionStarted(self.getProfileName(profile_key)) 404 return self.isSessionStarted(self.getProfileName(profile_key))
405 405
406 def isSessionStarted(self, profile): 406 def isSessionStarted(self, profile):
426 # submitting a form with empty passwords is restricted to local frontends. 426 # submitting a form with empty passwords is restricted to local frontends.
427 return defer.succeed(None) 427 return defer.succeed(None)
428 428
429 def check_result(result): 429 def check_result(result):
430 if not result: 430 if not result:
431 log.warning(u"Authentication failure of profile {}".format(profile)) 431 log.warning("Authentication failure of profile {}".format(profile))
432 raise failure.Failure( 432 raise failure.Failure(
433 exceptions.PasswordError( 433 exceptions.PasswordError(
434 u"The provided profile password doesn't match." 434 "The provided profile password doesn't match."
435 ) 435 )
436 ) 436 )
437 if ( 437 if (
438 not session_data 438 not session_data
439 ): # avoid to create two profile sessions when password if specified 439 ): # avoid to create two profile sessions when password if specified
458 def gotPersonalKey(personal_key): 458 def gotPersonalKey(personal_key):
459 """Create the session for this profile and store the personal key""" 459 """Create the session for this profile and store the personal key"""
460 self.auth_sessions.newSession( 460 self.auth_sessions.newSession(
461 {C.MEMORY_CRYPTO_KEY: personal_key}, profile=profile 461 {C.MEMORY_CRYPTO_KEY: personal_key}, profile=profile
462 ) 462 )
463 log.debug(u"auth session created for profile %s" % profile) 463 log.debug("auth session created for profile %s" % profile)
464 464
465 d = PersistentDict(C.MEMORY_CRYPTO_NAMESPACE, profile).load() 465 d = PersistentDict(C.MEMORY_CRYPTO_NAMESPACE, profile).load()
466 d.addCallback(lambda data: BlockCipher.decrypt(key, data[C.MEMORY_CRYPTO_KEY])) 466 d.addCallback(lambda data: BlockCipher.decrypt(key, data[C.MEMORY_CRYPTO_KEY]))
467 return d.addCallback(gotPersonalKey) 467 return d.addCallback(gotPersonalKey)
468 468
474 try: 474 try:
475 del self._entities_cache[profile] 475 del self._entities_cache[profile]
476 except KeyError: 476 except KeyError:
477 log.error( 477 log.error(
478 _( 478 _(
479 u"Trying to purge roster status cache for a profile not in memory: [%s]" 479 "Trying to purge roster status cache for a profile not in memory: [%s]"
480 ) 480 )
481 % profile 481 % profile
482 ) 482 )
483 483
484 def getProfilesList(self, clients=True, components=False): 484 def getProfilesList(self, clients=True, components=False):
487 @param clients(bool): if True return clients profiles 487 @param clients(bool): if True return clients profiles
488 @param components(bool): if True return components profiles 488 @param components(bool): if True return components profiles
489 @return (list[unicode]): selected profiles 489 @return (list[unicode]): selected profiles
490 """ 490 """
491 if not clients and not components: 491 if not clients and not components:
492 log.warning(_(u"requesting no profiles at all")) 492 log.warning(_("requesting no profiles at all"))
493 return [] 493 return []
494 profiles = self.storage.getProfilesList() 494 profiles = self.storage.getProfilesList()
495 if clients and components: 495 if clients and components:
496 return sorted(profiles) 496 return sorted(profiles)
497 isComponent = self.storage.profileIsComponent 497 isComponent = self.storage.profileIsComponent
531 @param component(None, unicode): set to entry point if this is a component 531 @param component(None, unicode): set to entry point if this is a component
532 @return: Deferred 532 @return: Deferred
533 @raise exceptions.NotFound: component is not a known plugin import name 533 @raise exceptions.NotFound: component is not a known plugin import name
534 """ 534 """
535 if not name: 535 if not name:
536 raise ValueError(u"Empty profile name") 536 raise ValueError("Empty profile name")
537 if name[0] == "@": 537 if name[0] == "@":
538 raise ValueError(u"A profile name can't start with a '@'") 538 raise ValueError("A profile name can't start with a '@'")
539 if "\n" in name: 539 if "\n" in name:
540 raise ValueError(u"A profile name can't contain line feed ('\\n')") 540 raise ValueError("A profile name can't contain line feed ('\\n')")
541 541
542 if name in self._entities_cache: 542 if name in self._entities_cache:
543 raise exceptions.ConflictError(u"A session for this profile exists") 543 raise exceptions.ConflictError("A session for this profile exists")
544 544
545 if component: 545 if component:
546 if not component in self.host.plugins: 546 if not component in self.host.plugins:
547 raise exceptions.NotFound( 547 raise exceptions.NotFound(
548 _( 548 _(
549 u"Can't find component {component} entry point".format( 549 "Can't find component {component} entry point".format(
550 component=component 550 component=component
551 ) 551 )
552 ) 552 )
553 ) 553 )
554 # FIXME: PLUGIN_INFO is not currently accessible after import, but type shoul be tested here 554 # FIXME: PLUGIN_INFO is not currently accessible after import, but type shoul be tested here
662 662
663 ## Statuses ## 663 ## Statuses ##
664 664
665 def _getPresenceStatuses(self, profile_key): 665 def _getPresenceStatuses(self, profile_key):
666 ret = self.getPresenceStatuses(profile_key) 666 ret = self.getPresenceStatuses(profile_key)
667 return {entity.full(): data for entity, data in ret.iteritems()} 667 return {entity.full(): data for entity, data in ret.items()}
668 668
669 def getPresenceStatuses(self, profile_key): 669 def getPresenceStatuses(self, profile_key):
670 """Get all the presence statuses of a profile 670 """Get all the presence statuses of a profile
671 671
672 @param profile_key: %(doc_profile_key)s 672 @param profile_key: %(doc_profile_key)s
674 """ 674 """
675 client = self.host.getClient(profile_key) 675 client = self.host.getClient(profile_key)
676 profile_cache = self._getProfileCache(client) 676 profile_cache = self._getProfileCache(client)
677 entities_presence = {} 677 entities_presence = {}
678 678
679 for entity_jid, entity_data in profile_cache.iteritems(): 679 for entity_jid, entity_data in profile_cache.items():
680 for resource, resource_data in entity_data.iteritems(): 680 for resource, resource_data in entity_data.items():
681 full_jid = copy.copy(entity_jid) 681 full_jid = copy.copy(entity_jid)
682 full_jid.resource = resource 682 full_jid.resource = resource
683 try: 683 try:
684 presence_data = self.getEntityDatum(full_jid, "presence", profile_key) 684 presence_data = self.getEntityDatum(full_jid, "presence", profile_key)
685 except KeyError: 685 except KeyError:
734 profile_cache = self._getProfileCache(client) 734 profile_cache = self._getProfileCache(client)
735 try: 735 try:
736 entity_data = profile_cache[entity_jid.userhostJID()] 736 entity_data = profile_cache[entity_jid.userhostJID()]
737 except KeyError: 737 except KeyError:
738 raise exceptions.UnknownEntityError( 738 raise exceptions.UnknownEntityError(
739 u"Entity {} not in cache".format(entity_jid) 739 "Entity {} not in cache".format(entity_jid)
740 ) 740 )
741 resources = set(entity_data.keys()) 741 resources = set(entity_data.keys())
742 resources.discard(None) 742 resources.discard(None)
743 return resources 743 return resources
744 744
756 full_jid = copy.copy(entity_jid) 756 full_jid = copy.copy(entity_jid)
757 full_jid.resource = resource 757 full_jid.resource = resource
758 try: 758 try:
759 presence_data = self.getEntityDatum(full_jid, "presence", client.profile) 759 presence_data = self.getEntityDatum(full_jid, "presence", client.profile)
760 except KeyError: 760 except KeyError:
761 log.debug(u"Can't get presence data for {}".format(full_jid)) 761 log.debug("Can't get presence data for {}".format(full_jid))
762 else: 762 else:
763 if presence_data.show != C.PRESENCE_UNAVAILABLE: 763 if presence_data.show != C.PRESENCE_UNAVAILABLE:
764 available.append(resource) 764 available.append(resource)
765 return available 765 return available
766 766
785 except KeyError: # plugin not found 785 except KeyError: # plugin not found
786 pass 786 pass
787 try: 787 try:
788 resources = self.getAllResources(client, entity_jid) 788 resources = self.getAllResources(client, entity_jid)
789 except exceptions.UnknownEntityError: 789 except exceptions.UnknownEntityError:
790 log.warning(u"Entity is not in cache, we can't find any resource") 790 log.warning("Entity is not in cache, we can't find any resource")
791 return None 791 return None
792 priority_resources = [] 792 priority_resources = []
793 for resource in resources: 793 for resource in resources:
794 full_jid = copy.copy(entity_jid) 794 full_jid = copy.copy(entity_jid)
795 full_jid.resource = resource 795 full_jid.resource = resource
796 try: 796 try:
797 presence_data = self.getEntityDatum(full_jid, "presence", client.profile) 797 presence_data = self.getEntityDatum(full_jid, "presence", client.profile)
798 except KeyError: 798 except KeyError:
799 log.debug(u"No presence information for {}".format(full_jid)) 799 log.debug("No presence information for {}".format(full_jid))
800 continue 800 continue
801 priority_resources.append((resource, presence_data.priority)) 801 priority_resources.append((resource, presence_data.priority))
802 try: 802 try:
803 return max(priority_resources, key=lambda res_tuple: res_tuple[1])[0] 803 return max(priority_resources, key=lambda res_tuple: res_tuple[1])[0]
804 except ValueError: 804 except ValueError:
805 log.warning(u"No resource found at all for {}".format(entity_jid)) 805 log.warning("No resource found at all for {}".format(entity_jid))
806 return None 806 return None
807 807
808 ## Entities data ## 808 ## Entities data ##
809 809
810 def _getProfileCache(self, client): 810 def _getProfileCache(self, client):
833 @param with_bare: if True, include bare jids 833 @param with_bare: if True, include bare jids
834 @return (list[unicode]): list of jids 834 @return (list[unicode]): list of jids
835 """ 835 """
836 profile_cache = self._getProfileCache(client) 836 profile_cache = self._getProfileCache(client)
837 # we construct a list of all known full jids (bare jid of entities x resources) 837 # we construct a list of all known full jids (bare jid of entities x resources)
838 for bare_jid, entity_data in profile_cache.iteritems(): 838 for bare_jid, entity_data in profile_cache.items():
839 for resource in entity_data.iterkeys(): 839 for resource in entity_data.keys():
840 if resource is None: 840 if resource is None:
841 continue 841 continue
842 full_jid = copy.copy(bare_jid) 842 full_jid = copy.copy(bare_jid)
843 full_jid.resource = resource 843 full_jid.resource = resource
844 yield full_jid 844 yield full_jid
869 jid_.resource, {} 869 jid_.resource, {}
870 ) 870 )
871 871
872 entity_data[key] = value 872 entity_data[key] = value
873 if key in self._key_signals and not silent: 873 if key in self._key_signals and not silent:
874 if not isinstance(value, basestring): 874 if not isinstance(value, str):
875 log.error( 875 log.error(
876 u"Setting a non string value ({}) for a key ({}) which has a signal flag".format( 876 "Setting a non string value ({}) for a key ({}) which has a signal flag".format(
877 value, key 877 value, key
878 ) 878 )
879 ) 879 )
880 else: 880 else:
881 self.host.bridge.entityDataUpdated( 881 self.host.bridge.entityDataUpdated(
903 for jid_ in entities: 903 for jid_ in entities:
904 try: 904 try:
905 entity_data = profile_cache[jid_.userhostJID()][jid_.resource] 905 entity_data = profile_cache[jid_.userhostJID()][jid_.resource]
906 except KeyError: 906 except KeyError:
907 raise exceptions.UnknownEntityError( 907 raise exceptions.UnknownEntityError(
908 u"Entity {} not in cache".format(jid_) 908 "Entity {} not in cache".format(jid_)
909 ) 909 )
910 try: 910 try:
911 del entity_data[key] 911 del entity_data[key]
912 except KeyError as e: 912 except KeyError as e:
913 if entity_jid in (C.ENTITY_ALL_RESOURCES, C.ENTITY_ALL): 913 if entity_jid in (C.ENTITY_ALL_RESOURCES, C.ENTITY_ALL):
917 917
918 def _getEntitiesData(self, entities_jids, keys_list, profile_key): 918 def _getEntitiesData(self, entities_jids, keys_list, profile_key):
919 ret = self.getEntitiesData( 919 ret = self.getEntitiesData(
920 [jid.JID(jid_) for jid_ in entities_jids], keys_list, profile_key 920 [jid.JID(jid_) for jid_ in entities_jids], keys_list, profile_key
921 ) 921 )
922 return {jid_.full(): data for jid_, data in ret.iteritems()} 922 return {jid_.full(): data for jid_, data in ret.items()}
923 923
924 def getEntitiesData(self, entities_jids, keys_list=None, profile_key=C.PROF_KEY_NONE): 924 def getEntitiesData(self, entities_jids, keys_list=None, profile_key=C.PROF_KEY_NONE):
925 """Get a list of cached values for several entities at once 925 """Get a list of cached values for several entities at once
926 926
927 @param entities_jids: jids of the entities, or empty list for all entities in cache 927 @param entities_jids: jids of the entities, or empty list for all entities in cache
959 ] 959 ]
960 except KeyError: 960 except KeyError:
961 continue 961 continue
962 ret_data[entity.full()] = fillEntityData(entity_cache_data, keys_list) 962 ret_data[entity.full()] = fillEntityData(entity_cache_data, keys_list)
963 else: 963 else:
964 for bare_jid, data in profile_cache.iteritems(): 964 for bare_jid, data in profile_cache.items():
965 for resource, entity_cache_data in data.iteritems(): 965 for resource, entity_cache_data in data.items():
966 full_jid = copy.copy(bare_jid) 966 full_jid = copy.copy(bare_jid)
967 full_jid.resource = resource 967 full_jid.resource = resource
968 ret_data[full_jid] = fillEntityData(entity_cache_data) 968 ret_data[full_jid] = fillEntityData(entity_cache_data)
969 969
970 return ret_data 970 return ret_data
985 profile_cache = self._getProfileCache(client) 985 profile_cache = self._getProfileCache(client)
986 try: 986 try:
987 entity_data = profile_cache[entity_jid.userhostJID()][entity_jid.resource] 987 entity_data = profile_cache[entity_jid.userhostJID()][entity_jid.resource]
988 except KeyError: 988 except KeyError:
989 raise exceptions.UnknownEntityError( 989 raise exceptions.UnknownEntityError(
990 u"Entity {} not in cache (was requesting {})".format( 990 "Entity {} not in cache (was requesting {})".format(
991 entity_jid, keys_list 991 entity_jid, keys_list
992 ) 992 )
993 ) 993 )
994 if keys_list is None: 994 if keys_list is None:
995 return entity_data 995 return entity_data
1028 raise ValueError(_("Need a bare jid to delete all resources")) 1028 raise ValueError(_("Need a bare jid to delete all resources"))
1029 try: 1029 try:
1030 del profile_cache[entity_jid] 1030 del profile_cache[entity_jid]
1031 except KeyError: 1031 except KeyError:
1032 raise exceptions.UnknownEntityError( 1032 raise exceptions.UnknownEntityError(
1033 u"Entity {} not in cache".format(entity_jid) 1033 "Entity {} not in cache".format(entity_jid)
1034 ) 1034 )
1035 else: 1035 else:
1036 try: 1036 try:
1037 del profile_cache[entity_jid.userhostJID()][entity_jid.resource] 1037 del profile_cache[entity_jid.userhostJID()][entity_jid.resource]
1038 except KeyError: 1038 except KeyError:
1039 raise exceptions.UnknownEntityError( 1039 raise exceptions.UnknownEntityError(
1040 u"Entity {} not in cache".format(entity_jid) 1040 "Entity {} not in cache".format(entity_jid)
1041 ) 1041 )
1042 1042
1043 ## Encryption ## 1043 ## Encryption ##
1044 1044
1045 def encryptValue(self, value, profile): 1045 def encryptValue(self, value, profile):
1101 1101
1102 return d.addCallback(cb) 1102 return d.addCallback(cb)
1103 1103
1104 def done(__): 1104 def done(__):
1105 log.debug( 1105 log.debug(
1106 _(u"Personal data (%(ns)s, %(key)s) has been successfuly encrypted") 1106 _("Personal data (%(ns)s, %(key)s) has been successfuly encrypted")
1107 % {"ns": C.MEMORY_CRYPTO_NAMESPACE, "key": data_key} 1107 % {"ns": C.MEMORY_CRYPTO_NAMESPACE, "key": data_key}
1108 ) 1108 )
1109 1109
1110 d = PersistentDict(C.MEMORY_CRYPTO_NAMESPACE, profile).load() 1110 d = PersistentDict(C.MEMORY_CRYPTO_NAMESPACE, profile).load()
1111 return d.addCallback(gotIndMemory).addCallback(done) 1111 return d.addCallback(gotIndMemory).addCallback(done)
1223 peer_jid = peer_jid.userhostJID() 1223 peer_jid = peer_jid.userhostJID()
1224 if peer_jid == file_data["owner"]: 1224 if peer_jid == file_data["owner"]:
1225 # the owner has all rights 1225 # the owner has all rights
1226 return 1226 return
1227 if not C.ACCESS_PERMS.issuperset(perms_to_check): 1227 if not C.ACCESS_PERMS.issuperset(perms_to_check):
1228 raise exceptions.InternalError(_(u"invalid permission")) 1228 raise exceptions.InternalError(_("invalid permission"))
1229 1229
1230 for perm in perms_to_check: 1230 for perm in perms_to_check:
1231 # we check each perm and raise PermissionError as soon as one condition is not valid 1231 # we check each perm and raise PermissionError as soon as one condition is not valid
1232 # we must never return here, we only return after the loop if nothing was blocking the access 1232 # we must never return here, we only return after the loop if nothing was blocking the access
1233 try: 1233 try:
1234 perm_data = file_data[u"access"][perm] 1234 perm_data = file_data["access"][perm]
1235 perm_type = perm_data[u"type"] 1235 perm_type = perm_data["type"]
1236 except KeyError: 1236 except KeyError:
1237 raise exceptions.PermissionError() 1237 raise exceptions.PermissionError()
1238 if perm_type == C.ACCESS_TYPE_PUBLIC: 1238 if perm_type == C.ACCESS_TYPE_PUBLIC:
1239 continue 1239 continue
1240 elif perm_type == C.ACCESS_TYPE_WHITELIST: 1240 elif perm_type == C.ACCESS_TYPE_WHITELIST:
1241 try: 1241 try:
1242 jids = perm_data[u"jids"] 1242 jids = perm_data["jids"]
1243 except KeyError: 1243 except KeyError:
1244 raise exceptions.PermissionError() 1244 raise exceptions.PermissionError()
1245 if peer_jid.full() in jids: 1245 if peer_jid.full() in jids:
1246 continue 1246 continue
1247 else: 1247 else:
1248 raise exceptions.PermissionError() 1248 raise exceptions.PermissionError()
1249 else: 1249 else:
1250 raise exceptions.InternalError( 1250 raise exceptions.InternalError(
1251 _(u"unknown access type: {type}").format(type=perm_type) 1251 _("unknown access type: {type}").format(type=perm_type)
1252 ) 1252 )
1253 1253
1254 @defer.inlineCallbacks 1254 @defer.inlineCallbacks
1255 def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check): 1255 def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
1256 """do checkFilePermission on file_data and all its parents until root""" 1256 """do checkFilePermission on file_data and all its parents until root"""
1257 current = file_data 1257 current = file_data
1258 while True: 1258 while True:
1259 self.checkFilePermission(current, peer_jid, perms_to_check) 1259 self.checkFilePermission(current, peer_jid, perms_to_check)
1260 parent = current[u"parent"] 1260 parent = current["parent"]
1261 if not parent: 1261 if not parent:
1262 break 1262 break
1263 files_data = yield self.getFile( 1263 files_data = yield self.getFile(
1264 self, client, peer_jid=None, file_id=parent, perms_to_check=None 1264 self, client, peer_jid=None, file_id=parent, perms_to_check=None
1265 ) 1265 )
1266 try: 1266 try:
1267 current = files_data[0] 1267 current = files_data[0]
1268 except IndexError: 1268 except IndexError:
1269 raise exceptions.DataError(u"Missing parent") 1269 raise exceptions.DataError("Missing parent")
1270 1270
1271 @defer.inlineCallbacks 1271 @defer.inlineCallbacks
1272 def _getParentDir( 1272 def _getParentDir(
1273 self, client, path, parent, namespace, owner, peer_jid, perms_to_check 1273 self, client, path, parent, namespace, owner, peer_jid, perms_to_check
1274 ): 1274 ):
1281 (i.e. which don't exist) 1281 (i.e. which don't exist)
1282 """ 1282 """
1283 # if path is set, we have to retrieve parent directory of the file(s) from it 1283 # if path is set, we have to retrieve parent directory of the file(s) from it
1284 if parent is not None: 1284 if parent is not None:
1285 raise exceptions.ConflictError( 1285 raise exceptions.ConflictError(
1286 _(u"You can't use path and parent at the same time") 1286 _("You can't use path and parent at the same time")
1287 ) 1287 )
1288 path_elts = filter(None, path.split(u"/")) 1288 path_elts = [_f for _f in path.split("/") if _f]
1289 if {u"..", u"."}.intersection(path_elts): 1289 if {"..", "."}.intersection(path_elts):
1290 raise ValueError(_(u'".." or "." can\'t be used in path')) 1290 raise ValueError(_('".." or "." can\'t be used in path'))
1291 1291
1292 # we retrieve all directories from path until we get the parent container 1292 # we retrieve all directories from path until we get the parent container
1293 # non existing directories will be created 1293 # non existing directories will be created
1294 parent = u"" 1294 parent = ""
1295 for idx, path_elt in enumerate(path_elts): 1295 for idx, path_elt in enumerate(path_elts):
1296 directories = yield self.storage.getFiles( 1296 directories = yield self.storage.getFiles(
1297 client, 1297 client,
1298 parent=parent, 1298 parent=parent,
1299 type_=C.FILE_TYPE_DIRECTORY, 1299 type_=C.FILE_TYPE_DIRECTORY,
1304 if not directories: 1304 if not directories:
1305 defer.returnValue((parent, path_elts[idx:])) 1305 defer.returnValue((parent, path_elts[idx:]))
1306 # from this point, directories don't exist anymore, we have to create them 1306 # from this point, directories don't exist anymore, we have to create them
1307 elif len(directories) > 1: 1307 elif len(directories) > 1:
1308 raise exceptions.InternalError( 1308 raise exceptions.InternalError(
1309 _(u"Several directories found, this should not happen") 1309 _("Several directories found, this should not happen")
1310 ) 1310 )
1311 else: 1311 else:
1312 directory = directories[0] 1312 directory = directories[0]
1313 self.checkFilePermission(directory, peer_jid, perms_to_check) 1313 self.checkFilePermission(directory, peer_jid, perms_to_check)
1314 parent = directory[u"id"] 1314 parent = directory["id"]
1315 defer.returnValue((parent, [])) 1315 defer.returnValue((parent, []))
1316 1316
1317 @defer.inlineCallbacks 1317 @defer.inlineCallbacks
1318 def getFiles( 1318 def getFiles(
1319 self, client, peer_jid, file_id=None, version=None, parent=None, path=None, 1319 self, client, peer_jid, file_id=None, version=None, parent=None, path=None,
1355 the file 1355 the file
1356 on the path 1356 on the path
1357 """ 1357 """
1358 if peer_jid is None and perms_to_check or perms_to_check is None and peer_jid: 1358 if peer_jid is None and perms_to_check or perms_to_check is None and peer_jid:
1359 raise exceptions.InternalError( 1359 raise exceptions.InternalError(
1360 u"if you want to disable permission check, both peer_jid and " 1360 "if you want to disable permission check, both peer_jid and "
1361 u"perms_to_check must be None" 1361 "perms_to_check must be None"
1362 ) 1362 )
1363 if owner is not None: 1363 if owner is not None:
1364 owner = owner.userhostJID() 1364 owner = owner.userhostJID()
1365 if path is not None: 1365 if path is not None:
1366 # permission are checked by _getParentDir 1366 # permission are checked by _getParentDir
1376 # we need to check all the parents 1376 # we need to check all the parents
1377 parent_data = yield self.storage.getFiles(client, file_id=parent) 1377 parent_data = yield self.storage.getFiles(client, file_id=parent)
1378 try: 1378 try:
1379 parent_data = parent_data[0] 1379 parent_data = parent_data[0]
1380 except IndexError: 1380 except IndexError:
1381 raise exceptions.DataError(u"mising parent") 1381 raise exceptions.DataError("mising parent")
1382 yield self.checkPermissionToRoot( 1382 yield self.checkPermissionToRoot(
1383 client, parent_data, peer_jid, perms_to_check 1383 client, parent_data, peer_jid, perms_to_check
1384 ) 1384 )
1385 1385
1386 files = yield self.storage.getFiles( 1386 files = yield self.storage.getFiles(
1412 files.remove(file_data) 1412 files.remove(file_data)
1413 defer.returnValue(files) 1413 defer.returnValue(files)
1414 1414
1415 @defer.inlineCallbacks 1415 @defer.inlineCallbacks
1416 def setFile( 1416 def setFile(
1417 self, client, name, file_id=None, version=u"", parent=None, path=None, 1417 self, client, name, file_id=None, version="", parent=None, path=None,
1418 type_=C.FILE_TYPE_FILE, file_hash=None, hash_algo=None, size=None, 1418 type_=C.FILE_TYPE_FILE, file_hash=None, hash_algo=None, size=None,
1419 namespace=None, mime_type=None, created=None, modified=None, owner=None, 1419 namespace=None, mime_type=None, created=None, modified=None, owner=None,
1420 access=None, extra=None, peer_jid=None, perms_to_check=(C.ACCESS_PERM_WRITE,) 1420 access=None, extra=None, peer_jid=None, perms_to_check=(C.ACCESS_PERM_WRITE,)
1421 ): 1421 ):
1422 """Set a file metadata 1422 """Set a file metadata
1479 if namespace is not None: 1479 if namespace is not None:
1480 namespace = namespace.strip() or None 1480 namespace = namespace.strip() or None
1481 if type_ == C.FILE_TYPE_DIRECTORY: 1481 if type_ == C.FILE_TYPE_DIRECTORY:
1482 if any(version, file_hash, size, mime_type): 1482 if any(version, file_hash, size, mime_type):
1483 raise ValueError( 1483 raise ValueError(
1484 u"version, file_hash, size and mime_type can't be set for a directory" 1484 "version, file_hash, size and mime_type can't be set for a directory"
1485 ) 1485 )
1486 if owner is not None: 1486 if owner is not None:
1487 owner = owner.userhostJID() 1487 owner = owner.userhostJID()
1488 1488
1489 if path is not None: 1489 if path is not None:
1496 new_dir_id = shortuuid.uuid() 1496 new_dir_id = shortuuid.uuid()
1497 yield self.storage.setFile( 1497 yield self.storage.setFile(
1498 client, 1498 client,
1499 name=new_dir, 1499 name=new_dir,
1500 file_id=new_dir_id, 1500 file_id=new_dir_id,
1501 version=u"", 1501 version="",
1502 parent=parent, 1502 parent=parent,
1503 type_=C.FILE_TYPE_DIRECTORY, 1503 type_=C.FILE_TYPE_DIRECTORY,
1504 namespace=namespace, 1504 namespace=namespace,
1505 created=time.time(), 1505 created=time.time(),
1506 owner=owner, 1506 owner=owner,
1507 access=access, 1507 access=access,
1508 extra={}, 1508 extra={},
1509 ) 1509 )
1510 parent = new_dir_id 1510 parent = new_dir_id
1511 elif parent is None: 1511 elif parent is None:
1512 parent = u"" 1512 parent = ""
1513 1513
1514 yield self.storage.setFile( 1514 yield self.storage.setFile(
1515 client, 1515 client,
1516 file_id=file_id, 1516 file_id=file_id,
1517 version=version, 1517 version=version,
1550 to delete) 1550 to delete)
1551 @param recursive(boolean): True if recursive deletion is needed 1551 @param recursive(boolean): True if recursive deletion is needed
1552 @param files_path(unicode): path of the directory containing the actual files 1552 @param files_path(unicode): path of the directory containing the actual files
1553 @param file_data(dict): data of the file to delete 1553 @param file_data(dict): data of the file to delete
1554 """ 1554 """
1555 if file_data[u'owner'] != peer_jid: 1555 if file_data['owner'] != peer_jid:
1556 raise exceptions.PermissionError( 1556 raise exceptions.PermissionError(
1557 u"file {file_name} can't be deleted, {peer_jid} is not the owner" 1557 "file {file_name} can't be deleted, {peer_jid} is not the owner"
1558 .format(file_name=file_data[u'name'], peer_jid=peer_jid.full())) 1558 .format(file_name=file_data['name'], peer_jid=peer_jid.full()))
1559 if file_data[u'type'] == C.FILE_TYPE_DIRECTORY: 1559 if file_data['type'] == C.FILE_TYPE_DIRECTORY:
1560 sub_files = yield self.getFiles(client, peer_jid, parent=file_data[u'id']) 1560 sub_files = yield self.getFiles(client, peer_jid, parent=file_data['id'])
1561 if sub_files and not recursive: 1561 if sub_files and not recursive:
1562 raise exceptions.DataError(_(u"Can't delete directory, it is not empty")) 1562 raise exceptions.DataError(_("Can't delete directory, it is not empty"))
1563 # we first delete the sub-files 1563 # we first delete the sub-files
1564 for sub_file_data in sub_files: 1564 for sub_file_data in sub_files:
1565 yield self._deleteFile(client, peer_jid, recursive, sub_file_data) 1565 yield self._deleteFile(client, peer_jid, recursive, sub_file_data)
1566 # then the directory itself 1566 # then the directory itself
1567 yield self.storage.fileDelete(file_data[u'id']) 1567 yield self.storage.fileDelete(file_data['id'])
1568 elif file_data[u'type'] == C.FILE_TYPE_FILE: 1568 elif file_data['type'] == C.FILE_TYPE_FILE:
1569 log.info(_(u"deleting file {name} with hash {file_hash}").format( 1569 log.info(_("deleting file {name} with hash {file_hash}").format(
1570 name=file_data[u'name'], file_hash=file_data[u'file_hash'])) 1570 name=file_data['name'], file_hash=file_data['file_hash']))
1571 yield self.storage.fileDelete(file_data[u'id']) 1571 yield self.storage.fileDelete(file_data['id'])
1572 references = yield self.getFiles( 1572 references = yield self.getFiles(
1573 client, peer_jid, file_hash=file_data[u'file_hash']) 1573 client, peer_jid, file_hash=file_data['file_hash'])
1574 if references: 1574 if references:
1575 log.debug(u"there are still references to the file, we keep it") 1575 log.debug("there are still references to the file, we keep it")
1576 else: 1576 else:
1577 file_path = os.path.join(files_path, file_data[u'file_hash']) 1577 file_path = os.path.join(files_path, file_data['file_hash'])
1578 log.info(_(u"no reference left to {file_path}, deleting").format( 1578 log.info(_("no reference left to {file_path}, deleting").format(
1579 file_path=file_path)) 1579 file_path=file_path))
1580 os.unlink(file_path) 1580 os.unlink(file_path)
1581 else: 1581 else:
1582 raise exceptions.InternalError(u'Unexpected file type: {file_type}' 1582 raise exceptions.InternalError('Unexpected file type: {file_type}'
1583 .format(file_type=file_data[u'type'])) 1583 .format(file_type=file_data['type']))
1584 1584
1585 @defer.inlineCallbacks 1585 @defer.inlineCallbacks
1586 def fileDelete(self, client, peer_jid, file_id, recursive=False): 1586 def fileDelete(self, client, peer_jid, file_id, recursive=False):
1587 """Delete a single file or a directory and all its sub-files 1587 """Delete a single file or a directory and all its sub-files
1588 1588
1593 """ 1593 """
1594 # FIXME: we only allow owner of file to delete files for now, but WRITE access 1594 # FIXME: we only allow owner of file to delete files for now, but WRITE access
1595 # should be checked too 1595 # should be checked too
1596 files_data = yield self.getFiles(client, peer_jid, file_id) 1596 files_data = yield self.getFiles(client, peer_jid, file_id)
1597 if not files_data: 1597 if not files_data:
1598 raise exceptions.NotFound(u"Can't find the file with id {file_id}".format( 1598 raise exceptions.NotFound("Can't find the file with id {file_id}".format(
1599 file_id=file_id)) 1599 file_id=file_id))
1600 file_data = files_data[0] 1600 file_data = files_data[0]
1601 if file_data[u"type"] != C.FILE_TYPE_DIRECTORY and recursive: 1601 if file_data["type"] != C.FILE_TYPE_DIRECTORY and recursive:
1602 raise ValueError(u"recursive can only be set for directories") 1602 raise ValueError("recursive can only be set for directories")
1603 files_path = self.host.getLocalPath(None, C.FILES_DIR, profile=False) 1603 files_path = self.host.getLocalPath(None, C.FILES_DIR, profile=False)
1604 yield self._deleteFile(client, peer_jid, recursive, files_path, file_data) 1604 yield self._deleteFile(client, peer_jid, recursive, files_path, file_data)
1605 1605
1606 ## Misc ## 1606 ## Misc ##
1607 1607
1616 self.getAvailableResources(client, entity_jid) 1616 self.getAvailableResources(client, entity_jid)
1617 ) # is any resource is available, entity is available 1617 ) # is any resource is available, entity is available
1618 try: 1618 try:
1619 presence_data = self.getEntityDatum(entity_jid, "presence", client.profile) 1619 presence_data = self.getEntityDatum(entity_jid, "presence", client.profile)
1620 except KeyError: 1620 except KeyError:
1621 log.debug(u"No presence information for {}".format(entity_jid)) 1621 log.debug("No presence information for {}".format(entity_jid))
1622 return False 1622 return False
1623 return presence_data.show != C.PRESENCE_UNAVAILABLE 1623 return presence_data.show != C.PRESENCE_UNAVAILABLE