Mercurial > libervia-backend
changeset 432:31e8c48b5f5d
core: - memory refactoring (moved memory.py and sqlite.py from tools to memory)
- private values are now stored in storage (currently sqlite database) and splitted in normal/binary storage. Normal storage is for string key/values, binary storage is for string key and any python data for value
- PersistentDict and PersistentBinaryDict are sugar classes to manage the data in storage thought a python dictionary like class.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sun, 20 Nov 2011 15:34:37 +0100 |
| parents | 482b9bcf0ca4 |
| children | 75aa709c83d5 |
| files | src/memory/__init__.py src/memory/memory.py src/memory/persistent.py src/memory/sqlite.py |
| diffstat | 3 files changed, 1344 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/memory/memory.py Sun Nov 20 15:34:37 2011 +0100 @@ -0,0 +1,825 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +SAT: a jabber client +Copyright (C) 2009, 2010, 2011 Jérôme Poisson (goffi@goffi.org) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + +from __future__ import with_statement + +import os.path +import time +import cPickle as pickle +from ConfigParser import SafeConfigParser, NoOptionError, NoSectionError +from xml.dom import minidom +from logging import debug, info, warning, error +from twisted.internet import defer +from twisted.words.protocols.jabber import jid +from sat.tools.xml_tools import paramsXml2xmlUI +from sat.core.default_config import default_config +from sat.memory.sqlite import SqliteStorage +from sat.memory.persistent import PersistentDict + +SAVEFILE_PARAM_XML="/param" #xml parameters template +SAVEFILE_PRIVATE="/private" #file used to store misc values (mainly for plugins) +SAVEFILE_DATABASE="/sat.db" + +class ProfileNotInCacheError(Exception): + pass + +class ConnectedProfileError(Exception): + pass + +class Params(): + """This class manage parameters with xml""" + ### TODO: add desciption in params + + #TODO: move Watched in a plugin + default_xml = u""" + <params> + <general> + </general> + <individual> + <category name="Connection" label="%(category_connection)s"> + <param name="JabberID" value="name@example.org/SàT" type="string" /> + <param name="Password" value="" type="password" /> + <param name="Server" value="example.org" type="string" /> + <param name="NewAccount" value="%(label_NewAccount)s" type="button" callback_id="registerNewAccount"/> + <param name="autoconnect" label="%(label_autoconnect)s" value="true" type="bool" /> + <param name="autodisconnect" label="%(label_autodisconnect)s" value="false" type="bool" /> + </category> + <category name="Misc" label="%(category_misc)s"> + <param name="Watched" value="test@Jabber.goffi.int" type="string" /> + </category> + </individual> + </params> + """ % {'category_connection': _("Connection"), + 'label_NewAccount': _("Register new account"), + 'label_autoconnect': _('Connect on frontend startup'), + 'label_autodisconnect': _('Disconnect on frontend closure'), + 'category_misc': _("Misc") + } + + def load_default_params(self): + self.dom = minidom.parseString(Param.default_xml.encode('utf-8')) + + def load_xml(self, file): + """Load parameters template from file""" + self.dom = minidom.parse(file) + + def loadGenParams(self): + """Load general parameters data from storage + @return: deferred triggered once params are loaded""" + return self.storage.loadGenParams(self.params_gen) + + def loadIndParams(self, profile, cache=None): + """Load individual parameters + set self.params cache or a temporary cache + @param profile: profile to load (*must exist*) + @param cache: if not None, will be used to store the value, as a short time cache + @return: deferred triggered once params are loaded""" + if cache == None: + self.params[profile] = {} + return self.storage.loadIndParams(self.params[profile] if cache==None else cache, profile) + + def purgeProfile(self, profile): + """Remove cache data of a profile + @param profile: %(doc_profile)s""" + try: + del self.params[profile] + except KeyError: + error(_("Trying to purge cache of a profile not in memory: [%s]") % profile) + + def save_xml(self, file): + """Save parameters template to xml file""" + with open(file, 'wb') as xml_file: + xml_file.write(self.dom.toxml('utf-8')) + + def __init__(self, host, storage): + debug("Parameters init") + self.host = host + self.storage = storage + self.default_profile = None + self.params = {} + self.params_gen = {} + host.set_const('savefile_param_xml', SAVEFILE_PARAM_XML) + host.registerGeneralCB("registerNewAccount", host.registerNewAccountCB) + + def createProfile(self, profile): + """Create a new profile + @param profile: profile of the profile""" + #FIXME: must be asynchronous and call the callback once the profile actually exists + if self.storage.hasProfile(profile): + info (_('The profile profile already exists')) + return True + if not self.host.trigger.point("ProfileCreation", profile): + return False + self.storage.createProfile(profile) + return False + + def asyncCreateProfile(self, profile): + """Create a new profile + @param profile: name of the profile + @param callback: called when the profile actually exists in database and memory + @param errback: called with a string constant as parameter: + - CONFLICT: the profile already exists + - CANCELED: profile creation canceled + """ + if self.storage.hasProfile(profile): + info (_('The profile name already exists')) + return defer.fail("CONFLICT") + if not self.host.trigger.point("ProfileCreation", profile): + return defer.fail("CANCEL") + return self.storage.createProfile(profile) + + + def deleteProfile(self, profile): + """Delete an existing profile + @param profile: name of the profile""" + #TODO: async equivalent, like for createProfile + if not self.storage.hasProfile(profile): + error(_('Trying to delete an unknown profile')) + return True + if self.host.isConnected(profile): + error(_("Trying to delete a connected profile")) + raise ConnectedProfileError + self.storage.deleteProfile(profile) + return False + + def getProfileName(self, profile_key): + """return profile according to profile_key + @param profile_key: profile name or key which can be + @ALL@ for all profiles + @DEFAULT@ for default profile + @return: requested profile name or None if it doesn't exist""" + if profile_key=='@DEFAULT@': + if not self.params: + return "" + default = self.host.memory.getPrivate('Profile_default') + if not default or not default in self.params: + info(_('No default profile, returning first one')) #TODO: manage real default profile + default = self.storage.getProfilesList()[0] + self.host.memory.setPrivate('Profile_default', default) + return default #FIXME: temporary, must use real default value, and fallback to first one if it doesn't exists + if not self.storage.hasProfile(profile_key): + info (_('Trying to access an unknown profile')) + return "" + return profile_key + + def __get_unique_node(self, parent, tag, name): + """return node with given tag + @param parent: parent of nodes to check (e.g. documentElement) + @param tag: tag to check (e.g. "category") + @param name: name to check (e.g. "JID") + @return: node if it exist or None + """ + for node in parent.childNodes: + if node.nodeName == tag and node.getAttribute("name") == name: + #the node already exists + return node + #the node is new + return None + + def importParams(self, xml): + """import xml in parameters, do nothing if the param already exist + @param xml: parameters in xml form""" + src_dom = minidom.parseString(xml.encode('utf-8')) + + def import_node(tgt_parent, src_parent): + for child in src_parent.childNodes: + if child.nodeName == '#text': + continue + node = self.__get_unique_node(tgt_parent, child.nodeName, child.getAttribute("name")) + if not node: #The node is new + tgt_parent.appendChild(child) + else: + import_node(node, child) + + import_node(self.dom.documentElement, src_dom.documentElement) + + def __default_ok(self, value, name, category): + #FIXME: gof: will not work with individual parameters + self.setParam(name, value, category) #FIXME: better to set param xml value ??? + + def __default_ko(self, failure, name, category): + error (_("Can't determine default value for [%(category)s/%(name)s]: %(reason)s") % {'category':category, 'name':name, 'reason':str(failure.value)}) + + def setDefault(self, name, category, callback, errback=None): + """Set default value of parameter + 'default_cb' attibute of parameter must be set to 'yes' + @param name: name of the parameter + @param category: category of the parameter + @param callback: must return a string with the value (use deferred if needed) + @param errback: must manage the error with args failure, name, category + """ + #TODO: send signal param update if value changed + node = self.__getParamNode(name, category, '@ALL@') + if not node: + error(_("Requested param [%(name)s] in category [%(category)s] doesn't exist !") % {'name':name, 'category':category}) + return + if node[1].getAttribute('default_cb') == 'yes': + del node[1].attributes['default_cb'] + d = defer.maybeDeferred(callback) + d.addCallback(self.__default_ok, name, category) + d.addErrback(errback or self.__default_ko, name, category) + + def getParamA(self, name, category, attr="value", profile_key="@DEFAULT@"): + """Helper method to get a specific attribute + @param name: name of the parameter + @param category: category of the parameter + @param attr: name of the attribute (default: "value") + @param profile: owner of the param (@ALL@ for everyone) + + @return: attribute""" + node = self.__getParamNode(name, category) + if not node: + error(_("Requested param [%(name)s] in category [%(category)s] doesn't exist !") % {'name':name, 'category':category}) + return "" + + if node[0] == 'general': + value = self.__getParam(None, category, name, 'general') + return value if value!=None else node[1].getAttribute(attr) + + assert(node[0] == 'individual') + + profile = self.getProfileName(profile_key) + if not profile: + error(_('Requesting a param for an non-existant profile')) + return "" + + if profile not in self.params: + error(_('Requesting synchronous param for not connected profile')) + return "" + + if attr == "value": + value = self.__getParam(profile, category, name) + return value if value!=None else node[1].getAttribute(attr) + else: + return node[1].getAttribute(attr) + + def asyncGetParamA(self, name, category, attr="value", profile_key="@DEFAULT@"): + """Helper method to get a specific attribute + @param name: name of the parameter + @param category: category of the parameter + @param attr: name of the attribute (default: "value") + @param profile: owner of the param (@ALL@ for everyone)""" + node = self.__getParamNode(name, category) + if not node: + error(_("Requested param [%(name)s] in category [%(category)s] doesn't exist !") % {'name':name, 'category':category}) + return None + + if node[0] == 'general': + value = self.__getParam(None, category, name, 'general') + return defer.succeed(value if value!=None else node[1].getAttribute(attr)) + + assert(node[0] == 'individual') + + profile = self.getProfileName(profile_key) + if not profile: + error(_('Requesting a param for a non-existant profile')) + return defer.fail() + + if attr != "value": + return defer.succeed(node[1].getAttribute(attr)) + default = node[1].getAttribute(attr) + try: + value = self.__getParam(profile, category, name) + return defer.succeed(value if value!=None else default) + except ProfileNotInCacheError: + #We have to ask data to the storage manager + d = self.storage.getIndParam(category, name, profile) + return d.addCallback(lambda value: value if value!=None else default) + + def __getParam(self, profile, category, name, type='individual', cache=None): + """Return the param, or None if it doesn't exist + @param profile: the profile name (not profile key, i.e. name and not something like @DEFAULT@) + @param category: param category + @param name: param name + @param type: "general" or "individual" + @param cache: temporary cache, to use when profile is not logged + @return: param value or None if it doesn't exist + """ + if type == 'general': + if self.params_gen.has_key((category, name)): + return self.params_gen[(category, name)] + return None #This general param has the default value + assert (type == 'individual') + if self.params.has_key(profile): + cache = self.params[profile] # if profile is in main cache, we use it, + # ignoring the temporary cache + elif cache == None: #else we use the temporary cache if it exists, or raise an exception + raise ProfileNotInCacheError + if not cache.has_key((category, name)): + return None + return cache[(category, name)] + + def __constructProfileXml(self, profile): + """Construct xml for asked profile, filling values when needed + /!\ as noticed in doc, don't forget to unlink the minidom.Document + @param profile: profile name (not key !) + @return: a deferred that fire a minidom.Document of the profile xml (cf warning above) + """ + def constructProfile(ignore,profile_cache): + prof_xml = minidom.parseString('<params/>') + cache = {} + + for type_node in self.dom.documentElement.childNodes: + if type_node.nodeName == 'general' or type_node.nodeName == 'individual': #we use all params, general and individual + for cat_node in type_node.childNodes: + if cat_node.nodeName == 'category': + category = cat_node.getAttribute('name') + if not cache.has_key(category): + cache[category] = dest_cat = cat_node.cloneNode(True) #we make a copy for the new xml + new_node = True + else: + dest_cat = cache[category] + new_node = False #It's not a new node, we will merge information + params = cat_node.getElementsByTagName("param") + dest_params = {} + for node in dest_cat.childNodes: + if node.nodeName != "param": + continue + dest_params[node.getAttribute('name')] = node + + for param_node in params: + name = param_node.getAttribute('name') + + if name not in dest_params: + dest_params[name] = param_node.cloneNode(True) + dest_cat.appendChild(dest_params[name]) + + profile_value = self.__getParam(profile, category, name, type_node.nodeName, cache=profile_cache) + if profile_value!=None: #there is a value for this profile, we must change the default + dest_params[name].setAttribute('value', profile_value) + if new_node: + prof_xml.documentElement.appendChild(dest_cat) + return prof_xml + + + if self.params.has_key(profile): + d = defer.succeed(None) + profile_cache = self.params[profile] + else: + #profile is not in cache, we load values in a short time cache + profile_cache = {} + d = self.loadIndParams(profile, profile_cache) + + return d.addCallback(constructProfile, profile_cache) + + def getParamsUI(self, profile_key): + """Return a SàT XMLUI for parameters, with given profile""" + profile = self.getProfileName(profile_key) + if not profile: + error(_("Asking params for inexistant profile")) + return "" + d = self.getParams(profile) + return d.addCallback(lambda param_xml:paramsXml2xmlUI(param_xml)) + + def getParams(self, profile_key): + """Construct xml for asked profile + Take params xml as skeleton""" + profile = self.getProfileName(profile_key) + if not profile: + error(_("Asking params for inexistant profile")) + return "" + + def returnXML(prof_xml): + return_xml = prof_xml.toxml() + prof_xml.unlink() + return return_xml + + return self.__constructProfileXml(profile).addCallback(returnXML) + + def getParamsForCategory(self, category, profile_key): + """Return node's xml for selected category""" + #TODO: manage category of general type (without existant profile) + profile = self.getProfileName(profile_key) + if not profile: + error(_("Asking params for inexistant profile")) + return "" + + def returnCategoryXml(prof_xml): + for node in prof_xml.getElementsByTagName("category"): + if node.nodeName == "category" and node.getAttribute("name") == category: + result = node.toxml() + prof_xml.unlink() + return result + + prof_xml.unlink() + return "<category />" + + d = self.__constructProfileXml(profile) + return d.addCallback(returnCategoryXml) + + def __getParamNode(self, name, category, type="@ALL@"): #FIXME: is type useful ? + """Return a node from the param_xml + @param name: name of the node + @param category: category of the node + @type: keyword for search: + @ALL@ search everywhere + @GENERAL@ only search in general type + @INDIVIDUAL@ only search in individual type + @return: a tuple with the node type and the the node, or None if not found""" + + for type_node in self.dom.documentElement.childNodes: + if ( ((type == "@ALL@" or type == "@GENERAL@") and type_node.nodeName == 'general') + or ( (type == "@ALL@" or type == "@INDIVIDUAL@") and type_node.nodeName == 'individual') ): + for node in type_node.getElementsByTagName('category'): + if node.getAttribute("name") == category: + params = node.getElementsByTagName("param") + for param in params: + if param.getAttribute("name") == name: + return (type_node.nodeName, param) + return None + + def getParamsCategories(self): + """return the categories availables""" + categories=[] + for cat in self.dom.getElementsByTagName("category"): + name = cat.getAttribute("name") + if name not in categories: + categories.append(cat.getAttribute("name")) + return categories + + def setParam(self, name, value, category, profile_key='@NONE@'): + """Set a parameter, return None if the parameter is not in param xml""" + #TODO: use different behaviour depending of the data type (e.g. password encrypted) + if profile_key!="@NONE@": + profile = self.getProfileName(profile_key) + if not profile: + error(_('Trying to set parameter for an unknown profile')) + return #TODO: throw an error + + node = self.__getParamNode(name, category, '@ALL@') + if not node: + error(_('Requesting an unknown parameter (%(category)s/%(name)s)') % {'category':category, 'name':name}) + return + + if node[0] == 'general': + self.params_gen[(category, name)] = value + self.storage.setGenParam(category, name, value) + for profile in self.storage.getProfilesList(): + if self.host.isConnected(profile): + self.host.bridge.paramUpdate(name, value, category, profile) + return + + assert (node[0] == 'individual') + assert (profile_key != "@NONE@") + + type = node[1].getAttribute("type") + if type=="button": + print "clique",node.toxml() + else: + if self.host.isConnected(profile): #key can not exists if profile is not connected + self.params[profile][(category, name)] = value + self.host.bridge.paramUpdate(name, value, category, profile) + self.storage.setIndParam(category, name, value, profile) + +class Memory: + """This class manage all persistent informations""" + + def __init__(self, host): + info (_("Memory manager init")) + self.initialized = defer.Deferred() + self.host = host + self.contacts={} + self.presenceStatus={} + self.lastResource={} #tmp, will be refactored with bdd integration + self.subscriptions={} + self.private={} #used to store private value + self.server_features={} #used to store discovery's informations + self.server_identities={} + self.config = self.parseMainConf() + host.set_const('savefile_private', SAVEFILE_PRIVATE) + host.set_const('savefile_database', SAVEFILE_DATABASE) + database_file = os.path.expanduser(self.getConfig('','local_dir')+ + self.host.get_const('savefile_database')) + self.storage = SqliteStorage(database_file) + PersistentDict.storage = self.storage + self.params=Params(host, self.storage) + self.loadFiles() + d = self.storage.initialized.addCallback(self.load) + d.chainDeferred(self.initialized) + + def parseMainConf(self): + """look for main .ini configuration file, and parse it""" + _config = SafeConfigParser(defaults=default_config) + try: + _config.read(map(os.path.expanduser, ['/etc/sat.conf', '~/sat.conf', '~/.sat.conf', 'sat.conf', '.sat.conf'])) + except: + error (_("Can't read main config !")) + + return _config + + def getConfig(self, section, name): + """Get the main configuration option + @param section: section of the config file (None or '' for DEFAULT) + @param name: name of the option + """ + if not section: + section='DEFAULT' + try: + _value = self.config.get(section, name) + except NoOptionError, NoSectionError: + _value = '' + + return os.path.expanduser(_value) if name.endswith('_path') or name.endswith('_dir') else _value + + + def loadFiles(self): + """Load parameters and all memory things from file/db""" + param_file_xml = os.path.expanduser(self.getConfig('','local_dir')+ + self.host.get_const('savefile_param_xml')) + private_file = os.path.expanduser(self.getConfig('','local_dir')+ + self.host.get_const('savefile_private')) + + #parameters template + if os.path.exists(param_file_xml): + try: + self.params.load_xml(param_file_xml) + debug(_("params template loaded")) + except: + error (_("Can't load params template !")) + self.params.load_default_params() + else: + info (_("No params template, using default template")) + self.params.load_default_params() + + #private + if os.path.exists(private_file): + try: + with open(private_file, 'r') as private_pickle: + self.private=pickle.load(private_pickle) + debug(_("private values loaded")) + except: + error (_("Can't load private values !")) + + def load(self, ignore): + """Load parameters and all memory things from db""" + #parameters data + return self.params.loadGenParams() + + def loadIndividualParams(self, profile): + """Load individual parameters for a profile + @param profile: %(doc_profile)s""" + return self.params.loadIndParams(profile) + + def purgeProfile(self, profile): + """Delete cache of data of profile + @param profile: %(doc_profile)s""" + self.params.purgeProfile(profile) + + def save(self): + """Save parameters and all memory things to file/db""" + #TODO: need to encrypt files (at least passwords !) and set permissions + param_file_xml = os.path.expanduser(self.getConfig('','local_dir')+ + self.host.get_const('savefile_param_xml')) + private_file = os.path.expanduser(self.getConfig('','local_dir')+ + self.host.get_const('savefile_private')) + + self.params.save_xml(param_file_xml) + debug(_("params saved")) + with open(private_file, 'w') as private_pickle: + pickle.dump(self.private, private_pickle) + debug(_("private values saved")) + + def getProfilesList(self): + return self.storage.getProfilesList() + + + def getProfileName(self, profile_key): + """Return name of profile from keyword + @param profile_key: can be the profile name or a keywork (like @DEFAULT@) + @return: profile name or None if it doesn't exist""" + return self.params.getProfileName(profile_key) + + def createProfile(self, name): + """Create a new profile + @param name: Profile name + """ + return self.params.createProfile(name) + + def asyncCreateProfile(self, name): + """Create a new profile + @param name: Profile name + """ + return self.params.asyncCreateProfile(name) + + def deleteProfile(self, name): + """Delete an existing profile + @param name: Name of the profile""" + return self.params.deleteProfile(name) + + def addToHistory(self, from_jid, to_jid, message, timestamp=None, profile="@NONE@"): + assert(profile!="@NONE@") + return self.storage.addToHistory(from_jid, to_jid, message, timestamp, profile) + + def getHistory(self, from_jid, to_jid, limit=0, between=True): + return self.storage.getHistory(jid.JID(from_jid), jid.JID(to_jid), limit, between) + + def setPrivate(self, key, value): + """Save a misc private value (mainly useful for plugins)""" + self.private[key] = value + + def getPrivate(self, key): + """return a private value + @param key: name of wanted value + @return: value or None if value don't exist""" + if self.private.has_key(key): + return self.private[key] + return None + + def addServerFeature(self, feature, profile): + """Add a feature discovered from server + @param feature: string of the feature + @param profile: which profile is using this server ?""" + if not self.server_features.has_key(profile): + self.server_features[profile] = [] + self.server_features[profile].append(feature) + + def addServerIdentity(self, category, type, entity, profile): + """Add an identity discovered from server + @param feature: string of the feature + @param profile: which profile is using this server ?""" + if not self.server_identities.has_key(profile): + self.server_identities[profile] = {} + if not self.server_identities[profile].has_key((category, type)): + self.server_identities[profile][(category, type)]=set() + self.server_identities[profile][(category, type)].add(entity) + + def getServerServiceEntities(self, category, type, profile): + """Return all available entities for a service""" + if self.server_identities.has_key(profile): + return self.server_identities[profile].get((category, type), set()) + else: + return None + + def getServerServiceEntity(self, category, type, profile): + """Helper method to get first available entity for a service""" + entities = self.getServerServiceEntities(category, type, profile) + if entities == None: + warning(_("Entities not available, maybe they haven't been asked to server yet ?")) + return None + else: + return list(entities)[0] if entities else None + + def hasServerFeature(self, feature, profile_key): + """Tell if the server of the profile has the required feature""" + profile = self.getProfileName(profile_key) + if not profile: + error (_('Trying find server feature for a non-existant profile')) + return + assert(self.server_features.has_key(profile)) + return feature in self.server_features[profile] + + + def addContact(self, contact_jid, attributes, groups, profile_key): + debug("Memory addContact: %s",contact_jid.userhost()) + profile = self.getProfileName(profile_key) + if not profile: + error (_('Trying to add a contact to a non-existant profile')) + return + assert(isinstance(attributes,dict)) + assert(isinstance(groups,set)) + if not self.contacts.has_key(profile): + self.contacts[profile] = {} + self.contacts[profile][contact_jid.userhost()]=[attributes, groups] + + def delContact(self, contact_jid, profile_key): + debug("Memory delContact: %s",contact_jid.userhost()) + profile = self.getProfileName(profile_key) + if not profile: + error (_('Trying to delete a contact for a non-existant profile')) + return + if self.contacts.has_key(profile) and self.contacts[profile].has_key(contact_jid.userhost()): + del self.contacts[profile][contact_jid.userhost()] + + def getContact(self, contact_jid, profile_key): + profile = self.getProfileName(profile_key) + if not profile: + error(_('Asking a contact for a non-existant profile')) + return None + if self.contacts.has_key(profile) and self.contacts[profile].has_key(contact_jid.userhost()): + return self.contacts[profile][contact_jid.userhost()] + + def getContacts(self, profile_key): + """Return list of contacts for given profile + @param profile_key: profile key + @return list of [contact, attr, groups]""" + profile = self.getProfileName(profile_key) + if not profile: + error(_('Asking contacts for a non-existant profile')) + return [] + ret=[] + if self.contacts.has_key(profile): + for contact in self.contacts[profile]: + attr, groups = self.contacts[profile][contact] + ret.append([contact, attr, groups ]) + return ret + + def getLastResource(self, contact, profile_key): + """Return the last resource used by a contact + @param contact: contact jid (unicode) + @param profile_key: %(doc_profile_key)s""" + profile = self.getProfileName(profile_key) + if not profile: + error(_('Asking contacts for a non-existant profile')) + return "" + try: + return self.lastResource[profile][jid.JID(contact).userhost()] + except: + return "" + + def addPresenceStatus(self, contact_jid, show, priority, statuses, profile_key): + profile = self.getProfileName(profile_key) + if not profile: + error(_('Trying to add presence status to a non-existant profile')) + return + if not self.presenceStatus.has_key(profile): + self.presenceStatus[profile] = {} + if not self.lastResource.has_key(profile): + self.lastResource[profile] = {} + if not self.presenceStatus[profile].has_key(contact_jid.userhost()): + self.presenceStatus[profile][contact_jid.userhost()] = {} + resource = jid.parse(contact_jid.full())[2] or '' + if resource: + self.lastResource[profile][contact_jid.userhost()] = resource + + self.presenceStatus[profile][contact_jid.userhost()][resource] = (show, priority, statuses) + + def addWaitingSub(self, type, contact_jid, profile_key): + """Called when a subcription request is received""" + profile = self.getProfileName(profile_key) + assert(profile) + if not self.subscriptions.has_key(profile): + self.subscriptions[profile] = {} + self.subscriptions[profile][contact_jid] = type + + def delWaitingSub(self, contact_jid, profile_key): + """Called when a subcription request is finished""" + profile = self.getProfileName(profile_key) + assert(profile) + if self.subscriptions.has_key(profile) and self.subscriptions[profile].has_key(contact_jid): + del self.subscriptions[profile][contact_jid] + + def getWaitingSub(self, profile_key): + """Called to get a list of currently waiting subscription requests""" + profile = self.getProfileName(profile_key) + if not profile: + error(_('Asking waiting subscriptions for a non-existant profile')) + return {} + if not self.subscriptions.has_key(profile): + return {} + + return self.subscriptions[profile] + + def getPresenceStatus(self, profile_key): + profile = self.getProfileName(profile_key) + if not profile: + error(_('Asking contacts for a non-existant profile')) + return {} + if not self.presenceStatus.has_key(profile): + self.presenceStatus[profile] = {} + debug ("Memory getPresenceStatus (%s)", self.presenceStatus[profile]) + return self.presenceStatus[profile] + + def getParamA(self, name, category, attr="value", profile_key='@DEFAULT@'): + return self.params.getParamA(name, category, attr, profile_key) + + def asyncGetParamA(self, name, category, attr="value", profile_key='@DEFAULT@'): + return self.params.asyncGetParamA(name, category, attr, profile_key) + + def getParamsUI(self, profile_key): + return self.params.getParamsUI(profile_key) + + def getParams(self, profile_key): + return self.params.getParams(profile_key) + + def getParamsForCategory(self, category, profile_key): + return self.params.getParamsForCategory(category, profile_key) + + def getParamsCategories(self): + return self.params.getParamsCategories() + + def setParam(self, name, value, category, profile_key): + return self.params.setParam(name, value, category, profile_key) + + def importParams(self, xml): + return self.params.importParams(xml) + + def setDefault(self, name, category, callback, errback=None): + return self.params.setDefault(name, category, callback, errback)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/memory/persistent.py Sun Nov 20 15:34:37 2011 +0100 @@ -0,0 +1,146 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +SAT: a jabber client +Copyright (C) 2009, 2010, 2011 Jérôme Poisson (goffi@goffi.org) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + +from logging import debug, info, warning, error + +class MemoryNotInitializedError(Exception): + pass + +class PersistentDict: + """A dictionary which save persistently each value assigned + /!\ be careful, each assignment means a database write + /!\ Memory must be initialised before loading/setting value with instances of this class""" + storage = None + + def __init__(self, namespace, profile=None): + """@param namespace: unique namespace for this dictionary + @param profile: profile which *MUST* exists, or None for general values""" + if not self.storage: + error(_("PersistentDict can't be used before memory initialisation")); + raise MemoryNotInitializedError + self._cache = {} + self.namespace = namespace + self.profile = profile + + def load(self): + """load persistent data from storage + """ + if not self.profile: + return self.storage.loadGenPrivates(self._cache, self.namespace) + else: + return self.storage.loadIndPrivates(self._cache, self.namespace, self.profile) + + def __repr__(self): + return self._cache.__repr__() + + def __str__(self): + return self._cache.__str__() + + def __lt__(self, other): + return self._cache.__lt__(other) + + def __le__(self, other): + return self._cache.__le__(other) + + def __eq__(self, other): + return self._cache.__eq__(other) + + def __ne__(self, other): + return self._cache.__ne__(other) + + def __gt__(self, other): + return self._cache.__gt__(other) + + def __ge__(self, other): + return self._cache.__ge__(other) + + def __cmp__(self, other): + return self._cache.__cmp__(other) + + def __hash__(self): + return self._cache.__hash__() + + def __nonzero__(self): + return self._cache.__nonzero__() + + def __unicode__(self): + return self._cache.__unicode__() + + def __contains__(self, key): + return self._cache.__contains__(key) + + def __getitem__(self, key): + return self._cache.__getitem__(key) + + def __setitem__(self, key, value): + if not self.profile: + self.storage.setGenPrivate(self.namespace, key, value) + else: + self.storage.setIndPrivate(self.namespace, key, value, self.profile) + return self._cache.__setitem__(key, value) + + def __delitem__(self, key): + if not self.profile: + self.storage.delGenPrivate(self.namespace, key) + else: + self.storage.delIndPrivate(self.namespace, key, self.profile) + return self._cache.__delitem__(key) + + def force(self, name): + """Force saving of an attribute to storage + @return: deferred fired when data is actually saved""" + if not self.profile: + return self.storage.setGenPrivate(self.namespace, name, self._cache[name]) + return self.storage.setIndPrivate(self.namespace, name, self._cache[name], self.profile) + + +class PersistentBinaryDict(PersistentDict): + """Persistent dict where value can be any python data (instead of string only)""" + + def load(self): + """load persistent data from storage + """ + if not self.profile: + return self.storage.loadGenPrivatesBinary(self._cache, self.namespace) + else: + return self.storage.loadIndPrivatesBinary(self._cache, self.namespace, self.profile) + + def __setitem__(self, key, value): + if not self.profile: + self.storage.setGenPrivateBinary(self.namespace, key, value) + else: + self.storage.setIndPrivateBinary(self.namespace, key, value, self.profile) + return self._cache.__setitem__(key, value) + + def __delitem__(self, key): + if not self.profile: + self.storage.delGenPrivateBinary(self.namespace, key) + else: + self.storage.delIndPrivateBinary(self.namespace, key, self.profile) + return self._cache.__delitem__(key) + + def force(self, name): + """Force saving of an attribute to storage + @return: deferred fired when data is actually saved""" + if not self.profile: + return self.storage.setGenPrivateBinary(self.namespace, name, self._cache[name]) + return self.storage.setIndPrivateBinary(self.namespace, name, self._cache[name], self.profile) +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/memory/sqlite.py Sun Nov 20 15:34:37 2011 +0100 @@ -0,0 +1,373 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +SAT: a jabber client +Copyright (C) 2009, 2010, 2011 Jérôme Poisson (goffi@goffi.org) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + + +from logging import debug, info, warning, error +from twisted.enterprise import adbapi +from twisted.internet import defer +import os.path +import time +import cPickle as pickle + +class SqliteStorage(): + """This class manage storage with Sqlite database""" + + + def __init__(self, db_filename): + """Connect to the given database + @param db_filename: full path to the Sqlite database""" + self.initialized = defer.Deferred() #triggered when memory is fully initialised and ready + init_defers = [] #list of deferred we have to wait to before initialisation is complete + self.profiles={} #we keep cache for the profiles (key: profile name, value: profile id) + + info(_("Connecting database")) + new_base = not os.path.exists(db_filename) #do we have to create the database ? + self.dbpool = adbapi.ConnectionPool("sqlite3", db_filename, check_same_thread=False) + init_defers.append(self.dbpool.runOperation("PRAGMA foreign_keys = ON").addErrback(lambda x: error(_("Can't activate foreign keys")))) + if new_base: + info(_("The database is new, creating the tables")) + database_creation = [ + "CREATE TABLE profiles (id INTEGER PRIMARY KEY ASC, name TEXT, UNIQUE (name))", + "CREATE TABLE history (id INTEGER PRIMARY KEY ASC, profile_id INTEGER, source TEXT, dest TEXT, source_res TEXT, dest_res TEXT, timestamp DATETIME, message TEXT, FOREIGN KEY(profile_id) REFERENCES profiles(id))", + "CREATE TABLE param_gen (category TEXT, name TEXT, value TEXT, PRIMARY KEY (category,name))", + "CREATE TABLE param_ind (category TEXT, name TEXT, profile_id INTEGER, value TEXT, PRIMARY KEY (category,name,profile_id), FOREIGN KEY(profile_id) REFERENCES profiles(id))", + "CREATE TABLE private_gen (namespace TEXT, key TEXT, value TEXT, PRIMARY KEY (namespace, key))", + "CREATE TABLE private_ind (namespace TEXT, key TEXT, profile_id INTEGER, value TEXT, PRIMARY KEY (namespace, key, profile_id), FOREIGN KEY(profile_id) REFERENCES profiles(id))", + "CREATE TABLE private_gen_bin (namespace TEXT, key TEXT, value BLOB, PRIMARY KEY (namespace, key))", + "CREATE TABLE private_ind_bin (namespace TEXT, key TEXT, profile_id INTEGER, value BLOB, PRIMARY KEY (namespace, key, profile_id), FOREIGN KEY(profile_id) REFERENCES profiles(id))", + ] + for op in database_creation: + d = self.dbpool.runOperation(op) + d.addErrback(lambda x: error(_("Error while creating tables in database [QUERY: %s]") % op )) + init_defers.append(d) + + def fillProfileCache(ignore): + d = self.dbpool.runQuery("SELECT name,id FROM profiles").addCallback(self._profilesCache) + d.chainDeferred(self.initialized) + + defer.DeferredList(init_defers).addCallback(fillProfileCache) + + #Profiles + def _profilesCache(self, profiles_result): + """Fill the profiles cache + @param profiles_result: result of the sql profiles query""" + for profile in profiles_result: + name, id = profile + self.profiles[name] = id + + def getProfilesList(self): + """"Return list of all registered profiles""" + return self.profiles.keys() + + def hasProfile(self, profile_name): + """return True if profile_name exists + @param profile_name: name of the profile to check""" + return self.profiles.has_key(profile_name) + + def createProfile(self, name): + """Create a new profile + @param name: name of the profile + @return: deferred triggered once profile is actually created""" + def getProfileId(ignore): + return self.dbpool.runQuery("SELECT (id) FROM profiles WHERE name = ?", (name,)) + + def profile_created(profile_id): + _id = profile_id[0][0] + self.profiles[name] = _id #we synchronise the cache + + d = self.dbpool.runQuery("INSERT INTO profiles(name) VALUES (?)", (name,)) + d.addCallback(getProfileId) + d.addCallback(profile_created) + return d + + def deleteProfile(self, name): + """Delete profile + @param name: name of the profile + @return: deferred triggered once profile is actually deleted""" + def deletionError(failure): + error(_("Can't delete profile [%s]") % name) + return failure + del self.profiles[name] + d = self.dbpool.runQuery("DELETE FROM profiles WHERE name = ?", (name,)) + d.addCallback(lambda ignore: info(_("Profile [%s] deleted") % name)) + return d + + + #Params + def loadGenParams(self, params_gen): + """Load general parameters + @param params_gen: dictionary to fill + @return: deferred""" + def fillParams(result): + for param in result: + category,name,value = param + params_gen[(category, name)] = value + debug(_("loading general parameters from database")) + return self.dbpool.runQuery("SELECT category,name,value FROM param_gen").addCallback(fillParams) + + def loadIndParams(self, params_ind, profile): + """Load individual parameters + @param params_ind: dictionary to fill + @param profile: a profile which *must* exist + @return: deferred""" + def fillParams(result): + for param in result: + category,name,value = param + params_ind[(category, name)] = value + debug(_("loading individual parameters from database")) + d = self.dbpool.runQuery("SELECT category,name,value FROM param_ind WHERE profile_id=?", (self.profiles[profile],)) + d.addCallback(fillParams) + return d + + def getIndParam(self, category, name, profile): + """Ask database for the value of one specific individual parameter + @param category: category of the parameter + @param name: name of the parameter + @param profile: %(doc_profile)s + @return: deferred""" + d = self.dbpool.runQuery("SELECT value FROM param_ind WHERE category=? AND name=? AND profile_id=?", (category,name,self.profiles[profile])) + d.addCallback(self.__getFirstResult) + return d + + def setGenParam(self, category, name, value): + """Save the general parameters in database + @param category: category of the parameter + @param name: name of the parameter + @param value: value to set + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO param_gen(category,name,value) VALUES (?,?,?)", (category, name, value)) + d.addErrback(lambda ignore: error(_("Can't set general parameter (%(category)s/%(name)s) in database" % {"category":category, "name":name}))) + return d + + def setIndParam(self, category, name, value, profile): + """Save the individual parameters in database + @param category: category of the parameter + @param name: name of the parameter + @param value: value to set + @param profile: a profile which *must* exist + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO param_ind(category,name,profile_id,value) VALUES (?,?,?,?)", (category, name, self.profiles[profile], value)) + d.addErrback(lambda ignore: error(_("Can't set individual parameter (%(category)s/%(name)s) for [%(profile)s] in database" % {"category":category, "name":name, "profile":profile}))) + return d + + #History + def addToHistory(self, from_jid, to_jid, message, timestamp=None, profile=None): + """Store a new message in history + @param from_jid: full source JID + @param to_jid: full dest JID + @param message: message + @param timestamp: timestamp in seconds since epoch, or None to use current time + """ + assert(profile!=None) + d = self.dbpool.runQuery("INSERT INTO history(source, source_res, dest, dest_res, timestamp, message, profile_id) VALUES (?,?,?,?,?,?,?)", + (from_jid.userhost(), from_jid.resource, to_jid.userhost(), to_jid.resource, timestamp or int(time.time()), + message, self.profiles[profile])) + d.addErrback(lambda ignore: error(_("Can't save following message in history: from [%(from_jid)s] to [%(to_jid)s] ==> [%(message)s]" % + {"from_jid":from_jid.full(), "to_jid":to_jid.full(), "message":message}))) + return d + + def getHistory(self, from_jid, to_jid, limit=0, between=True): + """Store a new message in history + @param from_jid: source JID (full, or bare for catchall + @param to_jid: dest JID (full, or bare for catchall + @param size: maximum number of messages to get, or 0 for unlimited + """ + def sqliteToDict(result): + result_dict = {} + for row in result: + timestamp, source, source_res, dest, dest_res, message = row + result_dict[timestamp] = ("%s/%s" % (source, source_res) if source_res else source, + "%s/%s" % (dest, dest_res) if dest_res else dest, + message) + return result_dict + + query_parts = ["SELECT timestamp, source, source_res, dest, dest_res, message FROM history WHERE"] + values = [] + + if between: + query_parts.append("(source=? OR source=?) AND (dest=? or dest=?)") + values.extend([from_jid.userhost(), to_jid.userhost(), to_jid.userhost(), from_jid.userhost()]) + else: + query_parts.append("source=? AND dest=?") + values.extend([from_jid.userhost(), to_jid.userhost()]) + if from_jid.resource: + query_parts.append("AND source_res=?") + values.append(from_jid.resource) + if to_jid.resource: + query_parts.append("AND dest_res=?") + values.append(to_jid.resource) + if limit: + query_parts.append("LIMIT ?") + values.append(limit) + d = self.dbpool.runQuery(" ".join(query_parts), values) + return d.addCallback(sqliteToDict) + + #Private values + def loadGenPrivates(self, private_gen, namespace): + """Load general private values + @param private_gen: dictionary to fill + @param namespace: namespace of the values + @return: deferred""" + def fillPrivates(result): + for private in result: + key,value = private + private_gen[key] = value + debug(_("loading general private values [namespace: %s] from database") % (namespace,)) + d = self.dbpool.runQuery("SELECT key,value FROM private_gen WHERE namespace=?", namespace).addCallback(fillPrivates) + return d.addErrback(lambda x: debug(_("No data present in database for namespace %s") % namespace)) + + def loadIndPrivates(self, private_ind, namespace, profile): + """Load individual private values + @param privates_ind: dictionary to fill + @param namespace: namespace of the values + @param profile: a profile which *must* exist + @return: deferred""" + def fillPrivates(result): + for private in result: + key,value = private + private_ind[key] = value + debug(_("loading individual private values [namespace: %s] from database") % (namespace,)) + d = self.dbpool.runQuery("SELECT key,value FROM private_ind WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile])) + d.addCallback(fillPrivates) + return d.addErrback(lambda x: debug(_("No data present in database for namespace %s") % namespace)) + + def setGenPrivate(self, namespace, key, value): + """Save the general private value in database + @param category: category of the privateeter + @param key: key of the private value + @param value: value to set + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO private_gen(namespace,key,value) VALUES (?,?,?)", (namespace,key,value)) + d.addErrback(lambda ignore: error(_("Can't set general private value (%(key)s) [namespace:%(namespace)s] in database" % + {"namespace":namespace, "key":key}))) + return d + + def setIndPrivate(self, namespace, key, value, profile): + """Save the individual private value in database + @param namespace: namespace of the value + @param key: key of the private value + @param value: value to set + @param profile: a profile which *must* exist + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO private_ind(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], value)) + d.addErrback(lambda ignore: error(_("Can't set individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % + {"namespace":namespace, "key":key, "profile":profile}))) + return d + + def delGenPrivate(self, namespace, key): + """Delete the general private value from database + @param category: category of the privateeter + @param key: key of the private value + @return: deferred""" + d = self.dbpool.runQuery("DELETE FROM private_gen WHERE namespace=? AND key=?", (namespace,key)) + d.addErrback(lambda ignore: error(_("Can't delete general private value (%(key)s) [namespace:%(namespace)s] in database" % + {"namespace":namespace, "key":key}))) + return d + + def delIndPrivate(self, namespace, key, profile): + """Delete the individual private value from database + @param namespace: namespace of the value + @param key: key of the private value + @param profile: a profile which *must* exist + @return: deferred""" + d = self.dbpool.runQuery("DELETE FROM private_ind WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile])) + d.addErrback(lambda ignore: error(_("Can't delete individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % + {"namespace":namespace, "key":key, "profile":profile}))) + return d + + + def loadGenPrivatesBinary(self, private_gen, namespace): + """Load general private binary values + @param private_gen: dictionary to fill + @param namespace: namespace of the values + @return: deferred""" + def fillPrivates(result): + for private in result: + key,value = private + private_gen[key] = pickle.loads(value,0) + debug(_("loading general private binary values [namespace: %s] from database") % (namespace,)) + d = self.dbpool.runQuery("SELECT key,value FROM private_gen_bin WHERE namespace=?", namespace).addCallback(fillPrivates) + return d.addErrback(lambda x: debug(_("No binary data present in database for namespace %s") % namespace)) + + def loadIndPrivatesBinary(self, private_ind, namespace, profile): + """Load individual private binary values + @param privates_ind: dictionary to fill + @param namespace: namespace of the values + @param profile: a profile which *must* exist + @return: deferred""" + def fillPrivates(result): + for private in result: + key,value = private + private_ind[key] = pickle.loads(value,0) + debug(_("loading individual private binary values [namespace: %s] from database") % (namespace,)) + d = self.dbpool.runQuery("SELECT key,value FROM private_ind_bin WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile])) + d.addCallback(fillPrivates) + return d.addErrback(lambda x: debug(_("No binary data present in database for namespace %s") % namespace)) + + def setGenPrivateBinary(self, namespace, key, value): + """Save the general private binary value in database + @param category: category of the privateeter + @param key: key of the private value + @param value: value to set + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO private_gen_bin(namespace,key,value) VALUES (?,?,?)", (namespace,key,pickle.dumps(value,0))) + d.addErrback(lambda ignore: error(_("Can't set general private binary value (%(key)s) [namespace:%(namespace)s] in database" % + {"namespace":namespace, "key":key}))) + return d + + def setIndPrivateBinary(self, namespace, key, value, profile): + """Save the individual private binary value in database + @param namespace: namespace of the value + @param key: key of the private value + @param value: value to set + @param profile: a profile which *must* exist + @return: deferred""" + d = self.dbpool.runQuery("REPLACE INTO private_ind_bin(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], pickle.dumps(value,0))) + d.addErrback(lambda ignore: error(_("Can't set individual binary private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % + {"namespace":namespace, "key":key, "profile":profile}))) + return d + + def delGenPrivateBinary(self, namespace, key): + """Delete the general private binary value from database + @param category: category of the privateeter + @param key: key of the private value + @return: deferred""" + d = self.dbpool.runQuery("DELETE FROM private_gen_bin WHERE namespace=? AND key=?", (namespace,key)) + d.addErrback(lambda ignore: error(_("Can't delete general private binary value (%(key)s) [namespace:%(namespace)s] in database" % + {"namespace":namespace, "key":key}))) + return d + + def delIndPrivateBinary(self, namespace, key, profile): + """Delete the individual private binary value from database + @param namespace: namespace of the value + @param key: key of the private value + @param profile: a profile which *must* exist + @return: deferred""" + d = self.dbpool.runQuery("DELETE FROM private_ind_bin WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile])) + d.addErrback(lambda ignore: error(_("Can't delete individual private binary value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % + {"namespace":namespace, "key":key, "profile":profile}))) + return d + ##Helper methods## + + def __getFirstResult(self, result): + """Return the first result of a database query + Useful when we are looking for one specific value""" + return None if not result else result[0][0]