diff src/memory/sqlite.py @ 2182:087eec4c6c07

memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict: - merged private values method handling in sqlite, and added a keys arguments to get only some keys - LazyPersistentBinaryDict allow to get values in database only when they are needed, saving memory for big data
author Goffi <goffi@goffi.org>
date Sun, 12 Mar 2017 19:33:17 +0100
parents 1d3f73e065e1
children ea41cf1e6d29
line wrap: on
line diff
--- a/src/memory/sqlite.py	Sun Mar 12 19:32:59 2017 +0100
+++ b/src/memory/sqlite.py	Sun Mar 12 19:33:17 2017 +0100
@@ -520,177 +520,119 @@
         return d
 
     #Private values
-    def loadGenPrivates(self, private_gen, namespace):
-        """Load general private values
 
-        @param private_gen: dictionary to fill
-        @param namespace: namespace of the values
-        @return: deferred
-        """
+    def _privateDataEb(self, failure_, operation, namespace, key=None, profile=None):
+        """generic errback for data queries"""
+        log.error(_(u"Can't {operation} data in database for namespace {namespace}{and_key}{for_profile}: {msg}").format(
+            operation = operation,
+            namespace = namespace,
+            and_key = (u" and key " + key) if key is not None else u"",
+            for_profile = (u' [' + profile + u']') if profile is not None else u'',
+            msg = failure_))
 
-        def fillPrivates(result):
-            for private in result:
-                key, value = private
-                private_gen[key] = value
-        log.debug(_(u"loading general private values [namespace: %s] from database") % (namespace,))
-        d = self.dbpool.runQuery("SELECT key,value FROM private_gen WHERE namespace=?", (namespace, )).addCallback(fillPrivates)
-        return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace))
+    def _generateDataDict(self, query_result, binary):
+        if binary:
+            return {k: pickle.loads(str(v)) for k,v in query_result}
+        else:
+            return dict(query_result)
+
+    def _getPrivateTable(self, binary, profile):
+        """Get table to use for private values"""
+        table = [u'private']
 
-    def loadIndPrivates(self, private_ind, namespace, profile):
-        """Load individual private values
+        if profile is None:
+            table.append(u'gen')
+        else:
+            table.append(u'ind')
+
+        if binary:
+            table.append(u'bin')
+
+        return u'_'.join(table)
 
-        @param privates_ind: dictionary to fill
-        @param namespace: namespace of the values
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
+    def getPrivates(self, namespace, keys=None, binary=False, profile=None):
+        """Get private value(s) from databases
 
-        def fillPrivates(result):
-            for private in result:
-                key, value = private
-                private_ind[key] = value
-        log.debug(_(u"loading individual private values [namespace: %s] from database") % (namespace,))
-        d = self.dbpool.runQuery("SELECT key,value FROM private_ind WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile]))
-        d.addCallback(fillPrivates)
-        return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace))
+        @param namespace(unicode): namespace of the values
+        @param keys(iterable, None): keys of the values to get
+            None to get all keys/values
+        @param binary(bool): True to deserialise binary values
+        @param profile(unicode, None): profile to use for individual values
+            None to use general values
+        @return (dict[unicode, object]): gotten keys/values
+        """
+        log.debug(_(u"getting {type}{binary} private values from database for namespace {namespace}{keys}".format(
+            type = u"general" if profile is None else "individual",
+            binary = u" binary" if binary else u"",
+            namespace = namespace,
+            keys = u" with keys {}".format(u", ".join(keys)) if keys is not None else u"")))
+        table = self._getPrivateTable(binary, profile)
+        query_parts = [u"SELECT key,value FROM", table, "WHERE namespace=?"]
+        args = [namespace]
 
-    def setGenPrivate(self, namespace, key, value):
-        """Save the general private value in database
+        if keys is not None:
+            query_parts.append(u'AND key IN ?')
+            args.append(keys)
 
-        @param category: category of the privateeter
-        @param key: key of the private value
-        @param value: value to set
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("REPLACE INTO private_gen(namespace,key,value) VALUES (?,?,?)", (namespace, key, value))
-        d.addErrback(lambda ignore: log.error(_(u"Can't set general private value (%(key)s) [namespace:%(namespace)s] in database" %
-                     {"namespace": namespace, "key": key})))
+        if profile is not None:
+            query_parts.append(u'AND profile_id=?')
+            args.append(self.profiles[profile])
+
+        d = self.dbpool.runQuery(u" ".join(query_parts), args)
+        d.addCallback(self._generateDataDict, binary)
+        d.addErrback(self._privateDataEb, u"get", namespace, profile=profile)
         return d
 
-    def setIndPrivate(self, namespace, key, value, profile):
-        """Save the individual private value in database
+    def setPrivateValue(self, namespace, key, value, binary=False, profile=None):
+        """Set a private value in database
+
+        @param namespace(unicode): namespace of the values
+        @param key(unicode): key of the value to set
+        @param value(object): value to set
+        @param binary(bool): True if it's a binary values
+            binary values need to be serialised, used for everything but strings
+        @param profile(unicode, None): profile to use for individual value
+            if None, it's a general value
+        """
+        table = self._getPrivateTable(binary, profile)
+        query_values_names = [u'namespace', u'key', u'value']
+        query_values = [namespace, key]
 
-        @param namespace: namespace of the value
-        @param key: key of the private value
-        @param value: value to set
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("REPLACE INTO private_ind(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], value))
-        d.addErrback(lambda ignore: log.error(_(u"Can't set individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
-                     {"namespace": namespace, "key": key, "profile": profile})))
+        if binary:
+            value = sqlite3.Binary(pickle.dumps(value, 0))
+
+        query_values.append(value)
+
+        if profile is not None:
+            query_values_names.append(u'profile_id')
+            query_values.append(self.profiles[profile])
+
+        query_parts = [u"REPLACE INTO", table, u'(', u','.join(query_values_names), u')',
+                       u"VALUES (", u",".join(u'?'*len(query_values_names)), u')']
+
+        d = self.dbpool.runQuery(u" ".join(query_parts), query_values)
+        d.addErrback(self._privateDataEb, u"set", namespace, key, profile=profile)
         return d
 
-    def delGenPrivate(self, namespace, key):
-        """Delete the general private value from database
+    def delPrivateValue(self, namespace, key, binary=False, profile=None):
+        """Delete private value from database
 
         @param category: category of the privateeter
         @param key: key of the private value
-        @return: deferred
+        @param binary(bool): True if it's a binary values
+        @param profile(unicode, None): profile to use for individual value
+            if None, it's a general value
         """
-        d = self.dbpool.runQuery("DELETE FROM private_gen WHERE namespace=? AND key=?", (namespace, key))
-        d.addErrback(lambda ignore: log.error(_(u"Can't delete general private value (%(key)s) [namespace:%(namespace)s] in database" %
-                     {"namespace": namespace, "key": key})))
-        return d
-
-    def delIndPrivate(self, namespace, key, profile):
-        """Delete the individual private value from database
-
-        @param namespace: namespace of the value
-        @param key: key of the private value
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("DELETE FROM private_ind WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile]))
-        d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
-                     {"namespace": namespace, "key": key, "profile": profile})))
+        table = self._getPrivateTable(binary, profile)
+        query_parts = [u"DELETE FROM", table, u"WHERE namespace=? AND key=?"]
+        args = [namespace, key]
+        if profile is not None:
+            query_parts.append(u"AND profile_id=?")
+            args.append(self.profiles[profile])
+        d = self.dbpool.runQuery(u" ".join(query_parts), args)
+        d.addErrback(self._privateDataEb, u"delete", namespace, key, profile=profile)
         return d
 
-    def loadGenPrivatesBinary(self, private_gen, namespace):
-        """Load general private binary values
-
-        @param private_gen: dictionary to fill
-        @param namespace: namespace of the values
-        @return: deferred
-        """
-
-        def fillPrivates(result):
-            for private in result:
-                key, value = private
-                private_gen[key] = pickle.loads(str(value))
-        log.debug(_(u"loading general private binary values [namespace: %s] from database") % (namespace,))
-        d = self.dbpool.runQuery("SELECT key,value FROM private_gen_bin WHERE namespace=?", (namespace, )).addCallback(fillPrivates)
-        return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace))
-
-    def loadIndPrivatesBinary(self, private_ind, namespace, profile):
-        """Load individual private binary values
-
-        @param privates_ind: dictionary to fill
-        @param namespace: namespace of the values
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
-
-        def fillPrivates(result):
-            for private in result:
-                key, value = private
-                private_ind[key] = pickle.loads(str(value))
-        log.debug(_(u"loading individual private binary values [namespace: %s] from database") % (namespace,))
-        d = self.dbpool.runQuery("SELECT key,value FROM private_ind_bin WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile]))
-        d.addCallback(fillPrivates)
-        return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace))
-
-    def setGenPrivateBinary(self, namespace, key, value):
-        """Save the general private binary value in database
-
-        @param category: category of the privateeter
-        @param key: key of the private value
-        @param value: value to set
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("REPLACE INTO private_gen_bin(namespace,key,value) VALUES (?,?,?)", (namespace, key, sqlite3.Binary(pickle.dumps(value, 0))))
-        d.addErrback(lambda ignore: log.error(_(u"Can't set general private binary value (%(key)s) [namespace:%(namespace)s] in database" %
-                     {"namespace": namespace, "key": key})))
-        return d
-
-    def setIndPrivateBinary(self, namespace, key, value, profile):
-        """Save the individual private binary value in database
-
-        @param namespace: namespace of the value
-        @param key: key of the private value
-        @param value: value to set
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("REPLACE INTO private_ind_bin(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], sqlite3.Binary(pickle.dumps(value, 0))))
-        d.addErrback(lambda ignore: log.error(_(u"Can't set individual binary private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
-                     {"namespace": namespace, "key": key, "profile": profile})))
-        return d
-
-    def delGenPrivateBinary(self, namespace, key):
-        """Delete the general private binary value from database
-
-        @param category: category of the privateeter
-        @param key: key of the private value
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("DELETE FROM private_gen_bin WHERE namespace=? AND key=?", (namespace, key))
-        d.addErrback(lambda ignore: log.error(_(u"Can't delete general private binary value (%(key)s) [namespace:%(namespace)s] in database" %
-                     {"namespace": namespace, "key": key})))
-        return d
-
-    def delIndPrivateBinary(self, namespace, key, profile):
-        """Delete the individual private binary value from database
-
-        @param namespace: namespace of the value
-        @param key: key of the private value
-        @param profile: a profile which *must* exist
-        @return: deferred
-        """
-        d = self.dbpool.runQuery("DELETE FROM private_ind_bin WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile]))
-        d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private binary value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
-                     {"namespace": namespace, "key": key, "profile": profile})))
-        return d
     ##Helper methods##
 
     def __getFirstResult(self, result):