Mercurial > libervia-backend
diff src/memory/sqlite.py @ 2182:087eec4c6c07
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
- merged private values method handling in sqlite, and added a keys arguments to get only some keys
- LazyPersistentBinaryDict allow to get values in database only when they are needed, saving memory for big data
author | Goffi <goffi@goffi.org> |
---|---|
date | Sun, 12 Mar 2017 19:33:17 +0100 |
parents | 1d3f73e065e1 |
children | ea41cf1e6d29 |
line wrap: on
line diff
--- a/src/memory/sqlite.py Sun Mar 12 19:32:59 2017 +0100 +++ b/src/memory/sqlite.py Sun Mar 12 19:33:17 2017 +0100 @@ -520,177 +520,119 @@ return d #Private values - def loadGenPrivates(self, private_gen, namespace): - """Load general private values - @param private_gen: dictionary to fill - @param namespace: namespace of the values - @return: deferred - """ + def _privateDataEb(self, failure_, operation, namespace, key=None, profile=None): + """generic errback for data queries""" + log.error(_(u"Can't {operation} data in database for namespace {namespace}{and_key}{for_profile}: {msg}").format( + operation = operation, + namespace = namespace, + and_key = (u" and key " + key) if key is not None else u"", + for_profile = (u' [' + profile + u']') if profile is not None else u'', + msg = failure_)) - def fillPrivates(result): - for private in result: - key, value = private - private_gen[key] = value - log.debug(_(u"loading general private values [namespace: %s] from database") % (namespace,)) - d = self.dbpool.runQuery("SELECT key,value FROM private_gen WHERE namespace=?", (namespace, )).addCallback(fillPrivates) - return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace)) + def _generateDataDict(self, query_result, binary): + if binary: + return {k: pickle.loads(str(v)) for k,v in query_result} + else: + return dict(query_result) + + def _getPrivateTable(self, binary, profile): + """Get table to use for private values""" + table = [u'private'] - def loadIndPrivates(self, private_ind, namespace, profile): - """Load individual private values + if profile is None: + table.append(u'gen') + else: + table.append(u'ind') + + if binary: + table.append(u'bin') + + return u'_'.join(table) - @param privates_ind: dictionary to fill - @param namespace: namespace of the values - @param profile: a profile which *must* exist - @return: deferred - """ + def getPrivates(self, namespace, keys=None, binary=False, profile=None): + """Get private value(s) from databases - def fillPrivates(result): - for private in result: - key, value = private - private_ind[key] = value - log.debug(_(u"loading individual private values [namespace: %s] from database") % (namespace,)) - d = self.dbpool.runQuery("SELECT key,value FROM private_ind WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile])) - d.addCallback(fillPrivates) - return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace)) + @param namespace(unicode): namespace of the values + @param keys(iterable, None): keys of the values to get + None to get all keys/values + @param binary(bool): True to deserialise binary values + @param profile(unicode, None): profile to use for individual values + None to use general values + @return (dict[unicode, object]): gotten keys/values + """ + log.debug(_(u"getting {type}{binary} private values from database for namespace {namespace}{keys}".format( + type = u"general" if profile is None else "individual", + binary = u" binary" if binary else u"", + namespace = namespace, + keys = u" with keys {}".format(u", ".join(keys)) if keys is not None else u""))) + table = self._getPrivateTable(binary, profile) + query_parts = [u"SELECT key,value FROM", table, "WHERE namespace=?"] + args = [namespace] - def setGenPrivate(self, namespace, key, value): - """Save the general private value in database + if keys is not None: + query_parts.append(u'AND key IN ?') + args.append(keys) - @param category: category of the privateeter - @param key: key of the private value - @param value: value to set - @return: deferred - """ - d = self.dbpool.runQuery("REPLACE INTO private_gen(namespace,key,value) VALUES (?,?,?)", (namespace, key, value)) - d.addErrback(lambda ignore: log.error(_(u"Can't set general private value (%(key)s) [namespace:%(namespace)s] in database" % - {"namespace": namespace, "key": key}))) + if profile is not None: + query_parts.append(u'AND profile_id=?') + args.append(self.profiles[profile]) + + d = self.dbpool.runQuery(u" ".join(query_parts), args) + d.addCallback(self._generateDataDict, binary) + d.addErrback(self._privateDataEb, u"get", namespace, profile=profile) return d - def setIndPrivate(self, namespace, key, value, profile): - """Save the individual private value in database + def setPrivateValue(self, namespace, key, value, binary=False, profile=None): + """Set a private value in database + + @param namespace(unicode): namespace of the values + @param key(unicode): key of the value to set + @param value(object): value to set + @param binary(bool): True if it's a binary values + binary values need to be serialised, used for everything but strings + @param profile(unicode, None): profile to use for individual value + if None, it's a general value + """ + table = self._getPrivateTable(binary, profile) + query_values_names = [u'namespace', u'key', u'value'] + query_values = [namespace, key] - @param namespace: namespace of the value - @param key: key of the private value - @param value: value to set - @param profile: a profile which *must* exist - @return: deferred - """ - d = self.dbpool.runQuery("REPLACE INTO private_ind(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], value)) - d.addErrback(lambda ignore: log.error(_(u"Can't set individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % - {"namespace": namespace, "key": key, "profile": profile}))) + if binary: + value = sqlite3.Binary(pickle.dumps(value, 0)) + + query_values.append(value) + + if profile is not None: + query_values_names.append(u'profile_id') + query_values.append(self.profiles[profile]) + + query_parts = [u"REPLACE INTO", table, u'(', u','.join(query_values_names), u')', + u"VALUES (", u",".join(u'?'*len(query_values_names)), u')'] + + d = self.dbpool.runQuery(u" ".join(query_parts), query_values) + d.addErrback(self._privateDataEb, u"set", namespace, key, profile=profile) return d - def delGenPrivate(self, namespace, key): - """Delete the general private value from database + def delPrivateValue(self, namespace, key, binary=False, profile=None): + """Delete private value from database @param category: category of the privateeter @param key: key of the private value - @return: deferred + @param binary(bool): True if it's a binary values + @param profile(unicode, None): profile to use for individual value + if None, it's a general value """ - d = self.dbpool.runQuery("DELETE FROM private_gen WHERE namespace=? AND key=?", (namespace, key)) - d.addErrback(lambda ignore: log.error(_(u"Can't delete general private value (%(key)s) [namespace:%(namespace)s] in database" % - {"namespace": namespace, "key": key}))) - return d - - def delIndPrivate(self, namespace, key, profile): - """Delete the individual private value from database - - @param namespace: namespace of the value - @param key: key of the private value - @param profile: a profile which *must* exist - @return: deferred - """ - d = self.dbpool.runQuery("DELETE FROM private_ind WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile])) - d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % - {"namespace": namespace, "key": key, "profile": profile}))) + table = self._getPrivateTable(binary, profile) + query_parts = [u"DELETE FROM", table, u"WHERE namespace=? AND key=?"] + args = [namespace, key] + if profile is not None: + query_parts.append(u"AND profile_id=?") + args.append(self.profiles[profile]) + d = self.dbpool.runQuery(u" ".join(query_parts), args) + d.addErrback(self._privateDataEb, u"delete", namespace, key, profile=profile) return d - def loadGenPrivatesBinary(self, private_gen, namespace): - """Load general private binary values - - @param private_gen: dictionary to fill - @param namespace: namespace of the values - @return: deferred - """ - - def fillPrivates(result): - for private in result: - key, value = private - private_gen[key] = pickle.loads(str(value)) - log.debug(_(u"loading general private binary values [namespace: %s] from database") % (namespace,)) - d = self.dbpool.runQuery("SELECT key,value FROM private_gen_bin WHERE namespace=?", (namespace, )).addCallback(fillPrivates) - return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace)) - - def loadIndPrivatesBinary(self, private_ind, namespace, profile): - """Load individual private binary values - - @param privates_ind: dictionary to fill - @param namespace: namespace of the values - @param profile: a profile which *must* exist - @return: deferred - """ - - def fillPrivates(result): - for private in result: - key, value = private - private_ind[key] = pickle.loads(str(value)) - log.debug(_(u"loading individual private binary values [namespace: %s] from database") % (namespace,)) - d = self.dbpool.runQuery("SELECT key,value FROM private_ind_bin WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile])) - d.addCallback(fillPrivates) - return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace)) - - def setGenPrivateBinary(self, namespace, key, value): - """Save the general private binary value in database - - @param category: category of the privateeter - @param key: key of the private value - @param value: value to set - @return: deferred - """ - d = self.dbpool.runQuery("REPLACE INTO private_gen_bin(namespace,key,value) VALUES (?,?,?)", (namespace, key, sqlite3.Binary(pickle.dumps(value, 0)))) - d.addErrback(lambda ignore: log.error(_(u"Can't set general private binary value (%(key)s) [namespace:%(namespace)s] in database" % - {"namespace": namespace, "key": key}))) - return d - - def setIndPrivateBinary(self, namespace, key, value, profile): - """Save the individual private binary value in database - - @param namespace: namespace of the value - @param key: key of the private value - @param value: value to set - @param profile: a profile which *must* exist - @return: deferred - """ - d = self.dbpool.runQuery("REPLACE INTO private_ind_bin(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], sqlite3.Binary(pickle.dumps(value, 0)))) - d.addErrback(lambda ignore: log.error(_(u"Can't set individual binary private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % - {"namespace": namespace, "key": key, "profile": profile}))) - return d - - def delGenPrivateBinary(self, namespace, key): - """Delete the general private binary value from database - - @param category: category of the privateeter - @param key: key of the private value - @return: deferred - """ - d = self.dbpool.runQuery("DELETE FROM private_gen_bin WHERE namespace=? AND key=?", (namespace, key)) - d.addErrback(lambda ignore: log.error(_(u"Can't delete general private binary value (%(key)s) [namespace:%(namespace)s] in database" % - {"namespace": namespace, "key": key}))) - return d - - def delIndPrivateBinary(self, namespace, key, profile): - """Delete the individual private binary value from database - - @param namespace: namespace of the value - @param key: key of the private value - @param profile: a profile which *must* exist - @return: deferred - """ - d = self.dbpool.runQuery("DELETE FROM private_ind_bin WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile])) - d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private binary value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" % - {"namespace": namespace, "key": key, "profile": profile}))) - return d ##Helper methods## def __getFirstResult(self, result):