Mercurial > libervia-backend

comparison src/memory/sqlite.py @ 2182:087eec4c6c07

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict: - merged private values method handling in sqlite, and added a keys arguments to get only some keys - LazyPersistentBinaryDict allow to get values in database only when they are needed, saving memory for big data
author Goffi <goffi@goffi.org>
date Sun, 12 Mar 2017 19:33:17 +0100
parents 1d3f73e065e1
children ea41cf1e6d29
comparison
equal deleted inserted replaced
2181:968b0d13bcc7 2182:087eec4c6c07
518 d.addCallback(self.sqliteHistoryToList) 518 d.addCallback(self.sqliteHistoryToList)
519 d.addCallback(self.listDict2listTuple) 519 d.addCallback(self.listDict2listTuple)
520 return d 520 return d
521 521
522 #Private values 522 #Private values
523 def loadGenPrivates(self, private_gen, namespace): 523
524 """Load general private values 524 def _privateDataEb(self, failure_, operation, namespace, key=None, profile=None):
525 525 """generic errback for data queries"""
526 @param private_gen: dictionary to fill 526 log.error(_(u"Can't {operation} data in database for namespace {namespace}{and_key}{for_profile}: {msg}").format(
527 @param namespace: namespace of the values 527 operation = operation,
528 @return: deferred 528 namespace = namespace,
529 """ 529 and_key = (u" and key " + key) if key is not None else u"",
530 530 for_profile = (u' [' + profile + u']') if profile is not None else u'',
531 def fillPrivates(result): 531 msg = failure_))
532 for private in result: 532
533 key, value = private 533 def _generateDataDict(self, query_result, binary):
534 private_gen[key] = value 534 if binary:
535 log.debug(_(u"loading general private values [namespace: %s] from database") % (namespace,)) 535 return {k: pickle.loads(str(v)) for k,v in query_result}
536 d = self.dbpool.runQuery("SELECT key,value FROM private_gen WHERE namespace=?", (namespace, )).addCallback(fillPrivates) 536 else:
537 return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace)) 537 return dict(query_result)
538 538
539 def loadIndPrivates(self, private_ind, namespace, profile): 539 def _getPrivateTable(self, binary, profile):
540 """Load individual private values 540 """Get table to use for private values"""
541 541 table = [u'private']
542 @param privates_ind: dictionary to fill 542
543 @param namespace: namespace of the values 543 if profile is None:
544 @param profile: a profile which *must* exist 544 table.append(u'gen')
545 @return: deferred 545 else:
546 """ 546 table.append(u'ind')
547 547
548 def fillPrivates(result): 548 if binary:
549 for private in result: 549 table.append(u'bin')
550 key, value = private 550
551 private_ind[key] = value 551 return u'_'.join(table)
552 log.debug(_(u"loading individual private values [namespace: %s] from database") % (namespace,)) 552
553 d = self.dbpool.runQuery("SELECT key,value FROM private_ind WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile])) 553 def getPrivates(self, namespace, keys=None, binary=False, profile=None):
554 d.addCallback(fillPrivates) 554 """Get private value(s) from databases
555 return d.addErrback(lambda x: log.debug(_(u"No data present in database for namespace %s") % namespace)) 555
556 556 @param namespace(unicode): namespace of the values
557 def setGenPrivate(self, namespace, key, value): 557 @param keys(iterable, None): keys of the values to get
558 """Save the general private value in database 558 None to get all keys/values
559 @param binary(bool): True to deserialise binary values
560 @param profile(unicode, None): profile to use for individual values
561 None to use general values
562 @return (dict[unicode, object]): gotten keys/values
563 """
564 log.debug(_(u"getting {type}{binary} private values from database for namespace {namespace}{keys}".format(
565 type = u"general" if profile is None else "individual",
566 binary = u" binary" if binary else u"",
567 namespace = namespace,
568 keys = u" with keys {}".format(u", ".join(keys)) if keys is not None else u"")))
569 table = self._getPrivateTable(binary, profile)
570 query_parts = [u"SELECT key,value FROM", table, "WHERE namespace=?"]
571 args = [namespace]
572
573 if keys is not None:
574 query_parts.append(u'AND key IN ?')
575 args.append(keys)
576
577 if profile is not None:
578 query_parts.append(u'AND profile_id=?')
579 args.append(self.profiles[profile])
580
581 d = self.dbpool.runQuery(u" ".join(query_parts), args)
582 d.addCallback(self._generateDataDict, binary)
583 d.addErrback(self._privateDataEb, u"get", namespace, profile=profile)
584 return d
585
586 def setPrivateValue(self, namespace, key, value, binary=False, profile=None):
587 """Set a private value in database
588
589 @param namespace(unicode): namespace of the values
590 @param key(unicode): key of the value to set
591 @param value(object): value to set
592 @param binary(bool): True if it's a binary values
593 binary values need to be serialised, used for everything but strings
594 @param profile(unicode, None): profile to use for individual value
595 if None, it's a general value
596 """
597 table = self._getPrivateTable(binary, profile)
598 query_values_names = [u'namespace', u'key', u'value']
599 query_values = [namespace, key]
600
601 if binary:
602 value = sqlite3.Binary(pickle.dumps(value, 0))
603
604 query_values.append(value)
605
606 if profile is not None:
607 query_values_names.append(u'profile_id')
608 query_values.append(self.profiles[profile])
609
610 query_parts = [u"REPLACE INTO", table, u'(', u','.join(query_values_names), u')',
611 u"VALUES (", u",".join(u'?'*len(query_values_names)), u')']
612
613 d = self.dbpool.runQuery(u" ".join(query_parts), query_values)
614 d.addErrback(self._privateDataEb, u"set", namespace, key, profile=profile)
615 return d
616
617 def delPrivateValue(self, namespace, key, binary=False, profile=None):
618 """Delete private value from database
559 619
560 @param category: category of the privateeter 620 @param category: category of the privateeter
561 @param key: key of the private value 621 @param key: key of the private value
562 @param value: value to set 622 @param binary(bool): True if it's a binary values
563 @return: deferred 623 @param profile(unicode, None): profile to use for individual value
564 """ 624 if None, it's a general value
565 d = self.dbpool.runQuery("REPLACE INTO private_gen(namespace,key,value) VALUES (?,?,?)", (namespace, key, value)) 625 """
566 d.addErrback(lambda ignore: log.error(_(u"Can't set general private value (%(key)s) [namespace:%(namespace)s] in database" % 626 table = self._getPrivateTable(binary, profile)
567 {"namespace": namespace, "key": key}))) 627 query_parts = [u"DELETE FROM", table, u"WHERE namespace=? AND key=?"]
568 return d 628 args = [namespace, key]
569 629 if profile is not None:
570 def setIndPrivate(self, namespace, key, value, profile): 630 query_parts.append(u"AND profile_id=?")
571 """Save the individual private value in database 631 args.append(self.profiles[profile])
572 632 d = self.dbpool.runQuery(u" ".join(query_parts), args)
573 @param namespace: namespace of the value 633 d.addErrback(self._privateDataEb, u"delete", namespace, key, profile=profile)
574 @param key: key of the private value 634 return d
575 @param value: value to set 635
576 @param profile: a profile which *must* exist
577 @return: deferred
578 """
579 d = self.dbpool.runQuery("REPLACE INTO private_ind(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], value))
580 d.addErrback(lambda ignore: log.error(_(u"Can't set individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
581 {"namespace": namespace, "key": key, "profile": profile})))
582 return d
583
584 def delGenPrivate(self, namespace, key):
585 """Delete the general private value from database
586
587 @param category: category of the privateeter
588 @param key: key of the private value
589 @return: deferred
590 """
591 d = self.dbpool.runQuery("DELETE FROM private_gen WHERE namespace=? AND key=?", (namespace, key))
592 d.addErrback(lambda ignore: log.error(_(u"Can't delete general private value (%(key)s) [namespace:%(namespace)s] in database" %
593 {"namespace": namespace, "key": key})))
594 return d
595
596 def delIndPrivate(self, namespace, key, profile):
597 """Delete the individual private value from database
598
599 @param namespace: namespace of the value
600 @param key: key of the private value
601 @param profile: a profile which *must* exist
602 @return: deferred
603 """
604 d = self.dbpool.runQuery("DELETE FROM private_ind WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile]))
605 d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
606 {"namespace": namespace, "key": key, "profile": profile})))
607 return d
608
609 def loadGenPrivatesBinary(self, private_gen, namespace):
610 """Load general private binary values
611
612 @param private_gen: dictionary to fill
613 @param namespace: namespace of the values
614 @return: deferred
615 """
616
617 def fillPrivates(result):
618 for private in result:
619 key, value = private
620 private_gen[key] = pickle.loads(str(value))
621 log.debug(_(u"loading general private binary values [namespace: %s] from database") % (namespace,))
622 d = self.dbpool.runQuery("SELECT key,value FROM private_gen_bin WHERE namespace=?", (namespace, )).addCallback(fillPrivates)
623 return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace))
624
625 def loadIndPrivatesBinary(self, private_ind, namespace, profile):
626 """Load individual private binary values
627
628 @param privates_ind: dictionary to fill
629 @param namespace: namespace of the values
630 @param profile: a profile which *must* exist
631 @return: deferred
632 """
633
634 def fillPrivates(result):
635 for private in result:
636 key, value = private
637 private_ind[key] = pickle.loads(str(value))
638 log.debug(_(u"loading individual private binary values [namespace: %s] from database") % (namespace,))
639 d = self.dbpool.runQuery("SELECT key,value FROM private_ind_bin WHERE namespace=? AND profile_id=?", (namespace, self.profiles[profile]))
640 d.addCallback(fillPrivates)
641 return d.addErrback(lambda x: log.debug(_(u"No binary data present in database for namespace %s") % namespace))
642
643 def setGenPrivateBinary(self, namespace, key, value):
644 """Save the general private binary value in database
645
646 @param category: category of the privateeter
647 @param key: key of the private value
648 @param value: value to set
649 @return: deferred
650 """
651 d = self.dbpool.runQuery("REPLACE INTO private_gen_bin(namespace,key,value) VALUES (?,?,?)", (namespace, key, sqlite3.Binary(pickle.dumps(value, 0))))
652 d.addErrback(lambda ignore: log.error(_(u"Can't set general private binary value (%(key)s) [namespace:%(namespace)s] in database" %
653 {"namespace": namespace, "key": key})))
654 return d
655
656 def setIndPrivateBinary(self, namespace, key, value, profile):
657 """Save the individual private binary value in database
658
659 @param namespace: namespace of the value
660 @param key: key of the private value
661 @param value: value to set
662 @param profile: a profile which *must* exist
663 @return: deferred
664 """
665 d = self.dbpool.runQuery("REPLACE INTO private_ind_bin(namespace,key,profile_id,value) VALUES (?,?,?,?)", (namespace, key, self.profiles[profile], sqlite3.Binary(pickle.dumps(value, 0))))
666 d.addErrback(lambda ignore: log.error(_(u"Can't set individual binary private value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
667 {"namespace": namespace, "key": key, "profile": profile})))
668 return d
669
670 def delGenPrivateBinary(self, namespace, key):
671 """Delete the general private binary value from database
672
673 @param category: category of the privateeter
674 @param key: key of the private value
675 @return: deferred
676 """
677 d = self.dbpool.runQuery("DELETE FROM private_gen_bin WHERE namespace=? AND key=?", (namespace, key))
678 d.addErrback(lambda ignore: log.error(_(u"Can't delete general private binary value (%(key)s) [namespace:%(namespace)s] in database" %
679 {"namespace": namespace, "key": key})))
680 return d
681
682 def delIndPrivateBinary(self, namespace, key, profile):
683 """Delete the individual private binary value from database
684
685 @param namespace: namespace of the value
686 @param key: key of the private value
687 @param profile: a profile which *must* exist
688 @return: deferred
689 """
690 d = self.dbpool.runQuery("DELETE FROM private_ind_bin WHERE namespace=? AND key=? AND profile=?)", (namespace, key, self.profiles[profile]))
691 d.addErrback(lambda ignore: log.error(_(u"Can't delete individual private binary value (%(key)s) [namespace: %(namespace)s] for [%(profile)s] in database" %
692 {"namespace": namespace, "key": key, "profile": profile})))
693 return d
694 ##Helper methods## 636 ##Helper methods##
695 637
696 def __getFirstResult(self, result): 638 def __getFirstResult(self, result):
697 """Return the first result of a database query 639 """Return the first result of a database query
698 Useful when we are looking for one specific value""" 640 Useful when we are looking for one specific value"""