Mercurial > libervia-backend

comparison src/memory/persistent.py @ 2182:087eec4c6c07

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict: - merged private values method handling in sqlite, and added a keys arguments to get only some keys - LazyPersistentBinaryDict allow to get values in database only when they are needed, saving memory for big data
author Goffi <goffi@goffi.org>
date Sun, 12 Mar 2017 19:33:17 +0100
parents 2daf7b4c6756
children ea41cf1e6d29
comparison
equal deleted inserted replaced
2181:968b0d13bcc7 2182:087eec4c6c07
26 pass 26 pass
27 27
28 28
29 class PersistentDict(object): 29 class PersistentDict(object):
30 r"""A dictionary which save persistently each value assigned 30 r"""A dictionary which save persistently each value assigned
31
31 /!\ be careful, each assignment means a database write 32 /!\ be careful, each assignment means a database write
32 /!\ Memory must be initialised before loading/setting value with instances of this class""" 33 /!\ Memory must be initialised before loading/setting value with instances of this class"""
33 storage = None 34 storage = None
35 binary = False
34 36
35 def __init__(self, namespace, profile=None): 37 def __init__(self, namespace, profile=None):
36 """@param namespace: unique namespace for this dictionary 38 """
37 @param profile: profile which *MUST* exists, or None for general values""" 39
40 @param namespace: unique namespace for this dictionary
41 @param profile(unicode, None): profile which *MUST* exists, or None for general values
42 """
38 if not self.storage: 43 if not self.storage:
39 log.error(_("PersistentDict can't be used before memory initialisation")) 44 log.error(_("PersistentDict can't be used before memory initialisation"))
40 raise MemoryNotInitializedError 45 raise MemoryNotInitializedError
41 self._cache = {} 46 self._cache = None
42 self.namespace = namespace 47 self.namespace = namespace
43 self.profile = profile 48 self.profile = profile
44 49
50 def _setCache(self, data):
51 self._cache = data
52
45 def load(self): 53 def load(self):
46 """Load persistent data from storage. 54 """Load persistent data from storage.
47 55
56 need to be called before any other operation
48 @return: defers the PersistentDict instance itself 57 @return: defers the PersistentDict instance itself
49 """ 58 """
50 if not self.profile: 59 d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
51 d = self.storage.loadGenPrivates(self._cache, self.namespace) 60 d.addCallback(self._setCache)
52 else: 61 d.addCallback(lambda dummy: self)
53 d = self.storage.loadIndPrivates(self._cache, self.namespace, self.profile) 62 return d
54 return d.addCallback(lambda dummy: self)
55 63
56 def iteritems(self): 64 def iteritems(self):
57 return self._cache.iteritems() 65 return self._cache.iteritems()
58 66
59 def items(self): 67 def items(self):
100 108
101 def __getitem__(self, key): 109 def __getitem__(self, key):
102 return self._cache.__getitem__(key) 110 return self._cache.__getitem__(key)
103 111
104 def __setitem__(self, key, value): 112 def __setitem__(self, key, value):
105 if not self.profile: 113 self.storage.setPrivateValue(self.namespace, key, value, self.binary, self.profile)
106 self.storage.setGenPrivate(self.namespace, key, value)
107 else:
108 self.storage.setIndPrivate(self.namespace, key, value, self.profile)
109 return self._cache.__setitem__(key, value) 114 return self._cache.__setitem__(key, value)
110 115
111 def __delitem__(self, key): 116 def __delitem__(self, key):
112 if not self.profile: 117 self.storage.detPrivateValue(self.namepace, key, self.binary, self.profile)
113 self.storage.delGenPrivate(self.namespace, key)
114 else:
115 self.storage.delIndPrivate(self.namespace, key, self.profile)
116 return self._cache.__delitem__(key) 118 return self._cache.__delitem__(key)
117 119
118 def get(self, key, default=None): 120 def get(self, key, default=None):
119 return self._cache.get(key, default) 121 return self._cache.get(key, default)
120 122
125 self.__setitem__(key, default) 127 self.__setitem__(key, default)
126 return default 128 return default
127 129
128 def force(self, name): 130 def force(self, name):
129 """Force saving of an attribute to storage 131 """Force saving of an attribute to storage
130 @return: deferred fired when data is actually saved""" 132
131 if not self.profile: 133 @return: deferred fired when data is actually saved
132 return self.storage.setGenPrivate(self.namespace, name, self._cache[name]) 134 """
133 return self.storage.setIndPrivate(self.namespace, name, self._cache[name], self.profile) 135 return self.storage.setPrivateValue(self.namespace, name, self._cache[name], self.binary, self.profile)
134 136
135 137
136 class PersistentBinaryDict(PersistentDict): 138 class PersistentBinaryDict(PersistentDict):
137 """Persistent dict where value can be any python data (instead of string only)""" 139 """Persistent dict where value can be any python data (instead of string only)"""
140 binary = True
141
142
143 class LazyPersistentBinaryDict(PersistentBinaryDict):
144 ur"""PersistentBinaryDict which get key/value when needed
145
146 This Persistent need more database access, it is suitable for largest data,
147 to save memory.
148 /!\ most of methods return a Deferred
149 """
150 # TODO: missing methods should be implemented using database access
151 # TODO: a cache would be useful (which is deleted after a timeout)
138 152
139 def load(self): 153 def load(self):
140 """load persistent data from storage 154 # we show a warning as calling load on LazyPersistentBinaryDict sounds like a code mistake
141 """ 155 log.warning(_(u"Calling load on LazyPersistentBinaryDict while it's not needed"))
142 if not self.profile: 156
143 return self.storage.loadGenPrivatesBinary(self._cache, self.namespace) 157 def iteritems(self):
144 else: 158 raise NotImplementedError
145 return self.storage.loadIndPrivatesBinary(self._cache, self.namespace, self.profile) 159
160 def items(self):
161 return self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
162
163 def __repr__(self):
164 raise NotImplementedError
165
166 def __str__(self):
167 return "LazyPersistentBinaryDict (namespace: {})".format(self.namespace)
168
169 def __lt__(self, other):
170 raise NotImplementedError
171
172 def __le__(self, other):
173 raise NotImplementedError
174
175 def __eq__(self, other):
176 raise NotImplementedError
177
178 def __ne__(self, other):
179 raise NotImplementedError
180
181 def __gt__(self, other):
182 raise NotImplementedError
183
184 def __ge__(self, other):
185 raise NotImplementedError
186
187 def __cmp__(self, other):
188 raise NotImplementedError
189
190 def __hash__(self):
191 return hash(unicode(self.__class__) + self.namespace + (self.profile or u''))
192
193 def __nonzero__(self):
194 raise NotImplementedError
195
196 def __contains__(self, key):
197 raise NotImplementedError
198
199 def __iter__(self):
200 raise NotImplementedError
201
202 def __getitem__(self, key):
203 """get the value as a Deferred"""
204 return self.storage.getPrivates(self.namespace, keys=[key], binary=self.binary, profile=self.profile)
146 205
147 def __setitem__(self, key, value): 206 def __setitem__(self, key, value):
148 if not self.profile: 207 self.storage.setPrivateValue(self.namespace, key, value, self.binary, self.profile)
149 self.storage.setGenPrivateBinary(self.namespace, key, value)
150 else:
151 self.storage.setIndPrivateBinary(self.namespace, key, value, self.profile)
152 return self._cache.__setitem__(key, value)
153 208
154 def __delitem__(self, key): 209 def __delitem__(self, key):
155 if not self.profile: 210 self.storage.detPrivateValue(self.namepace, key, self.binary, self.profile)
156 self.storage.delGenPrivateBinary(self.namespace, key) 211
157 else: 212 def _valueOrDefault(self, value, default):
158 self.storage.delIndPrivateBinary(self.namespace, key, self.profile) 213 if value is None:
159 return self._cache.__delitem__(key) 214 return default
160 215 return value
161 def force(self, name): 216
217 def get(self, key, default=None):
218 d = self.__getitem__(key)
219 d.addCallback(self._valueOrDefault)
220 return d
221
222 def setdefault(self, key, default):
223 raise NotImplementedError
224
225 def force(self, name, value):
162 """Force saving of an attribute to storage 226 """Force saving of an attribute to storage
163 @return: deferred fired when data is actually saved""" 227
164 if not self.profile: 228 @param value(object): value is needed for LazyPersistentBinaryDict
165 return self.storage.setGenPrivateBinary(self.namespace, name, self._cache[name]) 229 @return: deferred fired when data is actually saved
166 return self.storage.setIndPrivateBinary(self.namespace, name, self._cache[name], self.profile) 230 """
231 return self.storage.setPrivateValue(self.namespace, name, value, self.binary, self.profile)