Mercurial > libervia-backend

comparison sat/memory/cache.py @ 3188:a15773c6c273

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
memory(cache): extend EOL when a file metadata is retrieved
author Goffi <goffi@goffi.org>
date Wed, 26 Feb 2020 22:03:15 +0100
parents 554b3b632378
children 08151c103636
comparison
equal deleted inserted replaced
3187:d92a144f3589 3188:a15773c6c273
109 "invalid char found in file name, hack attempt? name:{}".format(filename) 109 "invalid char found in file name, hack attempt? name:{}".format(filename)
110 ) 110 )
111 raise exceptions.DataError("Invalid char found") 111 raise exceptions.DataError("Invalid char found")
112 return self.cache_dir / filename 112 return self.cache_dir / filename
113 113
114 def getMetadata(self, uid): 114 def getMetadata(self, uid, update_eol=True):
115 """Retrieve metadata for cached data 115 """Retrieve metadata for cached data
116 116
117 @param uid(unicode): unique identifier of file 117 @param uid(unicode): unique identifier of file
118 @param update_eol(bool): True if eol must extended
119 if True, max_age will be added to eol (only if it is not already expired)
118 @return (dict, None): metadata with following keys: 120 @return (dict, None): metadata with following keys:
119 see [cacheData] for data details, an additional "path" key is the full path to cached file. 121 see [cacheData] for data details, an additional "path" key is the full path to
122 cached file.
120 None if file is not in cache (or cache is invalid) 123 None if file is not in cache (or cache is invalid)
121 """ 124 """
122 125
123 uid = uid.strip() 126 uid = uid.strip()
124 if not uid: 127 if not uid:
125 raise exceptions.InternalError("uid must not be empty") 128 raise exceptions.InternalError("uid must not be empty")
126 cache_url = self.getPath(uid) 129 cache_url = self.getPath(uid)
127 if not cache_url.exists: 130 if not cache_url.exists():
128 return None 131 return None
129 132
130 try: 133 try:
131 with cache_url.open("rb") as f: 134 with cache_url.open("rb") as f:
132 cache_data = pickle.load(f) 135 cache_data = pickle.load(f)
133 except IOError: 136 except IOError as e:
134 log.warning("can't read cache at {}".format(cache_url)) 137 log.warning(f"can't read cache at {cache_url}: {e}")
135 return None 138 return None
136 except pickle.UnpicklingError: 139 except pickle.UnpicklingError:
137 log.warning("invalid cache found at {}".format(cache_url)) 140 log.warning(f"invalid cache found at {cache_url}")
138 return None 141 return None
139 142
140 try: 143 try:
141 eol = cache_data["eol"] 144 eol = cache_data["eol"]
142 except KeyError: 145 except KeyError:
145 if eol < time.time(): 148 if eol < time.time():
146 log.debug( 149 log.debug(
147 "removing expired cache (expired for {}s)".format(time.time() - eol) 150 "removing expired cache (expired for {}s)".format(time.time() - eol)
148 ) 151 )
149 return None 152 return None
153
154 if update_eol:
155 try:
156 max_age = cache_data["max_age"]
157 except KeyError:
158 log.warning(f"no max_age found for cache at {cache_url}, using default")
159 max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE
160 cache_data["eol"] = int(time.time()) + max_age
161 with cache_url.open("wb") as f:
162 pickle.dump(cache_data, f, protocol=2)
150 163
151 cache_data["path"] = self.getPath(cache_data["filename"]) 164 cache_data["path"] = self.getPath(cache_data["filename"])
152 return cache_data 165 return cache_data
153 166
154 def getFilePath(self, uid): 167 def getFilePath(self, uid):
197 max_age = C.DEFAULT_MAX_AGE 210 max_age = C.DEFAULT_MAX_AGE
198 cache_data = { 211 cache_data = {
199 "source": source, 212 "source": source,
200 "filename": filename, 213 "filename": filename,
201 "eol": int(time.time()) + max_age, 214 "eol": int(time.time()) + max_age,
215 # we also store max_age for updating eol
216 "max_age": max_age,
202 "mime_type": mime_type, 217 "mime_type": mime_type,
203 } 218 }
204 file_path = self.getPath(filename) 219 file_path = self.getPath(filename)
205 220
206 with open(cache_url, "wb") as f: 221 with open(cache_url, "wb") as f: