Mercurial > libervia-backend

comparison src/memory/cache.py @ 2109:85f3e12e984d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core (memory/cache): file caching handling, first draft: instead of having file caching handled individually by plugins, a generic module has been added in memory. - Cache can be global or associated to a profile. In the later case, client.cache can be used. - Cache are managed with unique ids (which can be any unique unicode, hash uuid, or something else). - To know if a file is in cache, getFilePath is used: if the file is in cache, its absolute path is returned, else None is returned. - To cache a file, cacheData is used with at list the source of cache (most of time plugin import name), and unique id. The method return file opened in binary writing mode (so cacheData can - and should - be used with "with" statement). - 2 files will be created: a metadata file (named after the unique id), and the actual file. - each file has a end of life time, after it, the cache is invalidated and the file must be requested again.
author Goffi <goffi@goffi.org>
date Thu, 05 Jan 2017 20:23:38 +0100
parents
children 766dbbec56f2
comparison
equal deleted inserted replaced
2108:70f23bc7859b 2109:85f3e12e984d
1 #!/usr/bin/env python2
2 # -*- coding: utf-8 -*-
3
4 # SAT: a jabber client
5 # Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org)
6
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
16
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
20 from sat.core.log import getLogger
21 log = getLogger(__name__)
22 from sat.core import exceptions
23 from sat.core.constants import Const as C
24 import cPickle as pickle
25 import mimetypes
26 import os.path
27 import time
28
29
30 class Cache(object):
31 """generic file caching"""
32
33 def __init__(self, host, profile=None):
34 host = host
35 self.profile = profile
36 self.cache_dir = os.path.join(
37 host.memory.getConfig('', 'local_dir'),
38 C.CACHE_DIR,
39 profile or '')
40 if not os.path.exists(self.cache_dir):
41 os.makedirs(self.cache_dir)
42
43 def getPath(self, filename):
44 """return cached file URL
45
46 @param filename(unicode): cached file name (cache data or actual file)
47 """
48 if not filename or u'/' in filename:
49 log.error(u"invalid char found in file name, hack attempt? name:{}".format(filename))
50 raise exceptions.DataError(u"Invalid char found")
51 return os.path.join(self.cache_dir, filename)
52
53 def getFilePath(self, uid):
54 """retrieve absolute path to file
55
56 @param uid(unicode): unique identifier of file
57 @return (unicode, None): absolute path to cached file
58 None if file is not in cache (or cache is invalid)
59 """
60 cache_url = self.getPath(uid)
61 if not os.path.exists(cache_url):
62 return None
63
64 try:
65 with open(cache_url, 'rb') as f:
66 cache_data = pickle.load(f)
67 except IOError:
68 log.warning(u"can't read cache at {}".format(cache_url))
69 return None
70
71 except pickle.UnpicklingError:
72 log.warning(u'invalid cache found at {}'.format(cache_url))
73 return None
74
75 try:
76 eol = cache_data['eol']
77 except KeyError:
78 log.warning(u'no End Of Life found for cached file {}'.format(uid))
79 eol = 0
80 if eol < time.time():
81 log.debug(u"removing expired cache (expired for {}s)".format(
82 time.time() - eol))
83 return None
84
85 return self.getPath(cache_data['filename'])
86
87 def cacheData(self, source, uid, mime_type=u'', max_age=None, filename=None):
88 """create cache metadata and file object to use for actual data
89
90 @param source(unicode): source of the cache (should be plugin's import_name)
91 @param uid(unicode): an identifier of the file which must be unique
92 @param mime_type(unicode): MIME type of the file to cache
93 it will be used notably to guess file extension
94 @param max_age(int, None): maximum age in seconds
95 the cache metadata will have an "eol" (end of life)
96 None to use default value
97 0 to ignore cache (file will be re-downloaded on each access)
98 @param filename: if not None, will be used as filename
99 else one will be generated from uid and guessed extension
100 @return(file): file object opened in write mode
101 you have to close it yourself (hint: use with statement)
102 """
103 # FIXME: is it needed to use a separate thread?
104 # probably not with the little data expected with BoB
105 cache_url = self.getPath(uid)
106 ext = mimetypes.guess_extension(mime_type, strict=False)
107 if ext is None:
108 log.warning(u"can't find extension for MIME type {}".format(mime_type))
109 ext = '.dump'
110 if filename is None:
111 filename = uid + ext
112 if max_age is None:
113 max_age = C.DEFAULT_MAX_AGE
114 cache_data = {'source': source,
115 'filename': filename,
116 'eol': int(time.time()) + max_age,
117 'mime_type': mime_type,
118 }
119 file_path = self.getPath(filename)
120
121 with open(cache_url, 'wb') as f:
122 pickle.dump(cache_data, f, protocol=2)
123
124 return open(file_path, 'wb')