Mercurial > libervia-backend
annotate src/memory/cache.py @ 2109:85f3e12e984d
core (memory/cache): file caching handling, first draft:
instead of having file caching handled individually by plugins, a generic module has been added in memory.
- Cache can be global or associated to a profile. In the later case, client.cache can be used.
- Cache are managed with unique ids (which can be any unique unicode, hash uuid, or something else).
- To know if a file is in cache, getFilePath is used: if the file is in cache, its absolute path is returned, else None is returned.
- To cache a file, cacheData is used with at list the source of cache (most of time plugin import name), and unique id. The method return file opened in binary writing mode (so cacheData can - and should - be used with "with" statement).
- 2 files will be created: a metadata file (named after the unique id), and the actual file.
- each file has a end of life time, after it, the cache is invalidated and the file must be requested again.
author | Goffi <goffi@goffi.org> |
---|---|
date | Thu, 05 Jan 2017 20:23:38 +0100 |
parents | |
children | 766dbbec56f2 |
rev | line source |
---|---|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python2 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SAT: a jabber client |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 # Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
20 from sat.core.log import getLogger |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 log = getLogger(__name__) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.core import exceptions |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from sat.core.constants import Const as C |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 import cPickle as pickle |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 import mimetypes |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 import os.path |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 import time |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 class Cache(object): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 """generic file caching""" |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 def __init__(self, host, profile=None): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 host = host |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 self.profile = profile |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 self.cache_dir = os.path.join( |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 host.memory.getConfig('', 'local_dir'), |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 C.CACHE_DIR, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 profile or '') |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 if not os.path.exists(self.cache_dir): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 os.makedirs(self.cache_dir) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 def getPath(self, filename): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
44 """return cached file URL |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 @param filename(unicode): cached file name (cache data or actual file) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 """ |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 if not filename or u'/' in filename: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 log.error(u"invalid char found in file name, hack attempt? name:{}".format(filename)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 raise exceptions.DataError(u"Invalid char found") |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 return os.path.join(self.cache_dir, filename) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 def getFilePath(self, uid): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 """retrieve absolute path to file |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
55 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 @param uid(unicode): unique identifier of file |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
57 @return (unicode, None): absolute path to cached file |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
58 None if file is not in cache (or cache is invalid) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
59 """ |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 cache_url = self.getPath(uid) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
61 if not os.path.exists(cache_url): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 try: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 with open(cache_url, 'rb') as f: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 cache_data = pickle.load(f) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 except IOError: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 log.warning(u"can't read cache at {}".format(cache_url)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 except pickle.UnpicklingError: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 log.warning(u'invalid cache found at {}'.format(cache_url)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
73 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
74 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
75 try: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 eol = cache_data['eol'] |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
77 except KeyError: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
78 log.warning(u'no End Of Life found for cached file {}'.format(uid)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
79 eol = 0 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
80 if eol < time.time(): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
81 log.debug(u"removing expired cache (expired for {}s)".format( |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
82 time.time() - eol)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
83 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
85 return self.getPath(cache_data['filename']) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
86 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
87 def cacheData(self, source, uid, mime_type=u'', max_age=None, filename=None): |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
88 """create cache metadata and file object to use for actual data |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
89 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
90 @param source(unicode): source of the cache (should be plugin's import_name) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
91 @param uid(unicode): an identifier of the file which must be unique |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
92 @param mime_type(unicode): MIME type of the file to cache |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
93 it will be used notably to guess file extension |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
94 @param max_age(int, None): maximum age in seconds |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
95 the cache metadata will have an "eol" (end of life) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
96 None to use default value |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
97 0 to ignore cache (file will be re-downloaded on each access) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
98 @param filename: if not None, will be used as filename |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
99 else one will be generated from uid and guessed extension |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
100 @return(file): file object opened in write mode |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
101 you have to close it yourself (hint: use with statement) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
102 """ |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
103 # FIXME: is it needed to use a separate thread? |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
104 # probably not with the little data expected with BoB |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
105 cache_url = self.getPath(uid) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
106 ext = mimetypes.guess_extension(mime_type, strict=False) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
107 if ext is None: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
108 log.warning(u"can't find extension for MIME type {}".format(mime_type)) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
109 ext = '.dump' |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
110 if filename is None: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
111 filename = uid + ext |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 if max_age is None: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 max_age = C.DEFAULT_MAX_AGE |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
114 cache_data = {'source': source, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
115 'filename': filename, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 'eol': int(time.time()) + max_age, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
117 'mime_type': mime_type, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
118 } |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
119 file_path = self.getPath(filename) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
120 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
121 with open(cache_url, 'wb') as f: |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
122 pickle.dump(cache_data, f, protocol=2) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
123 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
124 return open(file_path, 'wb') |