Mercurial > libervia-backend

annotate src/memory/cache.py @ 2503:c0bec8bac2b5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
XEP-0329: implementation of File Information Sharing: the plugin do an implementation for client and component. For client, any directory from local file can be shared. For component file are stored using new file handling mechanism from SàT. Permission is handled, using "public" and "whitelist" for now.
author Goffi <goffi@goffi.org>
date Wed, 28 Feb 2018 18:28:39 +0100
parents 7ad5f2c4e34a
children 516bf5309517
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python2
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SAT: a jabber client
2483
0046283a285d dates update
Goffi <goffi@goffi.org>
parents: 2414
diff changeset
5 # Copyright (C) 2009-2018 Jérôme Poisson (goffi@goffi.org)
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
20 from sat.core.log import getLogger
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
21 log = getLogger(__name__)
2166
1b3fbb76984b core (memory/cache): escape profile when creating cache dir
Goffi <goffi@goffi.org>
parents: 2116
diff changeset
22 from sat.tools.common import regex
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 from sat.core import exceptions
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 from sat.core.constants import Const as C
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 import cPickle as pickle
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
26 import mimetypes
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 import os.path
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 import time
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
29
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
30
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
31 class Cache(object):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
32 """generic file caching"""
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
33
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 def __init__(self, host, profile=None):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 self.profile = profile
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 self.cache_dir = os.path.join(
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 host.memory.getConfig('', 'local_dir'),
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 C.CACHE_DIR,
2166
1b3fbb76984b core (memory/cache): escape profile when creating cache dir
Goffi <goffi@goffi.org>
parents: 2116
diff changeset
39 regex.pathEscape(profile or ''))
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
40 if not os.path.exists(self.cache_dir):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
41 os.makedirs(self.cache_dir)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
42
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
43 def getPath(self, filename):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
44 """return cached file URL
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
45
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
46 @param filename(unicode): cached file name (cache data or actual file)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 """
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 if not filename or u'/' in filename:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 log.error(u"invalid char found in file name, hack attempt? name:{}".format(filename))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 raise exceptions.DataError(u"Invalid char found")
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
51 return os.path.join(self.cache_dir, filename)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
52
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
53 def getFilePath(self, uid):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
54 """retrieve absolute path to file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
55
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
56 @param uid(unicode): unique identifier of file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
57 @return (unicode, None): absolute path to cached file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
58 None if file is not in cache (or cache is invalid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
59 """
2116
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
60 uid = uid.strip()
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
61 if not uid:
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
62 return None
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
63 cache_url = self.getPath(uid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 if not os.path.exists(cache_url):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
66
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
67 try:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 with open(cache_url, 'rb') as f:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 cache_data = pickle.load(f)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 except IOError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 log.warning(u"can't read cache at {}".format(cache_url))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
72 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
73
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
74 except pickle.UnpicklingError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
75 log.warning(u'invalid cache found at {}'.format(cache_url))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
76 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
77
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
78 try:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
79 eol = cache_data['eol']
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
80 except KeyError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
81 log.warning(u'no End Of Life found for cached file {}'.format(uid))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
82 eol = 0
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
83 if eol < time.time():
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
84 log.debug(u"removing expired cache (expired for {}s)".format(
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
85 time.time() - eol))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
86 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
87
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
88 return self.getPath(cache_data['filename'])
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
89
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
90 def cacheData(self, source, uid, mime_type=u'', max_age=None, filename=None):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
91 """create cache metadata and file object to use for actual data
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
92
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
93 @param source(unicode): source of the cache (should be plugin's import_name)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
94 @param uid(unicode): an identifier of the file which must be unique
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
95 @param mime_type(unicode): MIME type of the file to cache
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
96 it will be used notably to guess file extension
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
97 @param max_age(int, None): maximum age in seconds
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
98 the cache metadata will have an "eol" (end of life)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
99 None to use default value
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
100 0 to ignore cache (file will be re-downloaded on each access)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
101 @param filename: if not None, will be used as filename
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
102 else one will be generated from uid and guessed extension
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
103 @return(file): file object opened in write mode
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
104 you have to close it yourself (hint: use with statement)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
105 """
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
106 # FIXME: is it needed to use a separate thread?
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
107 # probably not with the little data expected with BoB
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
108 cache_url = self.getPath(uid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
109 ext = mimetypes.guess_extension(mime_type, strict=False)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
110 if ext is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
111 log.warning(u"can't find extension for MIME type {}".format(mime_type))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
112 ext = '.dump'
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
113 if filename is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
114 filename = uid + ext
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
115 if max_age is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
116 max_age = C.DEFAULT_MAX_AGE
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
117 cache_data = {'source': source,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
118 'filename': filename,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
119 'eol': int(time.time()) + max_age,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
120 'mime_type': mime_type,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
121 }
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
122 file_path = self.getPath(filename)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
123
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
124 with open(cache_url, 'wb') as f:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 pickle.dump(cache_data, f, protocol=2)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
126
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
127 return open(file_path, 'wb')