Mercurial > libervia-backend

annotate src/memory/cache.py @ 2420:03da3ef5fb5b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
plugin tickets: added ticketsSet and ticketsSchemaGet methods: those methods are high level methods specialised for tickets. ticketsSet will use default tickets node if node is not set, set "created" and "updated" fields, create comments node if the ticket is new, and associate it with "comments_uri" field. ticketsSchemaGet is like getUISchema with node defaulting to tickets default node.
author Goffi <goffi@goffi.org>
date Sun, 05 Nov 2017 15:36:06 +0100
parents 8b37a62336c3
children 0046283a285d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
1 #!/usr/bin/env python2
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
3
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
4 # SAT: a jabber client
2414
8b37a62336c3 misc: date update (yes it's a bit late :p )
Goffi <goffi@goffi.org>
parents: 2166
diff changeset
5 # Copyright (C) 2009-2017 Jérôme Poisson (goffi@goffi.org)
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
6
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
7 # This program is free software: you can redistribute it and/or modify
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
8 # it under the terms of the GNU Affero General Public License as published by
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
9 # the Free Software Foundation, either version 3 of the License, or
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
10 # (at your option) any later version.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
11
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
12 # This program is distributed in the hope that it will be useful,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
15 # GNU Affero General Public License for more details.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
16
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
17 # You should have received a copy of the GNU Affero General Public License
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
19
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
20 from sat.core.log import getLogger
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
21 log = getLogger(__name__)
2166
1b3fbb76984b core (memory/cache): escape profile when creating cache dir
Goffi <goffi@goffi.org>
parents: 2116
diff changeset
22 from sat.tools.common import regex
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
23 from sat.core import exceptions
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
24 from sat.core.constants import Const as C
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
25 import cPickle as pickle
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
26 import mimetypes
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
27 import os.path
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
28 import time
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
29
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
30
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
31 class Cache(object):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
32 """generic file caching"""
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
33
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
34 def __init__(self, host, profile=None):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
35 host = host
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
36 self.profile = profile
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
37 self.cache_dir = os.path.join(
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
38 host.memory.getConfig('', 'local_dir'),
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
39 C.CACHE_DIR,
2166
1b3fbb76984b core (memory/cache): escape profile when creating cache dir
Goffi <goffi@goffi.org>
parents: 2116
diff changeset
40 regex.pathEscape(profile or ''))
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
41 if not os.path.exists(self.cache_dir):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
42 os.makedirs(self.cache_dir)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
43
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
44 def getPath(self, filename):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
45 """return cached file URL
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
46
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
47 @param filename(unicode): cached file name (cache data or actual file)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
48 """
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
49 if not filename or u'/' in filename:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
50 log.error(u"invalid char found in file name, hack attempt? name:{}".format(filename))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
51 raise exceptions.DataError(u"Invalid char found")
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
52 return os.path.join(self.cache_dir, filename)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
53
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
54 def getFilePath(self, uid):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
55 """retrieve absolute path to file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
56
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
57 @param uid(unicode): unique identifier of file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
58 @return (unicode, None): absolute path to cached file
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
59 None if file is not in cache (or cache is invalid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
60 """
2116
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
61 uid = uid.strip()
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
62 if not uid:
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents: 2109
diff changeset
63 return None
2109
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
64 cache_url = self.getPath(uid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
65 if not os.path.exists(cache_url):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
66 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
67
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
68 try:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
69 with open(cache_url, 'rb') as f:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
70 cache_data = pickle.load(f)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
71 except IOError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
72 log.warning(u"can't read cache at {}".format(cache_url))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
73 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
74
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
75 except pickle.UnpicklingError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
76 log.warning(u'invalid cache found at {}'.format(cache_url))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
77 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
78
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
79 try:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
80 eol = cache_data['eol']
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
81 except KeyError:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
82 log.warning(u'no End Of Life found for cached file {}'.format(uid))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
83 eol = 0
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
84 if eol < time.time():
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
85 log.debug(u"removing expired cache (expired for {}s)".format(
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
86 time.time() - eol))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
87 return None
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
88
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
89 return self.getPath(cache_data['filename'])
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
90
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
91 def cacheData(self, source, uid, mime_type=u'', max_age=None, filename=None):
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
92 """create cache metadata and file object to use for actual data
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
93
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
94 @param source(unicode): source of the cache (should be plugin's import_name)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
95 @param uid(unicode): an identifier of the file which must be unique
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
96 @param mime_type(unicode): MIME type of the file to cache
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
97 it will be used notably to guess file extension
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
98 @param max_age(int, None): maximum age in seconds
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
99 the cache metadata will have an "eol" (end of life)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
100 None to use default value
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
101 0 to ignore cache (file will be re-downloaded on each access)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
102 @param filename: if not None, will be used as filename
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
103 else one will be generated from uid and guessed extension
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
104 @return(file): file object opened in write mode
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
105 you have to close it yourself (hint: use with statement)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
106 """
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
107 # FIXME: is it needed to use a separate thread?
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
108 # probably not with the little data expected with BoB
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
109 cache_url = self.getPath(uid)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
110 ext = mimetypes.guess_extension(mime_type, strict=False)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
111 if ext is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
112 log.warning(u"can't find extension for MIME type {}".format(mime_type))
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
113 ext = '.dump'
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
114 if filename is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
115 filename = uid + ext
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
116 if max_age is None:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
117 max_age = C.DEFAULT_MAX_AGE
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
118 cache_data = {'source': source,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
119 'filename': filename,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
120 'eol': int(time.time()) + max_age,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
121 'mime_type': mime_type,
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
122 }
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
123 file_path = self.getPath(filename)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
124
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
125 with open(cache_url, 'wb') as f:
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
126 pickle.dump(cache_data, f, protocol=2)
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
127
85f3e12e984d core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff changeset
128 return open(file_path, 'wb')