Mercurial > libervia-backend
annotate sat/memory/cache.py @ 3837:56720561f45f
test (unit/AP gateway): tests for XMPP reference/mention <=> AP mention conversion:
this patch add 5 tests:
- 2 to mention AP mention to XMPP conversion, with direct addressing or using "tag" field
- 2 to test automatic mention parsing of body
- 1 to test XEP-0372 Reference to AP mention conversion
rel 369
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sun, 10 Jul 2022 16:15:09 +0200 |
| parents | 2863345c9bbb |
| children | 524856bd7b19 |
| rev | line source |
|---|---|
| 3028 | 1 #!/usr/bin/env python3 |
| 3137 | 2 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SAT: a jabber client |
| 3479 | 5 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
20 from io import BufferedIOBase |
| 3185 | 21 import mimetypes |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
22 from pathlib import Path |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
23 import pickle as pickle |
| 3185 | 24 import time |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
25 from typing import Any, Dict, Optional |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
26 |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
27 from sat.core import exceptions |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
28 from sat.core.constants import Const as C |
| 3185 | 29 from sat.core.i18n import _ |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 from sat.core.log import getLogger |
| 3185 | 31 from sat.tools.common import regex |
| 32 | |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
33 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 log = getLogger(__name__) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
36 DEFAULT_EXT = ".raw" |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
37 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 class Cache(object): |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 """generic file caching""" |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
42 def __init__(self, host, profile): |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
43 """ |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
44 @param profile(unicode, None): name of the profile to set the cache for |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
45 if None, the cache will be common for all profiles |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
46 """ |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 self.profile = profile |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
48 path_elts = [host.memory.getConfig("", "local_dir"), C.CACHE_DIR] |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
49 if profile: |
| 3028 | 50 path_elts.extend(["profiles", regex.pathEscape(profile)]) |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
51 else: |
| 3028 | 52 path_elts.append("common") |
| 3185 | 53 self.cache_dir = Path(*path_elts) |
| 54 | |
| 55 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True) | |
| 56 self.purge() | |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
57 |
| 3185 | 58 def purge(self): |
| 59 # remove expired files from cache | |
| 60 # TODO: this should not be called only on startup, but at regular interval | |
| 61 # (e.g. once a day) | |
| 62 purged = set() | |
| 63 # we sort files to have metadata files first | |
| 64 for cache_file in sorted(self.cache_dir.iterdir()): | |
| 65 if cache_file in purged: | |
| 66 continue | |
| 67 try: | |
| 68 with cache_file.open('rb') as f: | |
| 69 cache_data = pickle.load(f) | |
| 70 except IOError: | |
| 71 log.warning( | |
| 72 _("Can't read metadata file at {path}") | |
| 73 .format(path=cache_file)) | |
| 74 continue | |
| 75 except (pickle.UnpicklingError, EOFError): | |
| 76 log.debug(f"File at {cache_file} is not a metadata file") | |
| 77 continue | |
| 78 try: | |
| 79 eol = cache_data['eol'] | |
| 80 filename = cache_data['filename'] | |
| 81 except KeyError: | |
| 82 log.warning( | |
| 83 _("Invalid cache metadata at {path}") | |
| 84 .format(path=cache_file)) | |
| 85 continue | |
| 86 | |
|
3209
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
87 filepath = self.getPath(filename) |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
88 |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
89 if not filepath.exists(): |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
90 log.warning(_( |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
91 "cache {cache_file!r} references an inexisting file: {filepath!r}" |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
92 ).format(cache_file=str(cache_file), filepath=str(filepath))) |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
93 log.debug("purging cache with missing file") |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
94 cache_file.unlink() |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
95 elif eol < time.time(): |
| 3185 | 96 log.debug( |
| 97 "purging expired cache {filepath!r} (expired for {time}s)" | |
| 98 .format(filepath=str(filepath), time=int(time.time() - eol)) | |
| 99 ) | |
| 100 cache_file.unlink() | |
| 101 try: | |
| 102 filepath.unlink() | |
| 103 except FileNotFoundError: | |
| 104 log.warning( | |
| 105 _("following file is missing while purging cache: {path}") | |
| 106 .format(path=filepath) | |
| 107 ) | |
| 108 purged.add(cache_file) | |
| 109 purged.add(filepath) | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
110 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
111 def getPath(self, filename: str) -> Path: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 """return cached file URL |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
114 @param filename: cached file name (cache data or actual file) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
115 @return: path to the cached file |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 """ |
| 3028 | 117 if not filename or "/" in filename: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
118 log.error( |
| 3028 | 119 "invalid char found in file name, hack attempt? name:{}".format(filename) |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
120 ) |
| 3028 | 121 raise exceptions.DataError("Invalid char found") |
| 3185 | 122 return self.cache_dir / filename |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
123 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
124 def getMetadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]: |
| 3185 | 125 """Retrieve metadata for cached data |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
126 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
127 @param uid(unicode): unique identifier of file |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
128 @param update_eol(bool): True if eol must extended |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
129 if True, max_age will be added to eol (only if it is not already expired) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
130 @return (dict, None): metadata with following keys: |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
131 see [cacheData] for data details, an additional "path" key is the full path to |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
132 cached file. |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
133 None if file is not in cache (or cache is invalid) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
134 """ |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
135 |
|
2116
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
136 uid = uid.strip() |
|
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
137 if not uid: |
| 3028 | 138 raise exceptions.InternalError("uid must not be empty") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
139 cache_url = self.getPath(uid) |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
140 if not cache_url.exists(): |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
141 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
142 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
143 try: |
| 3185 | 144 with cache_url.open("rb") as f: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
145 cache_data = pickle.load(f) |
|
3572
b3fa179417e7
core (memory/cache): don't crash on EOFError in getMetadata
Goffi <goffi@goffi.org>
parents:
3479
diff
changeset
|
146 except (IOError, EOFError) as e: |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
147 log.warning(f"can't read cache at {cache_url}: {e}") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
148 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
149 except pickle.UnpicklingError: |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
150 log.warning(f"invalid cache found at {cache_url}") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
152 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
153 try: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
154 eol = cache_data["eol"] |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
155 except KeyError: |
| 3028 | 156 log.warning("no End Of Life found for cached file {}".format(uid)) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 eol = 0 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 if eol < time.time(): |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
159 log.debug( |
| 3028 | 160 "removing expired cache (expired for {}s)".format(time.time() - eol) |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
161 ) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
162 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
163 |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
164 if update_eol: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
165 try: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
166 max_age = cache_data["max_age"] |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
167 except KeyError: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
168 log.warning(f"no max_age found for cache at {cache_url}, using default") |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
169 max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
170 now = int(time.time()) |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
171 cache_data["last_access"] = now |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
172 cache_data["eol"] = now + max_age |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
173 with cache_url.open("wb") as f: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
174 pickle.dump(cache_data, f, protocol=2) |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
175 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
176 cache_data["path"] = self.getPath(cache_data["filename"]) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
177 return cache_data |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
178 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
179 def getFilePath(self, uid: str) -> Path: |
| 3185 | 180 """Retrieve absolute path to file |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
181 |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
182 @param uid(unicode): unique identifier of file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
183 @return (unicode, None): absolute path to cached file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
184 None if file is not in cache (or cache is invalid) |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
185 """ |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
186 metadata = self.getMetadata(uid) |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
187 if metadata is not None: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
188 return metadata["path"] |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
189 |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
190 def removeFromCache(self, uid, metadata=None): |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
191 """Remove data from cache |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
192 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
193 @param uid(unicode): unique identifier cache file |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
194 """ |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
195 cache_data = self.getMetadata(uid, update_eol=False) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
196 if cache_data is None: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
197 log.debug(f"cache with uid {uid!r} has already expired or been removed") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
198 return |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
199 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
200 try: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
201 filename = cache_data['filename'] |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
202 except KeyError: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
203 log.warning(_("missing filename for cache {uid!r}") .format(uid=uid)) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
204 else: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
205 filepath = self.getPath(filename) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
206 try: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
207 filepath.unlink() |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
208 except FileNotFoundError: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
209 log.warning( |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
210 _("missing file referenced in cache {uid!r}: {filename}") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
211 .format(uid=uid, filename=filename) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
212 ) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
213 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
214 cache_file = self.getPath(uid) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
215 cache_file.unlink() |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
216 log.debug(f"cache with uid {uid!r} has been removed") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
217 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
218 def cacheData( |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
219 self, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
220 source: str, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
221 uid: str, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
222 mime_type: Optional[str] = None, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
223 max_age: Optional[int] = None, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
224 original_filename: Optional[str] = None |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
225 ) -> BufferedIOBase: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 """create cache metadata and file object to use for actual data |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
228 @param source: source of the cache (should be plugin's import_name) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
229 @param uid: an identifier of the file which must be unique |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
230 @param mime_type: MIME type of the file to cache |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
231 it will be used notably to guess file extension |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
232 It may be autogenerated if filename is specified |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
233 @param max_age: maximum age in seconds |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
234 the cache metadata will have an "eol" (end of life) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
235 None to use default value |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 0 to ignore cache (file will be re-downloaded on each access) |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
237 @param original_filename: if not None, will be used to retrieve file extension and |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
238 guess |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
239 mime type, and stored in "original_filename" |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
240 @return: file object opened in write mode |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
241 you have to close it yourself (hint: use ``with`` statement) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
242 """ |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 if max_age is None: |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
244 max_age = C.DEFAULT_MAX_AGE |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
245 cache_data = { |
| 3028 | 246 "source": source, |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
247 # we also store max_age for updating eol |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
248 "max_age": max_age, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
249 } |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
250 cache_url = self.getPath(uid) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
251 if original_filename is not None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
252 cache_data["original_filename"] = original_filename |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
253 if mime_type is None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
254 # we have original_filename but not MIME type, we try to guess the later |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
255 mime_type = mimetypes.guess_type(original_filename, strict=False)[0] |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
256 if mime_type: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
257 ext = mimetypes.guess_extension(mime_type, strict=False) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
258 if ext is None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
259 log.warning( |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
260 "can't find extension for MIME type {}".format(mime_type) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
261 ) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
262 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
263 elif ext == ".jpe": |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
264 ext = ".jpg" |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
265 else: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
266 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
267 mime_type = None |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
268 filename = uid + ext |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
269 now = int(time.time()) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
270 cache_data.update({ |
| 3028 | 271 "filename": filename, |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
272 "creation": now, |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
273 "eol": now + max_age, |
| 3028 | 274 "mime_type": mime_type, |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
275 }) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
276 file_path = self.getPath(filename) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
277 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
278 with open(cache_url, "wb") as f: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
279 pickle.dump(cache_data, f, protocol=2) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
280 |
| 3185 | 281 return file_path.open("wb") |