Mercurial > libervia-backend
annotate libervia/backend/memory/cache.py @ 4230:314d3c02bb67
core (xmpp): Add a timeout for messages processing to avoid blocking the queue.
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Sat, 06 Apr 2024 12:21:04 +0200 |
| parents | 5f2d496c633f |
| children | e11b13418ba6 |
| rev | line source |
|---|---|
| 3028 | 1 #!/usr/bin/env python3 |
| 3137 | 2 |
| 4212 | 3 # Libervia: an XMPP client |
| 4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 # This program is free software: you can redistribute it and/or modify |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # it under the terms of the GNU Affero General Public License as published by |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # the Free Software Foundation, either version 3 of the License, or |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # (at your option) any later version. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 # This program is distributed in the hope that it will be useful, |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # GNU Affero General Public License for more details. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 # You should have received a copy of the GNU Affero General Public License |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
19 from io import BufferedIOBase |
| 3185 | 20 import mimetypes |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
21 from pathlib import Path |
| 3185 | 22 import time |
| 4212 | 23 from typing import Any |
| 24 | |
| 25 from pydantic import BaseModel, ValidationError | |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
26 |
|
4071
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
27 from libervia.backend.core import exceptions |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
28 from libervia.backend.core.constants import Const as C |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
29 from libervia.backend.core.i18n import _ |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
30 from libervia.backend.core.log import getLogger |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
31 from libervia.backend.tools.common import regex |
| 3185 | 32 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
33 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 log = getLogger(__name__) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
| 4212 | 36 CACHE_METADATA_EXT = ".cache.json" |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
37 DEFAULT_EXT = ".raw" |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
38 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 |
| 4212 | 40 class CacheMetadata(BaseModel): |
| 41 source: str | |
| 42 uid: str | |
| 43 filename: str | |
| 44 creation: int | |
| 45 eol: int | |
| 46 max_age: int = C.DEFAULT_MAX_AGE | |
| 47 original_filename: str | None = None | |
| 48 mime_type: str | None = None | |
| 49 last_access: int | None = None | |
| 50 | |
| 51 | |
| 52 class Cache: | |
| 53 """Generic file caching.""" | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
55 def __init__(self, host, profile): |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
56 """ |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
57 @param profile(unicode, None): name of the profile to set the cache for |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
58 if None, the cache will be common for all profiles |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
59 """ |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 self.profile = profile |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
61 path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR] |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
62 if profile: |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
63 path_elts.extend(["profiles", regex.path_escape(profile)]) |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
64 else: |
| 3028 | 65 path_elts.append("common") |
| 3185 | 66 self.cache_dir = Path(*path_elts) |
| 67 | |
| 68 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True) | |
| 69 self.purge() | |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
70 |
| 3185 | 71 def purge(self): |
| 4212 | 72 # Remove expired, unreadable, and unrelated files from cache |
| 3185 | 73 # TODO: this should not be called only on startup, but at regular interval |
| 74 # (e.g. once a day) | |
| 4212 | 75 to_delete = set() |
| 76 seen = set() | |
| 77 now = time.time() | |
| 78 for cache_data_file in self.cache_dir.glob(f"*{CACHE_METADATA_EXT}"): | |
| 3185 | 79 try: |
| 4212 | 80 with cache_data_file.open("r") as f: |
| 81 cache_data = CacheMetadata.model_validate_json(f.read()) | |
| 82 except (IOError, ValidationError): | |
| 3185 | 83 log.warning( |
| 4212 | 84 _("Can't read metadata file at {path}, deleting it.").format( |
| 85 path=cache_data_file | |
| 86 ) | |
| 87 ) | |
| 88 to_delete.add(cache_data_file) | |
| 3185 | 89 continue |
| 4212 | 90 else: |
| 91 cached_file = self.get_path(cache_data.filename) | |
| 92 if not cached_file.exists(): | |
| 93 log.warning( | |
| 94 f"Cache file {cache_data_file!r} references a non-existent file " | |
| 95 f"and will be deleted: {cache_data_file!r}." | |
| 96 ) | |
| 97 to_delete.add(cache_data_file) | |
| 98 elif cache_data.eol < now: | |
| 99 log.debug( | |
| 100 "Purging expired cache file {cache_data_file!r} (expired for " | |
| 101 "{time}s)".format(time=int(time.time() - cache_data.eol)) | |
| 102 ) | |
| 103 to_delete.add(cache_data_file) | |
| 104 seen.add(cached_file) | |
| 105 seen.add(cache_data_file) | |
| 3185 | 106 |
| 4212 | 107 for file in to_delete: |
| 108 log.debug(f"Deleting cache file: {file}") | |
| 109 file.unlink() | |
|
3209
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
110 |
| 4212 | 111 for file in self.cache_dir.iterdir(): |
| 112 if file not in seen: | |
| 113 log.debug(f"Deleting irrelevant file in cache dir: {file}") | |
| 114 file.unlink() | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
115 |
| 4212 | 116 def get_path(self, filename: str) -> Path: |
| 117 """Return cached file URL. | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
118 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
119 @param filename: cached file name (cache data or actual file) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
120 @return: path to the cached file |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
121 """ |
| 3028 | 122 if not filename or "/" in filename: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
123 log.error( |
| 3028 | 124 "invalid char found in file name, hack attempt? name:{}".format(filename) |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
125 ) |
| 3028 | 126 raise exceptions.DataError("Invalid char found") |
| 3185 | 127 return self.cache_dir / filename |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
128 |
| 4212 | 129 def get_metadata(self, uid: str, update_eol: bool = True) -> dict[str, Any] | None: |
| 130 """Retrieve metadata for cached data. | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
131 |
| 4212 | 132 @param uid: unique identifier of cache metadata. |
| 133 @param update_eol: True if eol must extended | |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
134 if True, max_age will be added to eol (only if it is not already expired) |
| 4212 | 135 @return: metadata, see [cache_data] for data details, an additional "path" key is |
| 136 the full path to cached file. | |
| 137 None if file is not in cache (or cache is invalid). | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
138 """ |
|
2116
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
139 uid = uid.strip() |
|
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
140 if not uid: |
| 3028 | 141 raise exceptions.InternalError("uid must not be empty") |
| 4212 | 142 cache_url = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
143 if not cache_url.exists(): |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
144 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
145 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
146 try: |
| 4212 | 147 with cache_url.open("r") as f: |
| 148 cache_data = CacheMetadata.model_validate_json(f.read()) | |
|
3572
b3fa179417e7
core (memory/cache): don't crash on EOFError in getMetadata
Goffi <goffi@goffi.org>
parents:
3479
diff
changeset
|
149 except (IOError, EOFError) as e: |
| 4212 | 150 log.warning(f"Can't read cache at {cache_url}: {e}") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 return None |
| 4212 | 152 except ValidationError: |
| 153 log.warning(f"Invalid cache found at {cache_url}") | |
| 154 return None | |
| 155 except UnicodeDecodeError as e: | |
| 156 log.warning(f"Invalid encoding, this is not a cache metadata file.") | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 |
| 4212 | 159 if cache_data.eol < time.time(): |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
160 log.debug( |
| 4212 | 161 "removing expired cache (expired for {}s)".format( |
| 162 time.time() - cache_data.eol | |
| 163 ) | |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
164 ) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
165 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
166 |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
167 if update_eol: |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
168 now = int(time.time()) |
| 4212 | 169 cache_data.last_access = now |
| 170 cache_data.eol = now + cache_data.max_age | |
| 171 with cache_url.open("w") as f: | |
| 172 f.write(cache_data.model_dump_json(exclude_none=True)) | |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
173 |
| 4212 | 174 # FIXME: we convert to dict to be compatible with former method (pre Pydantic). |
| 175 # All call to get_metadata should use directly the Pydantic model in the future. | |
| 176 cache_data_dict = cache_data.model_dump() | |
| 177 cache_data_dict["path"] = self.get_path(cache_data.filename) | |
| 178 return cache_data_dict | |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
179 |
| 4212 | 180 def get_file_path(self, uid: str) -> Path | None: |
| 3185 | 181 """Retrieve absolute path to file |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
182 |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
183 @param uid(unicode): unique identifier of file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
184 @return (unicode, None): absolute path to cached file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
185 None if file is not in cache (or cache is invalid) |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
186 """ |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
187 metadata = self.get_metadata(uid) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
188 if metadata is not None: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
189 return metadata["path"] |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
190 |
| 4212 | 191 def remove_from_cache(self, uid: str, metadata=None) -> None: |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
192 """Remove data from cache |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
193 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
194 @param uid(unicode): unique identifier cache file |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
195 """ |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
196 cache_data = self.get_metadata(uid, update_eol=False) |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
197 if cache_data is None: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
198 log.debug(f"cache with uid {uid!r} has already expired or been removed") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
199 return |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
200 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
201 try: |
| 4212 | 202 filename = cache_data["filename"] |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
203 except KeyError: |
| 4212 | 204 log.warning(_("missing filename for cache {uid!r}").format(uid=uid)) |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
205 else: |
| 4212 | 206 filepath = self.get_path(filename) |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
207 try: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
208 filepath.unlink() |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
209 except FileNotFoundError: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
210 log.warning( |
| 4212 | 211 _("missing file referenced in cache {uid!r}: {filename}").format( |
| 212 uid=uid, filename=filename | |
| 213 ) | |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
214 ) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
215 |
| 4212 | 216 cache_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
217 cache_file.unlink() |
| 4212 | 218 log.debug(f"Cache with uid {uid!r} has been removed.") |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
219 |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
220 def cache_data( |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
221 self, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
222 source: str, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
223 uid: str, |
| 4212 | 224 mime_type: str | None = None, |
| 225 max_age: int = C.DEFAULT_MAX_AGE, | |
| 226 original_filename: str | None = None, | |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
227 ) -> BufferedIOBase: |
| 4212 | 228 """Create cache metadata and file object to use for actual data. |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
229 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
230 @param source: source of the cache (should be plugin's import_name) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
231 @param uid: an identifier of the file which must be unique |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
232 @param mime_type: MIME type of the file to cache |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
233 it will be used notably to guess file extension |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
234 It may be autogenerated if filename is specified |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
235 @param max_age: maximum age in seconds |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 the cache metadata will have an "eol" (end of life) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
237 None to use default value |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
238 0 to ignore cache (file will be re-downloaded on each access) |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
239 @param original_filename: if not None, will be used to retrieve file extension and |
| 4212 | 240 guess mime type, and stored in "original_filename" |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
241 @return: file object opened in write mode |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
242 you have to close it yourself (hint: use ``with`` statement) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 """ |
| 4212 | 244 if original_filename is not None and mime_type is None: |
| 245 # we have original_filename but not MIME type, we try to guess the later | |
| 246 mime_type = mimetypes.guess_type(original_filename, strict=False)[0] | |
| 247 | |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
248 if mime_type: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
249 ext = mimetypes.guess_extension(mime_type, strict=False) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
250 if ext is None: |
| 4212 | 251 log.warning("can't find extension for MIME type {}".format(mime_type)) |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
252 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
253 elif ext == ".jpe": |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
254 ext = ".jpg" |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
255 else: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
256 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
257 mime_type = None |
| 4212 | 258 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
259 filename = uid + ext |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
260 now = int(time.time()) |
| 4212 | 261 metadata = CacheMetadata( |
| 262 source=source, | |
| 263 uid=uid, | |
| 264 mime_type=mime_type, | |
| 265 max_age=max_age, | |
| 266 original_filename=original_filename, | |
| 267 filename=filename, | |
| 268 creation=now, | |
| 269 eol=now + max_age, | |
| 270 ) | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
271 |
| 4212 | 272 cache_metadata_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
| 273 file_path = self.get_path(filename) | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
274 |
| 4212 | 275 with open(cache_metadata_file, "w") as f: |
| 276 f.write(metadata.model_dump_json(exclude_none=True)) | |
| 277 | |
| 278 return open(file_path, "wb") |