Mercurial > libervia-backend
annotate libervia/backend/memory/cache.py @ 4294:a0ed5c976bf8 default tip @
component conferences, plugin XEP-0167, XEP-0298: add stream user metadata:
A/V conference now adds user metadata about the stream it is forwarding through XEP-0298.
This is parsed and added to metadata during confirmation on client side.
rel 448
author | Goffi <goffi@goffi.org> |
---|---|
date | Tue, 06 Aug 2024 23:43:11 +0200 |
parents | e11b13418ba6 |
children |
rev | line source |
---|---|
3028 | 1 #!/usr/bin/env python3 |
3137 | 2 |
4212 | 3 # Libervia: an XMPP client |
4 # Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 # This program is free software: you can redistribute it and/or modify |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # it under the terms of the GNU Affero General Public License as published by |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # the Free Software Foundation, either version 3 of the License, or |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # (at your option) any later version. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 # This program is distributed in the hope that it will be useful, |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # GNU Affero General Public License for more details. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 # You should have received a copy of the GNU Affero General Public License |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
19 from io import BufferedIOBase |
3185 | 20 import mimetypes |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
21 from pathlib import Path |
3185 | 22 import time |
4212 | 23 from typing import Any |
24 | |
25 from pydantic import BaseModel, ValidationError | |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
26 |
4071
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
27 from libervia.backend.core import exceptions |
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
28 from libervia.backend.core.constants import Const as C |
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
29 from libervia.backend.core.i18n import _ |
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
30 from libervia.backend.core.log import getLogger |
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
31 from libervia.backend.tools.common import regex |
3185 | 32 |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
33 |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 log = getLogger(__name__) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
4212 | 36 CACHE_METADATA_EXT = ".cache.json" |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
37 DEFAULT_EXT = ".raw" |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
38 |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 |
4212 | 40 class CacheMetadata(BaseModel): |
41 source: str | |
42 uid: str | |
43 filename: str | |
44 creation: int | |
45 eol: int | |
46 max_age: int = C.DEFAULT_MAX_AGE | |
47 original_filename: str | None = None | |
48 mime_type: str | None = None | |
49 last_access: int | None = None | |
50 | |
51 | |
52 class Cache: | |
53 """Generic file caching.""" | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
55 def __init__(self, host, profile): |
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
56 """ |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
57 @param profile(unicode, None): name of the profile to set the cache for |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
58 if None, the cache will be common for all profiles |
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
59 """ |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 self.profile = profile |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
61 path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR] |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
62 if profile: |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
63 path_elts.extend(["profiles", regex.path_escape(profile)]) |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
64 else: |
3028 | 65 path_elts.append("common") |
3185 | 66 self.cache_dir = Path(*path_elts) |
67 | |
68 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True) | |
69 self.purge() | |
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
70 |
3185 | 71 def purge(self): |
4212 | 72 # Remove expired, unreadable, and unrelated files from cache |
3185 | 73 # TODO: this should not be called only on startup, but at regular interval |
74 # (e.g. once a day) | |
4212 | 75 to_delete = set() |
76 seen = set() | |
77 now = time.time() | |
78 for cache_data_file in self.cache_dir.glob(f"*{CACHE_METADATA_EXT}"): | |
3185 | 79 try: |
4212 | 80 with cache_data_file.open("r") as f: |
81 cache_data = CacheMetadata.model_validate_json(f.read()) | |
82 except (IOError, ValidationError): | |
3185 | 83 log.warning( |
4212 | 84 _("Can't read metadata file at {path}, deleting it.").format( |
85 path=cache_data_file | |
86 ) | |
87 ) | |
88 to_delete.add(cache_data_file) | |
3185 | 89 continue |
4212 | 90 else: |
91 cached_file = self.get_path(cache_data.filename) | |
92 if not cached_file.exists(): | |
93 log.warning( | |
94 f"Cache file {cache_data_file!r} references a non-existent file " | |
95 f"and will be deleted: {cache_data_file!r}." | |
96 ) | |
97 to_delete.add(cache_data_file) | |
98 elif cache_data.eol < now: | |
99 log.debug( | |
4231
e11b13418ba6
plugin XEP-0353, XEP-0234, jingle: WebRTC data channel signaling implementation:
Goffi <goffi@goffi.org>
parents:
4212
diff
changeset
|
100 f"Purging expired cache file {cache_data_file} (expired for " |
e11b13418ba6
plugin XEP-0353, XEP-0234, jingle: WebRTC data channel signaling implementation:
Goffi <goffi@goffi.org>
parents:
4212
diff
changeset
|
101 f"{int(time.time() - cache_data.eol)}s)" |
4212 | 102 ) |
103 to_delete.add(cache_data_file) | |
104 seen.add(cached_file) | |
105 seen.add(cache_data_file) | |
3185 | 106 |
4212 | 107 for file in to_delete: |
108 log.debug(f"Deleting cache file: {file}") | |
109 file.unlink() | |
3209
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
110 |
4212 | 111 for file in self.cache_dir.iterdir(): |
112 if file not in seen: | |
113 log.debug(f"Deleting irrelevant file in cache dir: {file}") | |
114 file.unlink() | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
115 |
4212 | 116 def get_path(self, filename: str) -> Path: |
117 """Return cached file URL. | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
118 |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
119 @param filename: cached file name (cache data or actual file) |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
120 @return: path to the cached file |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
121 """ |
3028 | 122 if not filename or "/" in filename: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
123 log.error( |
3028 | 124 "invalid char found in file name, hack attempt? name:{}".format(filename) |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
125 ) |
3028 | 126 raise exceptions.DataError("Invalid char found") |
3185 | 127 return self.cache_dir / filename |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
128 |
4212 | 129 def get_metadata(self, uid: str, update_eol: bool = True) -> dict[str, Any] | None: |
130 """Retrieve metadata for cached data. | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
131 |
4212 | 132 @param uid: unique identifier of cache metadata. |
133 @param update_eol: True if eol must extended | |
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
134 if True, max_age will be added to eol (only if it is not already expired) |
4212 | 135 @return: metadata, see [cache_data] for data details, an additional "path" key is |
136 the full path to cached file. | |
137 None if file is not in cache (or cache is invalid). | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
138 """ |
2116
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
139 uid = uid.strip() |
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
140 if not uid: |
3028 | 141 raise exceptions.InternalError("uid must not be empty") |
4212 | 142 cache_url = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
143 if not cache_url.exists(): |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
144 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
145 |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
146 try: |
4212 | 147 with cache_url.open("r") as f: |
148 cache_data = CacheMetadata.model_validate_json(f.read()) | |
3572
b3fa179417e7
core (memory/cache): don't crash on EOFError in getMetadata
Goffi <goffi@goffi.org>
parents:
3479
diff
changeset
|
149 except (IOError, EOFError) as e: |
4212 | 150 log.warning(f"Can't read cache at {cache_url}: {e}") |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 return None |
4212 | 152 except ValidationError: |
153 log.warning(f"Invalid cache found at {cache_url}") | |
154 return None | |
155 except UnicodeDecodeError as e: | |
156 log.warning(f"Invalid encoding, this is not a cache metadata file.") | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 |
4212 | 159 if cache_data.eol < time.time(): |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
160 log.debug( |
4212 | 161 "removing expired cache (expired for {}s)".format( |
162 time.time() - cache_data.eol | |
163 ) | |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
164 ) |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
165 return None |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
166 |
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
167 if update_eol: |
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
168 now = int(time.time()) |
4212 | 169 cache_data.last_access = now |
170 cache_data.eol = now + cache_data.max_age | |
171 with cache_url.open("w") as f: | |
172 f.write(cache_data.model_dump_json(exclude_none=True)) | |
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
173 |
4212 | 174 # FIXME: we convert to dict to be compatible with former method (pre Pydantic). |
175 # All call to get_metadata should use directly the Pydantic model in the future. | |
176 cache_data_dict = cache_data.model_dump() | |
177 cache_data_dict["path"] = self.get_path(cache_data.filename) | |
178 return cache_data_dict | |
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
179 |
4212 | 180 def get_file_path(self, uid: str) -> Path | None: |
3185 | 181 """Retrieve absolute path to file |
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
182 |
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
183 @param uid(unicode): unique identifier of file |
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
184 @return (unicode, None): absolute path to cached file |
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
185 None if file is not in cache (or cache is invalid) |
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
186 """ |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
187 metadata = self.get_metadata(uid) |
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
188 if metadata is not None: |
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
189 return metadata["path"] |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
190 |
4212 | 191 def remove_from_cache(self, uid: str, metadata=None) -> None: |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
192 """Remove data from cache |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
193 |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
194 @param uid(unicode): unique identifier cache file |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
195 """ |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
196 cache_data = self.get_metadata(uid, update_eol=False) |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
197 if cache_data is None: |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
198 log.debug(f"cache with uid {uid!r} has already expired or been removed") |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
199 return |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
200 |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
201 try: |
4212 | 202 filename = cache_data["filename"] |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
203 except KeyError: |
4212 | 204 log.warning(_("missing filename for cache {uid!r}").format(uid=uid)) |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
205 else: |
4212 | 206 filepath = self.get_path(filename) |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
207 try: |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
208 filepath.unlink() |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
209 except FileNotFoundError: |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
210 log.warning( |
4212 | 211 _("missing file referenced in cache {uid!r}: {filename}").format( |
212 uid=uid, filename=filename | |
213 ) | |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
214 ) |
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
215 |
4212 | 216 cache_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
217 cache_file.unlink() |
4212 | 218 log.debug(f"Cache with uid {uid!r} has been removed.") |
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
219 |
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
220 def cache_data( |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
221 self, |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
222 source: str, |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
223 uid: str, |
4212 | 224 mime_type: str | None = None, |
225 max_age: int = C.DEFAULT_MAX_AGE, | |
226 original_filename: str | None = None, | |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
227 ) -> BufferedIOBase: |
4212 | 228 """Create cache metadata and file object to use for actual data. |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
229 |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
230 @param source: source of the cache (should be plugin's import_name) |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
231 @param uid: an identifier of the file which must be unique |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
232 @param mime_type: MIME type of the file to cache |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
233 it will be used notably to guess file extension |
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
234 It may be autogenerated if filename is specified |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
235 @param max_age: maximum age in seconds |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 the cache metadata will have an "eol" (end of life) |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
237 None to use default value |
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
238 0 to ignore cache (file will be re-downloaded on each access) |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
239 @param original_filename: if not None, will be used to retrieve file extension and |
4212 | 240 guess mime type, and stored in "original_filename" |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
241 @return: file object opened in write mode |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
242 you have to close it yourself (hint: use ``with`` statement) |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 """ |
4212 | 244 if original_filename is not None and mime_type is None: |
245 # we have original_filename but not MIME type, we try to guess the later | |
246 mime_type = mimetypes.guess_type(original_filename, strict=False)[0] | |
247 | |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
248 if mime_type: |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
249 ext = mimetypes.guess_extension(mime_type, strict=False) |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
250 if ext is None: |
4212 | 251 log.warning("can't find extension for MIME type {}".format(mime_type)) |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
252 ext = DEFAULT_EXT |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
253 elif ext == ".jpe": |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
254 ext = ".jpg" |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
255 else: |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
256 ext = DEFAULT_EXT |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
257 mime_type = None |
4212 | 258 |
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
259 filename = uid + ext |
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
260 now = int(time.time()) |
4212 | 261 metadata = CacheMetadata( |
262 source=source, | |
263 uid=uid, | |
264 mime_type=mime_type, | |
265 max_age=max_age, | |
266 original_filename=original_filename, | |
267 filename=filename, | |
268 creation=now, | |
269 eol=now + max_age, | |
270 ) | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
271 |
4212 | 272 cache_metadata_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") |
273 file_path = self.get_path(filename) | |
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
274 |
4212 | 275 with open(cache_metadata_file, "w") as f: |
276 f.write(metadata.model_dump_json(exclude_none=True)) | |
277 | |
278 return open(file_path, "wb") |