Mercurial > libervia-backend
annotate libervia/backend/memory/cache.py @ 4094:c3b68fdc2de7
component AP gateway: fix handling of XMPP comments authors:
the gateway was supposing that comments where emitted from PEP of author. While this is
the case for most blog posts, it's not for comments. Instead the component is now using
`author_jid` which is retrieved by XEP-0277 plugin, and reject the item if the auhor is
not verified (i.e. if `publisher` attribute is not set by XMPP service).
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Mon, 12 Jun 2023 14:50:43 +0200 |
| parents | 4b842c1fb686 |
| children | 5f2d496c633f |
| rev | line source |
|---|---|
| 3028 | 1 #!/usr/bin/env python3 |
| 3137 | 2 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SAT: a jabber client |
| 3479 | 5 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
20 from io import BufferedIOBase |
| 3185 | 21 import mimetypes |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
22 from pathlib import Path |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
23 import pickle as pickle |
| 3185 | 24 import time |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
25 from typing import Any, Dict, Optional |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
26 |
|
4071
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
27 from libervia.backend.core import exceptions |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
28 from libervia.backend.core.constants import Const as C |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
29 from libervia.backend.core.i18n import _ |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
30 from libervia.backend.core.log import getLogger |
|
4b842c1fb686
refactoring: renamed `sat` package to `libervia.backend`
Goffi <goffi@goffi.org>
parents:
4037
diff
changeset
|
31 from libervia.backend.tools.common import regex |
| 3185 | 32 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
33 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 log = getLogger(__name__) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
36 DEFAULT_EXT = ".raw" |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
37 |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 class Cache(object): |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 """generic file caching""" |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
42 def __init__(self, host, profile): |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
43 """ |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
44 @param profile(unicode, None): name of the profile to set the cache for |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
45 if None, the cache will be common for all profiles |
|
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
46 """ |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 self.profile = profile |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
48 path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR] |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
49 if profile: |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
50 path_elts.extend(["profiles", regex.path_escape(profile)]) |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
51 else: |
| 3028 | 52 path_elts.append("common") |
| 3185 | 53 self.cache_dir = Path(*path_elts) |
| 54 | |
| 55 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True) | |
| 56 self.purge() | |
|
2509
d485e9416493
core (memory/cache): common cache:
Goffi <goffi@goffi.org>
parents:
2506
diff
changeset
|
57 |
| 3185 | 58 def purge(self): |
| 59 # remove expired files from cache | |
| 60 # TODO: this should not be called only on startup, but at regular interval | |
| 61 # (e.g. once a day) | |
| 62 purged = set() | |
| 63 # we sort files to have metadata files first | |
| 64 for cache_file in sorted(self.cache_dir.iterdir()): | |
| 65 if cache_file in purged: | |
| 66 continue | |
| 67 try: | |
| 68 with cache_file.open('rb') as f: | |
| 69 cache_data = pickle.load(f) | |
| 70 except IOError: | |
| 71 log.warning( | |
| 72 _("Can't read metadata file at {path}") | |
| 73 .format(path=cache_file)) | |
| 74 continue | |
| 75 except (pickle.UnpicklingError, EOFError): | |
| 76 log.debug(f"File at {cache_file} is not a metadata file") | |
| 77 continue | |
| 78 try: | |
| 79 eol = cache_data['eol'] | |
| 80 filename = cache_data['filename'] | |
| 81 except KeyError: | |
| 82 log.warning( | |
| 83 _("Invalid cache metadata at {path}") | |
| 84 .format(path=cache_file)) | |
| 85 continue | |
| 86 | |
|
3209
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
87 filepath = self.getPath(filename) |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
88 |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
89 if not filepath.exists(): |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
90 log.warning(_( |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
91 "cache {cache_file!r} references an inexisting file: {filepath!r}" |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
92 ).format(cache_file=str(cache_file), filepath=str(filepath))) |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
93 log.debug("purging cache with missing file") |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
94 cache_file.unlink() |
|
f14eb24328d0
core (memory/cache): purge cache metadata when the referenced file doesn't exist
Goffi <goffi@goffi.org>
parents:
3198
diff
changeset
|
95 elif eol < time.time(): |
| 3185 | 96 log.debug( |
| 97 "purging expired cache {filepath!r} (expired for {time}s)" | |
| 98 .format(filepath=str(filepath), time=int(time.time() - eol)) | |
| 99 ) | |
| 100 cache_file.unlink() | |
| 101 try: | |
| 102 filepath.unlink() | |
| 103 except FileNotFoundError: | |
| 104 log.warning( | |
| 105 _("following file is missing while purging cache: {path}") | |
| 106 .format(path=filepath) | |
| 107 ) | |
| 108 purged.add(cache_file) | |
| 109 purged.add(filepath) | |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
110 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
111 def getPath(self, filename: str) -> Path: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
112 """return cached file URL |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
113 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
114 @param filename: cached file name (cache data or actual file) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
115 @return: path to the cached file |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
116 """ |
| 3028 | 117 if not filename or "/" in filename: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
118 log.error( |
| 3028 | 119 "invalid char found in file name, hack attempt? name:{}".format(filename) |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
120 ) |
| 3028 | 121 raise exceptions.DataError("Invalid char found") |
| 3185 | 122 return self.cache_dir / filename |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
123 |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
124 def get_metadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]: |
| 3185 | 125 """Retrieve metadata for cached data |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
126 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
127 @param uid(unicode): unique identifier of file |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
128 @param update_eol(bool): True if eol must extended |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
129 if True, max_age will be added to eol (only if it is not already expired) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
130 @return (dict, None): metadata with following keys: |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
131 see [cache_data] for data details, an additional "path" key is the full path to |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
132 cached file. |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
133 None if file is not in cache (or cache is invalid) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
134 """ |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
135 |
|
2116
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
136 uid = uid.strip() |
|
766dbbec56f2
core (memory/cache): geFilePath now return None when uid is empty
Goffi <goffi@goffi.org>
parents:
2109
diff
changeset
|
137 if not uid: |
| 3028 | 138 raise exceptions.InternalError("uid must not be empty") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
139 cache_url = self.getPath(uid) |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
140 if not cache_url.exists(): |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
141 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
142 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
143 try: |
| 3185 | 144 with cache_url.open("rb") as f: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
145 cache_data = pickle.load(f) |
|
3572
b3fa179417e7
core (memory/cache): don't crash on EOFError in getMetadata
Goffi <goffi@goffi.org>
parents:
3479
diff
changeset
|
146 except (IOError, EOFError) as e: |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
147 log.warning(f"can't read cache at {cache_url}: {e}") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
148 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
149 except pickle.UnpicklingError: |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
150 log.warning(f"invalid cache found at {cache_url}") |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
151 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
152 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
153 try: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
154 eol = cache_data["eol"] |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
155 except KeyError: |
| 3028 | 156 log.warning("no End Of Life found for cached file {}".format(uid)) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
157 eol = 0 |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
158 if eol < time.time(): |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
159 log.debug( |
| 3028 | 160 "removing expired cache (expired for {}s)".format(time.time() - eol) |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
161 ) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
162 return None |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
163 |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
164 if update_eol: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
165 try: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
166 max_age = cache_data["max_age"] |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
167 except KeyError: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
168 log.warning(f"no max_age found for cache at {cache_url}, using default") |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
169 max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
170 now = int(time.time()) |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
171 cache_data["last_access"] = now |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
172 cache_data["eol"] = now + max_age |
|
3188
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
173 with cache_url.open("wb") as f: |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
174 pickle.dump(cache_data, f, protocol=2) |
|
a15773c6c273
memory(cache): extend EOL when a file metadata is retrieved
Goffi <goffi@goffi.org>
parents:
3185
diff
changeset
|
175 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
176 cache_data["path"] = self.getPath(cache_data["filename"]) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
177 return cache_data |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
178 |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
179 def get_file_path(self, uid: str) -> Path: |
| 3185 | 180 """Retrieve absolute path to file |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
181 |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
182 @param uid(unicode): unique identifier of file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
183 @return (unicode, None): absolute path to cached file |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
184 None if file is not in cache (or cache is invalid) |
|
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
185 """ |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
186 metadata = self.get_metadata(uid) |
|
2517
cd7a53c31eb6
core (memory/cache): new getMetadata method to retrieve metadata without opening the file
Goffi <goffi@goffi.org>
parents:
2509
diff
changeset
|
187 if metadata is not None: |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
188 return metadata["path"] |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
189 |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
190 def remove_from_cache(self, uid, metadata=None): |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
191 """Remove data from cache |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
192 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
193 @param uid(unicode): unique identifier cache file |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
194 """ |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
195 cache_data = self.get_metadata(uid, update_eol=False) |
|
3210
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
196 if cache_data is None: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
197 log.debug(f"cache with uid {uid!r} has already expired or been removed") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
198 return |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
199 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
200 try: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
201 filename = cache_data['filename'] |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
202 except KeyError: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
203 log.warning(_("missing filename for cache {uid!r}") .format(uid=uid)) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
204 else: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
205 filepath = self.getPath(filename) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
206 try: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
207 filepath.unlink() |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
208 except FileNotFoundError: |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
209 log.warning( |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
210 _("missing file referenced in cache {uid!r}: {filename}") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
211 .format(uid=uid, filename=filename) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
212 ) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
213 |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
214 cache_file = self.getPath(uid) |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
215 cache_file.unlink() |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
216 log.debug(f"cache with uid {uid!r} has been removed") |
|
fedec192a83f
core (memory/cache): new removeFromCache method
Goffi <goffi@goffi.org>
parents:
3209
diff
changeset
|
217 |
|
4037
524856bd7b19
massive refactoring to switch from camelCase to snake_case:
Goffi <goffi@goffi.org>
parents:
3818
diff
changeset
|
218 def cache_data( |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
219 self, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
220 source: str, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
221 uid: str, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
222 mime_type: Optional[str] = None, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
223 max_age: Optional[int] = None, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
224 original_filename: Optional[str] = None |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
225 ) -> BufferedIOBase: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
226 """create cache metadata and file object to use for actual data |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
227 |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
228 @param source: source of the cache (should be plugin's import_name) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
229 @param uid: an identifier of the file which must be unique |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
230 @param mime_type: MIME type of the file to cache |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
231 it will be used notably to guess file extension |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
232 It may be autogenerated if filename is specified |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
233 @param max_age: maximum age in seconds |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
234 the cache metadata will have an "eol" (end of life) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
235 None to use default value |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
236 0 to ignore cache (file will be re-downloaded on each access) |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
237 @param original_filename: if not None, will be used to retrieve file extension and |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
238 guess |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
239 mime type, and stored in "original_filename" |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
240 @return: file object opened in write mode |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
241 you have to close it yourself (hint: use ``with`` statement) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
242 """ |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
243 if max_age is None: |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
244 max_age = C.DEFAULT_MAX_AGE |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
245 cache_data = { |
| 3028 | 246 "source": source, |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
247 # we also store max_age for updating eol |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
248 "max_age": max_age, |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
249 } |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
250 cache_url = self.getPath(uid) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
251 if original_filename is not None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
252 cache_data["original_filename"] = original_filename |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
253 if mime_type is None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
254 # we have original_filename but not MIME type, we try to guess the later |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
255 mime_type = mimetypes.guess_type(original_filename, strict=False)[0] |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
256 if mime_type: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
257 ext = mimetypes.guess_extension(mime_type, strict=False) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
258 if ext is None: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
259 log.warning( |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
260 "can't find extension for MIME type {}".format(mime_type) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
261 ) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
262 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
263 elif ext == ".jpe": |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
264 ext = ".jpg" |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
265 else: |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
266 ext = DEFAULT_EXT |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
267 mime_type = None |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
268 filename = uid + ext |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
269 now = int(time.time()) |
|
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
270 cache_data.update({ |
| 3028 | 271 "filename": filename, |
|
3198
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
272 "creation": now, |
|
08151c103636
core (memory/cache): added some metadata:
Goffi <goffi@goffi.org>
parents:
3188
diff
changeset
|
273 "eol": now + max_age, |
| 3028 | 274 "mime_type": mime_type, |
|
3818
2863345c9bbb
core (memory/cache): type hints + filename fix:
Goffi <goffi@goffi.org>
parents:
3572
diff
changeset
|
275 }) |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
276 file_path = self.getPath(filename) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
277 |
|
2624
56f94936df1e
code style reformatting using black
Goffi <goffi@goffi.org>
parents:
2562
diff
changeset
|
278 with open(cache_url, "wb") as f: |
|
2109
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
279 pickle.dump(cache_data, f, protocol=2) |
|
85f3e12e984d
core (memory/cache): file caching handling, first draft:
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
280 |
| 3185 | 281 return file_path.open("wb") |