libervia-backend: sat/memory/cache.py annotate

annotate sat/memory/cache.py @ 3971:9b1d74a6b48c

tests (e2e/cli): add tests for JET: rel 378

author	Goffi <goffi@goffi.org>
date	Mon, 31 Oct 2022 04:09:38 +0100
parents	2863345c9bbb
children	524856bd7b19

rev	line source
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	1 #!/usr/bin/env python3
3137 559a625a236b fixed shebangs Goffi <goffi@goffi.org> parents: 3136 diff changeset	2
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	3
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	4 # SAT: a jabber client
3479 be6d91572633 date update Goffi <goffi@goffi.org> parents: 3210 diff changeset	5 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	6
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	7 # This program is free software: you can redistribute it and/or modify
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	8 # it under the terms of the GNU Affero General Public License as published by
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	9 # the Free Software Foundation, either version 3 of the License, or
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	10 # (at your option) any later version.
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	11
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	12 # This program is distributed in the hope that it will be useful,
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	15 # GNU Affero General Public License for more details.
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	16
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	17 # You should have received a copy of the GNU Affero General Public License
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	19
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	20 from io import BufferedIOBase
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	21 import mimetypes
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	22 from pathlib import Path
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	23 import pickle as pickle
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	24 import time
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	25 from typing import Any, Dict, Optional
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	26
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	27 from sat.core import exceptions
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	28 from sat.core.constants import Const as C
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	29 from sat.core.i18n import _
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	30 from sat.core.log import getLogger
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	31 from sat.tools.common import regex
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	32
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	33
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	34 log = getLogger(__name__)
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	35
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	36 DEFAULT_EXT = ".raw"
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	37
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	38
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	39 class Cache(object):
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	40 """generic file caching"""
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	41
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	42 def __init__(self, host, profile):
d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	43 """
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	44 @param profile(unicode, None): name of the profile to set the cache for
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	45 if None, the cache will be common for all profiles
d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	46 """
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	47 self.profile = profile
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	48 path_elts = [host.memory.getConfig("", "local_dir"), C.CACHE_DIR]
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	49 if profile:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	50 path_elts.extend(["profiles", regex.pathEscape(profile)])
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	51 else:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	52 path_elts.append("common")
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	53 self.cache_dir = Path(*path_elts)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	54
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	55 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	56 self.purge()
2509 d485e9416493 core (memory/cache): common cache: Goffi <goffi@goffi.org> parents: 2506 diff changeset	57
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	58 def purge(self):
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	59 # remove expired files from cache
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	60 # TODO: this should not be called only on startup, but at regular interval
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	61 # (e.g. once a day)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	62 purged = set()
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	63 # we sort files to have metadata files first
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	64 for cache_file in sorted(self.cache_dir.iterdir()):
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	65 if cache_file in purged:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	66 continue
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	67 try:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	68 with cache_file.open('rb') as f:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	69 cache_data = pickle.load(f)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	70 except IOError:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	71 log.warning(
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	72 _("Can't read metadata file at {path}")
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	73 .format(path=cache_file))
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	74 continue
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	75 except (pickle.UnpicklingError, EOFError):
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	76 log.debug(f"File at {cache_file} is not a metadata file")
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	77 continue
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	78 try:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	79 eol = cache_data['eol']
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	80 filename = cache_data['filename']
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	81 except KeyError:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	82 log.warning(
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	83 _("Invalid cache metadata at {path}")
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	84 .format(path=cache_file))
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	85 continue
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	86
3209 f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	87 filepath = self.getPath(filename)
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	88
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	89 if not filepath.exists():
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	90 log.warning(_(
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	91 "cache {cache_file!r} references an inexisting file: {filepath!r}"
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	92 ).format(cache_file=str(cache_file), filepath=str(filepath)))
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	93 log.debug("purging cache with missing file")
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	94 cache_file.unlink()
f14eb24328d0 core (memory/cache): purge cache metadata when the referenced file doesn't exist Goffi <goffi@goffi.org> parents: 3198 diff changeset	95 elif eol < time.time():
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	96 log.debug(
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	97 "purging expired cache {filepath!r} (expired for {time}s)"
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	98 .format(filepath=str(filepath), time=int(time.time() - eol))
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	99 )
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	100 cache_file.unlink()
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	101 try:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	102 filepath.unlink()
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	103 except FileNotFoundError:
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	104 log.warning(
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	105 _("following file is missing while purging cache: {path}")
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	106 .format(path=filepath)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	107 )
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	108 purged.add(cache_file)
554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	109 purged.add(filepath)
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	110
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	111 def getPath(self, filename: str) -> Path:
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	112 """return cached file URL
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	113
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	114 @param filename: cached file name (cache data or actual file)
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	115 @return: path to the cached file
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	116 """
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	117 if not filename or "/" in filename:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	118 log.error(
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	119 "invalid char found in file name, hack attempt? name:{}".format(filename)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	120 )
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	121 raise exceptions.DataError("Invalid char found")
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	122 return self.cache_dir / filename
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	123
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	124 def getMetadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]:
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	125 """Retrieve metadata for cached data
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	126
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	127 @param uid(unicode): unique identifier of file
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	128 @param update_eol(bool): True if eol must extended
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	129 if True, max_age will be added to eol (only if it is not already expired)
2517 cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	130 @return (dict, None): metadata with following keys:
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	131 see [cacheData] for data details, an additional "path" key is the full path to
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	132 cached file.
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	133 None if file is not in cache (or cache is invalid)
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	134 """
2517 cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	135
2116 766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty Goffi <goffi@goffi.org> parents: 2109 diff changeset	136 uid = uid.strip()
766dbbec56f2 core (memory/cache): geFilePath now return None when uid is empty Goffi <goffi@goffi.org> parents: 2109 diff changeset	137 if not uid:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	138 raise exceptions.InternalError("uid must not be empty")
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	139 cache_url = self.getPath(uid)
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	140 if not cache_url.exists():
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	141 return None
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	142
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	143 try:
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	144 with cache_url.open("rb") as f:
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	145 cache_data = pickle.load(f)
3572 b3fa179417e7 core (memory/cache): don't crash on EOFError in getMetadata Goffi <goffi@goffi.org> parents: 3479 diff changeset	146 except (IOError, EOFError) as e:
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	147 log.warning(f"can't read cache at {cache_url}: {e}")
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	148 return None
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	149 except pickle.UnpicklingError:
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	150 log.warning(f"invalid cache found at {cache_url}")
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	151 return None
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	152
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	153 try:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	154 eol = cache_data["eol"]
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	155 except KeyError:
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	156 log.warning("no End Of Life found for cached file {}".format(uid))
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	157 eol = 0
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	158 if eol < time.time():
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	159 log.debug(
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	160 "removing expired cache (expired for {}s)".format(time.time() - eol)
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	161 )
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	162 return None
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	163
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	164 if update_eol:
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	165 try:
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	166 max_age = cache_data["max_age"]
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	167 except KeyError:
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	168 log.warning(f"no max_age found for cache at {cache_url}, using default")
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	169 max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE
3198 08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	170 now = int(time.time())
08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	171 cache_data["last_access"] = now
08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	172 cache_data["eol"] = now + max_age
3188 a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	173 with cache_url.open("wb") as f:
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	174 pickle.dump(cache_data, f, protocol=2)
a15773c6c273 memory(cache): extend EOL when a file metadata is retrieved Goffi <goffi@goffi.org> parents: 3185 diff changeset	175
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	176 cache_data["path"] = self.getPath(cache_data["filename"])
2517 cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	177 return cache_data
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	178
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	179 def getFilePath(self, uid: str) -> Path:
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	180 """Retrieve absolute path to file
2517 cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	181
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	182 @param uid(unicode): unique identifier of file
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	183 @return (unicode, None): absolute path to cached file
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	184 None if file is not in cache (or cache is invalid)
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	185 """
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	186 metadata = self.getMetadata(uid)
cd7a53c31eb6 core (memory/cache): new getMetadata method to retrieve metadata without opening the file Goffi <goffi@goffi.org> parents: 2509 diff changeset	187 if metadata is not None:
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	188 return metadata["path"]
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	189
3210 fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	190 def removeFromCache(self, uid, metadata=None):
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	191 """Remove data from cache
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	192
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	193 @param uid(unicode): unique identifier cache file
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	194 """
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	195 cache_data = self.getMetadata(uid, update_eol=False)
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	196 if cache_data is None:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	197 log.debug(f"cache with uid {uid!r} has already expired or been removed")
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	198 return
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	199
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	200 try:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	201 filename = cache_data['filename']
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	202 except KeyError:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	203 log.warning(_("missing filename for cache {uid!r}") .format(uid=uid))
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	204 else:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	205 filepath = self.getPath(filename)
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	206 try:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	207 filepath.unlink()
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	208 except FileNotFoundError:
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	209 log.warning(
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	210 _("missing file referenced in cache {uid!r}: {filename}")
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	211 .format(uid=uid, filename=filename)
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	212 )
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	213
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	214 cache_file = self.getPath(uid)
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	215 cache_file.unlink()
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	216 log.debug(f"cache with uid {uid!r} has been removed")
fedec192a83f core (memory/cache): new removeFromCache method Goffi <goffi@goffi.org> parents: 3209 diff changeset	217
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	218 def cacheData(
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	219 self,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	220 source: str,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	221 uid: str,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	222 mime_type: Optional[str] = None,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	223 max_age: Optional[int] = None,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	224 original_filename: Optional[str] = None
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	225 ) -> BufferedIOBase:
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	226 """create cache metadata and file object to use for actual data
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	227
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	228 @param source: source of the cache (should be plugin's import_name)
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	229 @param uid: an identifier of the file which must be unique
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	230 @param mime_type: MIME type of the file to cache
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	231 it will be used notably to guess file extension
3198 08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	232 It may be autogenerated if filename is specified
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	233 @param max_age: maximum age in seconds
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	234 the cache metadata will have an "eol" (end of life)
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	235 None to use default value
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	236 0 to ignore cache (file will be re-downloaded on each access)
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	237 @param original_filename: if not None, will be used to retrieve file extension and
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	238 guess
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	239 mime type, and stored in "original_filename"
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	240 @return: file object opened in write mode
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	241 you have to close it yourself (hint: use ``with`` statement)
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	242 """
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	243 if max_age is None:
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	244 max_age = C.DEFAULT_MAX_AGE
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	245 cache_data = {
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	246 "source": source,
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	247 # we also store max_age for updating eol
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	248 "max_age": max_age,
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	249 }
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	250 cache_url = self.getPath(uid)
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	251 if original_filename is not None:
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	252 cache_data["original_filename"] = original_filename
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	253 if mime_type is None:
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	254 # we have original_filename but not MIME type, we try to guess the later
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	255 mime_type = mimetypes.guess_type(original_filename, strict=False)[0]
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	256 if mime_type:
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	257 ext = mimetypes.guess_extension(mime_type, strict=False)
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	258 if ext is None:
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	259 log.warning(
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	260 "can't find extension for MIME type {}".format(mime_type)
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	261 )
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	262 ext = DEFAULT_EXT
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	263 elif ext == ".jpe":
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	264 ext = ".jpg"
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	265 else:
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	266 ext = DEFAULT_EXT
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	267 mime_type = None
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	268 filename = uid + ext
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	269 now = int(time.time())
2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	270 cache_data.update({
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	271 "filename": filename,
3198 08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	272 "creation": now,
08151c103636 core (memory/cache): added some metadata: Goffi <goffi@goffi.org> parents: 3188 diff changeset	273 "eol": now + max_age,
3028 ab2696e34d29 Python 3 port: Goffi <goffi@goffi.org> parents: 2771 diff changeset	274 "mime_type": mime_type,
3818 2863345c9bbb core (memory/cache): type hints + filename fix: Goffi <goffi@goffi.org> parents: 3572 diff changeset	275 })
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	276 file_path = self.getPath(filename)
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	277
2624 56f94936df1e code style reformatting using black Goffi <goffi@goffi.org> parents: 2562 diff changeset	278 with open(cache_url, "wb") as f:
2109 85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	279 pickle.dump(cache_data, f, protocol=2)
85f3e12e984d core (memory/cache): file caching handling, first draft: Goffi <goffi@goffi.org> parents: diff changeset	280
3185 554b3b632378 memory (cache): purge + pathlib: Goffi <goffi@goffi.org> parents: 3137 diff changeset	281 return file_path.open("wb")

Mercurial > libervia-backend

annotate sat/memory/cache.py @ 3971:9b1d74a6b48c