libervia-backend: libervia/backend/memory/cache.py comparison

comparison libervia/backend/memory/cache.py @ 4212:5f2d496c633f

core: get rid of `pickle`: Use of `pickle` to serialise data was a technical legacy that was causing trouble to store in database, to update (if a class was serialised, a change could break update), and to security (pickle can lead to code execution). This patch remove all use of Pickle in favour in JSON, notably: - for caching data, a Pydantic model is now used instead - for SQLAlchemy model, the LegacyPickle is replaced by JSON serialisation - in XEP-0373 a class `PublicKeyMetadata` was serialised. New method `from_dict` and `to_dict` method have been implemented to do serialisation. - new methods to (de)serialise data can now be specified with Identity data types. It is notably used to (de)serialise `path` of avatars. A migration script has been created to convert data (for upgrade or downgrade), with special care for XEP-0373 case. Depending of size of database, this migration script can be long to run. rel 443

author	Goffi <goffi@goffi.org>
date	Fri, 23 Feb 2024 13:31:04 +0100
parents	4b842c1fb686
children	e11b13418ba6

comparison

equal deleted inserted replaced

-:be89ab1cbca4
+:5f2d496c633f
 #!/usr/bin/env python3
+# Libervia: an XMPP client
-# SAT: a jabber client
+# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)
-# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from io import BufferedIOBase
 import mimetypes
 from pathlib import Path
-import pickle as pickle
 import time
-from typing import Any, Dict, Optional
+from typing import Any
+from pydantic import BaseModel, ValidationError
 from libervia.backend.core import exceptions
 from libervia.backend.core.constants import Const as C
 from libervia.backend.core.i18n import _
 from libervia.backend.core.log import getLogger
 from libervia.backend.tools.common import regex
 log = getLogger(__name__)
+CACHE_METADATA_EXT = ".cache.json"
 DEFAULT_EXT = ".raw"
-class Cache(object):
+class CacheMetadata(BaseModel):
-"""generic file caching"""
+source: str
+uid: str
+filename: str
+creation: int
+eol: int
+max_age: int = C.DEFAULT_MAX_AGE
+original_filename: str | None = None
+mime_type: str | None = None
+last_access: int | None = None
+class Cache:
+"""Generic file caching."""
 def __init__(self, host, profile):
 """
 @param profile(unicode, None): name of the profile to set the cache for
 if None, the cache will be common for all profiles
 self.cache_dir.mkdir(0o700, parents=True, exist_ok=True)
 self.purge()
 def purge(self):
-# remove expired files from cache
+# Remove expired, unreadable, and unrelated files from cache
 # TODO: this should not be called only on startup, but at regular interval
 #   (e.g. once a day)
-purged = set()
+to_delete = set()
-# we sort files to have metadata files first
+seen = set()
-for cache_file in sorted(self.cache_dir.iterdir()):
+now = time.time()
-if cache_file in purged:
+for cache_data_file in self.cache_dir.glob(f"*{CACHE_METADATA_EXT}"):
+try:
+with cache_data_file.open("r") as f:
+cache_data = CacheMetadata.model_validate_json(f.read())
+except (IOError, ValidationError):
+log.warning(
+_("Can't read metadata file at {path}, deleting it.").format(
+path=cache_data_file
+)
+)
+to_delete.add(cache_data_file)
 continue
-try:
+else:
-with cache_file.open('rb') as f:
+cached_file = self.get_path(cache_data.filename)
-cache_data = pickle.load(f)
+if not cached_file.exists():
-except IOError:
-log.warning(
-_("Can't read metadata file at {path}")
-.format(path=cache_file))
-continue
-except (pickle.UnpicklingError, EOFError):
-log.debug(f"File at {cache_file} is not a metadata file")
-continue
-try:
-eol = cache_data['eol']
-filename = cache_data['filename']
-except KeyError:
-log.warning(
-_("Invalid cache metadata at {path}")
-.format(path=cache_file))
-continue
-filepath = self.getPath(filename)
-if not filepath.exists():
-log.warning(_(
-"cache {cache_file!r} references an inexisting file: {filepath!r}"
-).format(cache_file=str(cache_file), filepath=str(filepath)))
-log.debug("purging cache with missing file")
-cache_file.unlink()
-elif eol < time.time():
-log.debug(
-"purging expired cache {filepath!r} (expired for {time}s)"
-.format(filepath=str(filepath), time=int(time.time() - eol))
-)
-cache_file.unlink()
-try:
-filepath.unlink()
-except FileNotFoundError:
 log.warning(
-_("following file is missing while purging cache: {path}")
+f"Cache file {cache_data_file!r} references a non-existent file "
-.format(path=filepath)
+f"and will be deleted: {cache_data_file!r}."
 )
-purged.add(cache_file)
+to_delete.add(cache_data_file)
-purged.add(filepath)
+elif cache_data.eol < now:
+log.debug(
-def getPath(self, filename: str) -> Path:
+"Purging expired cache file {cache_data_file!r} (expired for "
-"""return cached file URL
+"{time}s)".format(time=int(time.time() - cache_data.eol))
+)
+to_delete.add(cache_data_file)
+seen.add(cached_file)
+seen.add(cache_data_file)
+for file in to_delete:
+log.debug(f"Deleting cache file: {file}")
+file.unlink()
+for file in self.cache_dir.iterdir():
+if file not in seen:
+log.debug(f"Deleting irrelevant file in cache dir: {file}")
+file.unlink()
+def get_path(self, filename: str) -> Path:
+"""Return cached file URL.
 @param filename: cached file name (cache data or actual file)
 @return: path to the cached file
 """
 if not filename or "/" in filename:
 "invalid char found in file name, hack attempt? name:{}".format(filename)
 )
 raise exceptions.DataError("Invalid char found")
 return self.cache_dir / filename
-def get_metadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]:
+def get_metadata(self, uid: str, update_eol: bool = True) -> dict[str, Any] | None:
-"""Retrieve metadata for cached data
+"""Retrieve metadata for cached data.
-@param uid(unicode): unique identifier of file
+@param uid: unique identifier of cache metadata.
-@param update_eol(bool): True if eol must extended
+@param update_eol: True if eol must extended
 if True, max_age will be added to eol (only if it is not already expired)
-@return (dict, None): metadata with following keys:
+@return: metadata, see [cache_data] for data details, an additional "path" key is
-see [cache_data] for data details, an additional "path" key is the full path to
+the full path to cached file.
-cached file.
+None if file is not in cache (or cache is invalid).
-None if file is not in cache (or cache is invalid)
+"""
-"""
 uid = uid.strip()
 if not uid:
 raise exceptions.InternalError("uid must not be empty")
-cache_url = self.getPath(uid)
+cache_url = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
 if not cache_url.exists():
 return None
 try:
-with cache_url.open("rb") as f:
+with cache_url.open("r") as f:
-cache_data = pickle.load(f)
+cache_data = CacheMetadata.model_validate_json(f.read())
 except (IOError, EOFError) as e:
-log.warning(f"can't read cache at {cache_url}: {e}")
+log.warning(f"Can't read cache at {cache_url}: {e}")
 return None
-except pickle.UnpicklingError:
+except ValidationError:
-log.warning(f"invalid cache found at {cache_url}")
+log.warning(f"Invalid cache found at {cache_url}")
 return None
+except UnicodeDecodeError as e:
-try:
+log.warning(f"Invalid encoding, this is not a cache metadata file.")
-eol = cache_data["eol"]
+return None
-except KeyError:
-log.warning("no End Of Life found for cached file {}".format(uid))
+if cache_data.eol < time.time():
-eol = 0
-if eol < time.time():
 log.debug(
-"removing expired cache (expired for {}s)".format(time.time() - eol)
+"removing expired cache (expired for {}s)".format(
+time.time() - cache_data.eol
+)
 )
 return None
 if update_eol:
-try:
-max_age = cache_data["max_age"]
-except KeyError:
-log.warning(f"no max_age found for cache at {cache_url}, using default")
-max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE
 now = int(time.time())
-cache_data["last_access"] = now
+cache_data.last_access = now
-cache_data["eol"] = now + max_age
+cache_data.eol = now + cache_data.max_age
-with cache_url.open("wb") as f:
+with cache_url.open("w") as f:
-pickle.dump(cache_data, f, protocol=2)
+f.write(cache_data.model_dump_json(exclude_none=True))
-cache_data["path"] = self.getPath(cache_data["filename"])
+# FIXME: we convert to dict to be compatible with former method (pre Pydantic).
-return cache_data
+#   All call to get_metadata should use directly the Pydantic model in the future.
+cache_data_dict = cache_data.model_dump()
-def get_file_path(self, uid: str) -> Path:
+cache_data_dict["path"] = self.get_path(cache_data.filename)
+return cache_data_dict
+def get_file_path(self, uid: str) -> Path | None:
 """Retrieve absolute path to file
 @param uid(unicode): unique identifier of file
 @return (unicode, None): absolute path to cached file
 None if file is not in cache (or cache is invalid)
 """
 metadata = self.get_metadata(uid)
 if metadata is not None:
 return metadata["path"]
-def remove_from_cache(self, uid, metadata=None):
+def remove_from_cache(self, uid: str, metadata=None) -> None:
 """Remove data from cache
 @param uid(unicode): unique identifier cache file
 """
 cache_data = self.get_metadata(uid, update_eol=False)
 if cache_data is None:
 log.debug(f"cache with uid {uid!r} has already expired or been removed")
 return
 try:
-filename = cache_data['filename']
+filename = cache_data["filename"]
 except KeyError:
-log.warning(_("missing filename for cache {uid!r}") .format(uid=uid))
+log.warning(_("missing filename for cache {uid!r}").format(uid=uid))
 else:
-filepath = self.getPath(filename)
+filepath = self.get_path(filename)
 try:
 filepath.unlink()
 except FileNotFoundError:
 log.warning(
-_("missing file referenced in cache {uid!r}: {filename}")
+_("missing file referenced in cache {uid!r}: {filename}").format(
-.format(uid=uid, filename=filename)
+uid=uid, filename=filename
+)
 )
-cache_file = self.getPath(uid)
+cache_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
 cache_file.unlink()
-log.debug(f"cache with uid {uid!r} has been removed")
+log.debug(f"Cache with uid {uid!r} has been removed.")
 def cache_data(
 self,
 source: str,
 uid: str,
-mime_type: Optional[str] = None,
+mime_type: str | None = None,
-max_age: Optional[int] = None,
+max_age: int = C.DEFAULT_MAX_AGE,
-original_filename: Optional[str] = None
+original_filename: str | None = None,
 ) -> BufferedIOBase:
-"""create cache metadata and file object to use for actual data
+"""Create cache metadata and file object to use for actual data.
 @param source: source of the cache (should be plugin's import_name)
 @param uid: an identifier of the file which must be unique
 @param mime_type: MIME type of the file to cache
 it will be used notably to guess file extension
 @param max_age: maximum age in seconds
 the cache metadata will have an "eol" (end of life)
 None to use default value
 0 to ignore cache (file will be re-downloaded on each access)
 @param original_filename: if not None, will be used to retrieve file extension and
-guess
+guess mime type, and stored in "original_filename"
-mime type, and stored in "original_filename"
 @return: file object opened in write mode
 you have to close it yourself (hint: use ``with`` statement)
 """
-if max_age is None:
+if original_filename is not None and mime_type is None:
-max_age = C.DEFAULT_MAX_AGE
+# we have original_filename but not MIME type, we try to guess the later
-cache_data = {
+mime_type = mimetypes.guess_type(original_filename, strict=False)[0]
-"source": source,
-# we also store max_age for updating eol
-"max_age": max_age,
-}
-cache_url = self.getPath(uid)
-if original_filename is not None:
-cache_data["original_filename"] = original_filename
-if mime_type is None:
-# we have original_filename but not MIME type, we try to guess the later
-mime_type = mimetypes.guess_type(original_filename, strict=False)[0]
 if mime_type:
 ext = mimetypes.guess_extension(mime_type, strict=False)
 if ext is None:
-log.warning(
+log.warning("can't find extension for MIME type {}".format(mime_type))
-"can't find extension for MIME type {}".format(mime_type)
-)
 ext = DEFAULT_EXT
 elif ext == ".jpe":
 ext = ".jpg"
 else:
 ext = DEFAULT_EXT
 mime_type = None
 filename = uid + ext
 now = int(time.time())
-cache_data.update({
+metadata = CacheMetadata(
-"filename": filename,
+source=source,
-"creation": now,
+uid=uid,
-"eol": now + max_age,
+mime_type=mime_type,
-"mime_type": mime_type,
+max_age=max_age,
-})
+original_filename=original_filename,
-file_path = self.getPath(filename)
+filename=filename,
+creation=now,
-with open(cache_url, "wb") as f:
+eol=now + max_age,
-pickle.dump(cache_data, f, protocol=2)
+)
-return file_path.open("wb")
+cache_metadata_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
+file_path = self.get_path(filename)
+with open(cache_metadata_file, "w") as f:
+f.write(metadata.model_dump_json(exclude_none=True))
+return open(file_path, "wb")

Mercurial > libervia-backend

comparison libervia/backend/memory/cache.py @ 4212:5f2d496c633f