view libervia/backend/memory/cache.py @ 4227:dfccc90cacc6

tests: fix and update tests: - update code to newer version of `sh` - update code to newer version of `selenium` - update names of OMEMO algorithms - fix paths and names used to mount source to the test container - skip modules using `Gst` and `PyQt` when they are not available.
author Goffi <goffi@goffi.org>
date Tue, 05 Mar 2024 17:31:56 +0100
parents 5f2d496c633f
children e11b13418ba6
line wrap: on
line source

#!/usr/bin/env python3

# Libervia: an XMPP client
# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from io import BufferedIOBase
import mimetypes
from pathlib import Path
import time
from typing import Any

from pydantic import BaseModel, ValidationError

from libervia.backend.core import exceptions
from libervia.backend.core.constants import Const as C
from libervia.backend.core.i18n import _
from libervia.backend.core.log import getLogger
from libervia.backend.tools.common import regex


log = getLogger(__name__)

CACHE_METADATA_EXT = ".cache.json"
DEFAULT_EXT = ".raw"


class CacheMetadata(BaseModel):
    source: str
    uid: str
    filename: str
    creation: int
    eol: int
    max_age: int = C.DEFAULT_MAX_AGE
    original_filename: str | None = None
    mime_type: str | None = None
    last_access: int | None = None


class Cache:
    """Generic file caching."""

    def __init__(self, host, profile):
        """
        @param profile(unicode, None): name of the profile to set the cache for
            if None, the cache will be common for all profiles
        """
        self.profile = profile
        path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR]
        if profile:
            path_elts.extend(["profiles", regex.path_escape(profile)])
        else:
            path_elts.append("common")
        self.cache_dir = Path(*path_elts)

        self.cache_dir.mkdir(0o700, parents=True, exist_ok=True)
        self.purge()

    def purge(self):
        # Remove expired, unreadable, and unrelated files from cache
        # TODO: this should not be called only on startup, but at regular interval
        #   (e.g. once a day)
        to_delete = set()
        seen = set()
        now = time.time()
        for cache_data_file in self.cache_dir.glob(f"*{CACHE_METADATA_EXT}"):
            try:
                with cache_data_file.open("r") as f:
                    cache_data = CacheMetadata.model_validate_json(f.read())
            except (IOError, ValidationError):
                log.warning(
                    _("Can't read metadata file at {path}, deleting it.").format(
                        path=cache_data_file
                    )
                )
                to_delete.add(cache_data_file)
                continue
            else:
                cached_file = self.get_path(cache_data.filename)
                if not cached_file.exists():
                    log.warning(
                        f"Cache file {cache_data_file!r} references a non-existent file "
                        f"and will be deleted: {cache_data_file!r}."
                    )
                    to_delete.add(cache_data_file)
                elif cache_data.eol < now:
                    log.debug(
                        "Purging expired cache file {cache_data_file!r} (expired for "
                        "{time}s)".format(time=int(time.time() - cache_data.eol))
                    )
                    to_delete.add(cache_data_file)
                seen.add(cached_file)
            seen.add(cache_data_file)

        for file in to_delete:
            log.debug(f"Deleting cache file: {file}")
            file.unlink()

        for file in self.cache_dir.iterdir():
            if file not in seen:
                log.debug(f"Deleting irrelevant file in cache dir: {file}")
                file.unlink()

    def get_path(self, filename: str) -> Path:
        """Return cached file URL.

        @param filename: cached file name (cache data or actual file)
        @return: path to the cached file
        """
        if not filename or "/" in filename:
            log.error(
                "invalid char found in file name, hack attempt? name:{}".format(filename)
            )
            raise exceptions.DataError("Invalid char found")
        return self.cache_dir / filename

    def get_metadata(self, uid: str, update_eol: bool = True) -> dict[str, Any] | None:
        """Retrieve metadata for cached data.

        @param uid: unique identifier of cache metadata.
        @param update_eol: True if eol must extended
            if True, max_age will be added to eol (only if it is not already expired)
        @return: metadata, see [cache_data] for data details, an additional "path" key is
            the full path to cached file.
            None if file is not in cache (or cache is invalid).
        """
        uid = uid.strip()
        if not uid:
            raise exceptions.InternalError("uid must not be empty")
        cache_url = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
        if not cache_url.exists():
            return None

        try:
            with cache_url.open("r") as f:
                cache_data = CacheMetadata.model_validate_json(f.read())
        except (IOError, EOFError) as e:
            log.warning(f"Can't read cache at {cache_url}: {e}")
            return None
        except ValidationError:
            log.warning(f"Invalid cache found at {cache_url}")
            return None
        except UnicodeDecodeError as e:
            log.warning(f"Invalid encoding, this is not a cache metadata file.")
            return None

        if cache_data.eol < time.time():
            log.debug(
                "removing expired cache (expired for {}s)".format(
                    time.time() - cache_data.eol
                )
            )
            return None

        if update_eol:
            now = int(time.time())
            cache_data.last_access = now
            cache_data.eol = now + cache_data.max_age
            with cache_url.open("w") as f:
                f.write(cache_data.model_dump_json(exclude_none=True))

        # FIXME: we convert to dict to be compatible with former method (pre Pydantic).
        #   All call to get_metadata should use directly the Pydantic model in the future.
        cache_data_dict = cache_data.model_dump()
        cache_data_dict["path"] = self.get_path(cache_data.filename)
        return cache_data_dict

    def get_file_path(self, uid: str) -> Path | None:
        """Retrieve absolute path to file

        @param uid(unicode): unique identifier of file
        @return (unicode, None): absolute path to cached file
            None if file is not in cache (or cache is invalid)
        """
        metadata = self.get_metadata(uid)
        if metadata is not None:
            return metadata["path"]

    def remove_from_cache(self, uid: str, metadata=None) -> None:
        """Remove data from cache

        @param uid(unicode): unique identifier cache file
        """
        cache_data = self.get_metadata(uid, update_eol=False)
        if cache_data is None:
            log.debug(f"cache with uid {uid!r} has already expired or been removed")
            return

        try:
            filename = cache_data["filename"]
        except KeyError:
            log.warning(_("missing filename for cache {uid!r}").format(uid=uid))
        else:
            filepath = self.get_path(filename)
            try:
                filepath.unlink()
            except FileNotFoundError:
                log.warning(
                    _("missing file referenced in cache {uid!r}: {filename}").format(
                        uid=uid, filename=filename
                    )
                )

        cache_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
        cache_file.unlink()
        log.debug(f"Cache with uid {uid!r} has been removed.")

    def cache_data(
        self,
        source: str,
        uid: str,
        mime_type: str | None = None,
        max_age: int = C.DEFAULT_MAX_AGE,
        original_filename: str | None = None,
    ) -> BufferedIOBase:
        """Create cache metadata and file object to use for actual data.

        @param source: source of the cache (should be plugin's import_name)
        @param uid: an identifier of the file which must be unique
        @param mime_type: MIME type of the file to cache
            it will be used notably to guess file extension
            It may be autogenerated if filename is specified
        @param max_age: maximum age in seconds
            the cache metadata will have an "eol" (end of life)
            None to use default value
            0 to ignore cache (file will be re-downloaded on each access)
        @param original_filename: if not None, will be used to retrieve file extension and
            guess mime type, and stored in "original_filename"
        @return: file object opened in write mode
            you have to close it yourself (hint: use ``with`` statement)
        """
        if original_filename is not None and mime_type is None:
            # we have original_filename but not MIME type, we try to guess the later
            mime_type = mimetypes.guess_type(original_filename, strict=False)[0]

        if mime_type:
            ext = mimetypes.guess_extension(mime_type, strict=False)
            if ext is None:
                log.warning("can't find extension for MIME type {}".format(mime_type))
                ext = DEFAULT_EXT
            elif ext == ".jpe":
                ext = ".jpg"
        else:
            ext = DEFAULT_EXT
            mime_type = None

        filename = uid + ext
        now = int(time.time())
        metadata = CacheMetadata(
            source=source,
            uid=uid,
            mime_type=mime_type,
            max_age=max_age,
            original_filename=original_filename,
            filename=filename,
            creation=now,
            eol=now + max_age,
        )

        cache_metadata_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}")
        file_path = self.get_path(filename)

        with open(cache_metadata_file, "w") as f:
            f.write(metadata.model_dump_json(exclude_none=True))

        return open(file_path, "wb")