Mercurial > libervia-backend
changeset 4212:5f2d496c633f
core: get rid of `pickle`:
Use of `pickle` to serialise data was a technical legacy that was causing trouble to store
in database, to update (if a class was serialised, a change could break update), and to
security (pickle can lead to code execution).
This patch remove all use of Pickle in favour in JSON, notably:
- for caching data, a Pydantic model is now used instead
- for SQLAlchemy model, the LegacyPickle is replaced by JSON serialisation
- in XEP-0373 a class `PublicKeyMetadata` was serialised. New method `from_dict` and
`to_dict` method have been implemented to do serialisation.
- new methods to (de)serialise data can now be specified with Identity data types. It is
notably used to (de)serialise `path` of avatars.
A migration script has been created to convert data (for upgrade or downgrade), with
special care for XEP-0373 case. Depending of size of database, this migration script can
be long to run.
rel 443
| author | Goffi <goffi@goffi.org> |
|---|---|
| date | Fri, 23 Feb 2024 13:31:04 +0100 |
| parents | be89ab1cbca4 |
| children | 716dd791be46 |
| files | libervia/backend/memory/cache.py libervia/backend/memory/migration/versions/fe3a02cb4bec_convert_legacypickle_columns_to_json.py libervia/backend/memory/sqla.py libervia/backend/memory/sqla_mapping.py libervia/backend/plugins/plugin_comp_ap_gateway/http_server.py libervia/backend/plugins/plugin_misc_identity.py libervia/backend/plugins/plugin_xep_0048.py libervia/backend/plugins/plugin_xep_0373.py |
| diffstat | 8 files changed, 340 insertions(+), 188 deletions(-) [+] |
line wrap: on
line diff
--- a/libervia/backend/memory/cache.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/memory/cache.py Fri Feb 23 13:31:04 2024 +0100 @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - -# SAT: a jabber client -# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) +# Libervia: an XMPP client +# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -20,9 +19,10 @@ from io import BufferedIOBase import mimetypes from pathlib import Path -import pickle as pickle import time -from typing import Any, Dict, Optional +from typing import Any + +from pydantic import BaseModel, ValidationError from libervia.backend.core import exceptions from libervia.backend.core.constants import Const as C @@ -33,11 +33,24 @@ log = getLogger(__name__) +CACHE_METADATA_EXT = ".cache.json" DEFAULT_EXT = ".raw" -class Cache(object): - """generic file caching""" +class CacheMetadata(BaseModel): + source: str + uid: str + filename: str + creation: int + eol: int + max_age: int = C.DEFAULT_MAX_AGE + original_filename: str | None = None + mime_type: str | None = None + last_access: int | None = None + + +class Cache: + """Generic file caching.""" def __init__(self, host, profile): """ @@ -56,60 +69,52 @@ self.purge() def purge(self): - # remove expired files from cache + # Remove expired, unreadable, and unrelated files from cache # TODO: this should not be called only on startup, but at regular interval # (e.g. once a day) - purged = set() - # we sort files to have metadata files first - for cache_file in sorted(self.cache_dir.iterdir()): - if cache_file in purged: - continue + to_delete = set() + seen = set() + now = time.time() + for cache_data_file in self.cache_dir.glob(f"*{CACHE_METADATA_EXT}"): try: - with cache_file.open('rb') as f: - cache_data = pickle.load(f) - except IOError: + with cache_data_file.open("r") as f: + cache_data = CacheMetadata.model_validate_json(f.read()) + except (IOError, ValidationError): log.warning( - _("Can't read metadata file at {path}") - .format(path=cache_file)) - continue - except (pickle.UnpicklingError, EOFError): - log.debug(f"File at {cache_file} is not a metadata file") + _("Can't read metadata file at {path}, deleting it.").format( + path=cache_data_file + ) + ) + to_delete.add(cache_data_file) continue - try: - eol = cache_data['eol'] - filename = cache_data['filename'] - except KeyError: - log.warning( - _("Invalid cache metadata at {path}") - .format(path=cache_file)) - continue + else: + cached_file = self.get_path(cache_data.filename) + if not cached_file.exists(): + log.warning( + f"Cache file {cache_data_file!r} references a non-existent file " + f"and will be deleted: {cache_data_file!r}." + ) + to_delete.add(cache_data_file) + elif cache_data.eol < now: + log.debug( + "Purging expired cache file {cache_data_file!r} (expired for " + "{time}s)".format(time=int(time.time() - cache_data.eol)) + ) + to_delete.add(cache_data_file) + seen.add(cached_file) + seen.add(cache_data_file) - filepath = self.getPath(filename) + for file in to_delete: + log.debug(f"Deleting cache file: {file}") + file.unlink() - if not filepath.exists(): - log.warning(_( - "cache {cache_file!r} references an inexisting file: {filepath!r}" - ).format(cache_file=str(cache_file), filepath=str(filepath))) - log.debug("purging cache with missing file") - cache_file.unlink() - elif eol < time.time(): - log.debug( - "purging expired cache {filepath!r} (expired for {time}s)" - .format(filepath=str(filepath), time=int(time.time() - eol)) - ) - cache_file.unlink() - try: - filepath.unlink() - except FileNotFoundError: - log.warning( - _("following file is missing while purging cache: {path}") - .format(path=filepath) - ) - purged.add(cache_file) - purged.add(filepath) + for file in self.cache_dir.iterdir(): + if file not in seen: + log.debug(f"Deleting irrelevant file in cache dir: {file}") + file.unlink() - def getPath(self, filename: str) -> Path: - """return cached file URL + def get_path(self, filename: str) -> Path: + """Return cached file URL. @param filename: cached file name (cache data or actual file) @return: path to the cached file @@ -121,62 +126,58 @@ raise exceptions.DataError("Invalid char found") return self.cache_dir / filename - def get_metadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]: - """Retrieve metadata for cached data + def get_metadata(self, uid: str, update_eol: bool = True) -> dict[str, Any] | None: + """Retrieve metadata for cached data. - @param uid(unicode): unique identifier of file - @param update_eol(bool): True if eol must extended + @param uid: unique identifier of cache metadata. + @param update_eol: True if eol must extended if True, max_age will be added to eol (only if it is not already expired) - @return (dict, None): metadata with following keys: - see [cache_data] for data details, an additional "path" key is the full path to - cached file. - None if file is not in cache (or cache is invalid) + @return: metadata, see [cache_data] for data details, an additional "path" key is + the full path to cached file. + None if file is not in cache (or cache is invalid). """ - uid = uid.strip() if not uid: raise exceptions.InternalError("uid must not be empty") - cache_url = self.getPath(uid) + cache_url = self.get_path(f"{uid}{CACHE_METADATA_EXT}") if not cache_url.exists(): return None try: - with cache_url.open("rb") as f: - cache_data = pickle.load(f) + with cache_url.open("r") as f: + cache_data = CacheMetadata.model_validate_json(f.read()) except (IOError, EOFError) as e: - log.warning(f"can't read cache at {cache_url}: {e}") + log.warning(f"Can't read cache at {cache_url}: {e}") return None - except pickle.UnpicklingError: - log.warning(f"invalid cache found at {cache_url}") + except ValidationError: + log.warning(f"Invalid cache found at {cache_url}") + return None + except UnicodeDecodeError as e: + log.warning(f"Invalid encoding, this is not a cache metadata file.") return None - try: - eol = cache_data["eol"] - except KeyError: - log.warning("no End Of Life found for cached file {}".format(uid)) - eol = 0 - if eol < time.time(): + if cache_data.eol < time.time(): log.debug( - "removing expired cache (expired for {}s)".format(time.time() - eol) + "removing expired cache (expired for {}s)".format( + time.time() - cache_data.eol + ) ) return None if update_eol: - try: - max_age = cache_data["max_age"] - except KeyError: - log.warning(f"no max_age found for cache at {cache_url}, using default") - max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE now = int(time.time()) - cache_data["last_access"] = now - cache_data["eol"] = now + max_age - with cache_url.open("wb") as f: - pickle.dump(cache_data, f, protocol=2) + cache_data.last_access = now + cache_data.eol = now + cache_data.max_age + with cache_url.open("w") as f: + f.write(cache_data.model_dump_json(exclude_none=True)) - cache_data["path"] = self.getPath(cache_data["filename"]) - return cache_data + # FIXME: we convert to dict to be compatible with former method (pre Pydantic). + # All call to get_metadata should use directly the Pydantic model in the future. + cache_data_dict = cache_data.model_dump() + cache_data_dict["path"] = self.get_path(cache_data.filename) + return cache_data_dict - def get_file_path(self, uid: str) -> Path: + def get_file_path(self, uid: str) -> Path | None: """Retrieve absolute path to file @param uid(unicode): unique identifier of file @@ -187,7 +188,7 @@ if metadata is not None: return metadata["path"] - def remove_from_cache(self, uid, metadata=None): + def remove_from_cache(self, uid: str, metadata=None) -> None: """Remove data from cache @param uid(unicode): unique identifier cache file @@ -198,32 +199,33 @@ return try: - filename = cache_data['filename'] + filename = cache_data["filename"] except KeyError: - log.warning(_("missing filename for cache {uid!r}") .format(uid=uid)) + log.warning(_("missing filename for cache {uid!r}").format(uid=uid)) else: - filepath = self.getPath(filename) + filepath = self.get_path(filename) try: filepath.unlink() except FileNotFoundError: log.warning( - _("missing file referenced in cache {uid!r}: {filename}") - .format(uid=uid, filename=filename) + _("missing file referenced in cache {uid!r}: {filename}").format( + uid=uid, filename=filename + ) ) - cache_file = self.getPath(uid) + cache_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") cache_file.unlink() - log.debug(f"cache with uid {uid!r} has been removed") + log.debug(f"Cache with uid {uid!r} has been removed.") def cache_data( self, source: str, uid: str, - mime_type: Optional[str] = None, - max_age: Optional[int] = None, - original_filename: Optional[str] = None + mime_type: str | None = None, + max_age: int = C.DEFAULT_MAX_AGE, + original_filename: str | None = None, ) -> BufferedIOBase: - """create cache metadata and file object to use for actual data + """Create cache metadata and file object to use for actual data. @param source: source of the cache (should be plugin's import_name) @param uid: an identifier of the file which must be unique @@ -235,47 +237,42 @@ None to use default value 0 to ignore cache (file will be re-downloaded on each access) @param original_filename: if not None, will be used to retrieve file extension and - guess - mime type, and stored in "original_filename" + guess mime type, and stored in "original_filename" @return: file object opened in write mode you have to close it yourself (hint: use ``with`` statement) """ - if max_age is None: - max_age = C.DEFAULT_MAX_AGE - cache_data = { - "source": source, - # we also store max_age for updating eol - "max_age": max_age, - } - cache_url = self.getPath(uid) - if original_filename is not None: - cache_data["original_filename"] = original_filename - if mime_type is None: - # we have original_filename but not MIME type, we try to guess the later - mime_type = mimetypes.guess_type(original_filename, strict=False)[0] + if original_filename is not None and mime_type is None: + # we have original_filename but not MIME type, we try to guess the later + mime_type = mimetypes.guess_type(original_filename, strict=False)[0] + if mime_type: ext = mimetypes.guess_extension(mime_type, strict=False) if ext is None: - log.warning( - "can't find extension for MIME type {}".format(mime_type) - ) + log.warning("can't find extension for MIME type {}".format(mime_type)) ext = DEFAULT_EXT elif ext == ".jpe": ext = ".jpg" else: ext = DEFAULT_EXT mime_type = None + filename = uid + ext now = int(time.time()) - cache_data.update({ - "filename": filename, - "creation": now, - "eol": now + max_age, - "mime_type": mime_type, - }) - file_path = self.getPath(filename) + metadata = CacheMetadata( + source=source, + uid=uid, + mime_type=mime_type, + max_age=max_age, + original_filename=original_filename, + filename=filename, + creation=now, + eol=now + max_age, + ) - with open(cache_url, "wb") as f: - pickle.dump(cache_data, f, protocol=2) + cache_metadata_file = self.get_path(f"{uid}{CACHE_METADATA_EXT}") + file_path = self.get_path(filename) - return file_path.open("wb") + with open(cache_metadata_file, "w") as f: + f.write(metadata.model_dump_json(exclude_none=True)) + + return open(file_path, "wb")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/libervia/backend/memory/migration/versions/fe3a02cb4bec_convert_legacypickle_columns_to_json.py Fri Feb 23 13:31:04 2024 +0100 @@ -0,0 +1,142 @@ +"""convert LegacyPickle columns to JSON + +Revision ID: fe3a02cb4bec +Revises: 610345f77e75 +Create Date: 2024-02-22 14:55:59.993983 + +""" +from alembic import op +import sqlalchemy as sa +import pickle +import json +from libervia.backend.plugins.plugin_xep_0373 import PublicKeyMetadata + +# revision identifiers, used by Alembic. +revision = "fe3a02cb4bec" +down_revision = "610345f77e75" +branch_labels = None +depends_on = None + + +def convert_pickle_to_json(value, table, primary_keys): + """Convert pickled data to JSON, handling potential errors.""" + if value is None: + return None + try: + # some values are converted to bytes with LegacyPickle + if isinstance(value, str): + value = value.encode() + try: + deserialized = pickle.loads(value, encoding="utf-8") + except ModuleNotFoundError: + deserialized = pickle.loads( + value.replace(b"sat.plugins", b"libervia.backend.plugins"), + encoding="utf-8", + ) + if ( + table == "private_ind_bin" + and primary_keys[0] == "XEP-0373" + and not primary_keys[1].startswith("/trust") + and isinstance(deserialized, set) + and deserialized + and isinstance(next(iter(deserialized)), PublicKeyMetadata) + ): + # XEP-0373 plugin was pickling an internal class, this can't be converted + # directly to JSON, so we do a special treatment with the add `to_dict` and + # `from_dict` methods. + deserialized = [pkm.to_dict() for pkm in deserialized] + + ret = json.dumps(deserialized, ensure_ascii=False, default=str) + if table == 'history' and ret == "{}": + # For history, we can remove empty data, but for other tables it may be + # significant. + ret = None + return ret + except Exception as e: + print( + f"Warning: Failed to convert pickle to JSON, using NULL instead. Error: {e}" + ) + return None + + +def upgrade(): + print( + "This migration may take very long, please be patient and don't stop the process." + ) + connection = op.get_bind() + + tables_and_columns = [ + ("history", "extra", "uid"), + ("private_gen_bin", "value", "namespace", "key"), + ("private_ind_bin", "value", "namespace", "key", "profile_id"), + ] + + for table, column, *primary_keys in tables_and_columns: + primary_key_clause = " AND ".join(f"{pk} = :{pk}" for pk in primary_keys) + select_stmt = sa.text(f"SELECT {', '.join(primary_keys)}, {column} FROM {table}") + update_stmt = sa.text( + f"UPDATE {table} SET {column} = :{column} WHERE {primary_key_clause}" + ) + + result = connection.execute(select_stmt) + for row in result: + value = row[-1] + if value is None: + continue + data = {pk: row[idx] for idx, pk in enumerate(primary_keys)} + data[column] = convert_pickle_to_json(value, table, row[:-1]) + connection.execute(update_stmt.bindparams(**data)) + + +def convert_json_to_pickle(value, table, primary_keys): + """Convert JSON data back to pickled data, handling potential errors.""" + if value is None: + return None + try: + deserialized = json.loads(value) + # Check for the specific table and primary key conditions that require special + # handling + if ( + table == "private_ind_bin" + and primary_keys[0] == "XEP-0373" + and not primary_keys[1].startswith("/trust") + ): + # Convert list of dicts back to set of PublicKeyMetadata objects + if isinstance(deserialized, list): + deserialized = {PublicKeyMetadata.from_dict(d) for d in deserialized} + return pickle.dumps(deserialized, 0) + except Exception as e: + print( + f"Warning: Failed to convert JSON to pickle, using NULL instead. Error: {e}" + ) + return None + + +def downgrade(): + print( + "Reverting JSON columns to LegacyPickle format. This may take a while, please be " + "patient." + ) + connection = op.get_bind() + + tables_and_columns = [ + ("history", "extra", "uid"), + ("private_gen_bin", "value", "namespace", "key"), + ("private_ind_bin", "value", "namespace", "key", "profile_id"), + ] + + for table, column, *primary_keys in tables_and_columns: + primary_key_clause = " AND ".join(f"{pk} = :{pk}" for pk in primary_keys) + select_stmt = sa.text(f"SELECT {', '.join(primary_keys)}, {column} FROM {table}") + update_stmt = sa.text( + f"UPDATE {table} SET {column} = :{column} WHERE {primary_key_clause}" + ) + + result = connection.execute(select_stmt) + for row in result: + value = row[-1] + if value is None: + continue + data = {pk: row[idx] for idx, pk in enumerate(primary_keys)} + data[column] = convert_json_to_pickle(value, table, row[:-1]) + connection.execute(update_stmt.bindparams(**data))
--- a/libervia/backend/memory/sqla.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/memory/sqla.py Fri Feb 23 13:31:04 2024 +0100 @@ -20,6 +20,7 @@ from asyncio.subprocess import PIPE import copy from datetime import datetime +import json from pathlib import Path import sys import time @@ -214,6 +215,7 @@ engine = create_async_engine( db_config["url"], future=True, + json_serializer=lambda obj: json.dumps(obj, ensure_ascii=False) ) new_base = not db_config["path"].exists()
--- a/libervia/backend/memory/sqla_mapping.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/memory/sqla_mapping.py Fri Feb 23 13:31:04 2024 +0100 @@ -19,7 +19,6 @@ from datetime import datetime import enum import json -import pickle import time from typing import Any, Dict @@ -132,42 +131,6 @@ URGENT = 40 -class LegacyPickle(TypeDecorator): - """Handle troubles with data pickled by former version of SàT - - This type is temporary until we do migration to a proper data type - """ - - # Blob is used on SQLite but gives errors when used here, while Text works fine - impl = Text - cache_ok = True - - def process_bind_param(self, value, dialect): - if value is None: - return None - return pickle.dumps(value, 0) - - def process_result_value(self, value, dialect): - if value is None: - return None - # value types are inconsistent (probably a consequence of Python 2/3 port - # and/or SQLite dynamic typing) - try: - value = value.encode() - except AttributeError: - pass - # "utf-8" encoding is needed to handle Python 2 pickled data - try: - return pickle.loads(value, encoding="utf-8") - except ModuleNotFoundError: - # FIXME: workaround due to package renaming, need to move all pickle code to - # JSON - return pickle.loads( - value.replace(b"sat.plugins", b"libervia.backend.plugins"), - encoding="utf-8", - ) - - class Json(TypeDecorator): """Handle JSON field in DB independant way""" @@ -178,7 +141,7 @@ def process_bind_param(self, value, dialect): if value is None: return None - return json.dumps(value) + return json.dumps(value, ensure_ascii=False) def process_result_value(self, value, dialect): if value is None: @@ -296,7 +259,7 @@ ), nullable=False, ) - extra = Column(LegacyPickle) + extra = Column(JSON) profile = relationship("Profile") messages = relationship( @@ -573,7 +536,7 @@ namespace = Column(Text, primary_key=True) key = Column(Text, primary_key=True) - value = Column(LegacyPickle) + value = Column(JSON) class PrivateIndBin(Base): @@ -582,7 +545,7 @@ namespace = Column(Text, primary_key=True) key = Column(Text, primary_key=True) profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True) - value = Column(LegacyPickle) + value = Column(JSON) profile = relationship("Profile", back_populates="private_bin_data")
--- a/libervia/backend/plugins/plugin_comp_ap_gateway/http_server.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/plugins/plugin_comp_ap_gateway/http_server.py Fri Feb 23 13:31:04 2024 +0100 @@ -1017,7 +1017,7 @@ if len(extra_args) != 1: raise exceptions.DataError("avatar argument expected in URL") avatar_filename = extra_args[0] - avatar_path = self.apg.host.common_cache.getPath(avatar_filename) + avatar_path = self.apg.host.common_cache.get_path(avatar_filename) return static.File(str(avatar_path)).render(request) elif request_type == "item": ret_data = await self.apg.ap_get_local_object(ap_url)
--- a/libervia/backend/plugins/plugin_misc_identity.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/plugins/plugin_misc_identity.py Fri Feb 23 13:31:04 2024 +0100 @@ -85,6 +85,8 @@ # we store the metadata in database, to restore it on next connection # (it is stored only for roster entities) "store": True, + "store_serialisation": self._avatar_ser, + "store_deserialisation": self._avatar_deser }, "nicknames": { "type": list, @@ -167,10 +169,16 @@ for key, value in stored_data.items(): entity_s, name = key.split('\n') - if name not in self.metadata.keys(): + try: + metadata = self.metadata[name] + except KeyError: log.debug(f"removing {key} from storage: not an allowed metadata name") to_delete.append(key) continue + if value is not None: + deser_method = metadata.get("store_deserialisation") + if deser_method is not None: + value = deser_method(value) entity = jid.JID(entity_s) if name == 'avatar': @@ -365,6 +373,10 @@ client, entity, metadata_name, data) if metadata.get('store', False): + if data is not None: + ser_method = metadata.get("store_serialisation") + if ser_method is not None: + data = ser_method(data) key = f"{entity}\n{metadata_name}" await client._identity_storage.aset(key, data) @@ -488,6 +500,10 @@ if metadata.get('store', False): key = f"{entity}\n{metadata_name}" + if data is not None: + ser_method = metadata.get("store_serialisation") + if ser_method is not None: + data = ser_method(data) await client._identity_storage.aset(key, data) def default_update_is_new_data(self, client, entity, cached_data, new_data): @@ -633,6 +649,18 @@ raise ValueError(f"missing avatar data keys: {mandatory_keys - data.keys()}") return data + def _avatar_ser(self, data: dict) -> dict: + if data.get("path"): + # Path instance can't be stored + data = data.copy() + data["path"] = str(data["path"]) + return data + + def _avatar_deser(self, data: dict) -> dict: + if data.get("path"): + data["path"] = Path(data["path"]) + return data + async def nicknames_get_post_treatment(self, client, entity, plugin_nicknames): """Prepend nicknames from core locations + set default nickname
--- a/libervia/backend/plugins/plugin_xep_0048.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/plugins/plugin_xep_0048.py Fri Feb 23 13:31:04 2024 +0100 @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from typing import cast from libervia.backend.core.i18n import _, D_ from libervia.backend.core import exceptions from libervia.backend.core.constants import Const as C @@ -106,9 +107,12 @@ NS_BOOKMARKS, client.profile ) await local.load() + local = cast(dict[str, dict|None]|None, local) if not local: - local[XEP_0048.MUC_TYPE] = dict() - local[XEP_0048.URL_TYPE] = dict() + local = { + XEP_0048.MUC_TYPE: {}, + XEP_0048.URL_TYPE: {} + } private = await self._get_server_bookmarks("private", client.profile) pubsub = client.bookmarks_pubsub = None
--- a/libervia/backend/plugins/plugin_xep_0373.py Fri Feb 16 18:46:06 2024 +0100 +++ b/libervia/backend/plugins/plugin_xep_0373.py Fri Feb 23 13:31:04 2024 +0100 @@ -20,12 +20,13 @@ import base64 from datetime import datetime, timezone import enum +import json import secrets import string from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, cast from xml.sax.saxutils import quoteattr -from typing_extensions import Final, NamedTuple, Never, assert_never +from typing import Final, NamedTuple, Never, assert_never from wokkel import muc, pubsub from wokkel.disco import DiscoFeature, DiscoInfo import xmlschema @@ -812,10 +813,21 @@ """ Metadata about a published public key. """ - fingerprint: str timestamp: datetime + def to_dict(self) -> dict: + # Convert the instance to a dictionary and handle datetime serialization + data = self._asdict() + data['timestamp'] = self.timestamp.isoformat() + return data + + @staticmethod + def from_dict(data: dict) -> 'PublicKeyMetadata': + # Load a serialised dictionary + data['timestamp'] = datetime.fromisoformat(data['timestamp']) + return PublicKeyMetadata(**data) + @enum.unique class TrustLevel(enum.Enum): @@ -1102,10 +1114,10 @@ storage_key = STR_KEY_PUBLIC_KEYS_METADATA.format(sender.userhost()) - local_public_keys_metadata = cast( - Set[PublicKeyMetadata], - await self.__storage[profile].get(storage_key, set()) - ) + local_public_keys_metadata = { + PublicKeyMetadata.from_dict(pkm) + for pkm in await self.__storage[profile].get(storage_key, []) + } unchanged_keys = new_public_keys_metadata & local_public_keys_metadata changed_or_new_keys = new_public_keys_metadata - unchanged_keys @@ -1149,7 +1161,10 @@ await self.publish_public_keys_list(client, new_public_keys_metadata) - await self.__storage[profile].force(storage_key, new_public_keys_metadata) + await self.__storage[profile].force( + storage_key, + [pkm.to_dict() for pkm in new_public_keys_metadata] + ) def list_public_keys(self, client: SatXMPPClient, jid: jid.JID) -> Set[GPGPublicKey]: """List GPG public keys available for a JID. @@ -1191,10 +1206,10 @@ storage_key = STR_KEY_PUBLIC_KEYS_METADATA.format(client.jid.userhost()) - public_keys_list = cast( - Set[PublicKeyMetadata], - await self.__storage[client.profile].get(storage_key, set()) - ) + public_keys_list = { + PublicKeyMetadata.from_dict(pkm) + for pkm in await self.__storage[client.profile].get(storage_key, []) + } public_keys_list.add(PublicKeyMetadata( fingerprint=secret_key.public_key.fingerprint, @@ -1508,10 +1523,10 @@ storage_key = STR_KEY_PUBLIC_KEYS_METADATA.format(entity_jid.userhost()) - public_keys_metadata = cast( - Set[PublicKeyMetadata], - await self.__storage[client.profile].get(storage_key, set()) - ) + public_keys_metadata = { + PublicKeyMetadata.from_dict(pkm) + for pkm in await self.__storage[client.profile].get(storage_key, []) + } if not public_keys_metadata: public_keys_metadata = await self.download_public_keys_list( client, entity_jid @@ -1522,7 +1537,8 @@ ) else: await self.__storage[client.profile].aset( - storage_key, public_keys_metadata + storage_key, + [pkm.to_dict() for pkm in public_keys_metadata] )