Mercurial > libervia-backend

diff libervia/backend/memory/sqla_mapping.py @ 4212:5f2d496c633f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
core: get rid of `pickle`: Use of `pickle` to serialise data was a technical legacy that was causing trouble to store in database, to update (if a class was serialised, a change could break update), and to security (pickle can lead to code execution). This patch remove all use of Pickle in favour in JSON, notably: - for caching data, a Pydantic model is now used instead - for SQLAlchemy model, the LegacyPickle is replaced by JSON serialisation - in XEP-0373 a class `PublicKeyMetadata` was serialised. New method `from_dict` and `to_dict` method have been implemented to do serialisation. - new methods to (de)serialise data can now be specified with Identity data types. It is notably used to (de)serialise `path` of avatars. A migration script has been created to convert data (for upgrade or downgrade), with special care for XEP-0373 case. Depending of size of database, this migration script can be long to run. rel 443
author Goffi <goffi@goffi.org>
date Fri, 23 Feb 2024 13:31:04 +0100
parents 2074b2bbe616
children
line wrap: on
line diff
--- a/libervia/backend/memory/sqla_mapping.py	Fri Feb 16 18:46:06 2024 +0100
+++ b/libervia/backend/memory/sqla_mapping.py	Fri Feb 23 13:31:04 2024 +0100
@@ -19,7 +19,6 @@
 from datetime import datetime
 import enum
 import json
-import pickle
 import time
 from typing import Any, Dict
 
@@ -132,42 +131,6 @@
     URGENT = 40
 
 
-class LegacyPickle(TypeDecorator):
-    """Handle troubles with data pickled by former version of SàT
-
-    This type is temporary until we do migration to a proper data type
-    """
-
-    # Blob is used on SQLite but gives errors when used here, while Text works fine
-    impl = Text
-    cache_ok = True
-
-    def process_bind_param(self, value, dialect):
-        if value is None:
-            return None
-        return pickle.dumps(value, 0)
-
-    def process_result_value(self, value, dialect):
-        if value is None:
-            return None
-        # value types are inconsistent (probably a consequence of Python 2/3 port
-        # and/or SQLite dynamic typing)
-        try:
-            value = value.encode()
-        except AttributeError:
-            pass
-        # "utf-8" encoding is needed to handle Python 2 pickled data
-        try:
-            return pickle.loads(value, encoding="utf-8")
-        except ModuleNotFoundError:
-            # FIXME: workaround due to package renaming, need to move all pickle code to
-            #   JSON
-            return pickle.loads(
-                value.replace(b"sat.plugins", b"libervia.backend.plugins"),
-                encoding="utf-8",
-            )
-
-
 class Json(TypeDecorator):
     """Handle JSON field in DB independant way"""
 
@@ -178,7 +141,7 @@
     def process_bind_param(self, value, dialect):
         if value is None:
             return None
-        return json.dumps(value)
+        return json.dumps(value, ensure_ascii=False)
 
     def process_result_value(self, value, dialect):
         if value is None:
@@ -296,7 +259,7 @@
         ),
         nullable=False,
     )
-    extra = Column(LegacyPickle)
+    extra = Column(JSON)
 
     profile = relationship("Profile")
     messages = relationship(
@@ -573,7 +536,7 @@
 
     namespace = Column(Text, primary_key=True)
     key = Column(Text, primary_key=True)
-    value = Column(LegacyPickle)
+    value = Column(JSON)
 
 
 class PrivateIndBin(Base):
@@ -582,7 +545,7 @@
     namespace = Column(Text, primary_key=True)
     key = Column(Text, primary_key=True)
     profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True)
-    value = Column(LegacyPickle)
+    value = Column(JSON)
 
     profile = relationship("Profile", back_populates="private_bin_data")