Mercurial > libervia-backend
view libervia/backend/plugins/plugin_exp_lang_detect.py @ 4212:5f2d496c633f
core: get rid of `pickle`:
Use of `pickle` to serialise data was a technical legacy that was causing trouble to store
in database, to update (if a class was serialised, a change could break update), and to
security (pickle can lead to code execution).
This patch remove all use of Pickle in favour in JSON, notably:
- for caching data, a Pydantic model is now used instead
- for SQLAlchemy model, the LegacyPickle is replaced by JSON serialisation
- in XEP-0373 a class `PublicKeyMetadata` was serialised. New method `from_dict` and
`to_dict` method have been implemented to do serialisation.
- new methods to (de)serialise data can now be specified with Identity data types. It is
notably used to (de)serialise `path` of avatars.
A migration script has been created to convert data (for upgrade or downgrade), with
special care for XEP-0373 case. Depending of size of database, this migration script can
be long to run.
rel 443
author | Goffi <goffi@goffi.org> |
---|---|
date | Fri, 23 Feb 2024 13:31:04 +0100 |
parents | 4b842c1fb686 |
children | 0d7bb4df2343 |
line wrap: on
line source
#!/usr/bin/env python3 # SAT plugin to detect language (experimental) # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from libervia.backend.core.i18n import _, D_ from libervia.backend.core.constants import Const as C from libervia.backend.core.log import getLogger log = getLogger(__name__) from libervia.backend.core import exceptions try: from langid.langid import LanguageIdentifier, model except ImportError: raise exceptions.MissingModule( 'Missing module langid, please download/install it with "pip install langid")' ) identifier = LanguageIdentifier.from_modelstring(model, norm_probs=False) PLUGIN_INFO = { C.PI_NAME: "Language detection plugin", C.PI_IMPORT_NAME: "EXP-LANG-DETECT", C.PI_TYPE: "EXP", C.PI_PROTOCOLS: [], C.PI_DEPENDENCIES: [], C.PI_MAIN: "LangDetect", C.PI_HANDLER: "no", C.PI_DESCRIPTION: _("""Detect and set message language when unknown"""), } CATEGORY = D_("Misc") NAME = "lang_detect" LABEL = D_("language detection") PARAMS = """ <params> <individual> <category name="{category_name}"> <param name="{name}" label="{label}" type="bool" value="true" /> </category> </individual> </params> """.format( category_name=CATEGORY, name=NAME, label=_(LABEL) ) class LangDetect(object): def __init__(self, host): log.info(_("Language detection plugin initialization")) self.host = host host.memory.update_params(PARAMS) host.trigger.add("message_received", self.message_received_trigger) host.trigger.add("sendMessage", self.message_send_trigger) def add_language(self, mess_data): message = mess_data["message"] if len(message) == 1 and list(message.keys())[0] == "": msg = list(message.values())[0].strip() if msg: lang = identifier.classify(msg)[0] mess_data["message"] = {lang: msg} return mess_data def message_received_trigger(self, client, message_elt, post_treat): """ Check if source is linked and repeat message, else do nothing """ lang_detect = self.host.memory.param_get_a( NAME, CATEGORY, profile_key=client.profile ) if lang_detect: post_treat.addCallback(self.add_language) return True def message_send_trigger(self, client, data, pre_xml_treatments, post_xml_treatments): lang_detect = self.host.memory.param_get_a( NAME, CATEGORY, profile_key=client.profile ) if lang_detect: self.add_language(data) return True