Mercurial > libervia-backend
annotate src/plugins/plugin_exp_lang_detect.py @ 2088:c02f96756d5c
core: bridge can now be changed in conf
by default DBus bridge is used, but this can be changed in sat.conf's DEFAULT section using "bridge" option.
bridge can be "dbus" (or equivalent "dbus_bridge") or embedded at the moment.
Bridge name used is saved in SAT.bridge_name, and load_bridge method can be used by frontends.
author | Goffi <goffi@goffi.org> |
---|---|
date | Sun, 04 Dec 2016 18:16:37 +0100 |
parents | d95a6d553bec |
children | 1d3f73e065e1 |
rev | line source |
---|---|
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
1 #!/usr/bin/env python2 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SAT plugin to detect language (experimental) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
5 # Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
2011
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
20 from sat.core.i18n import _, D_ |
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 from sat.core.log import getLogger |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 log = getLogger(__name__) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from sat.core import exceptions |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
24 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
25 try: |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 from langid.langid import LanguageIdentifier, model |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 except ImportError: |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 raise exceptions.MissingModule(u'Missing module langid, please download/install it with "pip install langid")') |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
30 identifier = LanguageIdentifier.from_modelstring(model, norm_probs=False) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 PLUGIN_INFO = { |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 "name": "Language detection plugin", |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 "import_name": "EXP-LANG-DETECT", |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 "type": "EXP", |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 "protocols": [], |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 "dependencies": [], |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 "main": "LangDetect", |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 "handler": "no", |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 "description": _("""Detect and set message language when unknown""") |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 } |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 |
2011
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
44 CATEGORY = D_(u"Misc") |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
45 NAME = u"lang_detect" |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
46 LABEL = D_(u"language detection") |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
47 PARAMS = """ |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
48 <params> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
49 <individual> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
50 <category name="{category_name}"> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
51 <param name="{name}" label="{label}" type="bool" value="true" /> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
52 </category> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
53 </individual> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
54 </params> |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
55 """.format(category_name=CATEGORY, |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
56 name=NAME, |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
57 label=_(LABEL), |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
58 ) |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
59 |
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
60 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
61 class LangDetect(object): |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
62 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
63 def __init__(self, host): |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
64 log.info(_(u"Language detection plugin initialization")) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
65 self.host = host |
2011
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
66 host.memory.updateParams(PARAMS) |
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 host.trigger.add("MessageReceived", self.MessageReceivedTrigger) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 host.trigger.add("messageSend", self.MessageSendTrigger) |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 def addLanguage(self, mess_data): |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 message = mess_data['message'] |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 if len(message) == 1 and message.keys()[0] == '': |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
73 msg = message.values()[0] |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
74 lang = identifier.classify(msg)[0] |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
75 mess_data["message"] = {lang: msg} |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 return mess_data |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
77 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
78 def MessageReceivedTrigger(self, client, message_elt, post_treat): |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
79 """ Check if source is linked and repeat message, else do nothing """ |
2011
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
80 |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
81 lang_detect = self.host.memory.getParamA(NAME, CATEGORY, profile_key=client.profile) |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
82 if lang_detect: |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
83 post_treat.addCallback(self.addLanguage) |
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 return True |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
85 |
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
86 def MessageSendTrigger(self, client, data, pre_xml_treatments, post_xml_treatments): |
2011
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
87 lang_detect = self.host.memory.getParamA(NAME, CATEGORY, profile_key=client.profile) |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
88 if lang_detect: |
d95a6d553bec
plugin lang detect: added a parameter to (de)activate the detection
Goffi <goffi@goffi.org>
parents:
1965
diff
changeset
|
89 self.addLanguage(data) |
1965
4c5d8cd35690
plugin exp_lang_detect: language detection plugin, first draft
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
90 return True |